Index: lib/Makefile =================================================================== --- lib/Makefile +++ lib/Makefile @@ -175,7 +175,12 @@ _libvgl= libvgl .endif +.if ${MACHINE_CPUARCH} == "aarch64" +SUBDIR.${MK_PMC}+= libopencsd +.endif + .if ${MACHINE_CPUARCH} == "amd64" +SUBDIR.${MK_PMC}+= libipt SUBDIR.${MK_BHYVE}+= libvmmapi .endif Index: lib/libipt/Makefile =================================================================== --- /dev/null +++ lib/libipt/Makefile @@ -0,0 +1,77 @@ +# $FreeBSD$ + +PACKAGE=lib${LIB} +SHLIBDIR?= /lib + +.include + +LIBIPTSRC= ${SRCTOP}/contrib/libipt + +.PATH: ${LIBIPTSRC}/libipt/src \ + ${LIBIPTSRC}/libipt/src/posix \ + ${LIBIPTSRC}/libipt/internal/include \ + ${LIBIPTSRC}/libipt/include \ + ${LIBIPTSRC}/include + +LIB= ipt +SHLIB_MAJOR=0 + +SRCS= \ + init.c \ + pt_asid.c \ + pt_block_cache.c \ + pt_block_decoder.c \ + pt_config.c \ + pt_cpu.c \ + pt_cpuid.c \ + pt_decoder_function.c \ + pt_encoder.c \ + pt_error.c \ + pt_event_queue.c \ + pt_ild.c \ + pt_image_section_cache.c \ + pt_image.c \ + pt_insn_decoder.c \ + pt_insn.c \ + pt_last_ip.c \ + pt_packet_decoder.c \ + pt_packet.c \ + pt_query_decoder.c \ + pt_retstack.c \ + pt_section_file.c \ + pt_section_posix.c \ + pt_section.c \ + pt_sync.c \ + pt_time.c \ + pt_tnt_cache.c \ + pt_version.c + +CFLAGS+= \ + -I${LIBIPTSRC}/libipt/internal/include/posix \ + -I${LIBIPTSRC}/libipt/internal/include \ + -I${LIBIPTSRC}/libipt/include \ + -I${LIBIPTSRC}/include \ + -I${.CURDIR} + +CFLAGS+= \ + -DPT_VERSION_BUILD=0 \ + -DPT_VERSION_EXT=\"\" \ + -DPT_VERSION_MAJOR=1 \ + -DPT_VERSION_MINOR=6 + +INCS= \ + intel-pt.h \ + pt_cpu.h \ + pt_last_ip.h \ + pt_time.h \ + pt_compiler.h + +INCSDIR=${INCLUDEDIR}/libipt + +LIBADD= + +WARNS?= 1 + +HAS_TESTS= + +.include Index: lib/libopencsd/Makefile =================================================================== --- /dev/null +++ lib/libopencsd/Makefile @@ -0,0 +1,180 @@ +# $FreeBSD$ + +PACKAGE=lib${LIB} +SHLIBDIR?= /lib + +.include + +OPENCSDSRC= ${SRCTOP}/contrib/opencsd + +.PATH: ${OPENCSDSRC}/decoder/source/etmv4/ \ + ${OPENCSDSRC}/decoder/source/etmv3/ \ + ${OPENCSDSRC}/decoder/source/pkt_printers/ \ + ${OPENCSDSRC}/decoder/source/mem_acc/ \ + ${OPENCSDSRC}/decoder/source/i_dec/ \ + ${OPENCSDSRC}/decoder/source/c_api/ \ + ${OPENCSDSRC}/decoder/source/ptm/ \ + ${OPENCSDSRC}/decoder/source/stm/ \ + ${OPENCSDSRC}/decoder/source/ \ + ${OPENCSDSRC}/decoder/include/opencsd/etmv4/ \ + ${OPENCSDSRC}/decoder/include/opencsd/etmv3/ \ + ${OPENCSDSRC}/decoder/include/opencsd/stm/ \ + ${OPENCSDSRC}/decoder/include/opencsd/ptm/ \ + ${OPENCSDSRC}/decoder/include/opencsd/c_api/ \ + ${OPENCSDSRC}/decoder/include/opencsd/ \ + ${OPENCSDSRC}/decoder/include + +LIB= opencsd +SHLIB_MAJOR=0 + +# Uncomment for debugging +#CFLAGS += -g -O0 -DDEBUG +#CPPFLAGS += -g -O0 -DDEBUG +#CXXFLAGS += -g -O0 -DDEBUG + +# ETMv3 +SRCS= \ + trc_cmp_cfg_etmv3.cpp \ + trc_pkt_decode_etmv3.cpp \ + trc_pkt_elem_etmv3.cpp \ + trc_pkt_proc_etmv3.cpp \ + trc_pkt_proc_etmv3_impl.cpp + +# ETMv4 +SRCS+= \ + trc_cmp_cfg_etmv4.cpp \ + trc_etmv4_stack_elem.cpp \ + trc_pkt_decode_etmv4i.cpp \ + trc_pkt_elem_etmv4d.cpp \ + trc_pkt_elem_etmv4i.cpp \ + trc_pkt_proc_etmv4.cpp \ + trc_pkt_proc_etmv4i_impl.cpp + +# PKT_PRINTERS +SRCS+= \ + raw_frame_printer.cpp \ + trc_print_fact.cpp + +# PTM +SRCS+= \ + trc_cmp_cfg_ptm.cpp \ + trc_pkt_decode_ptm.cpp \ + trc_pkt_elem_ptm.cpp \ + trc_pkt_proc_ptm.cpp + +# STM +SRCS+= \ + trc_pkt_decode_stm.cpp \ + trc_pkt_elem_stm.cpp \ + trc_pkt_proc_stm.cpp + +# C_API +SRCS+= \ + ocsd_c_api_custom_obj.cpp \ + ocsd_c_api.cpp + +# SRC +SRCS+= \ + ocsd_code_follower.cpp \ + ocsd_dcd_tree.cpp \ + ocsd_error.cpp \ + ocsd_error_logger.cpp \ + ocsd_gen_elem_list.cpp \ + ocsd_lib_dcd_register.cpp \ + ocsd_msg_logger.cpp \ + ocsd_version.cpp \ + trc_component.cpp \ + trc_core_arch_map.cpp \ + trc_frame_deformatter.cpp \ + trc_gen_elem.cpp \ + trc_printable_elem.cpp \ + trc_ret_stack.cpp + +# MEM_ACC +SRCS+= \ + trc_mem_acc_base.cpp \ + trc_mem_acc_cb.cpp \ + trc_mem_acc_mapper.cpp \ + trc_mem_acc_bufptr.cpp \ + trc_mem_acc_file.cpp + +# I_DEC +SRCS+= \ + trc_i_decode.cpp \ + trc_idec_arminst.cpp + +CFLAGS+= \ + -I${OPENCSDSRC}/decoder/include/ \ + -I${.CURDIR} + +INCS= \ + ocsd_if_types.h \ + trc_gen_elem_types.h \ + trc_pkt_types.h + +INCSDIR=${INCLUDEDIR}/opencsd + +APIINCS= \ + ocsd_c_api_cust_fact.h \ + ocsd_c_api_cust_impl.h \ + ocsd_c_api_custom.h \ + ocsd_c_api_types.h \ + opencsd_c_api.h + +APIINCSDIR=${INCLUDEDIR}/opencsd/c_api/ + +ETMV4INCS= \ + etmv4_decoder.h \ + trc_cmp_cfg_etmv4.h \ + trc_dcd_mngr_etmv4i.h \ + trc_etmv4_stack_elem.h \ + trc_pkt_decode_etmv4i.h \ + trc_pkt_elem_etmv4d.h \ + trc_pkt_elem_etmv4i.h \ + trc_pkt_proc_etmv4.h \ + trc_pkt_types_etmv4.h + +ETMV4INCSDIR=${INCLUDEDIR}/opencsd/etmv4/ + +ETMV3INCS= \ + etmv3_decoder.h \ + trc_cmp_cfg_etmv3.h \ + trc_dcd_mngr_etmv3.h \ + trc_pkt_decode_etmv3.h \ + trc_pkt_elem_etmv3.h \ + trc_pkt_proc_etmv3.h \ + trc_pkt_types_etmv3.h + +ETMV3INCSDIR=${INCLUDEDIR}/opencsd/etmv3/ + +PTMINCS= \ + ptm_decoder.h \ + trc_cmp_cfg_ptm.h \ + trc_dcd_mngr_ptm.h \ + trc_pkt_decode_ptm.h \ + trc_pkt_elem_ptm.h \ + trc_pkt_proc_ptm.h \ + trc_pkt_types_ptm.h + +PTMINCSDIR=${INCLUDEDIR}/opencsd/ptm/ + +STMINCS= \ + stm_decoder.h \ + trc_cmp_cfg_stm.h \ + trc_dcd_mngr_stm.h \ + trc_pkt_decode_stm.h \ + trc_pkt_elem_stm.h \ + trc_pkt_proc_stm.h \ + trc_pkt_types_stm.h + +STMINCSDIR=${INCLUDEDIR}/opencsd/stm/ + +INCSGROUPS=INCS APIINCS ETMV3INCS ETMV4INCS PTMINCS STMINCS + +LIBADD= cxxrt cplusplus + +WARNS?= 1 + +HAS_TESTS= + +.include Index: lib/libpmc/libpmc.c =================================================================== --- lib/libpmc/libpmc.c +++ lib/libpmc/libpmc.c @@ -76,6 +76,10 @@ static int tsc_allocate_pmc(enum pmc_event _pe, char *_ctrspec, struct pmc_op_pmcallocate *_pmc_config); #endif +#if defined(__amd64__) +static int pt_allocate_pmc(enum pmc_event _pe, char *_ctrspec, + struct pmc_op_pmcallocate *_pmc_config); +#endif #if defined(__arm__) #if defined(__XSCALE__) static int xscale_allocate_pmc(enum pmc_event _pe, char *_ctrspec, @@ -87,6 +91,8 @@ #if defined(__aarch64__) static int arm64_allocate_pmc(enum pmc_event _pe, char *_ctrspec, struct pmc_op_pmcallocate *_pmc_config); +static int coresight_allocate_pmc(enum pmc_event _pe, char *_ctrspec, + struct pmc_op_pmcallocate *_pmc_config); #endif #if defined(__mips__) static int mips_allocate_pmc(enum pmc_event _pe, char* ctrspec, @@ -239,6 +245,12 @@ __PMC_EV_ALIAS_SKYLAKE_XEON() }; +static const struct pmc_event_descr kabylake_event_table[] = +{ + /* Kabylake events are similar to Skylake */ + __PMC_EV_ALIAS_SKYLAKE() +}; + static const struct pmc_event_descr ivybridge_event_table[] = { __PMC_EV_ALIAS_IVYBRIDGE() @@ -336,6 +348,7 @@ PMC_MDEP_TABLE(broadwell_xeon, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP); PMC_MDEP_TABLE(skylake, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP); PMC_MDEP_TABLE(skylake_xeon, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC); +PMC_MDEP_TABLE(kabylake, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP, PMC_CLASS_PT); PMC_MDEP_TABLE(ivybridge, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC); PMC_MDEP_TABLE(ivybridge_xeon, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC); PMC_MDEP_TABLE(sandybridge, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP); @@ -350,8 +363,8 @@ PMC_MDEP_TABLE(xscale, XSCALE, PMC_CLASS_SOFT, PMC_CLASS_XSCALE); PMC_MDEP_TABLE(cortex_a8, ARMV7, PMC_CLASS_SOFT, PMC_CLASS_ARMV7); PMC_MDEP_TABLE(cortex_a9, ARMV7, PMC_CLASS_SOFT, PMC_CLASS_ARMV7); -PMC_MDEP_TABLE(cortex_a53, ARMV8, PMC_CLASS_SOFT, PMC_CLASS_ARMV8); -PMC_MDEP_TABLE(cortex_a57, ARMV8, PMC_CLASS_SOFT, PMC_CLASS_ARMV8); +PMC_MDEP_TABLE(cortex_a53, ARMV8, PMC_CLASS_SOFT, PMC_CLASS_ARMV8, PMC_CLASS_CORESIGHT); +PMC_MDEP_TABLE(cortex_a57, ARMV8, PMC_CLASS_SOFT, PMC_CLASS_ARMV8, PMC_CLASS_CORESIGHT); PMC_MDEP_TABLE(mips24k, MIPS24K, PMC_CLASS_SOFT, PMC_CLASS_MIPS24K); PMC_MDEP_TABLE(mips74k, MIPS74K, PMC_CLASS_SOFT, PMC_CLASS_MIPS74K); PMC_MDEP_TABLE(octeon, OCTEON, PMC_CLASS_SOFT, PMC_CLASS_OCTEON); @@ -365,6 +378,16 @@ __PMC_EV_TSC() }; +static const struct pmc_event_descr pt_event_table[] = +{ + __PMC_EV_PT() +}; + +static const struct pmc_event_descr coresight_event_table[] = +{ + __PMC_EV_CORESIGHT() +}; + #undef PMC_CLASS_TABLE_DESC #define PMC_CLASS_TABLE_DESC(NAME, CLASS, EVENTS, ALLOCATOR) \ static const struct pmc_class_descr NAME##_class_table_descr = \ @@ -392,6 +415,7 @@ PMC_CLASS_TABLE_DESC(broadwell_xeon, IAP, broadwell_xeon, iap); PMC_CLASS_TABLE_DESC(skylake, IAP, skylake, iap); PMC_CLASS_TABLE_DESC(skylake_xeon, IAP, skylake_xeon, iap); +PMC_CLASS_TABLE_DESC(kabylake, IAP, kabylake, iap); PMC_CLASS_TABLE_DESC(ivybridge, IAP, ivybridge, iap); PMC_CLASS_TABLE_DESC(ivybridge_xeon, IAP, ivybridge_xeon, iap); PMC_CLASS_TABLE_DESC(sandybridge, IAP, sandybridge, iap); @@ -419,6 +443,9 @@ #if defined(__i386__) || defined(__amd64__) PMC_CLASS_TABLE_DESC(tsc, TSC, tsc, tsc); #endif +#if defined(__amd64__) +PMC_CLASS_TABLE_DESC(pt, PT, pt, pt); +#endif #if defined(__arm__) #if defined(__XSCALE__) PMC_CLASS_TABLE_DESC(xscale, XSCALE, xscale, xscale); @@ -429,6 +456,7 @@ #if defined(__aarch64__) PMC_CLASS_TABLE_DESC(cortex_a53, ARMV8, cortex_a53, arm64); PMC_CLASS_TABLE_DESC(cortex_a57, ARMV8, cortex_a57, arm64); +PMC_CLASS_TABLE_DESC(coresight, CORESIGHT, coresight, coresight); #endif #if defined(__mips__) PMC_CLASS_TABLE_DESC(mips24k, MIPS24K, mips24k, mips); @@ -732,6 +760,8 @@ #define skylake_aliases_without_iaf core2_aliases_without_iaf #define skylake_xeon_aliases core2_aliases #define skylake_xeon_aliases_without_iaf core2_aliases_without_iaf +#define kabylake_aliases core2_aliases +#define kabylake_aliases_without_iaf core2_aliases_without_iaf #define ivybridge_aliases core2_aliases #define ivybridge_aliases_without_iaf core2_aliases_without_iaf #define ivybridge_xeon_aliases core2_aliases @@ -1049,7 +1079,8 @@ return (-1); } else if (cpu_info.pm_cputype == PMC_CPU_INTEL_SKYLAKE || - cpu_info.pm_cputype == PMC_CPU_INTEL_SKYLAKE_XEON) { + cpu_info.pm_cputype == PMC_CPU_INTEL_SKYLAKE_XEON || + cpu_info.pm_cputype == PMC_CPU_INTEL_KABYLAKE) { if (KWPREFIXMATCH(p, IAP_KW_RSP "=")) { n = pmc_parse_mask(iap_rsp_mask_skylake, p, &rsp); } else @@ -2495,6 +2526,84 @@ } #endif +#if defined(__amd64__) + +#define INTEL_PT_KW_BRANCHES "branches" +#define INTEL_PT_KW_TSC "tsc" +#define INTEL_PT_KW_MTC "mtc" +#define INTEL_PT_KW_DISRETC "disretc" +#define INTEL_PT_KW_ADDRA "addra" +#define INTEL_PT_KW_ADDRB "addrb" + +static int +pt_allocate_pmc(enum pmc_event pe, char *ctrspec, + struct pmc_op_pmcallocate *pmc_config) +{ + struct pmc_md_pt_op_pmcallocate *pm_pt; + uint64_t addr; + uint32_t addrn; + char *p, *q, *e; + + if (pe != PMC_EV_PT_PT) + return (-1); + + pm_pt = (struct pmc_md_pt_op_pmcallocate *)&pmc_config->pm_md.pm_pt; + + addrn = 0; + while ((p = strsep(&ctrspec, ",")) != NULL) { + if (KWMATCH(p, INTEL_PT_KW_BRANCHES)) { + pm_pt->flags |= INTEL_PT_FLAG_BRANCHES; + } + + if (KWMATCH(p, INTEL_PT_KW_TSC)) { + pm_pt->flags |= INTEL_PT_FLAG_TSC; + } + + if (KWMATCH(p, INTEL_PT_KW_MTC)) { + pm_pt->flags |= INTEL_PT_FLAG_MTC; + } + + if (KWMATCH(p, INTEL_PT_KW_DISRETC)) { + pm_pt->flags |= INTEL_PT_FLAG_DISRETC; + } + + if (KWPREFIXMATCH(p, INTEL_PT_KW_ADDRA "=")) { + q = strchr(p, '='); + if (*++q == '\0') /* skip '=' */ + return (-1); + + addr = strtoul(q, &e, 0); + if (e == q || *e != '\0') + return (-1); + pm_pt->ranges[addrn * 2] = addr; + } + + if (KWPREFIXMATCH(p, INTEL_PT_KW_ADDRB "=")) { + q = strchr(p, '='); + if (*++q == '\0') /* skip '=' */ + return (-1); + + addr = strtoul(q, &e, 0); + if (e == q || *e != '\0') + return (-1); + pm_pt->ranges[addrn * 2 + 1] = addr; + + if (pm_pt->ranges[addrn * 2 + 1] < pm_pt->ranges[addrn * 2]) + return (-1); + addrn += 1; + if (addrn > PT_NADDR) + return (-1); + } + }; + + pm_pt->nranges = addrn; + + pmc_config->pm_caps |= PMC_CAP_READ; + + return (0); +} +#endif + static struct pmc_event_alias generic_aliases[] = { EV_ALIAS("instructions", "SOFT-CLOCK.HARD"), EV_ALIAS(NULL, NULL) @@ -2583,6 +2692,61 @@ return (0); } + +#define ARM_CORESIGHT_KW_ADDRA "addra" +#define ARM_CORESIGHT_KW_ADDRB "addrb" + +static int +coresight_allocate_pmc(enum pmc_event pe, char *ctrspec, + struct pmc_op_pmcallocate *pmc_config) +{ + struct pmc_md_coresight_op_pmcallocate *pm_coresight; + uint64_t addr; + uint32_t addrn; + char *p, *q, *e; + + if (pe != PMC_EV_CORESIGHT_CORESIGHT) + return (-1); + + pm_coresight = (struct pmc_md_coresight_op_pmcallocate *)&pmc_config->pm_md.pm_coresight; + + addrn = 0; + while ((p = strsep(&ctrspec, ",")) != NULL) { + if (KWPREFIXMATCH(p, ARM_CORESIGHT_KW_ADDRA "=")) { + q = strchr(p, '='); + if (*++q == '\0') /* skip '=' */ + return (-1); + + addr = strtoul(q, &e, 0); + if (e == q || *e != '\0') + return (-1); + pm_coresight->ranges[addrn * 2] = addr; + } + + if (KWPREFIXMATCH(p, ARM_CORESIGHT_KW_ADDRB "=")) { + q = strchr(p, '='); + if (*++q == '\0') /* skip '=' */ + return (-1); + + addr = strtoul(q, &e, 0); + if (e == q || *e != '\0') + return (-1); + pm_coresight->ranges[addrn * 2 + 1] = addr; + + if (pm_coresight->ranges[addrn * 2 + 1] < pm_coresight->ranges[addrn * 2]) + return (-1); + addrn += 1; + if (addrn > CORESIGHT_NADDR) + return (-1); + } + }; + + pm_coresight->nranges = addrn; + + pmc_config->pm_caps |= PMC_CAP_READ; + + return (0); +} #endif #if defined(__mips__) @@ -2780,7 +2944,8 @@ retval = -1; if (mode != PMC_MODE_SS && mode != PMC_MODE_TS && - mode != PMC_MODE_SC && mode != PMC_MODE_TC) { + mode != PMC_MODE_SC && mode != PMC_MODE_TC && + mode != PMC_MODE_ST && mode != PMC_MODE_TT) { errno = EINVAL; goto out; } @@ -2903,6 +3068,7 @@ int pmc_cpuinfo(const struct pmc_cpuinfo **pci) { + if (pmc_syscall == -1) { errno = ENXIO; return (-1); @@ -3023,6 +3189,10 @@ ev = skylake_xeon_event_table; count = PMC_EVENT_TABLE_SIZE(skylake_xeon); break; + case PMC_CPU_INTEL_KABYLAKE: + ev = kabylake_event_table; + count = PMC_EVENT_TABLE_SIZE(kabylake); + break; case PMC_CPU_INTEL_IVYBRIDGE: ev = ivybridge_event_table; count = PMC_EVENT_TABLE_SIZE(ivybridge); @@ -3086,6 +3256,10 @@ ev = tsc_event_table; count = PMC_EVENT_TABLE_SIZE(tsc); break; + case PMC_CLASS_PT: + ev = pt_event_table; + count = PMC_EVENT_TABLE_SIZE(pt); + break; case PMC_CLASS_K7: ev = k7_event_table; count = PMC_EVENT_TABLE_SIZE(k7); @@ -3136,6 +3310,10 @@ break; } break; + case PMC_CLASS_CORESIGHT: + ev = coresight_event_table; + count = PMC_EVENT_TABLE_SIZE(coresight); + break; case PMC_CLASS_MIPS24K: ev = mips24k_event_table; count = PMC_EVENT_TABLE_SIZE(mips24k); @@ -3184,12 +3362,14 @@ int pmc_flush_logfile(void) { + return (PMC_CALL(FLUSHLOG,0)); } int pmc_close_logfile(void) { + return (PMC_CALL(CLOSELOG,0)); } @@ -3406,6 +3586,12 @@ case PMC_CPU_INTEL_SKYLAKE_XEON: PMC_MDEP_INIT_INTEL_V2(skylake_xeon); break; + case PMC_CPU_INTEL_KABYLAKE: +#if defined(__amd64__) + pmc_class_table[n++] = &pt_class_table_descr; +#endif + PMC_MDEP_INIT_INTEL_V2(kabylake); + break; case PMC_CPU_INTEL_IVYBRIDGE: PMC_MDEP_INIT_INTEL_V2(ivybridge); break; @@ -3455,11 +3641,13 @@ #if defined(__aarch64__) case PMC_CPU_ARMV8_CORTEX_A53: PMC_MDEP_INIT(cortex_a53); - pmc_class_table[n] = &cortex_a53_class_table_descr; + pmc_class_table[n++] = &cortex_a53_class_table_descr; + pmc_class_table[n++] = &coresight_class_table_descr; break; case PMC_CPU_ARMV8_CORTEX_A57: PMC_MDEP_INIT(cortex_a57); - pmc_class_table[n] = &cortex_a57_class_table_descr; + pmc_class_table[n++] = &cortex_a57_class_table_descr; + pmc_class_table[n++] = &coresight_class_table_descr; break; #endif #if defined(__mips__) @@ -3623,6 +3811,11 @@ evfence = skylake_xeon_event_table + PMC_EVENT_TABLE_SIZE(skylake_xeon); break; + case PMC_CPU_INTEL_KABYLAKE: + ev = kabylake_event_table; + evfence = kabylake_event_table + + PMC_EVENT_TABLE_SIZE(kabylake); + break; case PMC_CPU_INTEL_IVYBRIDGE: ev = ivybridge_event_table; evfence = ivybridge_event_table + PMC_EVENT_TABLE_SIZE(ivybridge); @@ -3715,6 +3908,9 @@ default: /* Unknown CPU type. */ break; } + } else if (pe == PMC_EV_CORESIGHT_CORESIGHT) { + ev = coresight_event_table; + evfence = coresight_event_table + PMC_EVENT_TABLE_SIZE(coresight); } else if (pe >= PMC_EV_MIPS24K_FIRST && pe <= PMC_EV_MIPS24K_LAST) { ev = mips24k_event_table; evfence = mips24k_event_table + PMC_EVENT_TABLE_SIZE(mips24k); @@ -3736,6 +3932,9 @@ } else if (pe == PMC_EV_TSC_TSC) { ev = tsc_event_table; evfence = tsc_event_table + PMC_EVENT_TABLE_SIZE(tsc); + } else if (pe == PMC_EV_PT_PT) { + ev = pt_event_table; + evfence = pt_event_table + PMC_EVENT_TABLE_SIZE(pt); } else if ((int)pe >= PMC_EV_SOFT_FIRST && (int)pe <= PMC_EV_SOFT_LAST) { ev = soft_event_table; evfence = soft_event_table + soft_event_info.pm_nevent; @@ -3852,6 +4051,68 @@ return (0); } +int +pmc_proc_unsuspend(pmc_id_t pmc, pid_t pid) +{ + struct pmc_op_proc_unsuspend u; + + u.pm_pmcid = pmc; + u.pm_pid = pid; + + return (PMC_CALL(THREAD_UNSUSPEND, &u)); +} + +int +pmc_read_trace(uint32_t cpu, pmc_id_t pmc, + pmc_value_t *cycle, pmc_value_t *offset) +{ + struct pmc_op_trace_read pmc_trace_read; + + pmc_trace_read.pm_pmcid = pmc; + pmc_trace_read.pm_cpu = cpu; + pmc_trace_read.pm_cycle = 0; + pmc_trace_read.pm_offset = 0; + + if (PMC_CALL(TRACE_READ, &pmc_trace_read) < 0) + return (-1); + + *cycle = pmc_trace_read.pm_cycle; + *offset = pmc_trace_read.pm_offset; + + return (0); +} + +int +pmc_trace_config(uint32_t cpu, pmc_id_t pmc, + uint64_t *ranges, uint32_t nranges) +{ + struct pmc_op_trace_config trc; + + trc.pm_pmcid = pmc; + trc.pm_cpu = cpu; + trc.nranges = nranges; + + if (nranges > PMC_FILTER_MAX_IP_RANGES) + return (-1); + + memcpy(&trc.ranges, ranges, sizeof(uint64_t) * 2 * nranges); + + if (PMC_CALL(TRACE_CONFIG, &trc) < 0) + return (-1); + + return (0); +} + +int +pmc_log_kmap(pmc_id_t pmc) +{ + struct pmc_op_simple pmc_log_km; + + pmc_log_km.pm_pmcid = pmc; + + return (PMC_CALL(LOG_KERNEL_MAP, &pmc_log_km)); +} + int pmc_release(pmc_id_t pmc) { Index: lib/libpmc/pmc.h =================================================================== --- lib/libpmc/pmc.h +++ lib/libpmc/pmc.h @@ -77,6 +77,7 @@ int pmc_allocate(const char *_ctrspec, enum pmc_mode _mode, uint32_t _flags, int _cpu, pmc_id_t *_pmcid); int pmc_attach(pmc_id_t _pmcid, pid_t _pid); +int pmc_proc_unsuspend(pmc_id_t pmc, pid_t pid); int pmc_capabilities(pmc_id_t _pmc, uint32_t *_caps); int pmc_configure_logfile(int _fd); int pmc_flush_logfile(void); @@ -88,7 +89,10 @@ int pmc_get_msr(pmc_id_t _pmc, uint32_t *_msr); int pmc_init(void); int pmc_read(pmc_id_t _pmc, pmc_value_t *_value); +int pmc_read_trace(uint32_t cpu, pmc_id_t pmc, pmc_value_t *cycle, pmc_value_t *offset); +int pmc_trace_config(uint32_t cpu, pmc_id_t pmc, uint64_t *ranges, uint32_t nranges); int pmc_release(pmc_id_t _pmc); +int pmc_log_kmap(pmc_id_t pmc); int pmc_rw(pmc_id_t _pmc, pmc_value_t _newvalue, pmc_value_t *_oldvalue); int pmc_set(pmc_id_t _pmc, pmc_value_t _value); int pmc_start(pmc_id_t _pmc); Index: lib/libpmcstat/libpmcstat_image.c =================================================================== --- lib/libpmcstat/libpmcstat_image.c +++ lib/libpmcstat/libpmcstat_image.c @@ -386,6 +386,21 @@ break; } } + } else if (eh.e_type == ET_DYN) { + for (i = 0; i < eh.e_phnum; i++) { + if (gelf_getphdr(e, i, &ph) != &ph) { + warnx( +"WARNING: Retrieval of PHDR entry #%ju in \"%s\" failed: %s.", + (uintmax_t) i, buffer, elf_errmsg(-1)); + goto done; + } + switch (ph.p_type) { + case PT_LOAD: + if ((ph.p_flags & PF_X) != 0) + image->pi_vaddr = ph.p_vaddr & (-ph.p_align); + break; + } + } } /* Index: sys/amd64/include/param.h =================================================================== --- sys/amd64/include/param.h +++ sys/amd64/include/param.h @@ -65,7 +65,7 @@ #define MACHINE_ARCH32 "i386" #endif -#if defined(SMP) || defined(KLD_MODULE) +#if defined(SMP) #ifndef MAXCPU #define MAXCPU 256 #endif Index: sys/amd64/include/pmc_mdep.h =================================================================== --- sys/amd64/include/pmc_mdep.h +++ sys/amd64/include/pmc_mdep.h @@ -45,6 +45,7 @@ #include #include #include +#include #include /* @@ -57,6 +58,7 @@ #define PMC_MDEP_CLASS_INDEX_P4 2 #define PMC_MDEP_CLASS_INDEX_IAP 2 #define PMC_MDEP_CLASS_INDEX_IAF 3 +#define PMC_MDEP_CLASS_INDEX_PT 4 #define PMC_MDEP_CLASS_INDEX_UCP 4 #define PMC_MDEP_CLASS_INDEX_UCF 5 @@ -70,6 +72,7 @@ * IAF Intel fixed-function PMCs in Core2 and later CPUs. * UCP Intel Uncore programmable PMCs. * UCF Intel Uncore fixed-function PMCs. + * PT Intel PT event. */ union pmc_md_op_pmcallocate { @@ -79,7 +82,8 @@ struct pmc_md_ucf_op_pmcallocate pm_ucf; struct pmc_md_ucp_op_pmcallocate pm_ucp; struct pmc_md_p4_op_pmcallocate pm_p4; - uint64_t __pad[4]; + struct pmc_md_pt_op_pmcallocate pm_pt; + uint64_t __pad[1]; }; /* Logging */ @@ -95,6 +99,7 @@ struct pmc_md_ucf_pmc pm_ucf; struct pmc_md_ucp_pmc pm_ucp; struct pmc_md_p4_pmc pm_p4; + struct pmc_md_pt_pmc pm_pt; }; #define PMC_TRAPFRAME_TO_PC(TF) ((TF)->tf_rip) Index: sys/arm64/coresight/coresight-cmd.c =================================================================== --- /dev/null +++ sys/arm64/coresight/coresight-cmd.c @@ -0,0 +1,167 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 + * ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include + +extern struct coresight_device_list cs_devs; + +#define CORESIGHT_DISABLE 0 +#define CORESIGHT_ENABLE 1 +#define CORESIGHT_READ 2 + +static struct coresight_device * +coresight_next_device(struct coresight_device *cs_dev, + struct coresight_event *event) +{ + struct coresight_device *out; + struct endpoint *out_endp; + struct endpoint *endp; + + TAILQ_FOREACH(endp, &cs_dev->pdata->endpoints, link) { + if (endp->slave != 0) + continue; + + out = coresight_get_output_device(endp, &out_endp); + if (out) { + if (LIST_EMPTY(&event->endplist)) { + /* Add source device */ + endp->cs_dev = cs_dev; + LIST_INSERT_HEAD(&event->endplist, endp, endplink); + } + + /* Add output device */ + out_endp->cs_dev = out; + LIST_INSERT_HEAD(&event->endplist, out_endp, endplink); + + return (out); + } + } + + return (NULL); +} + +static int +coresight_build_list(struct coresight_device *cs_dev, + struct coresight_event *event) +{ + struct coresight_device *out; + + out = cs_dev; + while (out) + out = coresight_next_device(out, event); + + return (0); +} + +int +coresight_init_event(int cpu, struct coresight_event *event) +{ + struct coresight_device *cs_dev; + + /* Start building path from source device */ + TAILQ_FOREACH(cs_dev, &cs_devs, link) { + if (cs_dev->dev_type == event->src && + cs_dev->pdata->cpu == cpu) { + LIST_INIT(&event->endplist); + coresight_build_list(cs_dev, event); + break; + } + } + + return (0); +} + +static int +coresight_cmd_one(struct coresight_device *out, + struct endpoint *out_endp, struct coresight_event *event, uint8_t cmd) +{ + + switch (cmd) { + case CORESIGHT_DISABLE: + if (out->ops->disable == NULL) + break; + out->ops->disable(out, out_endp, event); + break; + case CORESIGHT_ENABLE: + if (out->ops->enable == NULL) + break; + out->ops->enable(out, out_endp, event); + break; + case CORESIGHT_READ: + if (out->ops->read == NULL) + break; + out->ops->read(out, out_endp, event); + break; + }; + + return (0); +} + + +static void +coresight_cmd(int cpu, struct coresight_event *event, uint8_t cmd) +{ + struct endpoint *endp; + + LIST_FOREACH(endp, &event->endplist, endplink) + coresight_cmd_one(endp->cs_dev, endp, event, cmd); +} + +void +coresight_enable(int cpu, struct coresight_event *event) +{ + + coresight_cmd(cpu, event, CORESIGHT_ENABLE); +} + +void +coresight_disable(int cpu, struct coresight_event *event) +{ + + coresight_cmd(cpu, event, CORESIGHT_DISABLE); +} + +void +coresight_read(int cpu, struct coresight_event *event) +{ + + coresight_cmd(cpu, event, CORESIGHT_READ); +} Index: sys/arm64/coresight/coresight-cpu-debug.c =================================================================== --- /dev/null +++ sys/arm64/coresight/coresight-cpu-debug.c @@ -0,0 +1,145 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#define EDPCSR 0x0a0 +#define EDCIDSR 0x0a4 +#define EDVIDSR 0x0a8 +#define EDPCSR_HI 0x0ac +#define EDOSLAR 0x300 +#define EDPRCR 0x310 +#define EDPRCR_COREPURQ (1 << 3) +#define EDPRCR_CORENPDRQ (1 << 0) +#define EDPRSR 0x314 +#define EDDEVID1 0xfc4 +#define EDDEVID 0xfc8 + +static struct ofw_compat_data compat_data[] = { + { "arm,coresight-cpu-debug", 1 }, + { NULL, 0 } +}; + +struct debug_softc { + struct resource *res; +}; + +static struct resource_spec debug_spec[] = { + { SYS_RES_MEMORY, 0, RF_ACTIVE }, + { -1, 0 } +}; + +static int +debug_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) + return (ENXIO); + + device_set_desc(dev, "Coresight CPU Debug"); + + return (BUS_PROBE_DEFAULT); +} + +static int +debug_attach(device_t dev) +{ + struct debug_softc *sc; + uint32_t reg; + + sc = device_get_softc(dev); + + if (bus_alloc_resources(dev, debug_spec, &sc->res) != 0) { + device_printf(dev, "cannot allocate resources for device\n"); + return (ENXIO); + } + + /* Enable CPU debug for current CPU only */ + if (device_get_unit(dev) != 0) + return (0); + + /* Unlock Coresight */ + bus_write_4(sc->res, CORESIGHT_LAR, CORESIGHT_UNLOCK); + + wmb(); + + /* Unlock Debug */ + bus_write_4(sc->res, EDOSLAR, 0); + + wmb(); + + /* Enable power */ + reg = bus_read_4(sc->res, EDPRCR); + reg |= EDPRCR_COREPURQ; + bus_write_4(sc->res, EDPRCR, reg); + + do { + reg = bus_read_4(sc->res, EDPRSR); + } while ((reg & EDPRCR_CORENPDRQ) == 0); + + return (0); +} + +static device_method_t debug_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, debug_probe), + DEVMETHOD(device_attach, debug_attach), + DEVMETHOD_END +}; + +static driver_t debug_driver = { + "debug", + debug_methods, + sizeof(struct debug_softc), +}; + +static devclass_t debug_devclass; + +EARLY_DRIVER_MODULE(debug, simplebus, debug_driver, debug_devclass, + 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_LATE); +MODULE_VERSION(debug, 1); Index: sys/arm64/coresight/coresight-dynamic-replicator.c =================================================================== --- /dev/null +++ sys/arm64/coresight/coresight-dynamic-replicator.c @@ -0,0 +1,164 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include + +#define REPLICATOR_IDFILTER0 0x00 +#define REPLICATOR_IDFILTER1 0x04 + +#include +#include + +static struct ofw_compat_data compat_data[] = { + { "arm,coresight-dynamic-replicator", 1 }, + { NULL, 0 } +}; + +struct replicator_softc { + struct resource *res; + struct coresight_platform_data *pdata; +}; + +static struct resource_spec replicator_spec[] = { + { SYS_RES_MEMORY, 0, RF_ACTIVE }, + { -1, 0 } +}; + +static int +replicator_enable(struct coresight_device *out, struct endpoint *endp, + struct coresight_event *event) +{ + struct replicator_softc *sc; + + sc = device_get_softc(out->dev); + + /* Enable the port. Keep the other port disabled */ + + if (endp->reg == 0) { + bus_write_4(sc->res, REPLICATOR_IDFILTER0, 0x00); + bus_write_4(sc->res, REPLICATOR_IDFILTER1, 0xff); + } else { + bus_write_4(sc->res, REPLICATOR_IDFILTER0, 0xff); + bus_write_4(sc->res, REPLICATOR_IDFILTER1, 0x00); + } + + return (0); +} + +static void +replicator_disable(struct coresight_device *out, struct endpoint *endp, + struct coresight_event *event) +{ + struct replicator_softc *sc; + + sc = device_get_softc(out->dev); + + bus_write_4(sc->res, REPLICATOR_IDFILTER0, 0xff); + bus_write_4(sc->res, REPLICATOR_IDFILTER1, 0xff); +} + +static struct coresight_ops ops = { + .enable = &replicator_enable, + .disable = &replicator_disable, +}; + +static int +replicator_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) + return (ENXIO); + + device_set_desc(dev, "Coresight Dynamic Replicator"); + + return (BUS_PROBE_DEFAULT); +} + +static int +replicator_attach(device_t dev) +{ + struct replicator_softc *sc; + struct coresight_desc desc; + + sc = device_get_softc(dev); + + if (bus_alloc_resources(dev, replicator_spec, &sc->res) != 0) { + device_printf(dev, "cannot allocate resources for device\n"); + return (ENXIO); + } + + sc->pdata = coresight_get_platform_data(dev); + + desc.pdata = sc->pdata; + desc.dev = dev; + desc.dev_type = CORESIGHT_DYNAMIC_REPLICATOR; + desc.ops = &ops; + coresight_register(&desc); + + /* Unlock Coresight */ + bus_write_4(sc->res, CORESIGHT_LAR, CORESIGHT_UNLOCK); + + wmb(); + + return (0); +} + +static device_method_t replicator_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, replicator_probe), + DEVMETHOD(device_attach, replicator_attach), + DEVMETHOD_END +}; + +static driver_t replicator_driver = { + "replicator", + replicator_methods, + sizeof(struct replicator_softc), +}; + +static devclass_t replicator_devclass; + +DRIVER_MODULE(replicator, simplebus, replicator_driver, replicator_devclass, 0, 0); +MODULE_VERSION(replicator, 1); Index: sys/arm64/coresight/coresight-etm4x.h =================================================================== --- /dev/null +++ sys/arm64/coresight/coresight-etm4x.h @@ -0,0 +1,169 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARM64_CORESIGHT_ETM4X_H_ +#define _ARM64_CORESIGHT_ETM4X_H_ + +#define TRCPRGCTLR 0x004 /* Trace Programming Control Register */ +#define TRCPRGCTLR_EN (1 << 0) /* Trace unit enable bit */ +#define TRCPROCSELR 0x008 /* Trace PE Select Control Register */ +#define TRCSTATR 0x00C /* Trace Trace Status Register */ +#define TRCSTATR_PMSTABLE (1 << 1) /* The programmers' model is stable. */ +#define TRCSTATR_IDLE (1 << 0) /* The trace unit is idle. */ +#define TRCCONFIGR 0x010 /* Trace Trace Configuration Register */ +#define TRCCONFIGR_DV (1 << 17) /* Data value tracing is enabled when INSTP0 is not 0b00 */ +#define TRCCONFIGR_DA (1 << 16) /* Data address tracing is enabled when INSTP0 is not 0b00. */ +#define TRCCONFIGR_VMIDOPT (1 << 15) /* Control bit to configure the Virtual context identifier value */ +#define TRCCONFIGR_QE_S 13 /* Q element enable field */ +#define TRCCONFIGR_QE_M (0x3 << TRCCONFIGR_QE_S) +#define TRCCONFIGR_RS (1 << 12) /* Return stack enable bit */ +#define TRCCONFIGR_TS (1 << 11) /* Global timestamp tracing is enabled. */ +#define TRCCONFIGR_COND_S 8 /* Conditional instruction tracing bit. */ +#define TRCCONFIGR_COND_M (0x7 << TRCCONFIGR_COND_S) +#define TRCCONFIGR_COND_DIS 0 +#define TRCCONFIGR_COND_LDR (1 << TRCCONFIGR_COND_S) /* Conditional load instructions are traced. */ +#define TRCCONFIGR_COND_STR (2 << TRCCONFIGR_COND_S) /* Conditional store instructions are traced. */ +#define TRCCONFIGR_COND_LDRSTR (3 << TRCCONFIGR_COND_S) /* Conditional load and store instructions are traced. */ +#define TRCCONFIGR_COND_ALL (7 << TRCCONFIGR_COND_S) /* All conditional instructions are traced. */ +#define TRCCONFIGR_VMID (1 << 7) /* Virtual context identifier tracing is enabled. */ +#define TRCCONFIGR_CID (1 << 6) /* Context ID tracing is enabled. */ +#define TRCCONFIGR_CCI (1 << 4) /* Cycle counting in the instruction trace is enabled. */ +#define TRCCONFIGR_BB (1 << 3) /* Branch broadcast mode is enabled. */ +#define TRCCONFIGR_INSTP0_S 1 /* Instruction P0 field. */ +#define TRCCONFIGR_INSTP0_M (0x3 << TRCCONFIGR_INSTP0_S) +#define TRCCONFIGR_INSTP0_NONE 0 /* Do not trace load and store instructions as P0 instructions. */ +#define TRCCONFIGR_INSTP0_LDR (1 << TRCCONFIGR_INSTP0_S) /* Trace load instructions as P0 instructions. */ +#define TRCCONFIGR_INSTP0_STR (2 << TRCCONFIGR_INSTP0_S) /* Trace store instructions as P0 instructions. */ +#define TRCCONFIGR_INSTP0_LDRSTR (3 << TRCCONFIGR_INSTP0_S) /* Trace load and store instructions as P0 instr. */ +#define TRCAUXCTLR 0x018 /* Trace Auxiliary Control Register */ +#define TRCEVENTCTL0R 0x020 /* Trace Event Control 0 Register */ +#define TRCEVENTCTL1R 0x024 /* Trace Event Control 1 Register */ +#define TRCSTALLCTLR 0x02C /* Trace Stall Control Register */ +#define TRCTSCTLR 0x030 /* Trace Global Timestamp Control Register */ +#define TRCSYNCPR 0x034 /* Trace Synchronization Period Register */ +#define TRCCCCTLR 0x038 /* Trace Cycle Count Control Register */ +#define TRCBBCTLR 0x03C /* Trace Branch Broadcast Control Register */ +#define TRCTRACEIDR 0x040 /* Trace Trace ID Register */ +#define TRCQCTLR 0x044 /* Trace Q Element Control Register */ +#define TRCQCTLR_MODE_INC (1 << 8) /* Include mode. */ +#define TRCVICTLR 0x080 /* Trace ViewInst Main Control Register */ +#define TRCVICTLR_SSSTATUS (1 << 9) /* The start/stop logic is in the started state. */ +#define TRCVICTLR_EXLEVEL_NS_S 20 +#define TRCVICTLR_EXLEVEL_NS_M (0xf << TRCVICTLR_EXLEVEL_NS_S) +#define TRCVICTLR_EXLEVEL_NS(n) (0x1 << ((n) + TRCVICTLR_EXLEVEL_NS_S)) +#define TRCVICTLR_EXLEVEL_S_S 16 +#define TRCVICTLR_EXLEVEL_S_M (0xf << TRCVICTLR_EXLEVEL_S_S) +#define TRCVICTLR_EXLEVEL_S(n) (0x1 << ((n) + TRCVICTLR_EXLEVEL_S_S)) +#define EVENT_SEL_S 0 +#define EVENT_SEL_M (0x1f << EVENT_SEL_S) +#define TRCVIIECTLR 0x084 /* Trace ViewInst Include/Exclude Control Register */ +#define TRCVISSCTLR 0x088 /* Trace ViewInst Start/Stop Control Register */ +#define TRCVIPCSSCTLR 0x08C /* Trace ViewInst Start/Stop PE Comparator Control Register */ +#define TRCVDCTLR 0x0A0 /* Trace ViewData Main Control Register */ +#define TRCVDCTLR_TRCEXDATA (1 << 12) /* Exception and exception return data transfers are traced */ +#define TRCVDCTLR_TBI (1 << 11) /* The trace unit assigns bits[63:56] to have the same value as bits[63:56] of the data address. */ +#define TRCVDCTLR_PCREL (1 << 10) /* The trace unit does not trace the address or value portions of PC-relative transfers. */ +#define TRCVDCTLR_SPREL_S 8 +#define TRCVDCTLR_SPREL_M (0x3 << TRCVDCTLR_SPREL_S) +#define TRCVDCTLR_EVENT_S 0 +#define TRCVDCTLR_EVENT_M (0xff << TRCVDCTLR_EVENT_S) +#define TRCVDSACCTLR 0x0A4 /* Trace ViewData Include/Exclude Single Address Comparator Control Register */ +#define TRCVDARCCTLR 0x0A8 /* Trace ViewData Include/Exclude Address Range Comparator Control Register */ +#define TRCSEQEVR(n) (0x100 + (n) * 0x4) /* Trace Sequencer State Transition Control Register [n=0-2] */ +#define TRCSEQRSTEVR 0x118 /* Trace Sequencer Reset Control Register */ +#define TRCSEQSTR 0x11C /* Trace Sequencer State Register */ +#define TRCEXTINSELR 0x120 /* Trace External Input Select Register */ +#define TRCCNTRLDVR(n) (0x140 + (n) * 0x4) /* 32 Trace Counter Reload Value Register [n=0-3] */ +#define TRCCNTCTLR(n) (0x150 + (n) * 0x4) /* 32 Trace Counter Control Register [n=0-3] */ +#define TRCCNTVR(n) (0x160 + (n) * 0x4) /* 32 Trace Counter Value Register [n=0-3] */ +#define TRCIMSPEC(n) (0x1C0 + (n) * 0x4) /* Trace IMPLEMENTATION DEFINED register [n=0-7] */ + +#define TRCIDR0(n) (0x1E0 + 0x4 * (n)) +#define TRCIDR8(n) (0x180 + 0x4 * (n)) +#define TRCIDR(n) ((n > 7) ? TRCIDR8(n) : TRCIDR0(n)) +#define TRCIDR1_TRCARCHMAJ_S 8 +#define TRCIDR1_TRCARCHMAJ_M (0xf << TRCIDR1_TRCARCHMAJ_S) +#define TRCIDR1_TRCARCHMIN_S 4 +#define TRCIDR1_TRCARCHMIN_M (0xf << TRCIDR1_TRCARCHMIN_S) + +#define TRCRSCTLR(n) (0x200 + (n) * 0x4) /* Trace Resource Selection Control Register [n=2-31] */ +#define TRCSSCCR(n) (0x280 + (n) * 0x4) /* Trace Single-shot Comparator Control Register [n=0-7] */ +#define TRCSSCSR(n) (0x2A0 + (n) * 0x4) /* Trace Single-shot Comparator Status Register [n=0-7] */ +#define TRCSSPCICR(n) (0x2C0 + (n) * 0x4) /* Trace Single-shot PE Comparator Input Control [n=0-7] */ +#define TRCOSLAR 0x300 /* Management OS Lock Access Register */ +#define TRCOSLSR 0x304 /* Management OS Lock Status Register */ +#define TRCPDCR 0x310 /* Management PowerDown Control Register */ +#define TRCPDSR 0x314 /* Management PowerDown Status Register */ +#define TRCACVR(n) (0x400 + (n) * 0x8) /* Trace Address Comparator Value Register [n=0-15] */ +#define TRCACATR(n) (0x480 + (n) * 0x8) /* Trace Address Comparator Access Type Register [n=0-15] */ +#define TRCACATR_DTBM (1 << 21) +#define TRCACATR_DATARANGE (1 << 20) +#define TRCACATR_DATASIZE_S 18 +#define TRCACATR_DATASIZE_M (0x3 << TRCACATR_DATASIZE_S) +#define TRCACATR_DATASIZE_B (0x0 << TRCACATR_DATASIZE_S) +#define TRCACATR_DATASIZE_HW (0x1 << TRCACATR_DATASIZE_S) +#define TRCACATR_DATASIZE_W (0x2 << TRCACATR_DATASIZE_S) +#define TRCACATR_DATASIZE_DW (0x3 << TRCACATR_DATASIZE_S) +#define TRCACATR_DATAMATCH_S 16 +#define TRCACATR_DATAMATCH_M (0x3 << TRCACATR_DATAMATCH_S) +#define TRCACATR_EXLEVEL_S_S 8 +#define TRCACATR_EXLEVEL_S_M (0xf << TRCACATR_EXLEVEL_S_S) +#define TRCACATR_EXLEVEL_S(n) (0x1 << ((n) + TRCACATR_EXLEVEL_S_S)) +#define TRCACATR_EXLEVEL_NS_S 12 +#define TRCACATR_EXLEVEL_NS_M (0xf << TRCACATR_EXLEVEL_NS_S) +#define TRCACATR_EXLEVEL_NS(n) (0x1 << ((n) + TRCACATR_EXLEVEL_NS_S)) +#define TRCDVCVR(n) (0x500 + (n) * 0x8) /* Trace Data Value Comparator Value Register [n=0-7] */ +#define TRCDVCMR(n) (0x580 + (n) * 0x8) /* Trace Data Value Comparator Mask Register [n=0-7] */ +#define TRCCIDCVR(n) (0x600 + (n) * 0x8) /* Trace Context ID Comparator Value Register [n=0-7] */ +#define TRCVMIDCVR(n) (0x640 + (n) * 0x8) /* Trace Virtual context identifier Comparator Value [n=0-7] */ +#define TRCCIDCCTLR0 0x680 /* Trace Context ID Comparator Control Register 0 */ +#define TRCCIDCCTLR1 0x684 /* Trace Context ID Comparator Control Register 1 */ +#define TRCVMIDCCTLR0 0x688 /* Trace Virtual context identifier Comparator Control Register 0 */ +#define TRCVMIDCCTLR1 0x68C /* Trace Virtual context identifier Comparator Control Register 1 */ +#define TRCITCTRL 0xF00 /* Management Integration Mode Control register */ +#define TRCCLAIMSET 0xFA0 /* Trace Claim Tag Set register */ +#define TRCCLAIMCLR 0xFA4 /* Trace Claim Tag Clear register */ +#define TRCDEVAFF0 0xFA8 /* Management Device Affinity register 0 */ +#define TRCDEVAFF1 0xFAC /* Management Device Affinity register 1 */ +#define TRCLAR 0xFB0 /* Management Software Lock Access Register */ +#define TRCLSR 0xFB4 /* Management Software Lock Status Register */ +#define TRCAUTHSTATUS 0xFB8 /* Management Authentication Status register */ +#define TRCDEVARCH 0xFBC /* Management Device Architecture register */ +#define TRCDEVID 0xFC8 /* Management Device ID register */ +#define TRCDEVTYPE 0xFCC /* Management Device Type register */ +#define TRCPIDR4 0xFD0 /* Management Peripheral ID4 Register */ +#define TRCPIDR(n) (0xFE0 + (n) * 0x4) /* Management Peripheral IDn Register [n=0-3] */ +#define TRCPIDR567(n) (0xFD4 + ((n) - 5) * 0x4) /* Management Peripheral ID5 to Peripheral ID7 Registers */ +#define TRCCIDR(n) (0xFF0 + (n) * 0x4) /* Management Component IDn Register [n=0-4] */ + +#endif /* !_ARM64_CORESIGHT_ETM4X_H_ */ Index: sys/arm64/coresight/coresight-etm4x.c =================================================================== --- /dev/null +++ sys/arm64/coresight/coresight-etm4x.c @@ -0,0 +1,314 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +/* + * Typical trace flow: + * + * CPU0 -> ETM0 -> funnel1 -> funnel0 -> TMC(ETF) -> replicator -> TMC(ETR) -> DRAM + * CPU1 -> ETM1 -> funnel1 -^ + * CPU2 -> ETM2 -> funnel1 -^ + * CPU3 -> ETM3 -> funnel1 -^ + */ + +static struct ofw_compat_data compat_data[] = { + { "arm,coresight-etm4x", 1 }, + { NULL, 0 } +}; + +struct etm_softc { + struct resource *res; + struct coresight_platform_data *pdata; +}; + +static struct resource_spec etm_spec[] = { + { SYS_RES_MEMORY, 0, RF_ACTIVE }, + { -1, 0 } +}; + +static void +etm_print_version(struct etm_softc *sc) +{ + uint32_t reg; + + /* Unlocking Coresight */ + bus_write_4(sc->res, CORESIGHT_LAR, CORESIGHT_UNLOCK); + + isb(); + + /* Unlocking ETM */ + bus_write_4(sc->res, TRCOSLAR, 0); + + isb(); + + return; + + reg = bus_read_4(sc->res, TRCIDR(1)); + printf("ETM Version: %d.%d\n", + (reg & TRCIDR1_TRCARCHMAJ_M) >> TRCIDR1_TRCARCHMAJ_S, + (reg & TRCIDR1_TRCARCHMIN_M) >> TRCIDR1_TRCARCHMIN_S); +} + +static int +etm_start(device_t dev) +{ + struct etm_softc *sc; + uint32_t reg; + + sc = device_get_softc(dev); + + /* Enable the trace unit */ + bus_write_4(sc->res, TRCPRGCTLR, 1); + + /* Wait for an IDLE bit to be LOW */ + do { + reg = bus_read_4(sc->res, TRCSTATR); + } while ((reg & TRCSTATR_IDLE) == 1); + + if ((bus_read_4(sc->res, TRCPRGCTLR) & 1) == 0) + panic("etm is not enabled\n"); + + return (0); +} + +static int +etm_stop(device_t dev) +{ + struct etm_softc *sc; + uint32_t reg; + + sc = device_get_softc(dev); + + /* Disable the trace unit */ + bus_write_4(sc->res, TRCPRGCTLR, 0); + + /* Wait for an IDLE bit */ + do { + reg = bus_read_4(sc->res, TRCSTATR); + } while ((reg & TRCSTATR_IDLE) == 0); + + return (0); +} + +static int +etm_prepare(struct coresight_device *out, struct coresight_event *config) +{ + struct etm_softc *sc; + uint32_t reg; + + sc = device_get_softc(out->dev); + + etm_print_version(sc); + + /* Configure ETM */ + + /* Enable the return stack, global timestamping, Context ID, and Virtual context identifier tracing. */ + reg = TRCCONFIGR_RS | TRCCONFIGR_TS | TRCCONFIGR_CID | TRCCONFIGR_VMID; + reg |= TRCCONFIGR_INSTP0_LDRSTR; + reg |= TRCCONFIGR_COND_ALL; + bus_write_4(sc->res, TRCCONFIGR, reg); + + /* Disable all event tracing. */ + bus_write_4(sc->res, TRCEVENTCTL0R, 0); + bus_write_4(sc->res, TRCEVENTCTL1R, 0); + + /* Disable stalling, if implemented. */ + bus_write_4(sc->res, TRCSTALLCTLR, 0); + + /* Enable trace synchronization every 4096 bytes of trace. */ + bus_write_4(sc->res, TRCSYNCPR, 0xC); + + /* Set a value for the trace ID, with bit[0]=0. */ + bus_write_4(sc->res, TRCTRACEIDR, 0x10); + + /* + * Disable the timestamp event. The trace unit still generates + * timestamps due to other reasons such as trace synchronization. + */ + bus_write_4(sc->res, TRCTSCTLR, 0); + + /* Enable ViewInst to trace everything, with the start/stop logic started. */ + reg = TRCVICTLR_SSSTATUS; + + /* The number of the single resource used to activate the event. */ + reg |= (1 << EVENT_SEL_S); + + if (config->excp_level > 2) + return (-1); + + reg |= TRCVICTLR_EXLEVEL_NS_M; + reg &= ~TRCVICTLR_EXLEVEL_NS(config->excp_level); + reg |= TRCVICTLR_EXLEVEL_S_M; + reg &= ~TRCVICTLR_EXLEVEL_S(config->excp_level); + bus_write_4(sc->res, TRCVICTLR, reg); + + bus_write_4(sc->res, TRCRSCTLR(0), (5 << 16) | (1 << 0)); + + int i; + for (i = 0; i < config->naddr * 2; i++) { + printf("configure range %d, address %lx\n", i, config->addr[i]); + bus_write_8(sc->res, TRCACVR(i), config->addr[i]); + + reg = 0; + /* Secure state */ + reg |= TRCACATR_EXLEVEL_S_M; + reg &= ~TRCACATR_EXLEVEL_S(config->excp_level); + /* Non-secure state */ + reg |= TRCACATR_EXLEVEL_NS_M; + reg &= ~TRCACATR_EXLEVEL_NS(config->excp_level); + bus_write_4(sc->res, TRCACATR(i), reg); + } + + /* No address filtering for ViewData. */ + bus_write_4(sc->res, TRCVDARCCTLR, 0); + + /* Clear the STATUS bit to zero */ + bus_write_4(sc->res, TRCSSCSR(0), 0); + + /* No address range filtering for ViewInst. */ + if (config->naddr == 0) + bus_write_4(sc->res, TRCVIIECTLR, 0); + else + bus_write_4(sc->res, TRCVIIECTLR, (1 << 0)); + + /* No start or stop points for ViewInst. */ + bus_write_4(sc->res, TRCVISSCTLR, 0); + + /* Disable ViewData */ + bus_write_4(sc->res, TRCVDCTLR, 0); + + /* No address filtering for ViewData. */ + bus_write_4(sc->res, TRCVDSACCTLR, 0); + + return (0); +} + +static int +etm_enable(struct coresight_device *out, struct endpoint *endp, + struct coresight_event *event) +{ + + etm_prepare(out, event); + etm_start(out->dev); + + return (0); +} + +static void +etm_disable(struct coresight_device *out, struct endpoint *endp, + struct coresight_event *event) +{ + + etm_stop(out->dev); +} + +static struct coresight_ops ops = { + .enable = &etm_enable, + .disable = &etm_disable, +}; + +static int +etm_probe(device_t dev) +{ + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) + return (ENXIO); + + device_set_desc(dev, "AArch64 Embedded Trace Macrocell"); + + return (BUS_PROBE_DEFAULT); +} + +static int +etm_attach(device_t dev) +{ + struct coresight_desc desc; + struct etm_softc *sc; + + sc = device_get_softc(dev); + + if (bus_alloc_resources(dev, etm_spec, &sc->res) != 0) { + device_printf(dev, "cannot allocate resources for device\n"); + return (ENXIO); + } + + if (device_get_unit(dev) == 0) { + int i; + for (i = 0; i < 14; i++) + printf("TRCIDR%d: %x\n", i, bus_read_4(sc->res, TRCIDR(i))); + } + + sc->pdata = coresight_get_platform_data(dev); + + desc.pdata = sc->pdata; + desc.dev = dev; + desc.dev_type = CORESIGHT_ETMV4; + desc.ops = &ops; + coresight_register(&desc); + + return (0); +} + +static device_method_t etm_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, etm_probe), + DEVMETHOD(device_attach, etm_attach), + DEVMETHOD_END +}; + +static driver_t etm_driver = { + "etm", + etm_methods, + sizeof(struct etm_softc), +}; + +static devclass_t etm_devclass; + +DRIVER_MODULE(etm, simplebus, etm_driver, etm_devclass, 0, 0); +MODULE_VERSION(etm, 1); Index: sys/arm64/coresight/coresight-funnel.h =================================================================== --- /dev/null +++ sys/arm64/coresight/coresight-funnel.h @@ -0,0 +1,66 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARM64_CORESIGHT_CORESIGHT_FUNNEL_H_ +#define _ARM64_CORESIGHT_CORESIGHT_FUNNEL_H_ + +#define FUNNEL_FUNCTL 0x000 /* Funnel Control Register */ +#define FUNCTL_HOLDTIME_SHIFT 8 +#define FUNCTL_HOLDTIME_MASK (0xf << FUNCTL_HOLDTIME_SHIFT) +#define FUNNEL_PRICTL 0x004 /* Priority Control Register */ +#define FUNNEL_ITATBDATA0 0xEEC /* Integration Register, ITATBDATA0 */ +#define FUNNEL_ITATBCTR2 0xEF0 /* Integration Register, ITATBCTR2 */ +#define FUNNEL_ITATBCTR1 0xEF4 /* Integration Register, ITATBCTR1 */ +#define FUNNEL_ITATBCTR0 0xEF8 /* Integration Register, ITATBCTR0 */ +#define FUNNEL_IMCR 0xF00 /* Integration Mode Control Register */ +#define FUNNEL_CTSR 0xFA0 /* Claim Tag Set Register */ +#define FUNNEL_CTCR 0xFA4 /* Claim Tag Clear Register */ +#define FUNNEL_LOCKACCESS 0xFB0 /* Lock Access */ +#define FUNNEL_LOCKSTATUS 0xFB4 /* Lock Status */ +#define FUNNEL_AUTHSTATUS 0xFB8 /* Authentication status */ +#define FUNNEL_DEVICEID 0xFC8 /* Device ID */ +#define FUNNEL_DEVICETYPE 0xFCC /* Device Type Identifier */ +#define FUNNEL_PERIPH4 0xFD0 /* Peripheral ID4 */ +#define FUNNEL_PERIPH5 0xFD4 /* Peripheral ID5 */ +#define FUNNEL_PERIPH6 0xFD8 /* Peripheral ID6 */ +#define FUNNEL_PERIPH7 0xFDC /* Peripheral ID7 */ +#define FUNNEL_PERIPH0 0xFE0 /* Peripheral ID0 */ +#define FUNNEL_PERIPH1 0xFE4 /* Peripheral ID1 */ +#define FUNNEL_PERIPH2 0xFE8 /* Peripheral ID2 */ +#define FUNNEL_PERIPH3 0xFEC /* Peripheral ID3 */ +#define FUNNEL_COMP0 0xFF0 /* Component ID0 */ +#define FUNNEL_COMP1 0xFF4 /* Component ID1 */ +#define FUNNEL_COMP2 0xFF8 /* Component ID2 */ +#define FUNNEL_COMP3 0xFFC /* Component ID3 */ + +#endif /* !_ARM64_CORESIGHT_CORESIGHT_FUNNEL_H_ */ Index: sys/arm64/coresight/coresight-funnel.c =================================================================== --- /dev/null +++ sys/arm64/coresight/coresight-funnel.c @@ -0,0 +1,163 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +static struct ofw_compat_data compat_data[] = { + { "arm,coresight-funnel", 1 }, + { NULL, 0 } +}; + +struct funnel_softc { + struct resource *res; + struct coresight_platform_data *pdata; +}; + +static struct resource_spec funnel_spec[] = { + { SYS_RES_MEMORY, 0, RF_ACTIVE }, + { -1, 0 } +}; + +static int +funnel_enable(struct coresight_device *out, struct endpoint *endp, + struct coresight_event *event) +{ + struct funnel_softc *sc; + uint32_t reg; + + sc = device_get_softc(out->dev); + + reg = bus_read_4(sc->res, FUNNEL_FUNCTL); + reg &= ~(FUNCTL_HOLDTIME_MASK); + reg |= (7 << FUNCTL_HOLDTIME_SHIFT); + reg |= (1 << endp->reg); + bus_write_4(sc->res, FUNNEL_FUNCTL, reg); + + return (0); +} + +static void +funnel_disable(struct coresight_device *out, struct endpoint *endp, + struct coresight_event *event) +{ + struct funnel_softc *sc; + uint32_t reg; + + sc = device_get_softc(out->dev); + + reg = bus_read_4(sc->res, FUNNEL_FUNCTL); + reg &= ~(1 << endp->reg); + bus_write_4(sc->res, FUNNEL_FUNCTL, reg); +} + +static struct coresight_ops ops = { + .enable = &funnel_enable, + .disable = &funnel_disable, +}; + +static int +funnel_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) + return (ENXIO); + + device_set_desc(dev, "Coresight Funnel"); + + return (BUS_PROBE_DEFAULT); +} + +static int +funnel_attach(device_t dev) +{ + struct coresight_desc desc; + struct funnel_softc *sc; + + sc = device_get_softc(dev); + + if (bus_alloc_resources(dev, funnel_spec, &sc->res) != 0) { + device_printf(dev, "cannot allocate resources for device\n"); + return (ENXIO); + } + + sc->pdata = coresight_get_platform_data(dev); + + desc.pdata = sc->pdata; + desc.dev = dev; + desc.dev_type = CORESIGHT_FUNNEL; + desc.ops = &ops; + coresight_register(&desc); + + /* Unlock Coresight */ + bus_write_4(sc->res, CORESIGHT_LAR, CORESIGHT_UNLOCK); + + wmb(); + + printf("Device ID: %x\n", bus_read_4(sc->res, FUNNEL_DEVICEID)); + + return (0); +} + +static device_method_t funnel_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, funnel_probe), + DEVMETHOD(device_attach, funnel_attach), + DEVMETHOD_END +}; + +static driver_t funnel_driver = { + "funnel", + funnel_methods, + sizeof(struct funnel_softc), +}; + +static devclass_t funnel_devclass; + +DRIVER_MODULE(funnel, simplebus, funnel_driver, funnel_devclass, 0, 0); +MODULE_VERSION(funnel, 1); Index: sys/arm64/coresight/coresight-tmc.h =================================================================== --- /dev/null +++ sys/arm64/coresight/coresight-tmc.h @@ -0,0 +1,117 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 + * ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _ARM64_CORESIGHT_CORESIGHT_TMC_H_ +#define _ARM64_CORESIGHT_CORESIGHT_TMC_H_ + +#define TMC_RSZ 0x004 /* RAM Size Register */ +#define TMC_STS 0x00C /* Status Register */ +#define STS_MEMERR (1 << 5) +#define STS_EMPTY (1 << 4) +#define STS_FTEMPTY (1 << 3) +#define STS_TMCREADY (1 << 2) +#define STS_TRIGGERED (1 << 1) +#define STS_FULL (1 << 0) +#define TMC_RRD 0x010 /* RAM Read Data Register */ +#define TMC_RRP 0x014 /* RAM Read Pointer Register */ +#define TMC_RWP 0x018 /* RAM Write Pointer Register */ +#define TMC_TRG 0x01C /* Trigger Counter Register */ +#define TMC_CTL 0x020 /* Control Register */ +#define CTL_TRACECAPTEN (1 << 0) /* Controls trace capture. */ +#define TMC_RWD 0x024 /* RAM Write Data Register */ +#define TMC_MODE 0x028 /* Mode Register */ +#define MODE_HW_FIFO 2 +#define MODE_SW_FIFO 1 +#define MODE_CIRCULAR_BUFFER 0 +#define TMC_LBUFLEVEL 0x02C /* Latched Buffer Fill Level */ +#define TMC_CBUFLEVEL 0x030 /* Current Buffer Fill Level */ +#define TMC_BUFWM 0x034 /* Buffer Level Water Mark */ +#define TMC_RRPHI 0x038 /* RAM Read Pointer High Register */ +#define TMC_RWPHI 0x03C /* RAM Write Pointer High Register */ +#define TMC_AXICTL 0x110 /* AXI Control Register */ +#define AXICTL_WRBURSTLEN_S 8 +#define AXICTL_WRBURSTLEN_M (0xf << AXICTL_WRBURSTLEN_S) +#define AXICTL_WRBURSTLEN_16 (0xf << AXICTL_WRBURSTLEN_S) +#define AXICTL_SG_MODE (1 << 7) /* Scatter Gather Mode */ +#define AXICTL_CACHE_CTRL_BIT3 (1 << 5) +#define AXICTL_CACHE_CTRL_BIT2 (1 << 4) +#define AXICTL_CACHE_CTRL_BIT1 (1 << 3) +#define AXICTL_CACHE_CTRL_BIT0 (1 << 2) +#define AXICTL_AXCACHE_OS (0xf << 2) +#define AXICTL_PROT_CTRL_BIT1 (1 << 1) +#define AXICTL_PROT_CTRL_BIT0 (1 << 0) +#define TMC_DBALO 0x118 /* Data Buffer Address Low Register */ +#define TMC_DBAHI 0x11C /* Data Buffer Address High Register */ +#define TMC_FFSR 0x300 /* Formatter and Flush Status Register */ +#define TMC_FFCR 0x304 /* Formatter and Flush Control Register */ +#define FFCR_EN_FMT (1 << 0) +#define FFCR_EN_TI (1 << 1) +#define FFCR_FON_FLIN (1 << 4) +#define FFCR_FON_TRIG_EVT (1 << 5) +#define FFCR_FLUSH_MAN (1 << 6) +#define FFCR_TRIGON_TRIGIN (1 << 8) +#define TMC_PSCR 0x308 /* Periodic Synchronization Counter Register */ +#define TMC_ITATBMDATA0 0xED0 /* Integration Test ATB Master Data Register 0 */ +#define TMC_ITATBMCTR2 0xED4 /* Integration Test ATB Master Interface Control 2 Register */ +#define TMC_ITATBMCTR1 0xED8 /* Integration Test ATB Master Control Register 1 */ +#define TMC_ITATBMCTR0 0xEDC /* Integration Test ATB Master Interface Control 0 Register */ +#define TMC_ITMISCOP0 0xEE0 /* Integration Test Miscellaneous Output Register 0 */ +#define TMC_ITTRFLIN 0xEE8 /* Integration Test Trigger In and Flush In Register */ +#define TMC_ITATBDATA0 0xEEC /* Integration Test ATB Data Register 0 */ +#define TMC_ITATBCTR2 0xEF0 /* Integration Test ATB Control 2 Register */ +#define TMC_ITATBCTR1 0xEF4 /* Integration Test ATB Control 1 Register */ +#define TMC_ITATBCTR0 0xEF8 /* Integration Test ATB Control 0 Register */ +#define TMC_ITCTRL 0xF00 /* Integration Mode Control Register */ +#define TMC_CLAIMSET 0xFA0 /* Claim Tag Set Register */ +#define TMC_CLAIMCLR 0xFA4 /* Claim Tag Clear Register */ +#define TMC_LAR 0xFB0 /* Lock Access Register */ +#define TMC_LSR 0xFB4 /* Lock Status Register */ +#define TMC_AUTHSTATUS 0xFB8 /* Authentication Status Register */ +#define TMC_DEVID 0xFC8 /* Device Configuration Register */ +#define DEVID_CONFIGTYPE_S 6 +#define DEVID_CONFIGTYPE_M (0x3 << DEVID_CONFIGTYPE_S) +#define DEVID_CONFIGTYPE_ETB (0 << DEVID_CONFIGTYPE_S) +#define DEVID_CONFIGTYPE_ETR (1 << DEVID_CONFIGTYPE_S) +#define DEVID_CONFIGTYPE_ETF (2 << DEVID_CONFIGTYPE_S) +#define TMC_DEVTYPE 0xFCC /* Device Type Identifier Register */ +#define TMC_PERIPHID4 0xFD0 /* Peripheral ID4 Register */ +#define TMC_PERIPHID5 0xFD4 /* Peripheral ID5 Register */ +#define TMC_PERIPHID6 0xFD8 /* Peripheral ID6 Register */ +#define TMC_PERIPHID7 0xFDC /* Peripheral ID7 Register */ +#define TMC_PERIPHID0 0xFE0 /* Peripheral ID0 Register */ +#define TMC_PERIPHID1 0xFE4 /* Peripheral ID1 Register */ +#define TMC_PERIPHID2 0xFE8 /* Peripheral ID2 Register */ +#define TMC_PERIPHID3 0xFEC /* Peripheral ID3 Register */ +#define TMC_COMPID0 0xFF0 /* Component ID0 Register */ +#define TMC_COMPID1 0xFF4 /* Component ID1 Register */ +#define TMC_COMPID2 0xFF8 /* Component ID2 Register */ +#define TMC_COMPID3 0xFFC /* Component ID3 Register */ + +#endif /* !_ARM64_CORESIGHT_CORESIGHT_TMC_H_ */ Index: sys/arm64/coresight/coresight-tmc.c =================================================================== --- /dev/null +++ sys/arm64/coresight/coresight-tmc.c @@ -0,0 +1,349 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 + * ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +static struct ofw_compat_data compat_data[] = { + { "arm,coresight-tmc", 1 }, + { NULL, 0 } +}; + +struct tmc_softc { + struct resource *res; + device_t dev; + uint64_t cycle; + struct coresight_platform_data *pdata; +}; + +static struct resource_spec tmc_spec[] = { + { SYS_RES_MEMORY, 0, RF_ACTIVE }, + { -1, 0 } +}; + +static int +tmc_unlock(struct tmc_softc *sc) +{ + + /* Unlock Coresight */ + bus_write_4(sc->res, CORESIGHT_LAR, CORESIGHT_UNLOCK); + wmb(); + + /* Unlock TMC */ + bus_write_4(sc->res, TMC_LAR, CORESIGHT_UNLOCK); + wmb(); + + return (0); +} + +static int +tmc_start(device_t dev) +{ + struct tmc_softc *sc; + uint32_t reg; + + sc = device_get_softc(dev); + + if (bus_read_4(sc->res, TMC_CTL) & CTL_TRACECAPTEN) + return (-1); + + /* Enable TMC */ + bus_write_4(sc->res, TMC_CTL, CTL_TRACECAPTEN); + if ((bus_read_4(sc->res, TMC_CTL) & CTL_TRACECAPTEN) == 0) + panic("not enabled0\n"); + + do { + reg = bus_read_4(sc->res, TMC_STS); + } while ((reg & STS_TMCREADY) == 1); + + if ((bus_read_4(sc->res, TMC_CTL) & CTL_TRACECAPTEN) == 0) + panic("not enabled1\n"); + + return (0); +} + +static int +tmc_stop(device_t dev) +{ + struct tmc_softc *sc; + uint32_t reg; + + sc = device_get_softc(dev); + + reg = bus_read_4(sc->res, TMC_CTL); + reg &= ~CTL_TRACECAPTEN; + bus_write_4(sc->res, TMC_CTL, reg); + + do { + reg = bus_read_4(sc->res, TMC_STS); + } while ((reg & STS_TMCREADY) == 1); + + return (0); +} + +static int +tmc_read(struct coresight_device *out, struct endpoint *endp, + struct coresight_event *event) +{ + struct tmc_softc *sc; + uint32_t cur_ptr; + + sc = device_get_softc(out->dev); + + if (bus_read_4(sc->res, TMC_STS) & STS_FULL) { + event->etr.offset = 0; + event->etr.cycle++; + tmc_stop(out->dev); + tmc_start(out->dev); + } else { + cur_ptr = bus_read_4(sc->res, TMC_RWP); + event->etr.offset = (cur_ptr - event->etr.low); + } + + return (0); +} + +static int +tmc_configure_etf(device_t dev) +{ + struct tmc_softc *sc; + uint32_t reg; + + sc = device_get_softc(dev); + + tmc_unlock(sc); + + do { + reg = bus_read_4(sc->res, TMC_STS); + } while ((reg & STS_TMCREADY) == 0); + + bus_write_4(sc->res, TMC_MODE, MODE_HW_FIFO); + bus_write_4(sc->res, TMC_FFCR, FFCR_EN_FMT | FFCR_EN_TI); + bus_write_4(sc->res, TMC_BUFWM, 0x800-1); + + tmc_start(dev); + + printf("%s: STS %x, CTL %x, RSZ %x, RRP %x, RWP %x, LBUFLEVEL %x, CBUFLEVEL %x, \n", __func__, + bus_read_4(sc->res, TMC_STS), + bus_read_4(sc->res, TMC_CTL), + bus_read_4(sc->res, TMC_RSZ), + bus_read_4(sc->res, TMC_RRP), + bus_read_4(sc->res, TMC_RWP), + bus_read_4(sc->res, TMC_CBUFLEVEL), + bus_read_4(sc->res, TMC_LBUFLEVEL)); + + return (0); +} + +static int +tmc_configure_etr(struct coresight_device *out, struct endpoint *endp, + struct coresight_event *event) +{ + struct tmc_softc *sc; + uint32_t reg; + + sc = device_get_softc(out->dev); + + tmc_unlock(sc); + tmc_stop(out->dev); + + do { + reg = bus_read_4(sc->res, TMC_STS); + } while ((reg & STS_TMCREADY) == 0); + + /* Configure TMC */ + bus_write_4(sc->res, TMC_MODE, MODE_CIRCULAR_BUFFER); + + reg = AXICTL_PROT_CTRL_BIT1; + reg |= AXICTL_WRBURSTLEN_16; + + /* + * SG operation is broken on DragonBoard 410c + * reg |= AXICTL_SG_MODE; + */ + + reg |= AXICTL_AXCACHE_OS; + bus_write_4(sc->res, TMC_AXICTL, reg); + + reg = FFCR_EN_FMT | FFCR_EN_TI | FFCR_FON_FLIN | + FFCR_FON_TRIG_EVT | FFCR_TRIGON_TRIGIN; + bus_write_4(sc->res, TMC_FFCR, reg); + + bus_write_4(sc->res, TMC_TRG, 8); + + bus_write_4(sc->res, TMC_DBALO, event->etr.low); + bus_write_4(sc->res, TMC_DBAHI, event->etr.high); + bus_write_4(sc->res, TMC_RSZ, event->etr.bufsize / 4); + + bus_write_4(sc->res, TMC_RRP, event->etr.low); + bus_write_4(sc->res, TMC_RWP, event->etr.low); + + reg = bus_read_4(sc->res, TMC_STS); + reg &= ~STS_FULL; + bus_write_4(sc->res, TMC_STS, reg); + + tmc_start(out->dev); + + return (0); +} + +static int +tmc_enable(struct coresight_device *out, struct endpoint *endp, + struct coresight_event *event) +{ + struct tmc_softc *sc; + + sc = device_get_softc(out->dev); + + /* ETF configuration is static */ + switch (out->dev_type) { + case CORESIGHT_ETF: + return (0); + case CORESIGHT_ETR: + if (event->etr.started) + return (0); + tmc_unlock(sc); + tmc_stop(out->dev); + tmc_configure_etr(out, endp, event); + tmc_start(out->dev); + event->etr.started = 1; + default: + break; + } + + return (0); +} + +static void +tmc_disable(struct coresight_device *out, struct endpoint *endp, + struct coresight_event *event) +{ + + /* + * Can't restore the state: can't specify buffer offset + * to continue operation from. So we do not disable TMC here. + */ +} + +static struct coresight_ops ops = { + .read = &tmc_read, + .enable = &tmc_enable, + .disable = &tmc_disable, +}; + +static int +tmc_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) + return (ENXIO); + + device_set_desc(dev, "Coresight Trace Memory Controller (TMC)"); + + return (BUS_PROBE_DEFAULT); +} + +static int +tmc_attach(device_t dev) +{ + struct coresight_desc desc; + struct tmc_softc *sc; + uint32_t reg; + + sc = device_get_softc(dev); + + sc->dev = dev; + + if (bus_alloc_resources(dev, tmc_spec, &sc->res) != 0) { + device_printf(dev, "cannot allocate resources for device\n"); + return (ENXIO); + } + + sc->pdata = coresight_get_platform_data(dev); + + desc.pdata = sc->pdata; + desc.dev = dev; + desc.ops = &ops; + + reg = bus_read_4(sc->res, TMC_DEVID); + reg &= DEVID_CONFIGTYPE_M; + switch (reg) { + case DEVID_CONFIGTYPE_ETR: + desc.dev_type = CORESIGHT_ETR; + coresight_register(&desc); + device_printf(dev, "ETR configuration found\n"); + break; + case DEVID_CONFIGTYPE_ETF: + desc.dev_type = CORESIGHT_ETF; + coresight_register(&desc); + tmc_configure_etf(dev); + device_printf(dev, "ETF configuration found\n"); + break; + default: + break; + } + + return (0); +} + +static device_method_t tmc_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, tmc_probe), + DEVMETHOD(device_attach, tmc_attach), + DEVMETHOD_END +}; + +static driver_t tmc_driver = { + "tmc", + tmc_methods, + sizeof(struct tmc_softc), +}; + +static devclass_t tmc_devclass; + +DRIVER_MODULE(tmc, simplebus, tmc_driver, tmc_devclass, 0, 0); +MODULE_VERSION(tmc, 1); Index: sys/arm64/coresight/coresight.h =================================================================== --- /dev/null +++ sys/arm64/coresight/coresight.h @@ -0,0 +1,137 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 + * ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARM64_CORESIGHT_CORESIGHT_H_ +#define _ARM64_CORESIGHT_CORESIGHT_H_ + +#include + +#define CORESIGHT_ITCTRL 0xf00 +#define CORESIGHT_CLAIMSET 0xfa0 +#define CORESIGHT_CLAIMCLR 0xfa4 +#define CORESIGHT_LAR 0xfb0 +#define CORESIGHT_UNLOCK 0xc5acce55 +#define CORESIGHT_LSR 0xfb4 +#define CORESIGHT_AUTHSTATUS 0xfb8 +#define CORESIGHT_DEVID 0xfc8 +#define CORESIGHT_DEVTYPE 0xfcc + +enum cs_dev_type { + CORESIGHT_ETMV4, + CORESIGHT_ETR, + CORESIGHT_ETF, + CORESIGHT_DYNAMIC_REPLICATOR, + CORESIGHT_FUNNEL, +}; + +struct coresight_device { + TAILQ_ENTRY(coresight_device) link; + device_t dev; + phandle_t node; + enum cs_dev_type dev_type; + struct coresight_ops *ops; + struct coresight_platform_data *pdata; +}; + +struct endpoint { + TAILQ_ENTRY(endpoint) link; + phandle_t my_node; + phandle_t their_node; + phandle_t dev_node; + boolean_t slave; + int reg; + struct coresight_device *cs_dev; + LIST_ENTRY(endpoint) endplink; +}; + +struct coresight_platform_data { + int cpu; + int in_ports; + int out_ports; + struct mtx mtx_lock; + TAILQ_HEAD(endpoint_list, endpoint) endpoints; +}; + +struct coresight_desc { + struct coresight_platform_data *pdata; + device_t dev; + enum cs_dev_type dev_type; + struct coresight_ops *ops; +}; + +TAILQ_HEAD(coresight_device_list, coresight_device); + +#define ETM_N_COMPRATOR 16 + +struct etr_status { + boolean_t started; + uint32_t cycle; + uint32_t offset; + uint32_t low; + uint32_t high; + uint32_t bufsize; +}; + +struct coresight_event { + LIST_HEAD(, endpoint) endplist; + + uint64_t addr[ETM_N_COMPRATOR]; + uint32_t naddr; + uint8_t excp_level; + enum cs_dev_type src; + enum cs_dev_type sink; + + struct etr_status etr; +}; + +struct coresight_ops { + int (*read)(struct coresight_device *out, struct endpoint *endp, struct coresight_event *event); + int (*enable)(struct coresight_device *out, struct endpoint *endp, struct coresight_event *event); + void (*disable)(struct coresight_device *out, struct endpoint *endp, struct coresight_event *event); +}; + +struct etm_config { + uint64_t addr[ETM_N_COMPRATOR]; + uint32_t naddr; + uint8_t excp_level; +}; + +struct coresight_platform_data * coresight_get_platform_data(device_t dev); +struct endpoint * coresight_get_output_endpoint(struct coresight_platform_data *pdata); +struct coresight_device * coresight_get_output_device(struct endpoint *endp, struct endpoint **); +int coresight_register(struct coresight_desc *desc); +int coresight_init_event(int cpu, struct coresight_event *event); +void coresight_enable(int cpu, struct coresight_event *event); +void coresight_disable(int cpu, struct coresight_event *event); +void coresight_read(int cpu, struct coresight_event *event); + +#endif /* !_ARM64_CORESIGHT_CORESIGHT_H_ */ Index: sys/arm64/coresight/coresight.c =================================================================== --- /dev/null +++ sys/arm64/coresight/coresight.c @@ -0,0 +1,216 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 + * ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +MALLOC_DEFINE(M_CORESIGHT, "coresight", "ARM Coresight"); +static struct mtx cs_mtx; + +struct coresight_device_list cs_devs; + +static int +coresight_get_ports(phandle_t dev_node, + struct coresight_platform_data *pdata) +{ + phandle_t node, child; + pcell_t port_reg; + phandle_t xref; + char *name; + int ret; + phandle_t endpoint_child; + struct endpoint *endp; + + child = ofw_bus_find_child(dev_node, "ports"); + if (child) + node = child; + else + node = dev_node; + + for (child = OF_child(node); child != 0; child = OF_peer(child)) { + ret = OF_getprop_alloc(child, "name", sizeof(*name), (void **)&name); + if (ret == -1) + continue; + + if (strcasecmp(name, "port") || + strncasecmp(name, "port@", 6)) { + + port_reg = -1; + OF_getencprop(child, "reg", (void *)&port_reg, sizeof(port_reg)); + + endpoint_child = ofw_bus_find_child(child, "endpoint"); + if (endpoint_child) { + printf("endpoint found\n"); + if (OF_getencprop(endpoint_child, "remote-endpoint", &xref, + sizeof(xref)) == -1) { + printf("failed\n"); + continue; + } + endp = malloc(sizeof(struct endpoint), M_CORESIGHT, + M_WAITOK | M_ZERO); + endp->my_node = endpoint_child; + endp->their_node = OF_node_from_xref(xref); + endp->dev_node = dev_node; + endp->reg = port_reg; + if (OF_getproplen(endpoint_child, "slave-mode") >= 0) { + pdata->in_ports++; + endp->slave = 1; + } else { + pdata->out_ports++; + } + + mtx_lock(&pdata->mtx_lock); + TAILQ_INSERT_TAIL(&pdata->endpoints, endp, link); + mtx_unlock(&pdata->mtx_lock); + } + } + } + + return (0); +} + +int +coresight_register(struct coresight_desc *desc) +{ + struct coresight_device *cs_dev; + + cs_dev = malloc(sizeof(struct coresight_device), M_CORESIGHT, M_WAITOK | M_ZERO); + cs_dev->dev = desc->dev; + cs_dev->node = ofw_bus_get_node(desc->dev); + cs_dev->pdata = desc->pdata; + cs_dev->dev_type = desc->dev_type; + cs_dev->ops = desc->ops; + + mtx_lock(&cs_mtx); + TAILQ_INSERT_TAIL(&cs_devs, cs_dev, link); + mtx_unlock(&cs_mtx); + + return (0); +} + +struct endpoint * +coresight_get_output_endpoint(struct coresight_platform_data *pdata) +{ + struct endpoint *endp; + + if (pdata->out_ports != 1) { + return (NULL); + } + + TAILQ_FOREACH(endp, &pdata->endpoints, link) { + if (endp->slave == 0) + return (endp); + } + + return (NULL); +} + +struct coresight_device * +coresight_get_output_device(struct endpoint *endp, struct endpoint **out_endp) +{ + struct coresight_device *cs_dev; + struct endpoint *endp2; + + TAILQ_FOREACH(cs_dev, &cs_devs, link) { + TAILQ_FOREACH(endp2, &cs_dev->pdata->endpoints, link) { + if (endp->their_node == endp2->my_node) { + *out_endp = endp2; + return (cs_dev); + } + } + } + + return (NULL); +} + +static int +coresight_get_cpu(phandle_t node, + struct coresight_platform_data *pdata) +{ + phandle_t cpu_node; + pcell_t xref; + pcell_t cpu_reg; + + if (OF_getencprop(node, "cpu", &xref, sizeof(xref)) != -1) { + cpu_node = OF_node_from_xref(xref); + if (OF_getencprop(cpu_node, "reg", (void *)&cpu_reg, + sizeof(cpu_reg)) > 0) { + pdata->cpu = cpu_reg; + + return (0); + } + } + + return (-1); +} + +struct coresight_platform_data * +coresight_get_platform_data(device_t dev) +{ + struct coresight_platform_data *pdata; + phandle_t node; + + node = ofw_bus_get_node(dev); + + pdata = malloc(sizeof(struct coresight_platform_data), + M_CORESIGHT, M_WAITOK | M_ZERO); + mtx_init(&pdata->mtx_lock, "Coresight Platform Data", NULL, MTX_DEF); + TAILQ_INIT(&pdata->endpoints); + + coresight_get_cpu(node, pdata); + coresight_get_ports(node, pdata); + + printf("Total ports: in %d out %d\n", pdata->in_ports, pdata->out_ports); + + return (pdata); +} + +static void +coresight_init(void) +{ + + mtx_init(&cs_mtx, "ARM Coresight", NULL, MTX_DEF); + TAILQ_INIT(&cs_devs); +} + +SYSINIT(coresight, SI_SUB_DRIVERS, SI_ORDER_FIRST, coresight_init, NULL); Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -1800,6 +1800,7 @@ dev/hptiop/hptiop.c optional hptiop scbus dev/hwpmc/hwpmc_logging.c optional hwpmc dev/hwpmc/hwpmc_mod.c optional hwpmc +dev/hwpmc/hwpmc_vm.c optional hwpmc dev/hwpmc/hwpmc_soft.c optional hwpmc dev/ichiic/ig4_acpi.c optional ig4 acpi iicbus dev/ichiic/ig4_iic.c optional ig4 iicbus Index: sys/dev/hwpmc/hwpmc_arm64.c =================================================================== --- sys/dev/hwpmc/hwpmc_arm64.c +++ sys/dev/hwpmc/hwpmc_arm64.c @@ -480,6 +480,7 @@ struct pmc_mdep *pmc_mdep; struct pmc_classdep *pcd; int idcode; + int ncpus; int reg; reg = arm64_pmcr_read(); @@ -495,8 +496,8 @@ arm64_pcpu = malloc(sizeof(struct arm64_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK | M_ZERO); - /* Just one class */ - pmc_mdep = pmc_mdep_alloc(1); + /* CPU counters and ETM */ + pmc_mdep = pmc_mdep_alloc(2); switch (idcode) { case PMCR_IDCODE_CORTEX_A57: @@ -534,6 +535,9 @@ pmc_mdep->pmd_npmc += arm64_npmcs; + ncpus = pmc_cpu_max(); + pmc_coresight_initialize(pmc_mdep, ncpus); + return (pmc_mdep); } @@ -541,4 +545,5 @@ pmc_arm64_finalize(struct pmc_mdep *md) { + pmc_coresight_finalize(md); } Index: sys/dev/hwpmc/hwpmc_core.c =================================================================== --- sys/dev/hwpmc/hwpmc_core.c +++ sys/dev/hwpmc/hwpmc_core.c @@ -2287,6 +2287,7 @@ break; case PMC_CPU_INTEL_SKYLAKE: case PMC_CPU_INTEL_SKYLAKE_XEON: + case PMC_CPU_INTEL_KABYLAKE: case PMC_CPU_INTEL_BROADWELL: case PMC_CPU_INTEL_BROADWELL_XEON: case PMC_CPU_INTEL_SANDYBRIDGE: @@ -2325,6 +2326,7 @@ cpuflag = IAP_F_SLX; break; case PMC_CPU_INTEL_SKYLAKE: + case PMC_CPU_INTEL_KABYLAKE: cpuflag = IAP_F_SL; break; case PMC_CPU_INTEL_BROADWELL_XEON: @@ -2846,6 +2848,12 @@ struct core_cpu *cc; pmc_value_t v; + error = pmc_pt_intr(cpu, tf); + if (error) { + /* Found */ + return (1); + } + PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); Index: sys/dev/hwpmc/hwpmc_cs.h =================================================================== --- /dev/null +++ sys/dev/hwpmc/hwpmc_cs.h @@ -0,0 +1,71 @@ +/*- + * Copyright (c) 2017 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _DEV_HWPMC_CS_H_ +#define _DEV_HWPMC_CS_H_ + +#include +#include +#include + +#include + +#define CORESIGHT_NADDR 4 +#define CORESIGHT_NPMCS 1 + +struct pmc_md_coresight_op_pmcallocate { + uint32_t flags; + uint64_t ranges[2 * CORESIGHT_NADDR]; + int nranges; +}; + +#ifdef _KERNEL +struct coresight_buffer { + uint64_t cycle; + uint64_t phys_base; + vm_object_t obj; +}; + +/* MD extension for 'struct pmc' */ +struct pmc_md_coresight_pmc { + struct coresight_buffer coresight_buffers[MAXCPU]; +}; + +/* + * Prototypes. + */ + +int pmc_coresight_initialize(struct pmc_mdep *_md, int _maxcpu); +void pmc_coresight_finalize(struct pmc_mdep *_md); +int pmc_coresight_intr(int cpu, struct trapframe *tf); + +#endif /* !_KERNEL */ +#endif /* !_DEV_HWPMC_CS_H_ */ Index: sys/dev/hwpmc/hwpmc_cs.c =================================================================== --- /dev/null +++ sys/dev/hwpmc/hwpmc_cs.c @@ -0,0 +1,696 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +static MALLOC_DEFINE(M_CORESIGHT, "coresight", "CORESIGHT driver"); + +extern struct cdev *pmc_cdev[MAXCPU]; + +/* + * ARM CORESIGHT support. + */ + +#define CORESIGHT_CAPS (PMC_CAP_READ | PMC_CAP_INTERRUPT | PMC_CAP_SYSTEM | PMC_CAP_USER) + +#define PMC_CORESIGHT_DEBUG +#undef PMC_CORESIGHT_DEBUG + +#ifdef PMC_CORESIGHT_DEBUG +#define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define dprintf(fmt, ...) +#endif + +struct coresight_descr { + struct pmc_descr pm_descr; /* "base class" */ +}; + +static struct coresight_descr coresight_pmcdesc[CORESIGHT_NPMCS] = +{ + { + .pm_descr = + { + .pd_name = "CORESIGHT", + .pd_class = PMC_CLASS_CORESIGHT, + .pd_caps = CORESIGHT_CAPS, + .pd_width = 64 + } + } +}; + +/* + * Per-CPU data structure for PTs. + */ + +struct coresight_cpu { + struct pmc_hw tc_hw; + uint32_t l0_eax; + uint32_t l0_ebx; + uint32_t l0_ecx; + uint32_t l1_eax; + uint32_t l1_ebx; + struct pmc *pm_mmap; + uint32_t flags; +#define FLAG_CORESIGHT_ALLOCATED (1 << 0) + struct coresight_event event; +}; + +static struct coresight_cpu **coresight_pcpu; + +static int +coresight_buffer_allocate(uint32_t cpu, struct coresight_buffer *coresight_buf, + uint32_t bufsize) +{ + struct pmc_vm_map *map; + struct coresight_cpu *coresight_pc; + uint64_t phys_base; + struct cdev_cpu *cc; + vm_object_t obj; + vm_page_t m; + int npages; + int i; + + dprintf("%s\n", __func__); + + coresight_pc = coresight_pcpu[cpu]; + + coresight_buf->obj = obj = vm_pager_allocate(OBJT_PHYS, 0, bufsize, + PROT_READ, 0, curthread->td_ucred); + + npages = bufsize / PAGE_SIZE; + + VM_OBJECT_WLOCK(obj); + vm_object_reference_locked(obj); + m = vm_page_alloc_contig(obj, 0, VM_ALLOC_NOBUSY | VM_ALLOC_ZERO, + npages, 0, ~0, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); + if (m == NULL) { + VM_OBJECT_WUNLOCK(obj); + printf("%s: Can't allocate memory.\n", __func__); + vm_object_deallocate(obj); + return (-1); + } + for (i = 0; i < npages; i++) + m[i].valid = VM_PAGE_BITS_ALL; + phys_base = VM_PAGE_TO_PHYS(m); + VM_OBJECT_WUNLOCK(obj); + + map = malloc(sizeof(struct pmc_vm_map), M_CORESIGHT, M_WAITOK | M_ZERO); + map->t = curthread; + map->obj = obj; + map->buf = (void *)coresight_buf; + + cc = pmc_cdev[cpu]->si_drv1; + + mtx_lock(&cc->vm_mtx); + TAILQ_INSERT_HEAD(&cc->pmc_maplist, map, map_next); + mtx_unlock(&cc->vm_mtx); + + coresight_buf->phys_base = phys_base; + coresight_buf->cycle = 0; + + return (0); +} + +static int +coresight_buffer_deallocate(uint32_t cpu, struct coresight_buffer *coresight_buf) +{ + struct pmc_vm_map *map, *map_tmp; + struct cdev_cpu *cc; + + cc = pmc_cdev[cpu]->si_drv1; + + dprintf("%s\n", __func__); + + mtx_lock(&cc->vm_mtx); + TAILQ_FOREACH_SAFE(map, &cc->pmc_maplist, map_next, map_tmp) { + if (map->buf == (void *)coresight_buf) { + TAILQ_REMOVE(&cc->pmc_maplist, map, map_next); + free(map, M_CORESIGHT); + break; + } + } + mtx_unlock(&cc->vm_mtx); + + vm_object_deallocate(coresight_buf->obj); + + return (0); +} + +static int +coresight_buffer_prepare(uint32_t cpu, struct pmc *pm, + const struct pmc_op_pmcallocate *a) +{ + const struct pmc_md_coresight_op_pmcallocate *pm_coresighta; + struct coresight_cpu *coresight_pc; + struct pmc_md_coresight_pmc *pm_coresight; + struct coresight_buffer *coresight_buf; + uint32_t bufsize; + enum pmc_mode mode; + uint32_t phys_lo; + uint32_t phys_hi; + int error; + struct coresight_event *event; + + coresight_pc = coresight_pcpu[cpu]; + event = &coresight_pc->event; + + pm_coresighta = (const struct pmc_md_coresight_op_pmcallocate *)&a->pm_md.pm_coresight; + pm_coresight = (struct pmc_md_coresight_pmc *)&pm->pm_md; + coresight_buf = &pm_coresight->coresight_buffers[cpu]; + + bufsize = 16 * 1024 * 1024; + error = coresight_buffer_allocate(cpu, coresight_buf, bufsize); + if (error != 0) { + dprintf("%s: can't allocate buffers\n", __func__); + return (EINVAL); + } + + phys_lo = coresight_buf->phys_base & 0xffffffff; + phys_hi = (coresight_buf->phys_base >> 32) & 0xffffffff; + event->naddr = 0; + + event->etr.started = 0; + event->etr.low = phys_lo; + event->etr.high = phys_hi; + + mode = PMC_TO_MODE(pm); + if (mode == PMC_MODE_ST) + event->excp_level = 1; + else if (mode == PMC_MODE_TT) + event->excp_level = 0; + else { + dprintf("%s: unsupported mode %d\n", __func__, mode); + return (-1); + } + + event->src = CORESIGHT_ETMV4; + event->sink = CORESIGHT_ETR; + + coresight_init_event(cpu, event); + + return (0); +} + +static int +coresight_allocate_pmc(int cpu, int ri, struct pmc *pm, + const struct pmc_op_pmcallocate *a) +{ + struct coresight_cpu *coresight_pc; + int i; + + coresight_pc = coresight_pcpu[cpu]; + + dprintf("%s: curthread %lx, cpu %d (curcpu %d)\n", __func__, + (uint64_t)curthread, cpu, PCPU_GET(cpuid)); + dprintf("%s: cpu %d (curcpu %d)\n", __func__, + cpu, PCPU_GET(cpuid)); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[coresight,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < CORESIGHT_NPMCS, + ("[coresight,%d] illegal row index %d", __LINE__, ri)); + + if (a->pm_class != PMC_CLASS_CORESIGHT) + return (EINVAL); + + if (a->pm_ev != PMC_EV_CORESIGHT_CORESIGHT) + return (EINVAL); + + if ((pm->pm_caps & CORESIGHT_CAPS) == 0) + return (EINVAL); + + if ((pm->pm_caps & ~CORESIGHT_CAPS) != 0) + return (EPERM); + + if (a->pm_mode != PMC_MODE_ST && + a->pm_mode != PMC_MODE_TT) + return (EINVAL); + + /* Can't allocate multiple ST */ + if (a->pm_mode == PMC_MODE_ST && + coresight_pc->flags & FLAG_CORESIGHT_ALLOCATED) { + dprintf("error: coresight is already allocated for CPU %d\n", cpu); + return (EUSERS); + } + + if (a->pm_mode == PMC_MODE_TT) + for (i = 0; i < pmc_cpu_max(); i++) { + if (coresight_buffer_prepare(i, pm, a)) + return (EINVAL); + } + else + if (coresight_buffer_prepare(cpu, pm, a)) + return (EINVAL); + + if (a->pm_mode == PMC_MODE_ST) + coresight_pc->flags |= FLAG_CORESIGHT_ALLOCATED; + + return (0); +} + +static int +coresight_config_pmc(int cpu, int ri, struct pmc *pm) +{ + struct coresight_cpu *coresight_pc; + struct pmc_hw *phw; + + dprintf("%s: cpu %d (pm %lx)\n", __func__, cpu, (uint64_t)pm); + + PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[coresight,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[coresight,%d] illegal row-index %d", __LINE__, ri)); + + coresight_pc = coresight_pcpu[cpu]; + phw = &coresight_pc->tc_hw; + + KASSERT(pm == NULL || phw->phw_pmc == NULL, + ("[coresight,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, + pm, phw->phw_pmc)); + + phw->phw_pmc = pm; + + return (0); +} + +static int +coresight_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) +{ + const struct coresight_descr *pd; + struct pmc_hw *phw; + size_t copied; + int error; + + dprintf("%s\n", __func__); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[coresight,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[coresight,%d] illegal row-index %d", __LINE__, ri)); + + phw = &coresight_pcpu[cpu]->tc_hw; + pd = &coresight_pmcdesc[ri]; + + if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, + PMC_NAME_MAX, &copied)) != 0) + return (error); + + pi->pm_class = pd->pm_descr.pd_class; + + if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { + pi->pm_enabled = TRUE; + *ppmc = phw->phw_pmc; + } else { + pi->pm_enabled = FALSE; + *ppmc = NULL; + } + + return (0); +} + +static int +coresight_get_config(int cpu, int ri, struct pmc **ppm) +{ + struct coresight_cpu *coresight_pc; + struct pmc_hw *phw; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[coresight,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[coresight,%d] illegal row-index %d", __LINE__, ri)); + + coresight_pc = coresight_pcpu[cpu]; + phw = &coresight_pc->tc_hw; + + *ppm = phw->phw_pmc; + + return (0); +} + +static int +coresight_pcpu_init(struct pmc_mdep *md, int cpu) +{ + struct pmc_cpu *pc; + struct coresight_cpu *coresight_pc; + int ri; + + dprintf("%s: cpu %d\n", __func__, cpu); + + KASSERT(cpu == PCPU_GET(cpuid), ("Init on wrong CPU\n")); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[coresight,%d] illegal cpu %d", __LINE__, cpu)); + KASSERT(coresight_pcpu, ("[coresight,%d] null pcpu", __LINE__)); + KASSERT(coresight_pcpu[cpu] == NULL, ("[coresight,%d] non-null per-cpu", + __LINE__)); + + coresight_pc = malloc(sizeof(struct coresight_cpu), M_CORESIGHT, M_WAITOK | M_ZERO); + coresight_pc->tc_hw.phw_state = PMC_PHW_FLAG_IS_ENABLED | + PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(0) | + PMC_PHW_FLAG_IS_SHAREABLE; + + coresight_pcpu[cpu] = coresight_pc; + + ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_CORESIGHT].pcd_ri; + + KASSERT(pmc_pcpu, ("[coresight,%d] null generic pcpu", __LINE__)); + + pc = pmc_pcpu[cpu]; + + KASSERT(pc, ("[coresight,%d] null generic per-cpu", __LINE__)); + + pc->pc_hwpmcs[ri] = &coresight_pc->tc_hw; + + return (0); +} + +static int +coresight_pcpu_fini(struct pmc_mdep *md, int cpu) +{ + int ri; + struct pmc_cpu *pc; + struct coresight_cpu *coresight_pc; + + dprintf("%s: cpu %d\n", __func__, cpu); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[coresight,%d] illegal cpu %d", __LINE__, cpu)); + KASSERT(coresight_pcpu[cpu] != NULL, ("[coresight,%d] null pcpu", __LINE__)); + + coresight_pc = coresight_pcpu[cpu]; + + free(coresight_pcpu[cpu], M_CORESIGHT); + coresight_pcpu[cpu] = NULL; + + ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_CORESIGHT].pcd_ri; + + pc = pmc_pcpu[cpu]; + pc->pc_hwpmcs[ri] = NULL; + + return (0); +} + +static int +coresight_trace_config(int cpu, int ri, struct pmc *pm, + uint64_t *ranges, uint32_t nranges) +{ + struct coresight_event *event; + struct coresight_cpu *coresight_pc; + int i; + + dprintf("%s\n", __func__); + + coresight_pc = coresight_pcpu[cpu]; + event = &coresight_pc->event; + + KASSERT(cpu == PCPU_GET(cpuid), ("Configuring wrong CPU\n")); + + for (i = 0; i < nranges * 2; i++) + event->addr[i] = ranges[i]; + + event->naddr = nranges; + + enum pmc_mode mode; + mode = PMC_TO_MODE(pm); + if (mode == PMC_MODE_ST) + event->excp_level = 1; + else + event->excp_level = 0; + + event->src = CORESIGHT_ETMV4; + event->sink = CORESIGHT_ETR; + + return (0); +} + +static int +coresight_read_trace(int cpu, int ri, struct pmc *pm, + pmc_value_t *vcycle, pmc_value_t *voffset) +{ + struct pmc_md_coresight_pmc *pm_coresight; + struct coresight_event *event; + struct coresight_buffer *coresight_buf; + struct coresight_cpu *coresight_pc; + uint64_t offset; + uint64_t cycle; + + dprintf("%s\n", __func__); + + coresight_pc = coresight_pcpu[cpu]; + coresight_pc->pm_mmap = pm; + event = &coresight_pc->event; + + coresight_read(cpu, event); + + cycle = event->etr.cycle; + offset = event->etr.offset; + + pm_coresight = (struct pmc_md_coresight_pmc *)&pm->pm_md; + coresight_buf = &pm_coresight->coresight_buffers[cpu]; + + *vcycle = cycle; + *voffset = offset; + + return (0); +} + +static int +coresight_read_pmc(int cpu, int ri, pmc_value_t *v) +{ + + dprintf("%s\n", __func__); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[coresight,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[coresight,%d] illegal ri %d", __LINE__, ri)); + + *v = 0; + + return (0); +} + +static int +coresight_release_pmc(int cpu, int ri, struct pmc *pm) +{ + struct pmc_md_coresight_pmc *pm_coresight; + struct coresight_event *event; + struct coresight_cpu *coresight_pc; + enum pmc_mode mode; + struct pmc_hw *phw; + int i; + + pm_coresight = (struct pmc_md_coresight_pmc *)&pm->pm_md; + coresight_pc = coresight_pcpu[cpu]; + event = &coresight_pc->event; + + dprintf("%s: cpu %d (curcpu %d)\n", __func__, cpu, PCPU_GET(cpuid)); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[coresight,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, + ("[coresight,%d] illegal row-index %d", __LINE__, ri)); + + phw = &coresight_pcpu[cpu]->tc_hw; + phw->phw_pmc = NULL; + + KASSERT(phw->phw_pmc == NULL, + ("[coresight,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); + + coresight_disable(cpu, event); + + mode = PMC_TO_MODE(pm); + if (mode == PMC_MODE_TT) + for (i = 0; i < pmc_cpu_max(); i++) + coresight_buffer_deallocate(i, &pm_coresight->coresight_buffers[i]); + else + coresight_buffer_deallocate(cpu, &pm_coresight->coresight_buffers[cpu]); + + if (mode == PMC_MODE_ST) + coresight_pc->flags &= ~FLAG_CORESIGHT_ALLOCATED; + + return (0); +} + +static int +coresight_start_pmc(int cpu, int ri) +{ + struct coresight_event *event; + struct coresight_cpu *coresight_pc; + struct pmc_hw *phw; + + dprintf("%s: cpu %d (curcpu %d)\n", __func__, cpu, PCPU_GET(cpuid)); + + coresight_pc = coresight_pcpu[cpu]; + event = &coresight_pc->event; + phw = &coresight_pc->tc_hw; + if (phw == NULL || phw->phw_pmc == NULL) + return (-1); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[coresight,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[coresight,%d] illegal row-index %d", __LINE__, ri)); + + coresight_enable(cpu, event); + + return (0); +} + +static int +coresight_stop_pmc(int cpu, int ri) +{ + struct coresight_event *event; + struct coresight_cpu *coresight_pc; + + dprintf("%s\n", __func__); + + coresight_pc = coresight_pcpu[cpu]; + event = &coresight_pc->event; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[coresight,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[coresight,%d] illegal row-index %d", __LINE__, ri)); + + coresight_disable(cpu, event); + + return (0); +} + +static int +coresight_write_pmc(int cpu, int ri, pmc_value_t v) +{ + + dprintf("%s\n", __func__); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[coresight,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[coresight,%d] illegal row-index %d", __LINE__, ri)); + + return (0); +} + +int +pmc_coresight_initialize(struct pmc_mdep *md, int maxcpu) +{ + struct pmc_classdep *pcd; + + dprintf("%s\n", __func__); + + KASSERT(md != NULL, ("[coresight,%d] md is NULL", __LINE__)); + KASSERT(md->pmd_nclass >= 1, ("[coresight,%d] dubious md->nclass %d", + __LINE__, md->pmd_nclass)); + + coresight_pcpu = malloc(sizeof(struct coresight_cpu *) * maxcpu, M_CORESIGHT, + M_WAITOK | M_ZERO); + + pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_CORESIGHT]; + + pcd->pcd_caps = CORESIGHT_CAPS; + pcd->pcd_class = PMC_CLASS_CORESIGHT; + pcd->pcd_num = CORESIGHT_NPMCS; + pcd->pcd_ri = md->pmd_npmc; + pcd->pcd_width = 64; + + pcd->pcd_allocate_pmc = coresight_allocate_pmc; + pcd->pcd_config_pmc = coresight_config_pmc; + pcd->pcd_describe = coresight_describe; + pcd->pcd_get_config = coresight_get_config; + pcd->pcd_pcpu_init = coresight_pcpu_init; + pcd->pcd_pcpu_fini = coresight_pcpu_fini; + pcd->pcd_read_pmc = coresight_read_pmc; + pcd->pcd_read_trace = coresight_read_trace; + pcd->pcd_trace_config = coresight_trace_config; + pcd->pcd_release_pmc = coresight_release_pmc; + pcd->pcd_start_pmc = coresight_start_pmc; + pcd->pcd_stop_pmc = coresight_stop_pmc; + pcd->pcd_write_pmc = coresight_write_pmc; + + md->pmd_npmc += CORESIGHT_NPMCS; + + return (0); +} + +void +pmc_coresight_finalize(struct pmc_mdep *md) +{ + + dprintf("%s\n", __func__); + +#ifdef INVARIANTS + int i, ncpus; + + ncpus = pmc_cpu_max(); + for (i = 0; i < ncpus; i++) + KASSERT(coresight_pcpu[i] == NULL, ("[coresight,%d] non-null pcpu cpu %d", + __LINE__, i)); + + KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_CORESIGHT].pcd_class == + PMC_CLASS_CORESIGHT, ("[coresight,%d] class mismatch", __LINE__)); +#endif + + free(coresight_pcpu, M_CORESIGHT); + coresight_pcpu = NULL; +} Index: sys/dev/hwpmc/hwpmc_intel.c =================================================================== --- sys/dev/hwpmc/hwpmc_intel.c +++ sys/dev/hwpmc/hwpmc_intel.c @@ -181,12 +181,8 @@ cputype = PMC_CPU_INTEL_IVYBRIDGE_XEON; nclasses = 3; break; - /* Skylake */ case 0x4e: case 0x5e: - /* Kabylake */ - case 0x8E: /* Per Intel document 325462-063US July 2017. */ - case 0x9E: /* Per Intel document 325462-063US July 2017. */ cputype = PMC_CPU_INTEL_SKYLAKE; nclasses = 3; break; @@ -220,6 +216,11 @@ nclasses = 3; break; } + case 0x8E: /* Per Intel document 325462-063US July 2017. */ + case 0x9E: /* Per Intel document 325462-063US July 2017. */ + cputype = PMC_CPU_INTEL_KABYLAKE; + nclasses = 4; + break; break; #if defined(__i386__) || defined(__amd64__) case 0xF00: /* P4 */ @@ -237,7 +238,7 @@ /* Allocate base class and initialize machine dependent struct */ pmc_mdep = pmc_mdep_alloc(nclasses); - pmc_mdep->pmd_cputype = cputype; + pmc_mdep->pmd_cputype = cputype; pmc_mdep->pmd_switch_in = intel_switch_in; pmc_mdep->pmd_switch_out = intel_switch_out; @@ -256,6 +257,7 @@ case PMC_CPU_INTEL_BROADWELL_XEON: case PMC_CPU_INTEL_SKYLAKE_XEON: case PMC_CPU_INTEL_SKYLAKE: + case PMC_CPU_INTEL_KABYLAKE: case PMC_CPU_INTEL_CORE: case PMC_CPU_INTEL_CORE2: case PMC_CPU_INTEL_CORE2EXTREME: @@ -312,10 +314,10 @@ goto error; } +#if defined(__i386__) || defined(__amd64__) /* * Init the uncore class. */ -#if defined(__i386__) || defined(__amd64__) switch (cputype) { /* * Intel Corei7 and Westmere processors. @@ -330,7 +332,19 @@ default: break; } + + /* + * Intel Processor Tracing (PT). + */ + if (cputype == PMC_CPU_INTEL_KABYLAKE) { + error = pmc_pt_initialize(pmc_mdep, ncpus); + if (error) { + pmc_pt_finalize(pmc_mdep); + goto error; + } + } #endif + error: if (error) { pmc_mdep_free(pmc_mdep); @@ -353,6 +367,7 @@ case PMC_CPU_INTEL_BROADWELL_XEON: case PMC_CPU_INTEL_SKYLAKE_XEON: case PMC_CPU_INTEL_SKYLAKE: + case PMC_CPU_INTEL_KABYLAKE: case PMC_CPU_INTEL_CORE: case PMC_CPU_INTEL_CORE2: case PMC_CPU_INTEL_CORE2EXTREME: @@ -389,10 +404,10 @@ KASSERT(0, ("[intel,%d] unknown CPU type", __LINE__)); } +#if defined(__i386__) || defined(__amd64__) /* * Uncore. */ -#if defined(__i386__) || defined(__amd64__) switch (md->pmd_cputype) { case PMC_CPU_INTEL_BROADWELL: case PMC_CPU_INTEL_COREI7: @@ -404,5 +419,11 @@ default: break; } + + /* + * Intel Processor Tracing (PT). + */ + if (md->pmd_cputype == PMC_CPU_INTEL_KABYLAKE) + pmc_pt_finalize(md); #endif } Index: sys/dev/hwpmc/hwpmc_mod.c =================================================================== --- sys/dev/hwpmc/hwpmc_mod.c +++ sys/dev/hwpmc/hwpmc_mod.c @@ -74,6 +74,7 @@ #include #include "hwpmc_soft.h" +#include "hwpmc_vm.h" /* * Types @@ -1295,6 +1296,8 @@ pp->pp_pmcs[ri].pp_pmcval; pp->pp_pmcs[ri].pp_pmcval = pm->pm_sc.pm_reloadcount; mtx_pool_unlock_spin(pmc_mtxpool, pm); + } else if (PMC_TO_MODE(pm) == PMC_MODE_TT) { + /* Nothing */ } else { KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC, ("[pmc,%d] illegal mode=%d", __LINE__, @@ -1310,7 +1313,8 @@ pcd->pcd_write_pmc(cpu, adjri, newvalue); /* If a sampling mode PMC, reset stalled state. */ - if (PMC_TO_MODE(pm) == PMC_MODE_TS) + if (PMC_TO_MODE(pm) == PMC_MODE_TS || + PMC_TO_MODE(pm) == PMC_MODE_TT) CPU_CLR_ATOMIC(cpu, &pm->pm_stalled); /* Indicate that we desire this to run. */ @@ -1472,7 +1476,8 @@ pp->pp_pmcs[ri].pp_pmcval, pm->pm_sc.pm_reloadcount)); mtx_pool_unlock_spin(pmc_mtxpool, pm); - + } else if (mode == PMC_MODE_TT) { + /* Nothing */ } else { tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); @@ -1528,6 +1533,10 @@ const struct pmc *pm; struct pmc_owner *po; const struct pmc_process *pp; + struct proc *p; + bool pause_thread; + + sx_slock(&pmc_sx); freepath = fullpath = NULL; pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath); @@ -1539,17 +1548,42 @@ if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_map_in(po, pid, pkm->pm_address, fullpath); - if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL) + if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL) { + sx_sunlock(&pmc_sx); goto done; + } + + p = td->td_proc; + if ((p->p_flag & P_HWPMC) == 0) { + sx_sunlock(&pmc_sx); + goto done; + } + + pause_thread = 0; /* * Inform sampling PMC owners tracking this process. */ - for (ri = 0; ri < md->pmd_npmc; ri++) - if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && - PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) + for (ri = 0; ri < md->pmd_npmc; ri++) { + if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) + continue; + if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || + PMC_TO_MODE(pm) == PMC_MODE_TT) pmclog_process_map_in(pm->pm_owner, pid, pkm->pm_address, fullpath); + if (PMC_TO_MODE(pm) == PMC_MODE_TT) + pause_thread = 1; + } + + sx_sunlock(&pmc_sx); + + if (pause_thread) { + PROC_LOCK(td->td_proc); + PROC_SLOCK(td->td_proc); + thread_suspend_switch(td, td->td_proc); + PROC_SUNLOCK(td->td_proc); + PROC_UNLOCK(td->td_proc); + } done: if (freepath) @@ -1580,11 +1614,14 @@ if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL) return; - for (ri = 0; ri < md->pmd_npmc; ri++) - if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && - PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) + for (ri = 0; ri < md->pmd_npmc; ri++) { + if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) + continue; + if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || + PMC_TO_MODE(pm) == PMC_MODE_TT) pmclog_process_map_out(pm->pm_owner, pid, pkm->pm_address, pkm->pm_address + pkm->pm_size); + } } /* @@ -1598,7 +1635,8 @@ struct pmckern_map_in *km, *kmbase; sx_assert(&pmc_sx, SX_LOCKED); - KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), + KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || + PMC_TO_MODE(pm) == PMC_MODE_ST, ("[pmc,%d] non-sampling PMC (%p) desires mapping information", __LINE__, (void *) pm)); @@ -1999,7 +2037,6 @@ break; case PMC_FN_MMAP: - sx_assert(&pmc_sx, SX_LOCKED); pmc_process_mmap(td, (struct pmckern_map_in *) arg); break; @@ -2115,8 +2152,8 @@ mtx_lock_spin(&pmc_processhash_mtx); LIST_FOREACH(pp, pph, pp_next) - if (pp->pp_proc == p) - break; + if (pp->pp_proc == p) + break; if ((mode & PMC_FLAG_REMOVE) && pp != NULL) LIST_REMOVE(pp, pp_next); @@ -2652,7 +2689,8 @@ * If this is a sampling mode PMC, log mapping information for * the kernel modules that are currently loaded. */ - if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) + if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || + PMC_TO_MODE(pm) == PMC_MODE_ST) pmc_log_kernel_mappings(pm); if (PMC_IS_VIRTUAL_MODE(mode)) { @@ -3306,9 +3344,14 @@ mode = pa.pm_mode; cpu = pa.pm_cpu; - if ((mode != PMC_MODE_SS && mode != PMC_MODE_SC && - mode != PMC_MODE_TS && mode != PMC_MODE_TC) || - (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) { + if (mode != PMC_MODE_SS && mode != PMC_MODE_TS && + mode != PMC_MODE_SC && mode != PMC_MODE_TC && + mode != PMC_MODE_ST && mode != PMC_MODE_TT) { + error = EINVAL; + break; + } + + if (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max()) { error = EINVAL; break; } @@ -3755,6 +3798,175 @@ } break; + case PMC_OP_LOG_KERNEL_MAP: + { + struct pmc_op_simple sp; + struct pmc *pm; + + if ((error = copyin(arg, &sp, sizeof(sp))) != 0) + break; + + /* locate pmc descriptor */ + if ((error = pmc_find_pmc(sp.pm_pmcid, &pm)) != 0) + break; + + if (PMC_TO_MODE(pm) != PMC_MODE_ST) + break; + + if (pm->pm_state != PMC_STATE_ALLOCATED && + pm->pm_state != PMC_STATE_STOPPED && + pm->pm_state != PMC_STATE_RUNNING) { + error = EINVAL; + break; + } + + pmc_log_kernel_mappings(pm); + } + break; + + case PMC_OP_THREAD_UNSUSPEND: + { + struct pmc_op_proc_unsuspend u; + struct proc *p; + struct pmc *pm; + + if ((error = copyin(arg, &u, sizeof(u))) != 0) + break; + + /* locate pmc descriptor */ + if ((error = pmc_find_pmc(u.pm_pmcid, &pm)) != 0) + break; + + /* lookup pid */ + if ((p = pfind(u.pm_pid)) == NULL) { + error = ESRCH; + break; + } + + if ((p->p_flag & P_HWPMC) == 0) + break; + + PROC_SLOCK(p); + thread_unsuspend(p); + PROC_SUNLOCK(p); + PROC_UNLOCK(p); + } + break; + + case PMC_OP_TRACE_CONFIG: + { + struct pmc_op_trace_config trc; + uint64_t *ranges; + struct pmc *pm; + struct pmc_binding pb; + struct pmc_classdep *pcd; + uint32_t nranges; + uint32_t cpu; + uint32_t ri; + int adjri; + + if ((error = copyin(arg, &trc, sizeof(trc))) != 0) + break; + + /* locate pmc descriptor */ + if ((error = pmc_find_pmc(trc.pm_pmcid, &pm)) != 0) + break; + + if (PMC_TO_MODE(pm) != PMC_MODE_ST && + PMC_TO_MODE(pm) != PMC_MODE_TT) + break; + + /* Can't proceed with PMC that hasn't been started. */ + if (pm->pm_state != PMC_STATE_ALLOCATED && + pm->pm_state != PMC_STATE_STOPPED && + pm->pm_state != PMC_STATE_RUNNING) { + error = EINVAL; + break; + } + + cpu = trc.pm_cpu; + + ri = PMC_TO_ROWINDEX(pm); + pcd = pmc_ri_to_classdep(md, ri, &adjri); + if (pcd->pcd_trace_config == NULL) + break; + + /* switch to CPU 'cpu' */ + pmc_save_cpu_binding(&pb); + pmc_select_cpu(cpu); + + ranges = trc.ranges; + nranges = trc.nranges; + + mtx_pool_lock_spin(pmc_mtxpool, pm); + error = (*pcd->pcd_trace_config)(cpu, adjri, + pm, ranges, nranges); + mtx_pool_unlock_spin(pmc_mtxpool, pm); + + pmc_restore_cpu_binding(&pb); + } + break; + + /* + * Read a PMC trace buffer ptr. + */ + case PMC_OP_TRACE_READ: + { + struct pmc_op_trace_read trr; + struct pmc_op_trace_read *trr_ret; + struct pmc_binding pb; + struct pmc_classdep *pcd; + struct pmc *pm; + pmc_value_t cycle; + pmc_value_t offset; + uint32_t cpu; + uint32_t ri; + int adjri; + + if ((error = copyin(arg, &trr, sizeof(trr))) != 0) + break; + + /* locate pmc descriptor */ + if ((error = pmc_find_pmc(trr.pm_pmcid, &pm)) != 0) + break; + + if (PMC_TO_MODE(pm) != PMC_MODE_ST && + PMC_TO_MODE(pm) != PMC_MODE_TT) + break; + + /* Can't read a PMC that hasn't been started. */ + if (pm->pm_state != PMC_STATE_ALLOCATED && + pm->pm_state != PMC_STATE_STOPPED && + pm->pm_state != PMC_STATE_RUNNING) { + error = EINVAL; + break; + } + + cpu = trr.pm_cpu; + + ri = PMC_TO_ROWINDEX(pm); + pcd = pmc_ri_to_classdep(md, ri, &adjri); + + /* switch to CPU 'cpu' */ + pmc_save_cpu_binding(&pb); + pmc_select_cpu(cpu); + + mtx_pool_lock_spin(pmc_mtxpool, pm); + error = (*pcd->pcd_read_trace)(cpu, adjri, + pm, &cycle, &offset); + mtx_pool_unlock_spin(pmc_mtxpool, pm); + + pmc_restore_cpu_binding(&pb); + + trr_ret = (struct pmc_op_trace_read *)arg; + if ((error = copyout(&cycle, &trr_ret->pm_cycle, + sizeof(trr.pm_cycle)))) + break; + if ((error = copyout(&offset, &trr_ret->pm_offset, + sizeof(trr.pm_offset)))) + break; + } + break; /* * Read and/or write a PMC. @@ -3858,7 +4070,7 @@ /* save old value */ if (prw.pm_flags & PMC_F_OLDVALUE) if ((error = (*pcd->pcd_read_pmc)(cpu, adjri, - &oldvalue))) + &oldvalue))) goto error; /* write out new value */ if (prw.pm_flags & PMC_F_NEWVALUE) @@ -5029,6 +5241,8 @@ printf("\n"); } + pmc_vm_initialize(md); + return (error); } @@ -5181,6 +5395,7 @@ } pmclog_shutdown(); + pmc_vm_finalize(); sx_xunlock(&pmc_sx); /* we are done */ } Index: sys/dev/hwpmc/hwpmc_pt.h =================================================================== --- /dev/null +++ sys/dev/hwpmc/hwpmc_pt.h @@ -0,0 +1,107 @@ +/*- + * Copyright (c) 2017 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _DEV_HWPMC_PT_H_ +#define _DEV_HWPMC_PT_H_ + +#include +#include +#include + +#include + +#define PT_CPUID 0x14 +#define PT_NADDR 4 +#define PT_NPMCS 1 + +struct pmc_md_pt_op_pmcallocate { + uint32_t flags; +#define INTEL_PT_FLAG_BRANCHES (1 << 0) +#define INTEL_PT_FLAG_TSC (1 << 1) +#define INTEL_PT_FLAG_MTC (1 << 2) +#define INTEL_PT_FLAG_DISRETC (1 << 3) + uint64_t ranges[2 * PT_NADDR]; + int nranges; +}; + +#ifdef _KERNEL +struct xsave_header { + uint64_t xsave_bv; + uint64_t xcomp_bv; + uint8_t reserved[48]; +}; + +struct pt_ext_area { + uint64_t rtit_ctl; + uint64_t rtit_output_base; + uint64_t rtit_output_mask_ptrs; + uint64_t rtit_status; + uint64_t rtit_cr3_match; + uint64_t rtit_addr0_a; + uint64_t rtit_addr0_b; + uint64_t rtit_addr1_a; + uint64_t rtit_addr1_b; +}; + +struct pt_save_area { + uint8_t legacy_state[512]; + struct xsave_header header; + struct pt_ext_area pt_ext_area; +} __aligned(64); + +struct topa_entry { + uint64_t base; + uint64_t size; + uint64_t offset; +}; + +struct pt_buffer { + uint64_t *topa_hw; + struct topa_entry *topa_sw; + uint64_t cycle; + vm_object_t obj; +}; + +/* MD extension for 'struct pmc' */ +struct pmc_md_pt_pmc { + struct pt_buffer pt_buffers[MAXCPU]; +}; + +/* + * Prototypes. + */ + +int pmc_pt_initialize(struct pmc_mdep *_md, int _maxcpu); +void pmc_pt_finalize(struct pmc_mdep *_md); +int pmc_pt_intr(int cpu, struct trapframe *tf); + +#endif /* !_KERNEL */ +#endif /* !_DEV_HWPMC_PT_H */ Index: sys/dev/hwpmc/hwpmc_pt.c =================================================================== --- /dev/null +++ sys/dev/hwpmc/hwpmc_pt.c @@ -0,0 +1,952 @@ +/*- + * Copyright (c) 2017 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include + +static MALLOC_DEFINE(M_PT, "pt", "PT driver"); +static uint64_t pt_xsave_mask; + +extern struct cdev *pmc_cdev[MAXCPU]; + +/* + * Intel PT support. + */ + +#define PT_CAPS (PMC_CAP_READ | PMC_CAP_INTERRUPT | PMC_CAP_SYSTEM | PMC_CAP_USER) + +#define PMC_PT_DEBUG +#undef PMC_PT_DEBUG + +#ifdef PMC_PT_DEBUG +#define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define dprintf(fmt, ...) +#endif + +struct pt_descr { + struct pmc_descr pm_descr; /* "base class" */ +}; + +static struct pt_descr pt_pmcdesc[PT_NPMCS] = +{ + { + .pm_descr = + { + .pd_name = "PT", + .pd_class = PMC_CLASS_PT, + .pd_caps = PT_CAPS, + .pd_width = 64 + } + } +}; + +/* + * Per-CPU data structure for PTs. + */ + +struct pt_cpu { + struct pmc_hw tc_hw; + uint32_t l0_eax; + uint32_t l0_ebx; + uint32_t l0_ecx; + uint32_t l1_eax; + uint32_t l1_ebx; + struct pmc *pm_mmap; + uint32_t flags; +#define FLAG_PT_ALLOCATED (1 << 0) + struct pt_save_area save_area; +}; + +static struct pt_cpu **pt_pcpu; + +static __inline void +xrstors(char *addr, uint64_t mask) +{ + uint32_t low, hi; + + low = mask; + hi = mask >> 32; + __asm __volatile("xrstors %0" : : "m" (*addr), "a" (low), "d" (hi)); +} + +static __inline void +xsaves(char *addr, uint64_t mask) +{ + uint32_t low, hi; + + low = mask; + hi = mask >> 32; + __asm __volatile("xsaves %0" : "=m" (*addr) : "a" (low), "d" (hi) : + "memory"); +} + +static void +pt_save_restore(struct pt_cpu *pt_pc, bool save) +{ + uint64_t val; + + clts(); + val = rxcr(XCR0); + load_xcr(XCR0, pt_xsave_mask); + wrmsr(MSR_IA32_XSS, XFEATURE_ENABLED_PT); + if (save) { + KASSERT((rdmsr(MSR_IA32_RTIT_CTL) & RTIT_CTL_TRACEEN) != 0, + ("%s: PT is disabled", __func__)); + xsaves((char *)&pt_pc->save_area, XFEATURE_ENABLED_PT); + } else { + KASSERT((rdmsr(MSR_IA32_RTIT_CTL) & RTIT_CTL_TRACEEN) == 0, + ("%s: PT is enabled", __func__)); + xrstors((char *)&pt_pc->save_area, XFEATURE_ENABLED_PT); + } + load_xcr(XCR0, val); + load_cr0(rcr0() | CR0_TS); +} + +static void +pt_configure_ranges(struct pt_cpu *pt_pc, const uint64_t *ranges, + uint32_t nranges) +{ + struct pt_ext_area *pt_ext; + struct pt_save_area *save_area; + int nranges_supp; + int n; + + save_area = &pt_pc->save_area; + pt_ext = &save_area->pt_ext_area; + + if (pt_pc->l0_ebx & CPUPT_IPF) { + /* How many ranges CPU does support ? */ + nranges_supp = (pt_pc->l1_eax & CPUPT_NADDR_M) >> CPUPT_NADDR_S; + + /* xsave/xrstor supports two ranges only */ + if (nranges_supp > 2) + nranges_supp = 2; + + n = nranges > nranges_supp ? nranges_supp : nranges; + + switch (n) { + case 2: + pt_ext->rtit_ctl |= (1UL << RTIT_CTL_ADDR_CFG_S(1)); + pt_ext->rtit_addr1_a = ranges[2]; + pt_ext->rtit_addr1_b = ranges[3]; + case 1: + pt_ext->rtit_ctl |= (1UL << RTIT_CTL_ADDR_CFG_S(0)); + pt_ext->rtit_addr0_a = ranges[0]; + pt_ext->rtit_addr0_b = ranges[1]; + default: + break; + }; + } +} + +static int +pt_buffer_allocate(uint32_t cpu, struct pt_buffer *pt_buf) +{ + struct pmc_vm_map *map; + struct pt_cpu *pt_pc; + uint64_t topa_size; + uint64_t segsize; + uint64_t offset; + uint32_t size; + uint32_t bufsize; + struct cdev_cpu *cc; + vm_object_t obj; + vm_page_t m; + int npages; + int ntopa; + int req; + int i, j; + + pt_pc = pt_pcpu[cpu]; + + bufsize = 16 * 1024 * 1024; + + if (pt_pc->l0_ecx & CPUPT_TOPA_MULTI) + topa_size = TOPA_SIZE_4K; + else + topa_size = TOPA_SIZE_16M; + + segsize = PAGE_SIZE << (topa_size >> TOPA_SIZE_S); + ntopa = bufsize / segsize; + npages = segsize / PAGE_SIZE; + + pt_buf->obj = obj = vm_pager_allocate(OBJT_PHYS, 0, bufsize, + PROT_READ, 0, curthread->td_ucred); + + size = roundup2((ntopa + 1) * 8, PAGE_SIZE); + pt_buf->topa_hw = malloc(size, M_PT, M_WAITOK | M_ZERO); + pt_buf->topa_sw = malloc(ntopa * sizeof(struct topa_entry), M_PT, + M_WAITOK | M_ZERO); + + VM_OBJECT_WLOCK(obj); + vm_object_reference_locked(obj); + offset = 0; + for (i = 0; i < ntopa; i++) { + req = VM_ALLOC_NOBUSY | VM_ALLOC_ZERO; + if (npages == 1) + m = vm_page_alloc(obj, i, req); + else + m = vm_page_alloc_contig(obj, i, req, npages, 0, ~0, + bufsize, 0, VM_MEMATTR_DEFAULT); + if (m == NULL) { + VM_OBJECT_WUNLOCK(obj); + printf("%s: Can't allocate memory.\n", __func__); + goto error; + } + for (j = 0; j < npages; j++) + m[j].valid = VM_PAGE_BITS_ALL; + pt_buf->topa_sw[i].size = segsize; + pt_buf->topa_sw[i].offset = offset; + pt_buf->topa_hw[i] = VM_PAGE_TO_PHYS(m) | topa_size; + if (i == (ntopa - 1)) + pt_buf->topa_hw[i] |= TOPA_INT; + + offset += segsize; + } + VM_OBJECT_WUNLOCK(obj); + + /* The last entry is a pointer to the base table. */ + pt_buf->topa_hw[ntopa] = vtophys(pt_buf->topa_hw) | TOPA_END; + pt_buf->cycle = 0; + + map = malloc(sizeof(struct pmc_vm_map), M_PT, M_WAITOK | M_ZERO); + map->t = curthread; + map->obj = obj; + map->buf = pt_buf; + + cc = pmc_cdev[cpu]->si_drv1; + + mtx_lock(&cc->vm_mtx); + TAILQ_INSERT_HEAD(&cc->pmc_maplist, map, map_next); + mtx_unlock(&cc->vm_mtx); + + return (0); + +error: + free(pt_buf->topa_hw, M_PT); + free(pt_buf->topa_sw, M_PT); + vm_object_deallocate(obj); + + return (-1); +} + +static int +pt_buffer_deallocate(uint32_t cpu, struct pt_buffer *pt_buf) +{ + struct pmc_vm_map *map, *map_tmp; + struct cdev_cpu *cc; + + cc = pmc_cdev[cpu]->si_drv1; + + mtx_lock(&cc->vm_mtx); + TAILQ_FOREACH_SAFE(map, &cc->pmc_maplist, map_next, map_tmp) { + if (map->buf == pt_buf) { + TAILQ_REMOVE(&cc->pmc_maplist, map, map_next); + free(map, M_PT); + break; + } + } + mtx_unlock(&cc->vm_mtx); + + free(pt_buf->topa_hw, M_PT); + free(pt_buf->topa_sw, M_PT); + vm_object_deallocate(pt_buf->obj); + + return (0); +} + +static int +pt_buffer_prepare(uint32_t cpu, struct pmc *pm, + const struct pmc_op_pmcallocate *a) +{ + const struct pmc_md_pt_op_pmcallocate *pm_pta; + struct pt_cpu *pt_pc; + struct pmc_md_pt_pmc *pm_pt; + struct pt_buffer *pt_buf; + struct xsave_header *hdr; + struct pt_ext_area *pt_ext; + struct pt_save_area *save_area; + enum pmc_mode mode; + int error; + + pt_pc = pt_pcpu[cpu]; + if ((pt_pc->l0_ecx & CPUPT_TOPA) == 0) + return (ENXIO); /* We rely on TOPA support */ + + pm_pta = (const struct pmc_md_pt_op_pmcallocate *)&a->pm_md.pm_pt; + pm_pt = (struct pmc_md_pt_pmc *)&pm->pm_md; + pt_buf = &pm_pt->pt_buffers[cpu]; + + error = pt_buffer_allocate(cpu, pt_buf); + if (error != 0) { + dprintf("%s: can't allocate buffers\n", __func__); + return (EINVAL); + } + + save_area = &pt_pc->save_area; + bzero(save_area, sizeof(struct pt_save_area)); + + hdr = &save_area->header; + hdr->xsave_bv = XFEATURE_ENABLED_PT; + hdr->xcomp_bv = XFEATURE_ENABLED_PT | (1ULL << 63) /* compaction */; + + pt_ext = &save_area->pt_ext_area; + + pt_ext->rtit_ctl = RTIT_CTL_TOPA | RTIT_CTL_TRACEEN; + pt_ext->rtit_output_base = (uint64_t)vtophys(pt_buf->topa_hw); + pt_ext->rtit_output_mask_ptrs = 0x7f; + + pt_configure_ranges(pt_pc, pm_pta->ranges, pm_pta->nranges); + + /* + * TODO + * if (sc->l0_ebx & CPUPT_PRW) { + * reg |= RTIT_CTL_FUPONPTW; + * reg |= RTIT_CTL_PTWEN; + * } + */ + + mode = PMC_TO_MODE(pm); + if (mode == PMC_MODE_ST) + pt_ext->rtit_ctl |= RTIT_CTL_OS; + else if (mode == PMC_MODE_TT) + pt_ext->rtit_ctl |= RTIT_CTL_USER; + else { + dprintf("%s: unsupported mode %d\n", __func__, mode); + return (-1); + } + + /* Enable FUP, TIP, TIP.PGE, TIP.PGD, TNT, MODE.Exec and MODE.TSX packets */ + if (pm_pta->flags & INTEL_PT_FLAG_BRANCHES) + pt_ext->rtit_ctl |= RTIT_CTL_BRANCHEN; + + if (pm_pta->flags & INTEL_PT_FLAG_TSC) + pt_ext->rtit_ctl |= RTIT_CTL_TSCEN; + + if ((pt_pc->l0_ebx & CPUPT_MTC) && + (pm_pta->flags & INTEL_PT_FLAG_MTC)) + pt_ext->rtit_ctl |= RTIT_CTL_MTCEN; + + if (pm_pta->flags & INTEL_PT_FLAG_DISRETC) + pt_ext->rtit_ctl |= RTIT_CTL_DISRETC; + + /* + * TODO: specify MTC frequency + * Note: Check Bitmap of supported MTC Period Encodings + * pt_ext->rtit_ctl |= RTIT_CTL_MTC_FREQ(6); + */ + + return (0); +} + +static int +pt_allocate_pmc(int cpu, int ri, struct pmc *pm, + const struct pmc_op_pmcallocate *a) +{ + struct pt_cpu *pt_pc; + int i; + + if ((cpu_stdext_feature & CPUID_STDEXT_PROCTRACE) == 0) + return (ENXIO); + + pt_pc = pt_pcpu[cpu]; + + dprintf("%s: curthread %lx, cpu %d (curcpu %d)\n", __func__, + (uint64_t)curthread, cpu, PCPU_GET(cpuid)); + dprintf("%s: cpu %d (curcpu %d)\n", __func__, + cpu, PCPU_GET(cpuid)); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[pt,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < PT_NPMCS, + ("[pt,%d] illegal row index %d", __LINE__, ri)); + + if (a->pm_class != PMC_CLASS_PT) + return (EINVAL); + + if (a->pm_ev != PMC_EV_PT_PT) + return (EINVAL); + + if ((pm->pm_caps & PT_CAPS) == 0) + return (EINVAL); + + if ((pm->pm_caps & ~PT_CAPS) != 0) + return (EPERM); + + if (a->pm_mode != PMC_MODE_ST && + a->pm_mode != PMC_MODE_TT) + return (EINVAL); + + /* Can't allocate multiple ST */ + if (a->pm_mode == PMC_MODE_ST && + pt_pc->flags & FLAG_PT_ALLOCATED) { + dprintf("error: pt is already allocated for CPU %d\n", cpu); + return (EUSERS); + } + + if (a->pm_mode == PMC_MODE_TT) + for (i = 0; i < pmc_cpu_max(); i++) { + if (pt_buffer_prepare(i, pm, a)) + return (EINVAL); + } + else + if (pt_buffer_prepare(cpu, pm, a)) + return (EINVAL); + + if (a->pm_mode == PMC_MODE_ST) + pt_pc->flags |= FLAG_PT_ALLOCATED; + + return (0); +} + +int +pmc_pt_intr(int cpu, struct trapframe *tf) +{ + struct pmc_md_pt_pmc *pm_pt; + struct pt_buffer *pt_buf; + struct pt_cpu *pt_pc; + struct pmc_hw *phw; + struct pmc *pm; + + if (pt_pcpu == NULL) + return (0); + + pt_pc = pt_pcpu[cpu]; + if (pt_pc == NULL) + return (0); + + phw = &pt_pc->tc_hw; + if (phw == NULL || phw->phw_pmc == NULL) + return (0); + + pm = phw->phw_pmc; + if (pm == NULL) + return (0); + + KASSERT(pm != NULL, ("pm is NULL\n")); + + pm_pt = (struct pmc_md_pt_pmc *)&pm->pm_md; + pt_buf = &pm_pt->pt_buffers[cpu]; + + atomic_add_long(&pt_buf->cycle, 1); + + lapic_reenable_pmc(); + + return (1); +} + +static int +pt_config_pmc(int cpu, int ri, struct pmc *pm) +{ + struct pt_cpu *pt_pc; + struct pmc_hw *phw; + + dprintf("%s: cpu %d (pm %lx)\n", __func__, cpu, (uint64_t)pm); + + PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[pt,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[pt,%d] illegal row-index %d", __LINE__, ri)); + + pt_pc = pt_pcpu[cpu]; + phw = &pt_pc->tc_hw; + + KASSERT(pm == NULL || phw->phw_pmc == NULL, + ("[pt,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, + pm, phw->phw_pmc)); + + phw->phw_pmc = pm; + + return (0); +} + +static int +pt_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) +{ + const struct pt_descr *pd; + struct pmc_hw *phw; + size_t copied; + int error; + + dprintf("%s\n", __func__); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[pt,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[pt,%d] illegal row-index %d", __LINE__, ri)); + + phw = &pt_pcpu[cpu]->tc_hw; + pd = &pt_pmcdesc[ri]; + + if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, + PMC_NAME_MAX, &copied)) != 0) + return (error); + + pi->pm_class = pd->pm_descr.pd_class; + + if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { + pi->pm_enabled = TRUE; + *ppmc = phw->phw_pmc; + } else { + pi->pm_enabled = FALSE; + *ppmc = NULL; + } + + return (0); +} + +static int +pt_get_config(int cpu, int ri, struct pmc **ppm) +{ + struct pmc_hw *phw; + struct pt_cpu *pt_pc; + + dprintf("%s\n", __func__); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[pt,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[pt,%d] illegal row-index %d", __LINE__, ri)); + + pt_pc = pt_pcpu[cpu]; + phw = &pt_pc->tc_hw; + + *ppm = phw->phw_pmc; + + return (0); +} + +static void +pt_enumerate(struct pt_cpu *pt_pc) +{ + u_int cp[4]; + u_int *eax; + u_int *ebx; + u_int *ecx; + + eax = &cp[0]; + ebx = &cp[1]; + ecx = &cp[2]; + + dprintf("Enumerating part 1\n"); + + cpuid_count(PT_CPUID, 0, cp); + dprintf("%s: Maximum valid sub-leaf Index: %x\n", __func__, cp[0]); + dprintf("%s: ebx %x\n", __func__, cp[1]); + dprintf("%s: ecx %x\n", __func__, cp[2]); + + pt_pc->l0_eax = cp[0]; + pt_pc->l0_ebx = cp[1]; + pt_pc->l0_ecx = cp[2]; + + dprintf("Enumerating part 2\n"); + + cpuid_count(PT_CPUID, 1, cp); + dprintf("%s: eax %x\n", __func__, cp[0]); + dprintf("%s: ebx %x\n", __func__, cp[1]); + + pt_pc->l1_eax = cp[0]; + pt_pc->l1_ebx = cp[1]; +} + +static int +pt_pcpu_init(struct pmc_mdep *md, int cpu) +{ + struct pmc_cpu *pc; + struct pt_cpu *pt_pc; + u_int cp[4]; + int ri; + + dprintf("%s: cpu %d\n", __func__, cpu); + + /* We rely on XSAVE support */ + if ((cpu_feature2 & CPUID2_XSAVE) == 0) { + printf("Intel PT: XSAVE is not supported\n"); + return (ENXIO); + } + + cpuid_count(0xd, 0x0, cp); + if ((cp[0] & pt_xsave_mask) != pt_xsave_mask) { + printf("Intel PT: CPU0 does not support X87 or SSE: %x", cp[0]); + return (ENXIO); + } + + cpuid_count(0xd, 0x1, cp); + if ((cp[0] & (1 << 0)) == 0) { + printf("Intel PT: XSAVE compaction is not supported\n"); + return (ENXIO); + } + + if ((cp[0] & (1 << 3)) == 0) { + printf("Intel PT: XSAVES/XRSTORS are not supported\n"); + return (ENXIO); + } + + /* Enable XSAVE */ + load_cr4(rcr4() | CR4_XSAVE); + + KASSERT(cpu == PCPU_GET(cpuid), ("Init on wrong CPU\n")); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[pt,%d] illegal cpu %d", __LINE__, cpu)); + KASSERT(pt_pcpu, ("[pt,%d] null pcpu", __LINE__)); + KASSERT(pt_pcpu[cpu] == NULL, ("[pt,%d] non-null per-cpu", + __LINE__)); + + pt_pc = malloc(sizeof(struct pt_cpu), M_PT, M_WAITOK | M_ZERO); + + pt_pc->tc_hw.phw_state = PMC_PHW_FLAG_IS_ENABLED | + PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(0) | + PMC_PHW_FLAG_IS_SHAREABLE; + + pt_pcpu[cpu] = pt_pc; + + ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_PT].pcd_ri; + + KASSERT(pmc_pcpu, ("[pt,%d] null generic pcpu", __LINE__)); + + pc = pmc_pcpu[cpu]; + + KASSERT(pc, ("[pt,%d] null generic per-cpu", __LINE__)); + + pc->pc_hwpmcs[ri] = &pt_pc->tc_hw; + + pt_enumerate(pt_pc); + + return (0); +} + +static int +pt_pcpu_fini(struct pmc_mdep *md, int cpu) +{ + int ri; + struct pmc_cpu *pc; + struct pt_cpu *pt_pc; + + dprintf("%s: cpu %d\n", __func__, cpu); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[pt,%d] illegal cpu %d", __LINE__, cpu)); + KASSERT(pt_pcpu[cpu] != NULL, ("[pt,%d] null pcpu", __LINE__)); + + pt_pc = pt_pcpu[cpu]; + + free(pt_pcpu[cpu], M_PT); + pt_pcpu[cpu] = NULL; + + ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_PT].pcd_ri; + + pc = pmc_pcpu[cpu]; + pc->pc_hwpmcs[ri] = NULL; + + return (0); +} + +static int +pt_trace_config(int cpu, int ri, struct pmc *pm, + uint64_t *ranges, uint32_t nranges) +{ + struct pt_cpu *pt_pc; + uint64_t reg; + + dprintf("%s\n", __func__); + + pt_pc = pt_pcpu[cpu]; + + KASSERT(cpu == PCPU_GET(cpuid), ("Configuring wrong CPU\n")); + + /* Ensure tracing is turned off */ + reg = rdmsr(MSR_IA32_RTIT_CTL); + if (reg & RTIT_CTL_TRACEEN) + pt_save_restore(pt_pc, true); + + pt_configure_ranges(pt_pc, ranges, nranges); + + return (0); +} + +static int +pt_read_trace(int cpu, int ri, struct pmc *pm, + pmc_value_t *cycle, pmc_value_t *voffset) +{ + struct pt_ext_area *pt_ext; + struct pt_save_area *save_area; + struct pmc_md_pt_pmc *pm_pt; + struct pt_buffer *pt_buf; + struct pt_cpu *pt_pc; + uint64_t offset; + uint64_t reg; + uint32_t idx; + + pt_pc = pt_pcpu[cpu]; + pt_pc->pm_mmap = pm; + + pm_pt = (struct pmc_md_pt_pmc *)&pm->pm_md; + pt_buf = &pm_pt->pt_buffers[cpu]; + + save_area = &pt_pc->save_area; + pt_ext = &save_area->pt_ext_area; + + reg = rdmsr(MSR_IA32_RTIT_CTL); + if (reg & RTIT_CTL_TRACEEN) + reg = rdmsr(MSR_IA32_RTIT_OUTPUT_MASK_PTRS); + else + reg = pt_ext->rtit_output_mask_ptrs; + + idx = (reg & 0xffffffff) >> 7; + *cycle = pt_buf->cycle; + + offset = reg >> 32; + *voffset = pt_buf->topa_sw[idx].offset + offset; + + dprintf("%s: %lx\n", __func__, rdmsr(MSR_IA32_RTIT_OUTPUT_MASK_PTRS)); + dprintf("%s: cycle %ld offset %ld\n", __func__, pt_buf->cycle, offset); + + return (0); +} + +static int +pt_read_pmc(int cpu, int ri, pmc_value_t *v) +{ + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[pt,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[pt,%d] illegal ri %d", __LINE__, ri)); + + *v = 0; + + return (0); +} + +static int +pt_release_pmc(int cpu, int ri, struct pmc *pm) +{ + struct pmc_md_pt_pmc *pm_pt; + struct pt_cpu *pt_pc; + enum pmc_mode mode; + struct pmc_hw *phw; + int i; + + pm_pt = (struct pmc_md_pt_pmc *)&pm->pm_md; + pt_pc = pt_pcpu[cpu]; + + dprintf("%s: cpu %d (curcpu %d)\n", __func__, cpu, PCPU_GET(cpuid)); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[pt,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, + ("[pt,%d] illegal row-index %d", __LINE__, ri)); + + phw = &pt_pcpu[cpu]->tc_hw; + phw->phw_pmc = NULL; + + KASSERT(phw->phw_pmc == NULL, + ("[pt,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); + + dprintf("%s: cpu %d, output base %lx\n", + __func__, cpu, rdmsr(MSR_IA32_RTIT_OUTPUT_BASE)); + dprintf("%s: cpu %d, output base ptr %lx\n", + __func__, cpu, rdmsr(MSR_IA32_RTIT_OUTPUT_MASK_PTRS)); + + mode = PMC_TO_MODE(pm); + if (mode == PMC_MODE_TT) + for (i = 0; i < pmc_cpu_max(); i++) + pt_buffer_deallocate(i, &pm_pt->pt_buffers[i]); + else + pt_buffer_deallocate(cpu, &pm_pt->pt_buffers[cpu]); + + if (mode == PMC_MODE_ST) + pt_pc->flags &= ~FLAG_PT_ALLOCATED; + + return (0); +} + +static int +pt_start_pmc(int cpu, int ri) +{ + struct pt_cpu *pt_pc; + struct pmc_hw *phw; + + dprintf("%s: cpu %d (curcpu %d)\n", __func__, cpu, PCPU_GET(cpuid)); + + pt_pc = pt_pcpu[cpu]; + phw = &pt_pc->tc_hw; + if (phw == NULL || phw->phw_pmc == NULL) + return (-1); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[pt,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[pt,%d] illegal row-index %d", __LINE__, ri)); + + pt_save_restore(pt_pc, false); + + return (0); +} + +static int +pt_stop_pmc(int cpu, int ri) +{ + struct pt_cpu *pt_pc; + + pt_pc = pt_pcpu[cpu]; + + dprintf("%s: cpu %d, output base %lx, ptr %lx\n", __func__, cpu, + rdmsr(MSR_IA32_RTIT_OUTPUT_BASE), + rdmsr(MSR_IA32_RTIT_OUTPUT_MASK_PTRS)); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[pt,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[pt,%d] illegal row-index %d", __LINE__, ri)); + + /* + * Save the PT state to memory. + * This operation will disable tracing. + */ + pt_save_restore(pt_pc, true); + + return (0); +} + +static int +pt_write_pmc(int cpu, int ri, pmc_value_t v) +{ + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[pt,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[pt,%d] illegal row-index %d", __LINE__, ri)); + + return (0); +} + +int +pmc_pt_initialize(struct pmc_mdep *md, int maxcpu) +{ + struct pmc_classdep *pcd; + + dprintf("%s\n", __func__); + + pt_xsave_mask = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; + + KASSERT(md != NULL, ("[pt,%d] md is NULL", __LINE__)); + KASSERT(md->pmd_nclass >= 1, ("[pt,%d] dubious md->nclass %d", + __LINE__, md->pmd_nclass)); + + pt_pcpu = malloc(sizeof(struct pt_cpu *) * maxcpu, M_PT, + M_WAITOK | M_ZERO); + + pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_PT]; + + pcd->pcd_caps = PT_CAPS; + pcd->pcd_class = PMC_CLASS_PT; + pcd->pcd_num = PT_NPMCS; + pcd->pcd_ri = md->pmd_npmc; + pcd->pcd_width = 64; + + pcd->pcd_allocate_pmc = pt_allocate_pmc; + pcd->pcd_config_pmc = pt_config_pmc; + pcd->pcd_describe = pt_describe; + pcd->pcd_get_config = pt_get_config; + pcd->pcd_pcpu_init = pt_pcpu_init; + pcd->pcd_pcpu_fini = pt_pcpu_fini; + pcd->pcd_read_pmc = pt_read_pmc; + pcd->pcd_read_trace = pt_read_trace; + pcd->pcd_trace_config = pt_trace_config; + pcd->pcd_release_pmc = pt_release_pmc; + pcd->pcd_start_pmc = pt_start_pmc; + pcd->pcd_stop_pmc = pt_stop_pmc; + pcd->pcd_write_pmc = pt_write_pmc; + + md->pmd_npmc += PT_NPMCS; + + return (0); +} + +void +pmc_pt_finalize(struct pmc_mdep *md) +{ + + dprintf("%s\n", __func__); + +#ifdef INVARIANTS + int i, ncpus; + + ncpus = pmc_cpu_max(); + for (i = 0; i < ncpus; i++) + KASSERT(pt_pcpu[i] == NULL, ("[pt,%d] non-null pcpu cpu %d", + __LINE__, i)); + + KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_PT].pcd_class == + PMC_CLASS_PT, ("[pt,%d] class mismatch", __LINE__)); +#endif + + free(pt_pcpu, M_PT); + pt_pcpu = NULL; +} Index: sys/dev/hwpmc/hwpmc_vm.h =================================================================== --- /dev/null +++ sys/dev/hwpmc/hwpmc_vm.h @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2017 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _DEV_HWPMC_VM_H_ +#define _DEV_HWPMC_VM_H_ + +int pmc_vm_initialize(struct pmc_mdep *md); +int pmc_vm_finalize(void); + +struct pmc_vm_map { + TAILQ_ENTRY(pmc_vm_map) map_next; + struct thread *t; + vm_object_t obj; + void * buf; +}; + +struct cdev_cpu { + struct pmc_mdep *md; + struct mtx vm_mtx; + TAILQ_HEAD(, pmc_vm_map) pmc_maplist; + uint32_t cpu; +}; + +#endif /* !_DEV_HWPMC_VM_H_ */ Index: sys/dev/hwpmc/hwpmc_vm.c =================================================================== --- /dev/null +++ sys/dev/hwpmc/hwpmc_vm.c @@ -0,0 +1,134 @@ +/*- + * Copyright (c) 2017 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define PMC_VM_DEBUG +#undef PMC_VM_DEBUG + +#ifdef PMC_VM_DEBUG +#define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define dprintf(fmt, ...) +#endif + +#include "hwpmc_vm.h" + +struct cdev *pmc_cdev[MAXCPU]; + +static int +pmc_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, + vm_size_t mapsize, struct vm_object **objp, int nprot) +{ + struct pmc_vm_map *map, *map_tmp; + struct cdev_cpu *cc; + + cc = cdev->si_drv1; + + if (nprot != PROT_READ || *offset != 0) + return (ENXIO); + + mtx_lock(&cc->vm_mtx); + TAILQ_FOREACH_SAFE(map, &cc->pmc_maplist, map_next, map_tmp) { + if (map->t == curthread) { + mtx_unlock(&cc->vm_mtx); + *objp = map->obj; + return (0); + } + } + mtx_unlock(&cc->vm_mtx); + + return (ENXIO); +} + +static struct cdevsw pmc_cdevsw = { + .d_version = D_VERSION, + .d_mmap_single = pmc_mmap_single, + .d_name = "HWPMC", +}; + +int +pmc_vm_initialize(struct pmc_mdep *md) +{ + unsigned int maxcpu; + struct cdev_cpu *cc; + int cpu; + + maxcpu = pmc_cpu_max(); + + for (cpu = 0; cpu < maxcpu; cpu++) { + cc = malloc(sizeof(struct cdev_cpu), M_PMC, M_WAITOK | M_ZERO); + cc->cpu = cpu; + cc->md = md; + mtx_init(&cc->vm_mtx, "PMC VM", NULL, MTX_DEF); + TAILQ_INIT(&cc->pmc_maplist); + + pmc_cdev[cpu] = make_dev(&pmc_cdevsw, 0, UID_ROOT, GID_WHEEL, + 0666, "pmc%d", cpu); + pmc_cdev[cpu]->si_drv1 = cc; + } + + return (0); +} + +int +pmc_vm_finalize(void) +{ + unsigned int maxcpu; + struct cdev_cpu *cc; + int cpu; + + maxcpu = pmc_cpu_max(); + + for (cpu = 0; cpu < maxcpu; cpu++) { + cc = pmc_cdev[cpu]->si_drv1; + mtx_destroy(&cc->vm_mtx); + free(cc, M_PMC); + destroy_dev(pmc_cdev[cpu]); + } + + return (0); +} Index: sys/dev/hwpmc/pmc_events.h =================================================================== --- sys/dev/hwpmc/pmc_events.h +++ sys/dev/hwpmc/pmc_events.h @@ -4843,6 +4843,20 @@ #define PMC_EV_TSC_FIRST PMC_EV_TSC_TSC #define PMC_EV_TSC_LAST PMC_EV_TSC_TSC +/* Intel PT */ +#define __PMC_EV_PT() \ + __PMC_EV(PT, PT) + +#define PMC_EV_PT_FIRST PMC_EV_PT_PT +#define PMC_EV_PT_LAST PMC_EV_PT_PT + +/* ARM CORESIGHT */ +#define __PMC_EV_CORESIGHT() \ + __PMC_EV(CORESIGHT, CORESIGHT) + +#define PMC_EV_CORESIGHT_FIRST PMC_EV_CORESIGHT_CORESIGHT +#define PMC_EV_CORESIGHT_LAST PMC_EV_CORESIGHT_CORESIGHT + /* * Software events are dynamically defined. */ @@ -7141,6 +7155,7 @@ * START #EVENTS DESCRIPTION * 0 0x1000 Reserved * 0x1000 0x0001 TSC + * 0x1100 0x0001 PT * 0x2000 0x0080 AMD K7 events * 0x2080 0x0100 AMD K8 events * 0x10000 0x0080 INTEL architectural fixed-function events @@ -7157,11 +7172,14 @@ * 0x13300 0x00FF Freescale e500 events * 0x14000 0x0100 ARMv7 events * 0x14100 0x0100 ARMv8 events + * 0x14200 0x0001 ARM Coresight * 0x20000 0x1000 Software events */ #define __PMC_EVENTS() \ __PMC_EV_BLOCK(TSC, 0x01000) \ __PMC_EV_TSC() \ + __PMC_EV_BLOCK(PT, 0x1100) \ + __PMC_EV_PT() \ __PMC_EV_BLOCK(K7, 0x2000) \ __PMC_EV_K7() \ __PMC_EV_BLOCK(K8, 0x2080) \ @@ -7197,7 +7215,9 @@ __PMC_EV_BLOCK(ARMV7, 0x14000) \ __PMC_EV_ARMV7() \ __PMC_EV_BLOCK(ARMV8, 0x14100) \ - __PMC_EV_ARMV8() + __PMC_EV_ARMV8() \ + __PMC_EV_BLOCK(CORESIGHT, 0x14200) \ + __PMC_EV_CORESIGHT() #define PMC_EVENT_FIRST PMC_EV_TSC_TSC #define PMC_EVENT_LAST PMC_EV_SOFT_LAST Index: sys/kern/vfs_vnops.c =================================================================== --- sys/kern/vfs_vnops.c +++ sys/kern/vfs_vnops.c @@ -2489,7 +2489,7 @@ if ((prot & VM_PROT_EXECUTE) != 0 && error == 0) { pkm.pm_file = vp; pkm.pm_address = (uintptr_t) *addr; - PMC_CALL_HOOK(td, PMC_FN_MMAP, (void *) &pkm); + PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_MMAP, (void *) &pkm); } } #endif Index: sys/modules/hwpmc/Makefile =================================================================== --- sys/modules/hwpmc/Makefile +++ sys/modules/hwpmc/Makefile @@ -6,14 +6,16 @@ KMOD= hwpmc -SRCS= hwpmc_mod.c hwpmc_logging.c hwpmc_soft.c vnode_if.h +SRCS= hwpmc_mod.c hwpmc_logging.c hwpmc_soft.c hwpmc_vm.c vnode_if.h .if ${MACHINE_CPUARCH} == "aarch64" SRCS+= hwpmc_arm64.c hwpmc_arm64_md.c +SRCS+= hwpmc_cs.c .endif .if ${MACHINE_CPUARCH} == "amd64" SRCS+= hwpmc_amd.c hwpmc_core.c hwpmc_intel.c hwpmc_piv.c hwpmc_tsc.c +SRCS+= hwpmc_pt.c SRCS+= hwpmc_x86.c hwpmc_uncore.c SRCS+= device_if.h bus_if.h .endif Index: sys/sys/pmc.h =================================================================== --- sys/sys/pmc.h +++ sys/sys/pmc.h @@ -101,6 +101,7 @@ __PMC_CPU(INTEL_BROADWELL_XEON, 0x97, "Intel Broadwell Xeon") \ __PMC_CPU(INTEL_SKYLAKE, 0x98, "Intel Skylake") \ __PMC_CPU(INTEL_SKYLAKE_XEON, 0x99, "Intel Skylake Xeon") \ + __PMC_CPU(INTEL_KABYLAKE, 0x9A, "Intel Kabylake") \ __PMC_CPU(INTEL_XSCALE, 0x100, "Intel XScale") \ __PMC_CPU(MIPS_24K, 0x200, "MIPS 24K") \ __PMC_CPU(MIPS_OCTEON, 0x201, "Cavium Octeon") \ @@ -151,7 +152,9 @@ __PMC_CLASS(ARMV7, 0x10, "ARMv7") \ __PMC_CLASS(ARMV8, 0x11, "ARMv8") \ __PMC_CLASS(MIPS74K, 0x12, "MIPS 74K") \ - __PMC_CLASS(E500, 0x13, "Freescale e500 class") + __PMC_CLASS(E500, 0x13, "Freescale e500 class") \ + __PMC_CLASS(PT, 0x14, "Intel PT") \ + __PMC_CLASS(CORESIGHT, 0x15, "ARM Coresight") enum pmc_class { #undef __PMC_CLASS @@ -160,7 +163,7 @@ }; #define PMC_CLASS_FIRST PMC_CLASS_TSC -#define PMC_CLASS_LAST PMC_CLASS_E500 +#define PMC_CLASS_LAST PMC_CLASS_CORESIGHT /* * A PMC can be in the following states: @@ -231,7 +234,9 @@ __PMC_MODE(SS, 0) \ __PMC_MODE(SC, 1) \ __PMC_MODE(TS, 2) \ - __PMC_MODE(TC, 3) + __PMC_MODE(TC, 3) \ + __PMC_MODE(ST, 4) \ + __PMC_MODE(TT, 5) enum pmc_mode { #undef __PMC_MODE @@ -245,11 +250,11 @@ #define PMC_IS_COUNTING_MODE(mode) \ ((mode) == PMC_MODE_SC || (mode) == PMC_MODE_TC) #define PMC_IS_SYSTEM_MODE(mode) \ - ((mode) == PMC_MODE_SS || (mode) == PMC_MODE_SC) + ((mode) == PMC_MODE_SS || (mode) == PMC_MODE_SC || (mode) == PMC_MODE_ST) #define PMC_IS_SAMPLING_MODE(mode) \ ((mode) == PMC_MODE_SS || (mode) == PMC_MODE_TS) #define PMC_IS_VIRTUAL_MODE(mode) \ - ((mode) == PMC_MODE_TS || (mode) == PMC_MODE_TC) + ((mode) == PMC_MODE_TS || (mode) == PMC_MODE_TC || (mode) == PMC_MODE_TT) /* * PMC row disposition @@ -341,7 +346,11 @@ __PMC_OP(PMCSTOP, "Stop a PMC") \ __PMC_OP(WRITELOG, "Write a cookie to the log file") \ __PMC_OP(CLOSELOG, "Close log file") \ - __PMC_OP(GETDYNEVENTINFO, "Get dynamic events list") + __PMC_OP(GETDYNEVENTINFO, "Get dynamic events list") \ + __PMC_OP(LOG_KERNEL_MAP, "Log kernel mappings") \ + __PMC_OP(THREAD_UNSUSPEND, "Thread unsuspend") \ + __PMC_OP(TRACE_READ, "Read trace buffer pointer") \ + __PMC_OP(TRACE_CONFIG, "Setup trace IP ranges") enum pmc_ops { @@ -487,7 +496,6 @@ pmc_value_t pm_value; /* new&returned value */ }; - /* * OP GETPMCINFO * @@ -513,6 +521,40 @@ struct pmc_info pm_pmcs[]; /* space for 'npmc' structures */ }; +/* + * OP PROC_UNSUSPEND + * + * Unsuspend all threads of proc. + */ + +struct pmc_op_proc_unsuspend { + pmc_id_t pm_pmcid; + pid_t pm_pid; +}; + +/* + * OP TRACE_CONFIG + */ + +#define PMC_FILTER_MAX_IP_RANGES 4 + +struct pmc_op_trace_config { + pmc_id_t pm_pmcid; + uint32_t pm_cpu; /* CPU number or PMC_CPU_ANY */ + uint64_t ranges[2 * PMC_FILTER_MAX_IP_RANGES]; + uint32_t nranges; +}; + +/* + * OP TRACE_READ + */ + +struct pmc_op_trace_read { + pmc_id_t pm_pmcid; + uint32_t pm_cpu; + pmc_value_t pm_cycle; /* returned value */ + pmc_value_t pm_offset; /* returned value */ +}; /* * OP GETCPUINFO @@ -520,7 +562,6 @@ * Retrieve system CPU information. */ - struct pmc_classinfo { enum pmc_class pm_class; /* class id */ uint32_t pm_caps; /* counter capabilities */ @@ -951,6 +992,12 @@ int (*pcd_read_pmc)(int _cpu, int _ri, pmc_value_t *_value); int (*pcd_write_pmc)(int _cpu, int _ri, pmc_value_t _value); + /* trace */ + int (*pcd_read_trace)(int _cpu, int _ri, struct pmc *_pm, + pmc_value_t *_cycle, pmc_value_t *_offset); + int (*pcd_trace_config)(int _cpu, int _ri, struct pmc *_pm, + uint64_t *ranges, uint32_t nranges); + /* pmc allocation/release */ int (*pcd_allocate_pmc)(int _cpu, int _ri, struct pmc *_t, const struct pmc_op_pmcallocate *_a); @@ -978,7 +1025,7 @@ * Machine dependent bits needed per CPU type. */ -struct pmc_mdep { +struct pmc_mdep { uint32_t pmd_cputype; /* from enum pmc_cputype */ uint32_t pmd_npmc; /* number of PMCs per CPU */ uint32_t pmd_nclass; /* number of PMC classes present */ Index: sys/x86/include/specialreg.h =================================================================== --- sys/x86/include/specialreg.h +++ sys/x86/include/specialreg.h @@ -104,6 +104,7 @@ #define XFEATURE_ENABLED_OPMASK 0x00000020 #define XFEATURE_ENABLED_ZMM_HI256 0x00000040 #define XFEATURE_ENABLED_HI16_ZMM 0x00000080 +#define XFEATURE_ENABLED_PT 0x00000100 #define XFEATURE_AVX \ (XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE | XFEATURE_ENABLED_AVX) Index: usr.sbin/Makefile =================================================================== --- usr.sbin/Makefile +++ usr.sbin/Makefile @@ -183,6 +183,7 @@ SUBDIR.${MK_PMC}+= pmccontrol SUBDIR.${MK_PMC}+= pmcstat SUBDIR.${MK_PMC}+= pmcstudy +SUBDIR.${MK_PMC}+= pmctrace SUBDIR.${MK_PORTSNAP}+= portsnap SUBDIR.${MK_PPP}+= ppp SUBDIR.${MK_QUOTAS}+= edquota Index: usr.sbin/pmctrace/Makefile =================================================================== --- /dev/null +++ usr.sbin/pmctrace/Makefile @@ -0,0 +1,22 @@ +# @(#)Makefile 8.1 (Berkeley) 6/6/93 +# $FreeBSD$ + +PROG= pmctrace +SRCS= pmctrace.c +MAN= + +LIBADD= elf pmc pmcstat + +.if ${MACHINE_CPUARCH} == "amd64" +SRCS+= pmctrace_pt.c \ + pmctrace_pt.h +LIBADD+= ipt +.endif + +.if ${MACHINE_CPUARCH} == "aarch64" +SRCS+= pmctrace_cs.c \ + pmctrace_cs.h +LIBADD+= opencsd +.endif + +.include Index: usr.sbin/pmctrace/pmctrace.h =================================================================== --- /dev/null +++ usr.sbin/pmctrace/pmctrace.h @@ -0,0 +1,70 @@ +/*- + * Copyright (c) 2017 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _PMCTRACE_H_ +#define _PMCTRACE_H_ + +struct mtrace_data { + uint64_t ip; + int cpu; + struct pmcstat_process *pp; + uint32_t flags; +}; + +struct trace_cpu { + uint32_t cycle; + uint64_t offset; + struct mtrace_data mdata; + uint32_t bufsize; + void *base; + int fd; +}; + +struct trace_dev_methods { + int (*process)(struct trace_cpu *, struct pmcstat_process *, + uint32_t cpu, uint32_t cycle, uint64_t offset); + int (*init)(struct trace_cpu *tc); + int (*option)(int option); +}; + +struct trace_dev { + const char *ev_spec; + struct trace_dev_methods *methods; +}; + +struct pmctrace_config { + struct trace_dev *trace_dev; + uint32_t flags; +}; + +#endif /* !_PMCTRACE_H_ */ Index: usr.sbin/pmctrace/pmctrace.c =================================================================== --- /dev/null +++ usr.sbin/pmctrace/pmctrace.c @@ -0,0 +1,683 @@ +/*- + * Copyright (c) 2017 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "pmctrace.h" +#if defined(__amd64__) +#include "pmctrace_pt.h" +#endif +#if defined(__aarch64__) +#include "pmctrace_cs.h" +#endif + +#define MAX_CPU 4096 + +#define PMCTRACE_DEBUG +#undef PMCTRACE_DEBUG + +#ifdef PMCTRACE_DEBUG +#define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define dprintf(fmt, ...) +#endif + +static struct pmcstat_args args; +static struct kevent kev; +static struct pmcstat_process *pmcstat_kernproc; +static struct pmcstat_stats pmcstat_stats; +static struct trace_cpu *trace_cpus[MAX_CPU]; +static struct pmc_plugins plugins[] = {}; + +static int pmcstat_sockpair[NSOCKPAIRFD]; +static int pmcstat_kq; +static int pmcstat_npmcs; +static int pmcstat_mergepmc; +static int ps_samples_period; + +struct pmcstat_image_hash_list pmcstat_image_hash[PMCSTAT_NHASH]; +struct pmcstat_process_hash_list pmcstat_process_hash[PMCSTAT_NHASH]; +struct pmcstat_pmcs pmcstat_pmcs = LIST_HEAD_INITIALIZER(pmcstat_pmcs); + +static struct trace_dev trace_devs[] = { +#if defined(__amd64__) + { "pt", &ipt_methods }, +#elif defined(__aarch64__) + { "coresight", &cs_methods }, +#endif + { NULL, NULL } +}; + +static struct pmctrace_config pmctrace_cfg; + +static int +pmctrace_ncpu(void) +{ + size_t ncpu_size; + int error; + int ncpu; + + ncpu_size = sizeof(ncpu); + error = sysctlbyname("hw.ncpu", &ncpu, &ncpu_size, NULL, 0); + if (error) + return (-1); + + return (ncpu); +} + +static int +pmctrace_init_cpu(uint32_t cpu) +{ + struct trace_cpu *tc; + char filename[16]; + struct mtrace_data *mdata; + + tc = trace_cpus[cpu]; + mdata = &tc->mdata; + mdata->ip = 0; + mdata->cpu = cpu; + + sprintf(filename, "/dev/pmc%d", cpu); + + tc->fd = open(filename, O_RDWR); + if (tc->fd < 0) { + printf("Can't open %s\n", filename); + return (-1); + } + + tc->bufsize = 16 * 1024 * 1024; + tc->cycle = 0; + tc->offset = 0; + + tc->base = mmap(NULL, tc->bufsize, PROT_READ, MAP_SHARED, tc->fd, 0); + if (tc->base == MAP_FAILED) { + printf("mmap failed: err %d\n", errno); + return (-1); + } + dprintf("%s: tc->base %lx, *tc->base %lx\n", __func__, + (uint64_t)tc->base, *(uint64_t *)tc->base); + + if (pmctrace_cfg.trace_dev->methods->init != NULL) + pmctrace_cfg.trace_dev->methods->init(tc); + + return (0); +} + +static int +pmctrace_process_cpu(int cpu, struct pmcstat_ev *ev) +{ + struct pmcstat_process *pp; + struct pmcstat_target *pt; + pmc_value_t offset; + pmc_value_t cycle; + struct trace_cpu *tc; + struct trace_dev *trace_dev; + + trace_dev = pmctrace_cfg.trace_dev; + tc = trace_cpus[cpu]; + + pmc_read_trace(cpu, ev->ev_pmcid, &cycle, &offset); + + dprintf("cpu %d cycle %lx offset %lx\n", cpu, cycle, offset); + + pt = SLIST_FIRST(&args.pa_targets); + if (pt != NULL) + pp = pmcstat_process_lookup(pt->pt_pid, 0); + else + pp = pmcstat_kernproc; + + if (pp) + trace_dev->methods->process(tc, pp, cpu, cycle, offset); + else + dprintf("pp not found\n"); + + return (0); +} + +static int +pmctrace_process_all(int user_mode) +{ + struct pmcstat_ev *ev; + int ncpu; + int i; + + ncpu = pmctrace_ncpu(); + if (ncpu < 0) + errx(EX_SOFTWARE, "ERROR: Can't get cpus\n"); + + if (user_mode) { + ev = STAILQ_FIRST(&args.pa_events); + for (i = 0; i < ncpu; i++) + pmctrace_process_cpu(i, ev); + } else + STAILQ_FOREACH(ev, &args.pa_events, ev_next) + pmctrace_process_cpu(ev->ev_cpu, ev); + + return (0); +} + +static void +pmctrace_cleanup(void) +{ + struct pmcstat_ev *ev; + + /* release allocated PMCs. */ + STAILQ_FOREACH(ev, &args.pa_events, ev_next) + if (ev->ev_pmcid != PMC_ID_INVALID) { + if (pmc_stop(ev->ev_pmcid) < 0) + err(EX_OSERR, + "ERROR: cannot stop pmc 0x%x \"%s\"", + ev->ev_pmcid, ev->ev_name); + if (pmc_release(ev->ev_pmcid) < 0) + err(EX_OSERR, + "ERROR: cannot release pmc 0x%x \"%s\"", + ev->ev_pmcid, ev->ev_name); + } + + /* de-configure the log file if present. */ + if (args.pa_flags & (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE)) + (void) pmc_configure_logfile(-1); + + if (args.pa_logparser) { + pmclog_close(args.pa_logparser); + args.pa_logparser = NULL; + } + + pmcstat_shutdown_logging(&args, plugins, &pmcstat_stats); +} + +static void +pmctrace_start_pmcs(void) +{ + struct pmcstat_ev *ev; + + STAILQ_FOREACH(ev, &args.pa_events, ev_next) { + dprintf("starting ev->ev_cpu %d\n", ev->ev_cpu); + assert(ev->ev_pmcid != PMC_ID_INVALID); + if (pmc_start(ev->ev_pmcid) < 0) { + warn("ERROR: Cannot start pmc 0x%x \"%s\"", + ev->ev_pmcid, ev->ev_name); + pmctrace_cleanup(); + exit(EX_OSERR); + } + } +} + +static int +pmctrace_open_logfile(void) +{ + int pipefd[2]; + + /* + * process the log on the fly by reading it in + * through a pipe. + */ + if (pipe(pipefd) < 0) + err(EX_OSERR, "ERROR: pipe(2) failed"); + + if (fcntl(pipefd[READPIPEFD], F_SETFL, O_NONBLOCK) < 0) + err(EX_OSERR, "ERROR: fcntl(2) failed"); + + EV_SET(&kev, pipefd[READPIPEFD], EVFILT_READ, EV_ADD, + 0, 0, NULL); + + if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0) + err(EX_OSERR, "ERROR: Cannot register kevent"); + + args.pa_logfd = pipefd[WRITEPIPEFD]; + args.pa_flags |= FLAG_HAS_PIPE; + args.pa_logparser = pmclog_open(pipefd[READPIPEFD]); + + if (pmc_configure_logfile(args.pa_logfd) < 0) + err(EX_OSERR, "ERROR: Cannot configure log file"); + + return (0); +} + +static int +pmctrace_find_kernel(void) +{ + struct stat sb; + char buffer[PATH_MAX]; + size_t len; + char *tmp; + + /* Default to using the running system kernel. */ + len = 0; + if (sysctlbyname("kern.bootfile", NULL, &len, NULL, 0) == -1) + err(EX_OSERR, "ERROR: Cannot determine path of running kernel"); + args.pa_kernel = malloc(len); + if (args.pa_kernel == NULL) + errx(EX_SOFTWARE, "ERROR: Out of memory."); + if (sysctlbyname("kern.bootfile", args.pa_kernel, &len, NULL, 0) == -1) + err(EX_OSERR, "ERROR: Cannot determine path of running kernel"); + + /* + * Check if 'kerneldir' refers to a file rather than a + * directory. If so, use `dirname path` to determine the + * kernel directory. + */ + (void) snprintf(buffer, sizeof(buffer), "%s%s", args.pa_fsroot, + args.pa_kernel); + if (stat(buffer, &sb) < 0) + err(EX_OSERR, "ERROR: Cannot locate kernel \"%s\"", + buffer); + if (!S_ISREG(sb.st_mode) && !S_ISDIR(sb.st_mode)) + errx(EX_USAGE, "ERROR: \"%s\": Unsupported file type.", + buffer); + if (!S_ISDIR(sb.st_mode)) { + tmp = args.pa_kernel; + args.pa_kernel = strdup(dirname(args.pa_kernel)); + if (args.pa_kernel == NULL) + errx(EX_SOFTWARE, "ERROR: Out of memory"); + free(tmp); + (void) snprintf(buffer, sizeof(buffer), "%s%s", + args.pa_fsroot, args.pa_kernel); + if (stat(buffer, &sb) < 0) + err(EX_OSERR, "ERROR: Cannot stat \"%s\"", + buffer); + if (!S_ISDIR(sb.st_mode)) + errx(EX_USAGE, + "ERROR: \"%s\" is not a directory.", + buffer); + } + + return (0); +} + +static void +pmctrace_setup_cpumask(cpuset_t *cpumask) +{ + cpuset_t rootmask; + + /* + * The initial CPU mask specifies the root mask of this process + * which is usually all CPUs in the system. + */ + if (cpuset_getaffinity(CPU_LEVEL_ROOT, CPU_WHICH_PID, -1, + sizeof(rootmask), &rootmask) == -1) + err(EX_OSERR, "ERROR: Cannot determine the root set of CPUs"); + CPU_COPY(&rootmask, cpumask); +} + +static int +pmctrace_delayed_start(bool user_mode, char *func_name, char *func_image) +{ + uint64_t ranges[2]; + struct pmcstat_symbol *sym; + struct pmcstat_target *pt; + struct pmcstat_process *pp; + struct pmcstat_ev *ev; + uintptr_t addr_start; + uintptr_t addr_end; + int ncpu; + int i; + + if (func_name == NULL || func_image == NULL) + return (0); + + ncpu = pmctrace_ncpu(); + if (ncpu < 0) + errx(EX_SOFTWARE, "ERROR: Can't get cpus\n"); + + if (user_mode) { + pt = SLIST_FIRST(&args.pa_targets); + if (pt == NULL) + errx(EX_SOFTWARE, "ERROR: can't get target."); + pp = pmcstat_process_lookup(pt->pt_pid, 0); + if (pp == NULL) + errx(EX_SOFTWARE, "ERROR: pp is NULL, pid %d\n", + (uint32_t)pt->pt_pid); + } else + pp = pmcstat_kernproc; + + sym = pmcstat_symbol_search_by_name(pp, func_image, func_name, + &addr_start, &addr_end); + if (!sym) + return (0); + + dprintf("%s: SYM addr start %lx end %lx\n", + __func__, addr_start, addr_end); + + ranges[0] = addr_start; + ranges[1] = addr_end; + + if (user_mode) { + ev = STAILQ_FIRST(&args.pa_events); + for (i = 0; i < ncpu; i++) + pmc_trace_config(i, ev->ev_pmcid, &ranges[0], 1); + } else { + STAILQ_FOREACH(ev, &args.pa_events, ev_next) + pmc_trace_config(ev->ev_cpu, + ev->ev_pmcid, &ranges[0], 1); + } + + pmctrace_start_pmcs(); + + return (1); +} + + +static int +pmctrace_run(bool user_mode, char *func_name, char *func_image) +{ + struct pmcstat_target *pt; + struct pmcstat_process *pp; + struct pmcstat_ev *ev; + int stopping; + int running; + int started; + int c; + + stopping = 0; + running = 3; + started = 0; + + if (user_mode) { + pmcstat_create_process(pmcstat_sockpair, &args, pmcstat_kq); + pmcstat_attach_pmcs(&args); + if (func_name == NULL || func_image == NULL) { + pmctrace_start_pmcs(); + started = 1; + } + pmcstat_start_process(pmcstat_sockpair); + } else { + if (func_name == NULL || func_image == NULL) { + pmctrace_start_pmcs(); + started = 1; + } else { + ev = STAILQ_FIRST(&args.pa_events); + STAILQ_FOREACH(ev, &args.pa_events, ev_next) { + pmc_log_kmap(ev->ev_pmcid); + } + } + } + + do { + if ((c = kevent(pmcstat_kq, NULL, 0, &kev, 1, NULL)) <= 0) { + if (errno != EINTR) + err(EX_OSERR, "ERROR: kevent failed"); + else + continue; + } + + dprintf("%s: pmcstat event: filter %d, ident %ld\n", + __func__, kev.filter, kev.ident); + + if (kev.flags & EV_ERROR) + errc(EX_OSERR, kev.data, "ERROR: kevent failed"); + + switch (kev.filter) { + case EVFILT_PROC: + stopping = 1; + break; + case EVFILT_READ: + args.pa_flags |= FLAG_DO_ANALYSIS; + pmcstat_analyze_log(&args, plugins, &pmcstat_stats, + pmcstat_kernproc, pmcstat_mergepmc, &pmcstat_npmcs, + &ps_samples_period); + + if (started == 0 && + pmctrace_delayed_start(user_mode, func_name, func_image) == 1) + started = 1; + + if (user_mode) { + pt = SLIST_FIRST(&args.pa_targets); + ev = STAILQ_FIRST(&args.pa_events); + pmc_proc_unsuspend(ev->ev_pmcid, pt->pt_pid); + } + + break; + case EVFILT_TIMER: + pmc_flush_logfile(); + + pp = pmcstat_kernproc; + if (!user_mode && TAILQ_EMPTY(&pp->pp_map)) + break; + + pmctrace_process_all(user_mode); + + if (stopping) + running -= 1; + break; + } + } while (running > 0); + + return (0); +} + +static void +usage(void) +{ + + errx(EX_USAGE, + "[options] [commandline]\n" + "\t -s device\t\tTrace kernel\n" + "\t -u device\t\tTrace userspace\n" + ); +} + +int +main(int argc, char *argv[]) +{ + struct pmcstat_ev *ev; + bool user_mode; + bool supervisor_mode; + int option; + cpuset_t cpumask; + char *func_name; + char *func_image; + int ncpu; + int i; + + bzero(&args, sizeof(struct pmcstat_args)); + bzero(&pmctrace_cfg, sizeof(struct pmctrace_config)); + + func_name = NULL; + func_image = NULL; + + user_mode = 0; + supervisor_mode = 0; + + STAILQ_INIT(&args.pa_events); + SLIST_INIT(&args.pa_targets); + CPU_ZERO(&cpumask); + + args.pa_fsroot = strdup("/"); /* TODO */ + + pmctrace_find_kernel(); + pmctrace_setup_cpumask(&cpumask); + + while ((option = getopt(argc, argv, + "htu:s:i:f:")) != -1) + switch (option) { + case 'i': + func_image = strdup(optarg); + break; + case 'f': + func_name = strdup(optarg); + break; + case 'u': + case 's': + if (ev != NULL) + usage(); + + if ((ev = malloc(sizeof(struct pmcstat_ev))) == NULL) + errx(EX_SOFTWARE, "ERROR: Out of memory."); + if (option == 'u') { + user_mode = 1; + ev->ev_mode = PMC_MODE_TT; + args.pa_flags |= FLAG_HAS_PROCESS_PMCS; + } else { + ev->ev_mode = PMC_MODE_ST; + supervisor_mode = 1; + } + ev->ev_spec = strdup(optarg); + if (ev->ev_spec == NULL) + errx(EX_SOFTWARE, "ERROR: Out of memory."); + + for (i = 0; trace_devs[i].ev_spec != NULL; i++) { + if (strncmp(trace_devs[i].ev_spec, ev->ev_spec, + strlen(trace_devs[i].ev_spec)) == 0) { + /* found */ + pmctrace_cfg.trace_dev = &trace_devs[i]; + break; + } + } + + if (pmctrace_cfg.trace_dev == NULL) + errx(EX_SOFTWARE, "ERROR: trace device not found"); + break; + case 't': + if (ev == NULL) + usage(); + + if (pmctrace_cfg.trace_dev->methods->option != NULL) + pmctrace_cfg.trace_dev->methods->option(option); + break; + case 'h': + usage(); + default: + break; + }; + + if ((user_mode == 0 && supervisor_mode == 0) || + (user_mode == 1 && supervisor_mode == 1)) + errx(EX_USAGE, "ERROR: specify -u or -s"); + + if ((func_image == NULL && func_name != NULL) || + (func_image != NULL && func_name == NULL)) + errx(EX_USAGE, "ERROR: specify both or neither -i and -f"); + + args.pa_argc = (argc -= optind); + args.pa_argv = (argv += optind); + args.pa_cpumask = cpumask; + + if (user_mode && !argc) + errx(EX_USAGE, "ERROR: user mode requires command to be specified"); + if (supervisor_mode && argc) + errx(EX_USAGE, "ERROR: supervisor mode does not require command"); + + args.pa_required |= (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE); + + ev->ev_saved = 0LL; + ev->ev_pmcid = PMC_ID_INVALID; + ev->ev_name = strdup("pmctrace"); + ev->ev_flags = 0; + + if (!user_mode) + ev->ev_cpu = CPU_FFS(&cpumask) - 1; + else + ev->ev_cpu = PMC_CPU_ANY; + + STAILQ_INSERT_TAIL(&args.pa_events, ev, ev_next); + + if (!user_mode) { + CPU_CLR(ev->ev_cpu, &cpumask); + pmcstat_clone_event_descriptor(ev, &cpumask, &args); + CPU_SET(ev->ev_cpu, &cpumask); + } + + ncpu = pmctrace_ncpu(); + if (ncpu < 0) + errx(EX_SOFTWARE, "ERROR: Can't get cpus\n"); + + if (pmc_init() < 0) + err(EX_UNAVAILABLE, "ERROR: Initialization of the pmc(3) library failed"); + + if ((pmcstat_kq = kqueue()) < 0) + err(EX_OSERR, "ERROR: Cannot allocate kqueue"); + + pmctrace_open_logfile(); + + STAILQ_FOREACH(ev, &args.pa_events, ev_next) { + if (pmc_allocate(ev->ev_spec, ev->ev_mode, + ev->ev_flags, ev->ev_cpu, &ev->ev_pmcid) < 0) + err(EX_OSERR, + "ERROR: Cannot allocate %s-mode pmc with specification \"%s\"", + PMC_IS_SYSTEM_MODE(ev->ev_mode) ? + "system" : "process", ev->ev_spec); + } + + for (i = 0; i < ncpu; i++) { + trace_cpus[i] = malloc(sizeof(struct trace_cpu)); + pmctrace_init_cpu(i); + } + + EV_SET(&kev, 0, EVFILT_TIMER, EV_ADD, 0, 100, NULL); + + if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0) + err(EX_OSERR, "ERROR: Cannot register kevent for timer"); + + pmcstat_initialize_logging(&pmcstat_kernproc, + &args, plugins, &pmcstat_npmcs, &pmcstat_mergepmc); + + pmctrace_run(user_mode, func_name, func_image); + + return (0); +} Index: usr.sbin/pmctrace/pmctrace_cs.h =================================================================== --- /dev/null +++ usr.sbin/pmctrace/pmctrace_cs.h @@ -0,0 +1,39 @@ +/*- + * Copyright (c) 2017 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _PMCTRACE_CS_H_ +#define _PMCTRACE_CS_H_ + +extern struct trace_dev_methods cs_methods; + +#endif /* !_PMCTRACE_CS_H_ */ Index: usr.sbin/pmctrace/pmctrace_cs.c =================================================================== --- /dev/null +++ usr.sbin/pmctrace/pmctrace_cs.c @@ -0,0 +1,537 @@ +/*- + * Copyright (c) 2018 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "pmctrace.h" +#include "pmctrace_cs.h" + +#include +#include + +#define PMCTRACE_CS_DEBUG +#undef PMCTRACE_CS_DEBUG + +#ifdef PMCTRACE_CS_DEBUG +#define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define dprintf(fmt, ...) +#endif + +#define PACKET_STR_LEN 1024 +static char packet_str[PACKET_STR_LEN]; + +static dcd_tree_handle_t dcdtree_handle; + +static int cs_flags; +#define FLAG_FORMAT (1 << 0) +#define FLAG_FRAME_RAW_UNPACKED (1 << 1) +#define FLAG_FRAME_RAW_PACKED (1 << 2) +#define FLAG_CALLBACK_MEM_ACC (1 << 3) + +static struct pmcstat_symbol * +symbol_lookup(const struct mtrace_data *mdata, uint64_t ip, struct pmcstat_image **img) +{ + struct pmcstat_image *image; + struct pmcstat_symbol *sym; + struct pmcstat_pcmap *map; + uint64_t newpc; + + map = pmcstat_process_find_map(mdata->pp, ip); + if (map != NULL) { + image = map->ppm_image; + newpc = ip - (map->ppm_lowpc + + (image->pi_vaddr - image->pi_start)); + + sym = pmcstat_symbol_search(image, newpc); + *img = image; + + if (sym == NULL) + dprintf("cpu%d: symbol 0x%lx not found\n", mdata->cpu, newpc); + + return (sym); + } else { + dprintf("cpu%d: 0x%lx map not found\n", mdata->cpu, ip); + } + + return (NULL); +} + +static ocsd_err_t +attach_raw_printers(dcd_tree_handle_t dcd_tree_h) +{ + ocsd_err_t err; + int flags; + + flags = 0; + err = OCSD_OK; + + if (cs_flags & FLAG_FRAME_RAW_UNPACKED) + flags |= OCSD_DFRMTR_UNPACKED_RAW_OUT; + + if (cs_flags & FLAG_FRAME_RAW_PACKED) + flags |= OCSD_DFRMTR_PACKED_RAW_OUT; + + if (flags) + err = ocsd_dt_set_raw_frame_printer(dcd_tree_h, flags); + + return err; +} + +static int +print_data_array(const uint8_t *p_array, const int array_size, + char *p_buffer, int buf_size) +{ + int bytes_processed; + int chars_printed; + + chars_printed = 0; + p_buffer[0] = 0; + + if (buf_size > 9) { + strcat(p_buffer, "[ "); + chars_printed += 2; + + for (bytes_processed = 0; bytes_processed < array_size; + bytes_processed++) { + sprintf(p_buffer + chars_printed, "0x%02X ", + p_array[bytes_processed]); + chars_printed += 5; + if ((chars_printed + 5) > buf_size) + break; + } + + strcat(p_buffer, "];"); + chars_printed += 2; + } else if (buf_size >= 4) { + sprintf(p_buffer, "[];"); + chars_printed += 3; + } + + return (chars_printed); +} + +static void +packet_monitor(void *context __unused, + const ocsd_datapath_op_t op, + const ocsd_trc_index_t index_sop, + const void *p_packet_in, + const uint32_t size, + const uint8_t *p_data) +{ + int offset; + + offset = 0; + + switch(op) { + case OCSD_OP_DATA: + sprintf(packet_str, "Idx:%" OCSD_TRC_IDX_STR ";", index_sop); + offset = strlen(packet_str); + offset += print_data_array(p_data, size, packet_str + offset, + PACKET_STR_LEN - offset); + + /* + * Got a packet -- convert to string and use the libraries' + * message output to print to file and stdoout + */ + + if (ocsd_pkt_str(OCSD_PROTOCOL_ETMV4I, p_packet_in, packet_str + offset, + PACKET_STR_LEN - offset) == OCSD_OK) { + /* add in */ + if (strlen(packet_str) == PACKET_STR_LEN - 1) /* maximum length */ + packet_str[PACKET_STR_LEN - 2] = '\n'; + else + strcat(packet_str,"\n"); + + /* print it using the library output logger. */ + ocsd_def_errlog_msgout(packet_str); + } + break; + + case OCSD_OP_EOT: + sprintf(packet_str,"**** END OF TRACE ****\n"); + ocsd_def_errlog_msgout(packet_str); + break; + default: + printf("%s: unknown op %d\n", __func__, op); + break; + } +} + +static uint32_t +cs_cs_decoder__mem_access(const void *context __unused, + const ocsd_vaddr_t address __unused, const ocsd_mem_space_acc_t mem_space __unused, + const uint32_t req_size __unused, uint8_t *buffer __unused) +{ + + /* TODO */ + + return (0); +} + +static ocsd_err_t +create_test_memory_acc(dcd_tree_handle_t handle, uint64_t base, uint64_t start, uint64_t end) +{ + ocsd_vaddr_t address; + uint8_t *p_mem_buffer; + uint32_t mem_length; + int ret; + + dprintf("%s: base %lx start %lx end %lx\n", __func__, base, start, end); + + address = (ocsd_vaddr_t)base; + p_mem_buffer = (uint8_t *)(base + start); + mem_length = (end-start); + + if (cs_flags & FLAG_CALLBACK_MEM_ACC) + ret = ocsd_dt_add_callback_mem_acc(handle, base+start, base+end-1, + OCSD_MEM_SPACE_ANY, cs_cs_decoder__mem_access, NULL); + else + ret = ocsd_dt_add_buffer_mem_acc(handle, address, OCSD_MEM_SPACE_ANY, + p_mem_buffer, mem_length); + + if (ret != OCSD_OK) + printf("%s: can't create memory accessor: ret %d\n", __func__, ret); + + return (ret); +} + +static ocsd_err_t +create_generic_decoder(dcd_tree_handle_t handle, const char *p_name, const void *p_cfg, + const void *p_context __unused, uint64_t base, uint64_t start, uint64_t end) +{ + ocsd_err_t ret; + uint8_t CSID; + + CSID = 0; + + dprintf("%s\n", __func__); + + ret = ocsd_dt_create_decoder(handle, p_name, OCSD_CREATE_FLG_FULL_DECODER, + p_cfg, &CSID); + if(ret != OCSD_OK) + return (-1); + + if (cs_flags & FLAG_FORMAT) { + ret = ocsd_dt_attach_packet_callback(handle, CSID, + OCSD_C_API_CB_PKT_MON, packet_monitor, p_context); + if (ret != OCSD_OK) + return (-1); + } + + /* attach a memory accessor */ + ret = create_test_memory_acc(handle, base, start, end); + if(ret != OCSD_OK) + ocsd_dt_remove_decoder(handle,CSID); + + return (ret); +} + +static ocsd_err_t +create_decoder_etmv4(dcd_tree_handle_t dcd_tree_h, uint64_t base, + uint64_t start, uint64_t end) +{ + ocsd_etmv4_cfg trace_config; + ocsd_err_t ret; + + trace_config.arch_ver = ARCH_V8; + trace_config.core_prof = profile_CortexA; + + trace_config.reg_configr = 0x000000C1; + trace_config.reg_traceidr = 0x00000010; /* Trace ID */ + + trace_config.reg_idr0 = 0x28000EA1; + trace_config.reg_idr1 = 0x4100F403; + trace_config.reg_idr2 = 0x00000488; + trace_config.reg_idr8 = 0x0; + trace_config.reg_idr9 = 0x0; + trace_config.reg_idr10 = 0x0; + trace_config.reg_idr11 = 0x0; + trace_config.reg_idr12 = 0x0; + trace_config.reg_idr13 = 0x0; + + ret = create_generic_decoder(dcd_tree_h, OCSD_BUILTIN_DCD_ETMV4I, + (void *)&trace_config, 0, base, start, end); + return (ret); +} + +static ocsd_datapath_resp_t +gen_trace_elem_print_lookup(const void *p_context, const ocsd_trc_index_t index_sop __unused, + const uint8_t trc_chan_id __unused, const ocsd_generic_trace_elem *elem __unused) +{ + const struct mtrace_data *mdata; + ocsd_datapath_resp_t resp; + struct pmcstat_symbol *sym; + struct pmcstat_image *image; + + mdata = (const struct mtrace_data *)p_context; + + resp = OCSD_RESP_CONT; + +#if 0 + dprintf("%s: Idx:%d ELEM TYPE %d, st_addr %lx, en_addr %lx\n", + __func__, index_sop, elem->elem_type, elem->st_addr, elem->en_addr); +#endif + + if (elem->st_addr == 0) + return (0); + sym = symbol_lookup(mdata, elem->st_addr, &image); + if (sym) + printf("cpu%d: IP 0x%lx %s %s\n", mdata->cpu, elem->st_addr, + pmcstat_string_unintern(image->pi_name), + pmcstat_string_unintern(sym->ps_name)); + + switch (elem->elem_type) { + case OCSD_GEN_TRC_ELEM_UNKNOWN: + break; + case OCSD_GEN_TRC_ELEM_NO_SYNC: + /* Trace off */ + break; + case OCSD_GEN_TRC_ELEM_TRACE_ON: + break; + case OCSD_GEN_TRC_ELEM_INSTR_RANGE: + printf("range\n"); + break; + case OCSD_GEN_TRC_ELEM_EXCEPTION: + case OCSD_GEN_TRC_ELEM_EXCEPTION_RET: + case OCSD_GEN_TRC_ELEM_PE_CONTEXT: + case OCSD_GEN_TRC_ELEM_EO_TRACE: + case OCSD_GEN_TRC_ELEM_ADDR_NACC: + case OCSD_GEN_TRC_ELEM_TIMESTAMP: + case OCSD_GEN_TRC_ELEM_CYCLE_COUNT: + case OCSD_GEN_TRC_ELEM_ADDR_UNKNOWN: + case OCSD_GEN_TRC_ELEM_EVENT: + case OCSD_GEN_TRC_ELEM_SWTRACE: + case OCSD_GEN_TRC_ELEM_CUSTOM: + default: + break; + }; + + return (resp); +} + +static int +cs_process_chunk(struct mtrace_data *mdata __unused, uint64_t base, + uint64_t start, uint64_t end) +{ + uint32_t bytes_done; + uint32_t block_size; + uint8_t *p_block; + int bytes_this_time; + int block_index; + int dp_ret; + int ret; + + dprintf("%s: start %lx end %lx\n", __func__, start, end); + + bytes_this_time = 0; + block_index = 0; + bytes_done = 0; + block_size = (end - start); + p_block = (uint8_t *)(base + start); + + ret = OCSD_OK; + dp_ret = OCSD_RESP_CONT; + + while (bytes_done < (uint32_t)block_size && (ret == OCSD_OK)) { + + if (OCSD_DATA_RESP_IS_CONT(dp_ret)) { + dprintf("process data, block_size %d\n", block_size - bytes_done); + dp_ret = ocsd_dt_process_data(dcdtree_handle, OCSD_OP_DATA, + block_index + bytes_done, + block_size - bytes_done, + ((uint8_t *)p_block) + bytes_done, + &bytes_this_time); + bytes_done += bytes_this_time; + dprintf("BYTES DONE %d\n", bytes_done); + } else if (OCSD_DATA_RESP_IS_WAIT(dp_ret)) { + dp_ret = ocsd_dt_process_data(dcdtree_handle, OCSD_OP_FLUSH, + 0, 0, NULL, NULL); + } else { + ret = OCSD_ERR_DATA_DECODE_FATAL; + } + } + + ocsd_dt_process_data(dcdtree_handle, OCSD_OP_EOT, 0, 0, NULL, NULL); + + return (0); +} + +static int cs_init(struct trace_cpu *tc); + +static int +cs_process(struct trace_cpu *tc, struct pmcstat_process *pp, + uint32_t cpu, uint32_t cycle, uint64_t offset) +{ + struct mtrace_data *mdata; + + mdata = &tc->mdata; + mdata->pp = pp; + + cs_init(tc); + + dprintf("%s: cpu %d, cycle %d, offset %ld, tc->base %lx, *tc->base %lx\n", + __func__, cpu, cycle, offset, (uint64_t)tc->base, *(uint64_t *)tc->base); + + if (offset == tc->offset) + return (0); + + if (cycle == tc->cycle) { + if (offset > tc->offset) { + cs_process_chunk(mdata, (uint64_t)tc->base, tc->offset, offset); + tc->offset = offset; + } else if (offset < tc->offset) { + err(EXIT_FAILURE, "cpu%d: offset already processed %lx %lx", + cpu, offset, tc->offset); + } + } else if (cycle > tc->cycle) { + if ((cycle - tc->cycle) > 1) + err(EXIT_FAILURE, "cpu%d: trace buffers fills up faster than" + " we can process it (%d/%d). Consider setting trace filters", + cpu, cycle, tc->cycle); + cs_process_chunk(mdata, (uint64_t)tc->base, tc->offset, tc->bufsize); + tc->offset = 0; + tc->cycle += 1; + tc->offset = offset; + } + + return (0); +} + +static int +cs_init(struct trace_cpu *tc) +{ + uint64_t start; + uint64_t end; + int ret; + + ocsd_def_errlog_init(OCSD_ERR_SEV_INFO, 1); + ocsd_def_errlog_init(0, 0); + +#if 0 + ret = ocsd_def_errlog_config_output(C_API_MSGLOGOUT_FLG_FILE | + C_API_MSGLOGOUT_FLG_STDOUT, "c_api_test.log"); + if (ret != OCSD_OK) + return (-1); +#endif + + dcdtree_handle = ocsd_create_dcd_tree(OCSD_TRC_SRC_FRAME_FORMATTED, + OCSD_DFRMTR_FRAME_MEM_ALIGN); + if(dcdtree_handle == C_API_INVALID_TREE_HANDLE) { + printf("can't find dcd tree\n"); + return (-1); + } + + start = (uint64_t)tc->base; + end = (uint64_t)tc->base + tc->bufsize; + + ret = create_decoder_etmv4(dcdtree_handle, (uint64_t)tc->base, start, end); + if (ret != OCSD_OK) { + printf("can't create decoder: base %lx start %lx end %lx\n", + (uint64_t)tc->base, start, end); + return (-2); + } + +#ifdef PMCTRACE_CS_DEBUG + ocsd_tl_log_mapped_mem_ranges(dcdtree_handle); +#endif + + if (cs_flags & FLAG_FORMAT) + ocsd_dt_set_gen_elem_printer(dcdtree_handle); + else + ocsd_dt_set_gen_elem_outfn(dcdtree_handle, gen_trace_elem_print_lookup, + (const struct mtrace_data *)&tc->mdata); + + attach_raw_printers(dcdtree_handle); + + return (0); +} + +static int +cs_option(int option) +{ + + switch (option) { + case 't': + cs_flags |= FLAG_FORMAT; + break; + default: + break; + } + + return (0); +} + +struct trace_dev_methods cs_methods = { + .init = cs_init, + .process = cs_process, + .option = cs_option, +}; Index: usr.sbin/pmctrace/pmctrace_pt.h =================================================================== --- /dev/null +++ usr.sbin/pmctrace/pmctrace_pt.h @@ -0,0 +1,39 @@ +/*- + * Copyright (c) 2017 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _PMCTRACE_PT_H_ +#define _PMCTRACE_PT_H_ + +extern struct trace_dev_methods ipt_methods; + +#endif /* !_PMCTRACE_PT_H_ */ Index: usr.sbin/pmctrace/pmctrace_pt.c =================================================================== --- /dev/null +++ usr.sbin/pmctrace/pmctrace_pt.c @@ -0,0 +1,371 @@ +/*- + * Copyright (c) 2017 Ruslan Bukin + * All rights reserved. + * + * This software was developed by BAE Systems, the University of Cambridge + * Computer Laboratory, and Memorial University under DARPA/AFRL contract + * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing + * (TC) research program. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "pmctrace.h" +#include "pmctrace_pt.h" + +#include +#include +#include +#include +#include + +#define PMCTRACE_PT_DEBUG +#undef PMCTRACE_PT_DEBUG + +#ifdef PMCTRACE_PT_DEBUG +#define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define dprintf(fmt, ...) +#endif + +static int ipt_flags; +#define FLAG_BRANCH_TNT (1 << 0) /* Taken/Not Taken */ + +static struct pmcstat_symbol * +symbol_lookup(struct mtrace_data *mdata) +{ + struct pmcstat_image *image; + struct pmcstat_symbol *sym; + struct pmcstat_pcmap *map; + uint64_t newpc; + uint64_t ip; + + if (mdata->ip & (1UL << 47)) + ip = mdata->ip | 0xffffUL << 48; + else + ip = mdata->ip; + + map = pmcstat_process_find_map(mdata->pp, ip); + if (map != NULL) { + image = map->ppm_image; + newpc = ip - (map->ppm_lowpc + + (image->pi_vaddr - image->pi_start)); + sym = pmcstat_symbol_search(image, newpc); + return (sym); + } else + dprintf("cpu%d: 0x%lx map not found\n", mdata->cpu, ip); + + return (NULL); +} + +static int +print_tnt_payload(struct mtrace_data *mdata, uint64_t offset __unused, + const struct pt_packet_tnt *packet) +{ + char payload[48]; + uint64_t tnt; + uint8_t bits; + char *begin; + char *end; + + bits = packet->bit_size; + tnt = packet->payload; + begin = &payload[0]; + end = begin + bits; + + if (sizeof(payload) < bits) + end = begin + sizeof(payload); + + for (; begin < end; ++begin, --bits) + *begin = tnt & (1ull << (bits - 1)) ? '!' : '.'; + + printf("cpu%d: TNT %s\n", mdata->cpu, payload); + + return (0); +} + +static int +print_ip_payload(struct mtrace_data *mdata, uint64_t offset __unused, + const struct pt_packet_ip *packet) +{ + struct pmcstat_symbol *sym; + + switch (packet->ipc) { + case pt_ipc_suppressed: + break; + case pt_ipc_update_16: + mdata->ip &= ~0xffffUL; + mdata->ip |= (packet->ip & 0xffffUL); + break; + case pt_ipc_update_32: + mdata->ip &= ~0xffffffffUL; + mdata->ip |= (packet->ip & 0xffffffffUL); + break; + case pt_ipc_update_48: + mdata->ip &= ~0xffffffffffffUL; + mdata->ip |= (packet->ip & 0xffffffffffffUL); + break; + case pt_ipc_sext_48: + mdata->ip &= ~0xffffffffffffUL; + mdata->ip |= (packet->ip & 0xffffffffffffUL); + symbol_lookup(mdata); + case pt_ipc_full: + mdata->ip = packet->ip; + break; + default: + printf("unknown ipc: %d\n", packet->ipc); + return (0); + } + + sym = symbol_lookup(mdata); + if (sym) { + printf("cpu%d: IP 0x%lx %s\n", mdata->cpu, mdata->ip, + pmcstat_string_unintern(sym->ps_name)); + } else + dprintf("cpu%d: 0x%lx not found\n", mdata->cpu, mdata->ip); + + return (0); +} + +static int +dump_packets(struct mtrace_data *mdata, struct pt_packet_decoder *decoder, + const struct pt_config *config __unused) +{ + struct pt_packet packet; + uint64_t offset; + int error; + + dprintf("%s\n", __func__); + + while (1) { + error = pt_pkt_get_offset(decoder, &offset); + if (error < 0) + errx(EX_SOFTWARE, "ERROR: can't get offset, err %d\n", error); + + error = pt_pkt_next(decoder, &packet, sizeof(packet)); + if (error < 0) { + dprintf("%s: error %d\n", __func__, error); + break; + } + + switch (packet.type) { + case ppt_invalid: + case ppt_unknown: + case ppt_pad: + case ppt_psb: + case ppt_psbend: + break; + case ppt_fup: + case ppt_tip: + case ppt_tip_pge: + case ppt_tip_pgd: + print_ip_payload(mdata, offset, &packet.payload.ip); + break; + case ppt_tnt_8: + case ppt_tnt_64: + if (ipt_flags & FLAG_BRANCH_TNT) + print_tnt_payload(mdata, offset, &packet.payload.tnt); + break; + case ppt_mode: + case ppt_pip: + case ppt_vmcs: + case ppt_cbr: + break; + case ppt_tsc: + printf("cpu%d: TSC %ld\n", mdata->cpu, packet.payload.tsc.tsc); + break; + case ppt_tma: + break; + case ppt_mtc: + printf("cpu%d: MTC %x\n", mdata->cpu, packet.payload.mtc.ctc); + break; + case ppt_cyc: + case ppt_stop: + case ppt_ovf: + case ppt_mnt: + case ppt_exstop: + case ppt_mwait: + case ppt_pwre: + case ppt_pwrx: + case ppt_ptw: + default: + break; + } + } + + return (0); +} + +static int +ipt_process_chunk(struct mtrace_data *mdata, uint64_t base, + uint64_t start, uint64_t end) +{ + struct pt_packet_decoder *decoder; + struct pt_config config; + int error; + + dprintf("%s\n", __func__); + + memset(&config, 0, sizeof(config)); + pt_config_init(&config); + + error = pt_cpu_read(&config.cpu); + if (error < 0) + errx(EX_SOFTWARE, "ERROR: pt_cpu_read failed, err %d\n", error); + error = pt_cpu_errata(&config.errata, &config.cpu); + if (error < 0) + errx(EX_SOFTWARE, "ERROR: can't get errata, err %d\n", error); + + config.begin = (uint8_t *)(base + start); + config.end = (uint8_t *)(base + end); + + dprintf("%s: begin %lx end %lx\n", __func__, + (uint64_t)config.begin, (uint64_t)config.end); + + decoder = pt_pkt_alloc_decoder(&config); + if (decoder == NULL) { + printf("Can't allocate decoder\n"); + return (-1); + } + + error = pt_pkt_sync_set(decoder, 0ull); + if (error < 0) + errx(EX_SOFTWARE, "ERROR: sync_set failed, err %d\n", error); + error = pt_pkt_sync_forward(decoder); + if (error < 0 && error != -pte_eos) + errx(EX_SOFTWARE, "ERROR: sync_forward failed, err %d\n", error); + + while (1) { + error = dump_packets(mdata, decoder, &config); + if (error == 0) + break; + + error = pt_pkt_sync_forward(decoder); + if (error < 0) { + if (error == -pte_eos) + return (0); + } + } + + return (0); +} + +static int +ipt_process(struct trace_cpu *tc, struct pmcstat_process *pp, + uint32_t cpu, uint32_t cycle, uint64_t offset) +{ + struct mtrace_data *mdata; + + mdata = &tc->mdata; + mdata->pp = pp; + + dprintf("%s: cpu %d, cycle %d, offset %ld\n", + __func__, cpu, cycle, offset); + + if (offset == tc->offset) + return (0); + + if (cycle == tc->cycle) { + if (offset > tc->offset) { + ipt_process_chunk(mdata, (uint64_t)tc->base, tc->offset, offset); + tc->offset = offset; + } else if (offset < tc->offset) { + err(EXIT_FAILURE, "cpu%d: offset already processed %lx %lx", + cpu, offset, tc->offset); + } + } else if (cycle > tc->cycle) { + if ((cycle - tc->cycle) > 1) + err(EXIT_FAILURE, "cpu%d: trace buffers fills up faster than" + " we can process it (%d/%d). Consider setting trace filters", + cpu, cycle, tc->cycle); + ipt_process_chunk(mdata, (uint64_t)tc->base, tc->offset, tc->bufsize); + tc->offset = 0; + tc->cycle += 1; + ipt_process_chunk(mdata, (uint64_t)tc->base, tc->offset, offset); + tc->offset = offset; + } + + return (0); +} + +static int +ipt_option(int option) +{ + + switch (option) { + case 't': + /* Decode 'Taken/Not_Taken branch' packet. */ + ipt_flags |= FLAG_BRANCH_TNT; + break; + default: + break; + } + + return (0); +} + +struct trace_dev_methods ipt_methods = { + .process = ipt_process, + .option = ipt_option, +};