diff --git a/Makefile.inc1 b/Makefile.inc1 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -3092,8 +3092,8 @@ _cddl_lib= cddl/lib cddl/lib/libctf__L: lib/libz__L cddl/lib/libspl__L .endif -# cddl/lib/libdtrace requires lib/libproc and lib/librtld_db -_prebuild_libs+= lib/libprocstat lib/libproc lib/librtld_db +# cddl/lib/libdtrace libs +_prebuild_libs+= lib/libprocstat lib/libproc lib/librtld_db lib/libdwarf lib/libprocstat__L: lib/libelf__L lib/libkvm__L lib/libutil__L lib/libproc__L: lib/libprocstat__L lib/librtld_db__L: lib/libprocstat__L diff --git a/cddl/contrib/opensolaris/cmd/dtrace/test/tst/amd64/kinst/tst.dtracetest_fbtconvert.ksh b/cddl/contrib/opensolaris/cmd/dtrace/test/tst/amd64/kinst/tst.dtracetest_fbtconvert.ksh new file mode 100644 --- /dev/null +++ b/cddl/contrib/opensolaris/cmd/dtrace/test/tst/amd64/kinst/tst.dtracetest_fbtconvert.ksh @@ -0,0 +1,50 @@ +#!/usr/bin/ksh +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2023 Christos Margiolis +# + +script() +{ + loaded="$(kldstat | grep 'dtrace_test')" + + # Don't attempt to load it if it was already loaded. + test -n "${loaded}" || kldload dtrace_test + + $dtrace -q -n \ + 'kinst::kinst_test_fbtconvert:entry,kinst::kinst_test_fbtconvert:return' \ + -c "sysctl debug.dtracetest.kinst=2" + + # If it wasn't loaded by us, don't unload it. + test -n "${loaded}" || kldunload dtrace_test +} + +spin() +{ + while true; do + ls -la / >/dev/null 2>&1 + done +} + +if [ $# != 1 ]; then + echo expected one argument: '<'dtrace-path'>' + exit 2 +fi + +dtrace=$1 + +spin & +child=$! + +script +exit $? diff --git a/cddl/contrib/opensolaris/cmd/dtrace/test/tst/amd64/kinst/tst.dtracetest_inline.ksh b/cddl/contrib/opensolaris/cmd/dtrace/test/tst/amd64/kinst/tst.dtracetest_inline.ksh new file mode 100644 --- /dev/null +++ b/cddl/contrib/opensolaris/cmd/dtrace/test/tst/amd64/kinst/tst.dtracetest_inline.ksh @@ -0,0 +1,50 @@ +#!/usr/bin/ksh +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2023 Christos Margiolis +# + +script() +{ + loaded="$(kldstat | grep 'dtrace_test')" + + # Don't attempt to load it if it was already loaded. + test -n "${loaded}" || kldload dtrace_test + + $dtrace -q -n \ + 'kinst::kinst_test_inline:entry,kinst::kinst_test_inline:return' \ + -c "sysctl debug.dtracetest.kinst=1" + + # If it wasn't loaded by us, don't unload it. + test -n "${loaded}" || kldunload dtrace_test +} + +spin() +{ + while true; do + ls -la / >/dev/null 2>&1 + done +} + +if [ $# != 1 ]; then + echo expected one argument: '<'dtrace-path'>' + exit 2 +fi + +dtrace=$1 + +spin & +child=$! + +script +exit $? diff --git a/cddl/contrib/opensolaris/cmd/dtrace/test/tst/amd64/kinst/tst.inline_agg.ksh b/cddl/contrib/opensolaris/cmd/dtrace/test/tst/amd64/kinst/tst.inline_agg.ksh new file mode 100644 --- /dev/null +++ b/cddl/contrib/opensolaris/cmd/dtrace/test/tst/amd64/kinst/tst.inline_agg.ksh @@ -0,0 +1,54 @@ +#!/usr/bin/ksh +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2023 Christos Margiolis +# + +script() +{ + $dtrace -q -s /dev/stdin <<__EOF__ +kinst::critical_exit:entry +{ + self->ts = timestamp; +} + +kinst::critical_exit:return +/self->ts/ +{ + @[probefunc] = quantize(timestamp - self->ts); + self->ts = 0; +} + +tick-10s {exit(0);} +__EOF__ +} + +spin() +{ + while true; do + ls -la / >/dev/null 2>&1 + done +} + +if [ $# != 1 ]; then + echo expected one argument: '<'dtrace-path'>' + exit 2 +fi + +dtrace=$1 + +spin & +child=$! + +script +exit $? diff --git a/cddl/contrib/opensolaris/cmd/dtrace/test/tst/amd64/kinst/tst.inline_multiple.ksh b/cddl/contrib/opensolaris/cmd/dtrace/test/tst/amd64/kinst/tst.inline_multiple.ksh new file mode 100644 --- /dev/null +++ b/cddl/contrib/opensolaris/cmd/dtrace/test/tst/amd64/kinst/tst.inline_multiple.ksh @@ -0,0 +1,51 @@ +#!/usr/bin/ksh +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2023 Christos Margiolis +# + +script() +{ + $dtrace -q -s /dev/stdin <<__EOF__ +kinst::critical_exit:return, +kinst::bwrite:entry, +kinst::malloc:entry +/pid/ +{ + if (cpu == 1) + printf("\t%s\t0x%x", execname, regs[R_RDI]); +} + +tick-10s {exit(0);} +__EOF__ +} + +spin() +{ + while true; do + ls -la / >/dev/null 2>&1 + done +} + +if [ $# != 1 ]; then + echo expected one argument: '<'dtrace-path'>' + exit 2 +fi + +dtrace=$1 + +spin & +child=$! + +script +exit $? diff --git a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_sugar.c b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_sugar.c --- a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_sugar.c +++ b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_sugar.c @@ -38,12 +38,23 @@ #include #include #include +#include +#include #include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include #include @@ -54,14 +65,56 @@ #include #include +#include + +/* kinst-related */ +struct elf_info { + Elf *elf; + struct section { + Elf_Scn *scn; + GElf_Shdr sh; + const char *name; + } *sl; + size_t shnum; + int fd; +}; + +struct entry { + struct off { + const char *func; + uint64_t val; + } *off; + int noff; + TAILQ_ENTRY(entry) next; +}; + typedef struct dt_sugar_parse { dtrace_hdl_t *dtsp_dtp; /* dtrace handle */ dt_node_t *dtsp_pdescs; /* probe descriptions */ int dtsp_num_conditions; /* number of condition variables */ int dtsp_num_ifs; /* number of "if" statements */ dt_node_t *dtsp_clause_list; /* list of clauses */ + struct elf_info dtsp_elf_mod; /* ELF info of the kernel executable */ + struct elf_info dtsp_elf_dbg; /* ELF info of the kernel debug file */ + dtrace_probedesc_t *dtsp_desc; /* kinst pdesc to duplicate contents */ + Dwarf_Off dtsp_dieoff; /* DIE offset of kinst inline definition */ +#define DF_EXISTS 0x01 /* function exists in module */ +#define DF_REGULAR 0x02 /* function has regular definition */ +#define DF_INLINE 0x04 /* function has inline definition */ +#define DF_ENTRY 0x08 /* probe is entry */ +#define DF_RETURN 0x10 /* probe is return */ +#define DF_NEWPDESC 0x20 /* we're searching a new pdesc */ + int dtsp_flags; /* kinst-related flags */ + char dtsp_func[DTRACE_FUNCNAMELEN]; /* current function */ + char dtsp_name[DTRACE_NAMELEN]; /* current probe name */ + TAILQ_HEAD(, entry) dtsp_head; /* kinst inline copy entry TAILQ */ } dt_sugar_parse_t; +enum { + F_SUBPROGRAM, + F_INLINE_COPY, +}; + static void dt_sugar_visit_stmts(dt_sugar_parse_t *, dt_node_t *, int); /* @@ -186,8 +239,655 @@ } /* - * Visit the specified node and all of its descendants. Currently this is only - * used to count the number of "if" statements (dtsp_num_ifs). + * kinst-related + */ +static int +dt_sugar_elf_init(dtrace_hdl_t *dtp, struct elf_info *ei, const char *file) +{ + Elf_Scn *scn; + GElf_Shdr sh; + struct section *s; + const char *name; + size_t shstrndx, ndx; + + if (elf_version(EV_CURRENT) == EV_NONE) { + warnx("dt_sugar: elf_version(): %s", elf_errmsg(-1)); + return (-1); + } + if ((ei->fd = open(file, O_RDONLY)) < 0) { + dt_dprintf("dt_sugar: open(%s)", file); + return (-1); + } + if ((ei->elf = elf_begin(ei->fd, ELF_C_READ, NULL)) == NULL) { + warnx("dt_sugar: elf_begin(): %s", elf_errmsg(-1)); + return (-1); + } + if (elf_kind(ei->elf) == ELF_K_NONE) { + warnx("dt_sugar: not an ELF file: %s", file); + return (-1); + } + + /* Load ELF sections */ + if (!elf_getshnum(ei->elf, &ei->shnum)) { + warnx("dt_sugar: elf_getshnum(): %s", elf_errmsg(-1)); + return (-1); + } + ei->sl = dt_alloc(dtp, ei->shnum * sizeof(struct section)); + if (ei->sl == NULL) + longjmp(yypcb->pcb_jmpbuf, EDT_NOMEM); + if (!elf_getshstrndx(ei->elf, &shstrndx)) { + warnx("dt_sugar: elf_getshstrndx(): %s", elf_errmsg(-1)); + return (-1); + } + if ((scn = elf_getscn(ei->elf, 0)) == NULL) { + warnx("dt_sugar: elf_getscn(): %s", elf_errmsg(-1)); + return (-1); + } + (void) elf_errno(); + + do { + if (gelf_getshdr(scn, &sh) == NULL) { + warnx("dt_sugar: gelf_getshdr(): %s", elf_errmsg(-1)); + (void) elf_errno(); + continue; + } + if ((name = elf_strptr(ei->elf, shstrndx, sh.sh_name)) == NULL) + (void) elf_errno(); + if ((ndx = elf_ndxscn(scn)) == SHN_UNDEF && elf_errno() != 0) { + warnx("dt_sugar: elf_ndxscn(): %s", elf_errmsg(-1)); + continue; + } + if (ndx >= ei->shnum) + continue; + s = &ei->sl[ndx]; + s->scn = scn; + s->sh = sh; + s->name = name; + } while ((scn = elf_nextscn(ei->elf, scn)) != NULL); + if (elf_errno() != 0) + warnx("dt_sugar: elf_nextscn(): %s", elf_errmsg(-1)); + + return (0); +} + +static void +dt_sugar_elf_deinit(dtrace_hdl_t *dtp, struct elf_info *ei) +{ + dt_free(dtp, ei->sl); + close(ei->fd); + elf_end(ei->elf); +} + +/* + * Verify the checksum in the .gnu_debuglink section to make sure the debug + * file is up to date. + */ +static int +dt_sugar_elf_verify_debuglink(struct elf_info *ei, int dbgfd) +{ + Elf_Data *d; + struct section *s; + char buf[MAXPHYS]; + ssize_t nr; + uint32_t crc, compcrc; + int i; + + for (i = 1; i < ei->shnum; i++) { + s = &ei->sl[i]; + if (strcmp(s->name, ".gnu_debuglink") == 0) + break; + } + if ((d = elf_getdata(s->scn, NULL)) == NULL) { + if (elf_errno() != 0) + warnx("dt_sugar: elf_getdata(): %s", elf_errmsg(-1)); + return (-1); + } + if (d->d_size < sizeof(crc) + 1 || d->d_buf == NULL) + return (-1); + if (strnlen(d->d_buf, d->d_size) >= d->d_size - sizeof(crc)) + return (-1); + memcpy(&crc, (uint8_t *)d->d_buf + d->d_size - sizeof(crc), + sizeof(crc)); + + compcrc = crc32(0L, Z_NULL, 0); + while ((nr = read(dbgfd, buf, sizeof(buf))) > 0) + compcrc = crc32(compcrc, (char *)buf, nr); + + if (nr != 0 || crc != compcrc) + return (-1); + + return (0); +} + +/* + * Find the caller function and offset of an inline copy. Since we know the + * inline copy's boundaries (`addr_lo` and `addr_hi` arguments), the caller + * function is going to be the ELF symbol that the inline copy's boundaries are + * inside of. + */ +static void +dt_sugar_kinst_find_caller_func(dt_sugar_parse_t *dp, struct off *off, + uint64_t addr_lo, uint64_t addr_hi, int last) +{ + Elf_Data *d; + GElf_Sym sym; + struct section *s; + uint8_t *buf; + uint64_t addr, lo, hi; + uint32_t stab; + int len, i, j; + + /* Find the caller function's boundaries and name. */ + off->func = NULL; + for (i = 1; i < dp->dtsp_elf_mod.shnum; i++) { + s = &dp->dtsp_elf_mod.sl[i]; + if (s->sh.sh_type != SHT_SYMTAB && s->sh.sh_type != SHT_DYNSYM) + continue; + if (s->sh.sh_link >= dp->dtsp_elf_mod.shnum) + continue; + stab = s->sh.sh_link; + len = (int)(s->sh.sh_size / s->sh.sh_entsize); + (void) elf_errno(); + if ((d = elf_getdata(s->scn, NULL)) == NULL) { + if (elf_errno() != 0) + warnx("dt_sugar: elf_getdata(): %s", + elf_errmsg(-1)); + continue; + } + if (d->d_size <= 0) + continue; + if (s->sh.sh_entsize == 0) + continue; + else if (len > INT_MAX) + continue; + for (j = 0; j < len; j++) { + if (gelf_getsym(d, j, &sym) != &sym) { + warnx("dt_sugar: gelf_getsym(): %s", + elf_errmsg(-1)); + continue; + } + if (GELF_ST_TYPE(sym.st_info) != STT_FUNC) + continue; + lo = sym.st_value; + hi = sym.st_value + sym.st_size; + if (addr_lo < lo || addr_hi > hi) + continue; + if ((off->func = elf_strptr(dp->dtsp_elf_mod.elf, stab, + sym.st_name)) != NULL) + break; + } + } + + /* Find inline copy's return offset. */ + for (i = 1; i < dp->dtsp_elf_mod.shnum; i++) { + s = &dp->dtsp_elf_mod.sl[i]; + if (strcmp(s->name, ".text") != 0 || + s->sh.sh_type != SHT_PROGBITS) + continue; + (void) elf_errno(); + if ((d = elf_getdata(s->scn, NULL)) == NULL) { + if (elf_errno() != 0) + warnx("dt_sugar: elf_getdata(): %s", + elf_errmsg(-1)); + continue; + } + if (d->d_size <= 0 || d->d_buf == NULL) + continue; + + buf = d->d_buf; + addr = s->sh.sh_addr + d->d_off; + + while (addr != lo) { + addr++; + buf++; + } + + if (dp->dtsp_flags & DF_ENTRY) { + off->val = addr_lo - lo; + } else if (dp->dtsp_flags & DF_RETURN) { + if (addr_hi == hi || last) { + /* + * In this case the offset is one instruction + * *outside* the inline or the caller function, + * so we have to go back one instruction to + * stay within bounds. + */ + + /* + * Get to the inline copy's start manually to + * avoid potential dtrace_disx86() failures. + */ + while (addr != addr_lo) { + addr++; + buf++; + } + + while (addr != addr_hi) { + len = dtrace_instr_size(buf); + addr += len; + buf += len; + } + addr -= len; + off->val = addr - lo; + } else + off->val = addr_hi - lo; + } + break; + } +} + +/* + * Parse DWARF info recursively and create a TAILQ of entries that correspond + * to inline copies of the probe function. + */ +static void +dt_sugar_kinst_parse_die(dt_sugar_parse_t *dp, Dwarf_Debug dbg, Dwarf_Die die, + int level, int flag) +{ + static Dwarf_Die die_root; + Dwarf_Die die_next; + Dwarf_Ranges *ranges, *rp; + Dwarf_Attribute attp; + Dwarf_Addr base0, lowpc, highpc; + Dwarf_Off dieoff, cuoff, culen, v_off; + Dwarf_Unsigned nbytes, v_udata; + Dwarf_Signed nranges; + Dwarf_Half attr, tag; + Dwarf_Bool v_flag; + Dwarf_Error error; + struct entry *e; + struct off *off; + char *v_str; + int res, noff, i, found = 0; + + if (level == 0) + die_root = die; + + if (dwarf_dieoffset(die, &dieoff, &error) != DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + if (dwarf_die_CU_offset_range(die, &cuoff, &culen, &error) != DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + cuoff = 0; + } + if (dwarf_tag(die, &tag, &error) != DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + if (tag != DW_TAG_subprogram && tag != DW_TAG_inlined_subroutine) + goto cont; + + if (flag == F_SUBPROGRAM && tag == DW_TAG_subprogram) { + /* Find if the function exists in the current module. */ + if (dwarf_hasattr(die, DW_AT_name, &v_flag, &error) != + DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + if (!v_flag) + goto cont; + if (dwarf_diename(die, &v_str, &error) != DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + if (strcmp(v_str, dp->dtsp_func) == 0) + dp->dtsp_flags |= DF_EXISTS; + else + goto cont; + + if (dwarf_hasattr(die, DW_AT_inline, &v_flag, &error) != + DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + if (v_flag) { + dp->dtsp_flags |= DF_INLINE; + found = 1; + } else + dp->dtsp_flags |= DF_REGULAR; + goto cont; + } else if (flag == F_INLINE_COPY) { + res = dwarf_attr(die, DW_AT_abstract_origin, &attp, &error); + if (res != DW_DLV_OK) { + if (res == DW_DLV_ERROR) + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + if (dwarf_formref(attp, &v_off, &error) != DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + v_off += cuoff; + /* Doesn't point to the definition's DIE offset. */ + if (v_off != dp->dtsp_dieoff) + goto cont; + + if (dwarf_hasattr(die, DW_AT_ranges, &v_flag, &error) != + DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + if (v_flag) { + /* DIE has ranges */ + res = dwarf_attr(die, DW_AT_ranges, &attp, &error); + if (res != DW_DLV_OK) { + if (res == DW_DLV_ERROR) + warnx("dt_sugar: %s", + dwarf_errmsg(error)); + goto cont; + } + if (dwarf_global_formref(attp, &v_off, &error) != + DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + if (dwarf_get_ranges(dbg, v_off, &ranges, &nranges, + &nbytes, &error) != DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + + res = dwarf_lowpc(die_root, &lowpc, &error); + if (res != DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + base0 = lowpc; + + if (dp->dtsp_flags & DF_ENTRY) { + /* + * Trace the first instruction of the first + * range since this is the beginning of the + * inline copy. + */ + noff = 1; + } else if (dp->dtsp_flags & DF_RETURN) { + /* + * Trace the last instruction of every range in + * case the inline copy is split into multiple + * ranges (e.g if it has early `return`s). + */ + noff = nranges - 1; + } + off = dt_alloc(dp->dtsp_dtp, noff * sizeof(struct off)); + if (off == NULL) + longjmp(yypcb->pcb_jmpbuf, EDT_NOMEM); + for (i = 0; i < noff; i++) { + rp = &ranges[i]; + if (rp->dwr_type == DW_RANGES_ADDRESS_SELECTION) + base0 = rp->dwr_addr2; + dt_sugar_kinst_find_caller_func(dp, &off[i], + rp->dwr_addr1 + base0, + rp->dwr_addr2 + base0, + (dp->dtsp_flags & DF_RETURN) && + i == noff - 1); + } + dwarf_ranges_dealloc(dbg, ranges, nranges); + } else { + /* DIE has high/low PC boundaries */ + res = dwarf_lowpc(die, &lowpc, &error); + if (res != DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + res = dwarf_highpc(die, &highpc, &error); + if (res != DW_DLV_OK) { + warnx("dt_sugar: %s", dwarf_errmsg(error)); + goto cont; + } + noff = 1; + off = dt_alloc(dp->dtsp_dtp, noff * sizeof(struct off)); + if (off == NULL) + longjmp(yypcb->pcb_jmpbuf, EDT_NOMEM); + dt_sugar_kinst_find_caller_func(dp, off, lowpc, + lowpc + highpc, + dp->dtsp_flags & DF_RETURN); + } + } else + goto cont; + + e = dt_alloc(dp->dtsp_dtp, sizeof(struct entry)); + if (e == NULL) + longjmp(yypcb->pcb_jmpbuf, EDT_NOMEM); + e->noff = noff; + e->off = off; + TAILQ_INSERT_TAIL(&dp->dtsp_head, e, next); +cont: + /* + * Inline copies might appear before the declaration, so we need to + * re-parse the CU. + * + * The rationale for choosing to re-parse the CU instead of using a + * hash table of DIEs is that, because we re-parse only when an inline + * definition of the function we want is found, statistically, we won't + * have to re-parse many times at all considering that only a handful + * of CUs will define the same function, whereas if we have used a hash + * table, we would first need to parse the whole CU at once and store + * all DW_TAG_inlined_subroutine DIEs (so that we can match them + * afterwards). In this case, we always have to "parse" twice -- first + * the CU, then the DIE table -- and also, the program would use much + * more memory since we would have allocated DIEs, which most of them + * would never be used. + */ + if (found) { + die = die_root; + level = 0; + /* + * We'll be checking against the DIE offset of the definition + * to determine if the inline copy's DW_AT_abstract_origin + * points to it. + */ + dp->dtsp_dieoff = dieoff; + flag = F_INLINE_COPY; + } + + res = dwarf_child(die, &die_next, &error); + if (res == DW_DLV_ERROR) + warnx("dt_sugar: %s", dwarf_errmsg(error)); + else if (res == DW_DLV_OK) + dt_sugar_kinst_parse_die(dp, dbg, die_next, level + 1, flag); + + res = dwarf_siblingof(dbg, die, &die_next, &error); + if (res == DW_DLV_ERROR) + warnx("dt_sugar: %s", dwarf_errmsg(error)); + else if (res == DW_DLV_OK) + dt_sugar_kinst_parse_die(dp, dbg, die_next, level, flag); + + /* + * Deallocating on level 0 will attempt to double-free, since die_root + * points to the first DIE. We'll deallocate the root DIE in main(). + */ + if (level > 0) + dwarf_dealloc(dbg, die, DW_DLA_DIE); +} + +/* + * Append new clauses for each inline copy to the parse tree. + * + * If foo() is an inline function, and is called from functions bar() and baz() + * at offsets 10 and 20 respectively, we'll transform the parse tree from: + * + * kinst::foo: + * // + * { + * + * } + * + * To: + * + * kinst::bar:10, + * kinst::baz:20 + * // + * { + * + * } + */ +static void +dt_sugar_kinst_create_probes(dt_sugar_parse_t *dp) +{ + dt_node_t *dnp, *pdesc; + struct entry *e; + char buf[DTRACE_FULLNAMELEN]; + int i, n; + + dnp = dp->dtsp_clause_list->dn_pdescs; + n = dp->dtsp_flags & DF_NEWPDESC; + + /* We have found inline copies. Clean up as well */ + while (!TAILQ_EMPTY(&dp->dtsp_head)) { + e = TAILQ_FIRST(&dp->dtsp_head); + TAILQ_REMOVE(&dp->dtsp_head, e, next); + for (i = 0; i < e->noff; i++) { + if (dp->dtsp_flags & DF_NEWPDESC) { + /* We want to get here only once per-pdesc. */ + dp->dtsp_flags &= ~DF_NEWPDESC; + /* + * Since we're trying to trace inline copies of + * a given function by requesting a probe of + * the form + * `kinst:::`, + * the requested probe, by definition cannot be + * traced, and as a result DTrace will exit + * with an error because it cannot create a + * probe for this function. In order to get + * around this, we're overriding the requested + * probe's and fields with + * the very first inline copy's information. + */ + snprintf(buf, sizeof(buf), "%lu", e->off[i].val); + strlcpy(dp->dtsp_desc->dtpd_func, e->off[i].func, + sizeof(dp->dtsp_desc->dtpd_func)); + strlcpy(dp->dtsp_desc->dtpd_name, buf, + sizeof(dp->dtsp_desc->dtpd_name)); + } else { + /* + * Create a new probe description for each + * inline copy and append it to the main pdesc + * list. + */ + snprintf(buf, sizeof(buf), "%s:%s:%s:%lu", + dp->dtsp_desc->dtpd_provider, + dp->dtsp_desc->dtpd_mod, + e->off[i].func, e->off[i].val); + pdesc = dt_node_pdesc_by_name(strdup(buf)); + dnp = dt_node_link(dnp, pdesc); + } + } + dt_free(dp->dtsp_dtp, e->off); + dt_free(dp->dtsp_dtp, e); + } + if (!(dp->dtsp_flags & DF_INLINE)) { + /* + * Delegate non-inline function probes to FBT so that we don't + * duplicate FBT code in kinst. Regular kinst probes are not + * affected by this. + */ + strlcpy(dp->dtsp_desc->dtpd_provider, "fbt", + sizeof(dp->dtsp_desc->dtpd_provider)); + } else if ((~dp->dtsp_flags & (DF_REGULAR | DF_INLINE)) == 0 && n) { + /* + * If we have found a regular definition along with an inline + * one, create an FBT probe for the non-inline definition as + * well. + */ + snprintf(buf, sizeof(buf), "fbt:%s:%s:%s", + dp->dtsp_desc->dtpd_mod, dp->dtsp_func, dp->dtsp_name); + pdesc = dt_node_pdesc_by_name(strdup(buf)); + dnp = dt_node_link(dnp, pdesc); + } +} + +/* + * Initialize libelf and libdwarf and parse kernel.debug's DWARF info. + */ +static void +dt_sugar_do_kinst_inline(dt_sugar_parse_t *dp) +{ + dt_module_t *dmp; + Dwarf_Debug dbg; + Dwarf_Die die; + Dwarf_Error error; + char dbgfile[MAXPATHLEN]; + int res = DW_DLV_OK; + + strlcpy(dp->dtsp_func, dp->dtsp_desc->dtpd_func, sizeof(dp->dtsp_func)); + strlcpy(dp->dtsp_name, dp->dtsp_desc->dtpd_name, sizeof(dp->dtsp_func)); + + dp->dtsp_flags = 0; + + /* We only make entry and return probes for inline functions. */ + if (strcmp(dp->dtsp_name, "entry") == 0) + dp->dtsp_flags |= DF_ENTRY; + else if (strcmp(dp->dtsp_name, "return") == 0) + dp->dtsp_flags |= DF_RETURN; + else + return; + dp->dtsp_flags |= DF_NEWPDESC; + + for (dmp = dt_list_next(&dp->dtsp_dtp->dt_modlist); dmp != NULL; + dmp = dt_list_next(dmp)) { + if (strcmp(dmp->dm_name, "C") == 0 || + strcmp(dmp->dm_name, "D") == 0) + continue; + + (void) snprintf(dbgfile, sizeof(dbgfile), + "/usr/lib/debug/%s.debug", dmp->dm_file); + + if (dt_sugar_elf_init(dp->dtsp_dtp, &dp->dtsp_elf_mod, + dmp->dm_file) < 0) + continue; + if (dt_sugar_elf_init(dp->dtsp_dtp, &dp->dtsp_elf_dbg, + dbgfile) < 0) { + dt_sugar_elf_deinit(dp->dtsp_dtp, &dp->dtsp_elf_mod); + continue; + } + + if (dt_sugar_elf_verify_debuglink(&dp->dtsp_elf_mod, + dp->dtsp_elf_dbg.fd) < 0) { + warnx("dt_sugar: debug link mismatch: " + "make sure '%s' is up to date", dbgfile); + dt_sugar_elf_deinit(dp->dtsp_dtp, &dp->dtsp_elf_mod); + dt_sugar_elf_deinit(dp->dtsp_dtp, &dp->dtsp_elf_dbg); + continue; + } + + if (dwarf_elf_init(dp->dtsp_elf_dbg.elf, DW_DLC_READ, NULL, + NULL, &dbg, &error) != DW_DLV_OK) { + warnx("dt_sugar: dwarf_elf_init(): %s", + dwarf_errmsg(error)); + continue; + } + + TAILQ_INIT(&dp->dtsp_head); + dp->dtsp_flags &= ~(DF_INLINE | DF_REGULAR | DF_EXISTS); + + while ((res = dwarf_next_cu_header(dbg, NULL, NULL, NULL, NULL, + NULL, &error)) == DW_DLV_OK) { + die = NULL; + while (dwarf_siblingof(dbg, die, &die, &error) == + DW_DLV_OK) { + dt_sugar_kinst_parse_die(dp, dbg, die, 0, + F_SUBPROGRAM); + } + dwarf_dealloc(dbg, die, DW_DLA_DIE); + } + if (res == DW_DLV_ERROR) + warnx("dt_sugar: %s", dwarf_errmsg(error)); + + if (!(dp->dtsp_flags & DF_EXISTS)) + goto cont; + + dt_sugar_kinst_create_probes(dp); +cont: + dwarf_finish(dbg, &error); + dt_sugar_elf_deinit(dp->dtsp_dtp, &dp->dtsp_elf_mod); + dt_sugar_elf_deinit(dp->dtsp_dtp, &dp->dtsp_elf_dbg); + } +} + +/* + * Visit the specified node and all of its descendants. */ static void dt_sugar_visit_all(dt_sugar_parse_t *dp, dt_node_t *dnp) @@ -461,6 +1161,7 @@ dt_compile_sugar(dtrace_hdl_t *dtp, dt_node_t *clause) { dt_sugar_parse_t dp = { 0 }; + dt_node_t *dnp; int condid = 0; dp.dtsp_dtp = dtp; @@ -507,10 +1208,47 @@ dt_sugar_new_clearerror_clause(&dp)); } + /* + * This loop is a bit of a hack. What it does is iterate through all + * probe descriptions and handle kinst entry/return probes, but you + * will notice that in case we handle inline function probes, + * dt_sugar_kinst_create_probes() appends new elements to the list + * we're looping through, yet it works just fine! + * + * Consider the following initial `dn_pdescs` list: + * + * dn_pdescs = kinst::inlinefunc1:entry + * dn_pdescs->dn_list = kinst::inlinefunc2:return + * dn_pdescs->dn_list->dn_list = kinst::normalfunc1:0 + * dn_pdescs->dn_list->dn_list->dn_list = kinst::normalfunc2:entry + * + * The final list will look like this (read the comments in + * dt_sugar_kinst_create_probes()): + * + * dn_pdescs = kinst::callerfunc1: + * dn_pdescs->dn_list = kinst::callerfunc2: + * dn_pdescs->dn_list->dn_list = kinst::normalfunc1:0 + * dn_pdescs->dn_list->dn_list->dn_list = fbt::normalfunc2:entry + * ... = new probes are appended here + * + * Because it is guaranteed that any new probes appended to the list by + * dt_sugar_kinst_create_probes() will be regular kinst probes, the + * loop below *does* loop through them as well, but does nothing since + * regular kinst probes are skipped. + */ + for (dnp = dp.dtsp_clause_list->dn_pdescs; dnp != NULL; + dnp = dnp->dn_list) { + if (strcmp(dnp->dn_desc->dtpd_provider, "kinst") != 0) + continue; + dp.dtsp_desc = dnp->dn_desc; + dt_sugar_do_kinst_inline(&dp); + } + if (dp.dtsp_clause_list != NULL && dp.dtsp_clause_list->dn_list != NULL && !dtp->dt_has_sugar) { dtp->dt_has_sugar = B_TRUE; dt_sugar_prepend_clause(&dp, dt_sugar_makeerrorclause()); } + return (dp.dtsp_clause_list); } diff --git a/cddl/lib/libdtrace/Makefile b/cddl/lib/libdtrace/Makefile --- a/cddl/lib/libdtrace/Makefile +++ b/cddl/lib/libdtrace/Makefile @@ -133,7 +133,7 @@ YFLAGS+=-d -LIBADD= ctf elf proc pthread rtld_db +LIBADD= ctf dwarf elf proc pthread rtld_db CLEANFILES= dt_errtags.c dt_names.c diff --git a/cddl/usr.sbin/dtrace/tests/amd64/kinst/Makefile b/cddl/usr.sbin/dtrace/tests/amd64/kinst/Makefile --- a/cddl/usr.sbin/dtrace/tests/amd64/kinst/Makefile +++ b/cddl/usr.sbin/dtrace/tests/amd64/kinst/Makefile @@ -8,6 +8,10 @@ ${PACKAGE}FILES= \ tst.basic.ksh \ + tst.dtracetest_fbtconvert.ksh \ + tst.dtracetest_inline.ksh \ + tst.inline_agg.ksh \ + tst.inline_multiple.ksh \ TESTEXES= \ diff --git a/share/man/man4/dtrace_kinst.4 b/share/man/man4/dtrace_kinst.4 --- a/share/man/man4/dtrace_kinst.4 +++ b/share/man/man4/dtrace_kinst.4 @@ -24,7 +24,7 @@ .\" .\" $FreeBSD$ .\" -.Dd February 27, 2023 +.Dd March 18, 2023 .Dt DTRACE_KINST 4 .Os .Sh NAME @@ -36,9 +36,22 @@ The DTrace .Nm kinst provider allows the user to trace any instruction in a given kernel function. - corresponds to the function to be traced, and is the -offset to the specific instruction, and can be obtained from the function's -disassembly using kgdb from the gdb package. +.Cm +corresponds to the function to be traced, and +.Cm +is the offset to the specific instruction, and can be obtained from the +function's disassembly using kgdb from the gdb package. +.Pp +.Nm kinst +can also trace inline functions by requesting a probe of the form +.Cm kinst::: . +If the probe is +.Cm entry +or +.Cm return +but the function is not an inline one, the probe will be delegated to FBT, +otherwise DTrace will find all call sites of the inline function and create new +probes for each one of them. .Pp .Nm kinst creates probes on-demand, meaning it searches for and parses the function's @@ -72,13 +85,50 @@ 2 81500 vm_fault:4 0x1fab9bef0000 2 81500 vm_fault:4 0xe16cf749000 0 81500 vm_fault:4 0x13587c366000 - ... + ^C .Ed .Pp Trace all instructions in .Fn amd64_syscall : .Bd -literal -offset indent # dtrace -n 'kinst::amd64_syscall:' +dtrace: description 'kinst::amd64_syscall:' matched 458 probes +CPU ID FUNCTION:NAME + 2 80676 amd64_syscall:323 + 2 80677 amd64_syscall:326 + 2 80678 amd64_syscall:334 + 2 80679 amd64_syscall:339 + 2 80680 amd64_syscall:345 + 2 80681 amd64_syscall:353 + ^C +.Ed +.Pp +Trace the return point of +.Fn vm_page_mvqueue , +which is an inline function: +.Bd -literal -offset indent +# dtrace -n 'kinst::critical_enter:return' +dtrace: description 'kinst::critical_enter:return' matched 130 probes +CPU ID FUNCTION:NAME + 1 71024 spinlock_enter:53 + 0 71024 spinlock_enter:53 + 1 70992 uma_zalloc_arg:49 + 1 70925 malloc_type_zone_allocated:21 + 1 70994 uma_zfree_arg:365 + 1 70924 malloc_type_freed:21 + 1 71024 spinlock_enter:53 + 0 71024 spinlock_enter:53 + 0 71024 spinlock_enter:53 + 0 71024 spinlock_enter:53 + 1 71024 spinlock_enter:53 + 0 71024 spinlock_enter:53 + 0 70947 _epoch_enter_preempt:122 + 0 70949 _epoch_exit_preempt:28 + 0 71024 spinlock_enter:53 + 0 71024 spinlock_enter:53 + 0 70947 _epoch_enter_preempt:122 + 0 70949 _epoch_exit_preempt:28 + ^C .Ed .Sh SEE ALSO .Xr dtrace 1 diff --git a/share/mk/src.libnames.mk b/share/mk/src.libnames.mk --- a/share/mk/src.libnames.mk +++ b/share/mk/src.libnames.mk @@ -388,7 +388,7 @@ _DP_vmmapi= util _DP_opencsd= cxxrt _DP_ctf= spl z -_DP_dtrace= ctf elf proc pthread rtld_db +_DP_dtrace= ctf dwarf elf proc pthread rtld_db _DP_xo= util _DP_ztest= geom m nvpair umem zpool pthread avl zfs_core spl zutil zfs uutil icp # The libc dependencies are not strictly needed but are defined to make the diff --git a/sys/cddl/dev/dtrace/dtrace_test.c b/sys/cddl/dev/dtrace/dtrace_test.c --- a/sys/cddl/dev/dtrace/dtrace_test.c +++ b/sys/cddl/dev/dtrace/dtrace_test.c @@ -35,6 +35,7 @@ #include #include #include +#include #include SDT_PROVIDER_DEFINE(test); @@ -75,6 +76,56 @@ return (error); } +static __always_inline void +kinst_test_inline(void) +{ + struct timeval tv; + size_t len; + + /* + * TODO Modify the code so that the function is splitted into multiple + * DW_AT_ranges. + */ + len = sizeof(struct timeval); + if (kernel_sysctlbyname(curthread, "kern.boottime", &tv, &len, + NULL, 0, NULL, 0) != 0) + return; +} + +static __noinline void +kinst_test_fbtconvert(void) +{ + struct timeval tv; + size_t len; + + len = sizeof(struct timeval); + (void) kernel_sysctlbyname(curthread, "kern.boottime", &tv, &len, + NULL, 0, NULL, 0); +} + +static int +kinst_test(SYSCTL_HANDLER_ARGS) +{ + int val, error; + + val = 0; + error = sysctl_handle_int(oidp, &val, 0, req); + if (error || req->newptr == NULL) + return (error); + switch (val) { + case 1: + kinst_test_inline(); + break; + case 2: + kinst_test_fbtconvert(); + break; + default: + return (0); + } + + return (error); +} + static SYSCTL_NODE(_debug, OID_AUTO, dtracetest, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); @@ -83,6 +134,10 @@ CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, dtrace_test_sdttest, "I", "Trigger the SDT test probe"); +SYSCTL_PROC(_debug_dtracetest, OID_AUTO, kinst, + CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, kinst_test, + "I", "Trigger the kinst test functions"); + static int dtrace_test_modevent(module_t mod, int type, void *data) {