Index: head/sys/cddl/dev/fbt/aarch64/fbt_isa.c =================================================================== --- head/sys/cddl/dev/fbt/aarch64/fbt_isa.c (revision 338358) +++ head/sys/cddl/dev/fbt/aarch64/fbt_isa.c (revision 338359) @@ -1,177 +1,177 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org * Portions Copyright 2013 Howard Su howardsu@freebsd.org * Portions Copyright 2015 Ruslan Bukin * * $FreeBSD$ */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include "fbt.h" #define AARCH64_BRK 0xd4200000 #define AARCH64_BRK_IMM16_SHIFT 5 #define AARCH64_BRK_IMM16_VAL (0x40d << AARCH64_BRK_IMM16_SHIFT) #define FBT_PATCHVAL (AARCH64_BRK | AARCH64_BRK_IMM16_VAL) #define FBT_ENTRY "entry" #define FBT_RETURN "return" int fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) { solaris_cpu_t *cpu; fbt_probe_t *fbt; cpu = &solaris_cpu[curcpu]; fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { if ((uintptr_t)fbt->fbtp_patchpoint == addr) { cpu->cpu_dtrace_caller = addr; dtrace_probe(fbt->fbtp_id, frame->tf_x[0], frame->tf_x[1], frame->tf_x[2], frame->tf_x[3], frame->tf_x[4]); cpu->cpu_dtrace_caller = 0; return (fbt->fbtp_savedval); } } return (0); } void fbt_patch_tracepoint(fbt_probe_t *fbt, fbt_patchval_t val) { *fbt->fbtp_patchpoint = val; cpu_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); } int fbt_provide_module_function(linker_file_t lf, int symindx, linker_symval_t *symval, void *opaque) { fbt_probe_t *fbt, *retfbt; uint32_t *target, *start; uint32_t *instr, *limit; const char *name; char *modname; int offs; modname = opaque; name = symval->name; /* Check if function is excluded from instrumentation */ if (fbt_excluded(name)) return (0); instr = (uint32_t *)(symval->value); limit = (uint32_t *)(symval->value + symval->size); /* Look for stp (pre-indexed) operation */ for (; instr < limit; instr++) { if ((*instr & LDP_STP_MASK) == STP_64) break; } if (instr >= limit) return (0); fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_ENTRY, 3, fbt); fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_rval = DTRACE_INVOP_PUSHM; fbt->fbtp_symindx = symindx; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; retfbt = NULL; again: for (; instr < limit; instr++) { if (*instr == RET_INSTR) break; else if ((*instr & B_MASK) == B_INSTR) { offs = (*instr & B_DATA_MASK); offs *= 4; target = (instr + offs); start = (uint32_t *)symval->value; if (target >= limit || target < start) break; } } if (instr >= limit) return (0); /* * We have a winner! */ fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; if (retfbt == NULL) { fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_RETURN, 3, fbt); } else { - retfbt->fbtp_next = fbt; + retfbt->fbtp_probenext = fbt; fbt->fbtp_id = retfbt->fbtp_id; } retfbt = fbt; fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_symindx = symindx; if ((*instr & B_MASK) == B_INSTR) fbt->fbtp_rval = DTRACE_INVOP_B; else fbt->fbtp_rval = DTRACE_INVOP_RET; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; instr++; goto again; } Index: head/sys/cddl/dev/fbt/arm/fbt_isa.c =================================================================== --- head/sys/cddl/dev/fbt/arm/fbt_isa.c (revision 338358) +++ head/sys/cddl/dev/fbt/arm/fbt_isa.c (revision 338359) @@ -1,190 +1,190 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org * Portions Copyright 2013 Howard Su howardsu@freebsd.org * * $FreeBSD$ * */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include "fbt.h" #define FBT_PUSHM 0xe92d0000 #define FBT_POPM 0xe8bd0000 #define FBT_JUMP 0xea000000 #define FBT_SUBSP 0xe24dd000 #define FBT_ENTRY "entry" #define FBT_RETURN "return" int fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) { solaris_cpu_t *cpu = &solaris_cpu[curcpu]; fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; register_t fifthparam; for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { if ((uintptr_t)fbt->fbtp_patchpoint == addr) { cpu->cpu_dtrace_caller = addr; /* Get 5th parameter from stack */ DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); fifthparam = *(register_t *)frame->tf_svc_sp; DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); dtrace_probe(fbt->fbtp_id, frame->tf_r0, frame->tf_r1, frame->tf_r2, frame->tf_r3, fifthparam); cpu->cpu_dtrace_caller = 0; return (fbt->fbtp_rval | (fbt->fbtp_savedval << DTRACE_INVOP_SHIFT)); } } return (0); } void fbt_patch_tracepoint(fbt_probe_t *fbt, fbt_patchval_t val) { *fbt->fbtp_patchpoint = val; icache_sync((vm_offset_t)fbt->fbtp_patchpoint, sizeof(val)); } int fbt_provide_module_function(linker_file_t lf, int symindx, linker_symval_t *symval, void *opaque) { char *modname = opaque; const char *name = symval->name; fbt_probe_t *fbt, *retfbt; uint32_t *instr, *limit; int popm; if (fbt_excluded(name)) return (0); instr = (uint32_t *)symval->value; limit = (uint32_t *)(symval->value + symval->size); /* * va_arg functions has first instruction of * sub sp, sp, #? */ if ((*instr & 0xfffff000) == FBT_SUBSP) instr++; /* * check if insn is a pushm with LR */ if ((*instr & 0xffff0000) != FBT_PUSHM || (*instr & (1 << LR)) == 0) return (0); fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_ENTRY, 2, fbt); fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_BREAKPOINT; fbt->fbtp_rval = DTRACE_INVOP_PUSHM; fbt->fbtp_symindx = symindx; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; popm = FBT_POPM | ((*instr) & 0x3FFF) | 0x8000; retfbt = NULL; again: for (; instr < limit; instr++) { if (*instr == popm) break; else if ((*instr & 0xff000000) == FBT_JUMP) { uint32_t *target, *start; int offset; offset = (*instr & 0xffffff); offset <<= 8; offset /= 64; target = instr + (2 + offset); start = (uint32_t *)symval->value; if (target >= limit || target < start) break; } } if (instr >= limit) return (0); /* * We have a winner! */ fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; if (retfbt == NULL) { fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_RETURN, 2, fbt); } else { - retfbt->fbtp_next = fbt; + retfbt->fbtp_probenext = fbt; fbt->fbtp_id = retfbt->fbtp_id; } retfbt = fbt; fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_symindx = symindx; if ((*instr & 0xff000000) == FBT_JUMP) fbt->fbtp_rval = DTRACE_INVOP_B; else fbt->fbtp_rval = DTRACE_INVOP_POPM; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_BREAKPOINT; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; instr++; goto again; } Index: head/sys/cddl/dev/fbt/fbt.c =================================================================== --- head/sys/cddl/dev/fbt/fbt.c (revision 338358) +++ head/sys/cddl/dev/fbt/fbt.c (revision 338359) @@ -1,1169 +1,1197 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * * $FreeBSD$ * */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fbt.h" MALLOC_DEFINE(M_FBT, "fbt", "Function Boundary Tracing"); dtrace_provider_id_t fbt_id; fbt_probe_t **fbt_probetab; int fbt_probetab_mask; static d_open_t fbt_open; static int fbt_unload(void); static void fbt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *); static void fbt_provide_module(void *, modctl_t *); static void fbt_destroy(void *, dtrace_id_t, void *); static void fbt_enable(void *, dtrace_id_t, void *); static void fbt_disable(void *, dtrace_id_t, void *); static void fbt_load(void *); static void fbt_suspend(void *, dtrace_id_t, void *); static void fbt_resume(void *, dtrace_id_t, void *); static struct cdevsw fbt_cdevsw = { .d_version = D_VERSION, .d_open = fbt_open, .d_name = "fbt", }; static dtrace_pattr_t fbt_attr = { { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, }; static dtrace_pops_t fbt_pops = { .dtps_provide = NULL, .dtps_provide_module = fbt_provide_module, .dtps_enable = fbt_enable, .dtps_disable = fbt_disable, .dtps_suspend = fbt_suspend, .dtps_resume = fbt_resume, .dtps_getargdesc = fbt_getargdesc, .dtps_getargval = NULL, .dtps_usermode = NULL, .dtps_destroy = fbt_destroy }; static struct cdev *fbt_cdev; static int fbt_probetab_size; static int fbt_verbose = 0; int fbt_excluded(const char *name) { if (strncmp(name, "dtrace_", 7) == 0 && strncmp(name, "dtrace_safe_", 12) != 0) { /* * Anything beginning with "dtrace_" may be called * from probe context unless it explicitly indicates * that it won't be called from probe context by * using the prefix "dtrace_safe_". */ return (1); } /* * Lock owner methods may be called from probe context. */ if (strcmp(name, "owner_mtx") == 0 || strcmp(name, "owner_rm") == 0 || strcmp(name, "owner_rw") == 0 || strcmp(name, "owner_sx") == 0) return (1); /* * When DTrace is built into the kernel we need to exclude * the FBT functions from instrumentation. */ #ifndef _KLD_MODULE if (strncmp(name, "fbt_", 4) == 0) return (1); #endif return (0); } static void fbt_doubletrap(void) { fbt_probe_t *fbt; int i; for (i = 0; i < fbt_probetab_size; i++) { fbt = fbt_probetab[i]; - for (; fbt != NULL; fbt = fbt->fbtp_next) + for (; fbt != NULL; fbt = fbt->fbtp_probenext) fbt_patch_tracepoint(fbt, fbt->fbtp_savedval); } } static void fbt_provide_module(void *arg, modctl_t *lf) { char modname[MAXPATHLEN]; int i; size_t len; strlcpy(modname, lf->filename, sizeof(modname)); len = strlen(modname); if (len > 3 && strcmp(modname + len - 3, ".ko") == 0) modname[len - 3] = '\0'; /* * Employees of dtrace and their families are ineligible. Void * where prohibited. */ if (strcmp(modname, "dtrace") == 0) return; /* * To register with DTrace, a module must list 'dtrace' as a * dependency in order for the kernel linker to resolve * symbols like dtrace_register(). All modules with such a * dependency are ineligible for FBT tracing. */ for (i = 0; i < lf->ndeps; i++) if (strncmp(lf->deps[i]->filename, "dtrace", 6) == 0) return; if (lf->fbt_nentries) { /* * This module has some FBT entries allocated; we're afraid * to screw with it. */ return; } /* * List the functions in the module and the symbol values. */ (void) linker_file_function_listall(lf, fbt_provide_module_function, modname); } static void +fbt_destroy_one(fbt_probe_t *fbt) +{ + fbt_probe_t *hash, *hashprev, *next; + int ndx; + + ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint); + for (hash = fbt_probetab[ndx], hashprev = NULL; hash != NULL; + hash = hash->fbtp_hashnext, hashprev = hash) { + if (hash == fbt) { + if ((next = fbt->fbtp_tracenext) != NULL) + next->fbtp_hashnext = hash->fbtp_hashnext; + else + next = hash->fbtp_hashnext; + if (hashprev != NULL) + hashprev->fbtp_hashnext = next; + else + fbt_probetab[ndx] = next; + goto free; + } else if (hash->fbtp_patchpoint == fbt->fbtp_patchpoint) { + for (next = hash; next->fbtp_tracenext != NULL; + next = next->fbtp_tracenext) { + if (fbt == next->fbtp_tracenext) { + next->fbtp_tracenext = + fbt->fbtp_tracenext; + goto free; + } + } + } + } + panic("probe %p not found in hash table", fbt); +free: + free(fbt, M_FBT); +} + +static void fbt_destroy(void *arg, dtrace_id_t id, void *parg) { - fbt_probe_t *fbt = parg, *next, *hash, *last; + fbt_probe_t *fbt = parg, *next; modctl_t *ctl; - int ndx; do { ctl = fbt->fbtp_ctl; - ctl->fbt_nentries--; - /* - * Now we need to remove this probe from the fbt_probetab. - */ - ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint); - last = NULL; - hash = fbt_probetab[ndx]; - - while (hash != fbt) { - ASSERT(hash != NULL); - last = hash; - hash = hash->fbtp_hashnext; - } - - if (last != NULL) { - last->fbtp_hashnext = fbt->fbtp_hashnext; - } else { - fbt_probetab[ndx] = fbt->fbtp_hashnext; - } - - next = fbt->fbtp_next; - free(fbt, M_FBT); - + next = fbt->fbtp_probenext; + fbt_destroy_one(fbt); fbt = next; } while (fbt != NULL); } static void fbt_enable(void *arg, dtrace_id_t id, void *parg) { fbt_probe_t *fbt = parg; modctl_t *ctl = fbt->fbtp_ctl; ctl->nenabled++; /* * Now check that our modctl has the expected load count. If it * doesn't, this module must have been unloaded and reloaded -- and * we're not going to touch it. */ if (ctl->loadcnt != fbt->fbtp_loadcnt) { if (fbt_verbose) { printf("fbt is failing for probe %s " "(module %s reloaded)", fbt->fbtp_name, ctl->filename); } return; } - for (; fbt != NULL; fbt = fbt->fbtp_next) + for (; fbt != NULL; fbt = fbt->fbtp_probenext) { fbt_patch_tracepoint(fbt, fbt->fbtp_patchval); + fbt->fbtp_enabled++; + } } static void fbt_disable(void *arg, dtrace_id_t id, void *parg) { - fbt_probe_t *fbt = parg; + fbt_probe_t *fbt = parg, *hash; modctl_t *ctl = fbt->fbtp_ctl; ASSERT(ctl->nenabled > 0); ctl->nenabled--; if ((ctl->loadcnt != fbt->fbtp_loadcnt)) return; - for (; fbt != NULL; fbt = fbt->fbtp_next) - fbt_patch_tracepoint(fbt, fbt->fbtp_savedval); + for (; fbt != NULL; fbt = fbt->fbtp_probenext) { + fbt->fbtp_enabled--; + + for (hash = fbt_probetab[FBT_ADDR2NDX(fbt->fbtp_patchpoint)]; + hash != NULL; hash = hash->fbtp_hashnext) { + if (hash->fbtp_patchpoint == fbt->fbtp_patchpoint) { + for (; hash != NULL; hash = hash->fbtp_tracenext) + if (hash->fbtp_enabled > 0) + break; + break; + } + } + if (hash == NULL) + fbt_patch_tracepoint(fbt, fbt->fbtp_savedval); + } } static void fbt_suspend(void *arg, dtrace_id_t id, void *parg) { fbt_probe_t *fbt = parg; modctl_t *ctl = fbt->fbtp_ctl; ASSERT(ctl->nenabled > 0); if ((ctl->loadcnt != fbt->fbtp_loadcnt)) return; - for (; fbt != NULL; fbt = fbt->fbtp_next) + for (; fbt != NULL; fbt = fbt->fbtp_probenext) fbt_patch_tracepoint(fbt, fbt->fbtp_savedval); } static void fbt_resume(void *arg, dtrace_id_t id, void *parg) { fbt_probe_t *fbt = parg; modctl_t *ctl = fbt->fbtp_ctl; ASSERT(ctl->nenabled > 0); if ((ctl->loadcnt != fbt->fbtp_loadcnt)) return; - for (; fbt != NULL; fbt = fbt->fbtp_next) + for (; fbt != NULL; fbt = fbt->fbtp_probenext) fbt_patch_tracepoint(fbt, fbt->fbtp_patchval); } static int fbt_ctfoff_init(modctl_t *lf, linker_ctf_t *lc) { const Elf_Sym *symp = lc->symtab;; const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t); int i; uint32_t *ctfoff; uint32_t objtoff = hp->cth_objtoff; uint32_t funcoff = hp->cth_funcoff; ushort_t info; ushort_t vlen; /* Sanity check. */ if (hp->cth_magic != CTF_MAGIC) { printf("Bad magic value in CTF data of '%s'\n",lf->pathname); return (EINVAL); } if (lc->symtab == NULL) { printf("No symbol table in '%s'\n",lf->pathname); return (EINVAL); } ctfoff = malloc(sizeof(uint32_t) * lc->nsym, M_LINKER, M_WAITOK); *lc->ctfoffp = ctfoff; for (i = 0; i < lc->nsym; i++, ctfoff++, symp++) { if (symp->st_name == 0 || symp->st_shndx == SHN_UNDEF) { *ctfoff = 0xffffffff; continue; } switch (ELF_ST_TYPE(symp->st_info)) { case STT_OBJECT: if (objtoff >= hp->cth_funcoff || (symp->st_shndx == SHN_ABS && symp->st_value == 0)) { *ctfoff = 0xffffffff; break; } *ctfoff = objtoff; objtoff += sizeof (ushort_t); break; case STT_FUNC: if (funcoff >= hp->cth_typeoff) { *ctfoff = 0xffffffff; break; } *ctfoff = funcoff; info = *((const ushort_t *)(ctfdata + funcoff)); vlen = CTF_INFO_VLEN(info); /* * If we encounter a zero pad at the end, just skip it. * Otherwise skip over the function and its return type * (+2) and the argument list (vlen). */ if (CTF_INFO_KIND(info) == CTF_K_UNKNOWN && vlen == 0) funcoff += sizeof (ushort_t); /* skip pad */ else funcoff += sizeof (ushort_t) * (vlen + 2); break; default: *ctfoff = 0xffffffff; break; } } return (0); } static ssize_t fbt_get_ctt_size(uint8_t version, const ctf_type_t *tp, ssize_t *sizep, ssize_t *incrementp) { ssize_t size, increment; if (version > CTF_VERSION_1 && tp->ctt_size == CTF_LSIZE_SENT) { size = CTF_TYPE_LSIZE(tp); increment = sizeof (ctf_type_t); } else { size = tp->ctt_size; increment = sizeof (ctf_stype_t); } if (sizep) *sizep = size; if (incrementp) *incrementp = increment; return (size); } static int fbt_typoff_init(linker_ctf_t *lc) { const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; const ctf_type_t *tbuf; const ctf_type_t *tend; const ctf_type_t *tp; const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t); int ctf_typemax = 0; uint32_t *xp; ulong_t pop[CTF_K_MAX + 1] = { 0 }; /* Sanity check. */ if (hp->cth_magic != CTF_MAGIC) return (EINVAL); tbuf = (const ctf_type_t *) (ctfdata + hp->cth_typeoff); tend = (const ctf_type_t *) (ctfdata + hp->cth_stroff); int child = hp->cth_parname != 0; /* * We make two passes through the entire type section. In this first * pass, we count the number of each type and the total number of types. */ for (tp = tbuf; tp < tend; ctf_typemax++) { ushort_t kind = CTF_INFO_KIND(tp->ctt_info); ulong_t vlen = CTF_INFO_VLEN(tp->ctt_info); ssize_t size, increment; size_t vbytes; uint_t n; (void) fbt_get_ctt_size(hp->cth_version, tp, &size, &increment); switch (kind) { case CTF_K_INTEGER: case CTF_K_FLOAT: vbytes = sizeof (uint_t); break; case CTF_K_ARRAY: vbytes = sizeof (ctf_array_t); break; case CTF_K_FUNCTION: vbytes = sizeof (ushort_t) * (vlen + (vlen & 1)); break; case CTF_K_STRUCT: case CTF_K_UNION: if (size < CTF_LSTRUCT_THRESH) { ctf_member_t *mp = (ctf_member_t *) ((uintptr_t)tp + increment); vbytes = sizeof (ctf_member_t) * vlen; for (n = vlen; n != 0; n--, mp++) child |= CTF_TYPE_ISCHILD(mp->ctm_type); } else { ctf_lmember_t *lmp = (ctf_lmember_t *) ((uintptr_t)tp + increment); vbytes = sizeof (ctf_lmember_t) * vlen; for (n = vlen; n != 0; n--, lmp++) child |= CTF_TYPE_ISCHILD(lmp->ctlm_type); } break; case CTF_K_ENUM: vbytes = sizeof (ctf_enum_t) * vlen; break; case CTF_K_FORWARD: /* * For forward declarations, ctt_type is the CTF_K_* * kind for the tag, so bump that population count too. * If ctt_type is unknown, treat the tag as a struct. */ if (tp->ctt_type == CTF_K_UNKNOWN || tp->ctt_type >= CTF_K_MAX) pop[CTF_K_STRUCT]++; else pop[tp->ctt_type]++; /*FALLTHRU*/ case CTF_K_UNKNOWN: vbytes = 0; break; case CTF_K_POINTER: case CTF_K_TYPEDEF: case CTF_K_VOLATILE: case CTF_K_CONST: case CTF_K_RESTRICT: child |= CTF_TYPE_ISCHILD(tp->ctt_type); vbytes = 0; break; default: printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind); return (EIO); } tp = (ctf_type_t *)((uintptr_t)tp + increment + vbytes); pop[kind]++; } /* account for a sentinel value below */ ctf_typemax++; *lc->typlenp = ctf_typemax; xp = malloc(sizeof(uint32_t) * ctf_typemax, M_LINKER, M_ZERO | M_WAITOK); *lc->typoffp = xp; /* type id 0 is used as a sentinel value */ *xp++ = 0; /* * In the second pass, fill in the type offset. */ for (tp = tbuf; tp < tend; xp++) { ushort_t kind = CTF_INFO_KIND(tp->ctt_info); ulong_t vlen = CTF_INFO_VLEN(tp->ctt_info); ssize_t size, increment; size_t vbytes; uint_t n; (void) fbt_get_ctt_size(hp->cth_version, tp, &size, &increment); switch (kind) { case CTF_K_INTEGER: case CTF_K_FLOAT: vbytes = sizeof (uint_t); break; case CTF_K_ARRAY: vbytes = sizeof (ctf_array_t); break; case CTF_K_FUNCTION: vbytes = sizeof (ushort_t) * (vlen + (vlen & 1)); break; case CTF_K_STRUCT: case CTF_K_UNION: if (size < CTF_LSTRUCT_THRESH) { ctf_member_t *mp = (ctf_member_t *) ((uintptr_t)tp + increment); vbytes = sizeof (ctf_member_t) * vlen; for (n = vlen; n != 0; n--, mp++) child |= CTF_TYPE_ISCHILD(mp->ctm_type); } else { ctf_lmember_t *lmp = (ctf_lmember_t *) ((uintptr_t)tp + increment); vbytes = sizeof (ctf_lmember_t) * vlen; for (n = vlen; n != 0; n--, lmp++) child |= CTF_TYPE_ISCHILD(lmp->ctlm_type); } break; case CTF_K_ENUM: vbytes = sizeof (ctf_enum_t) * vlen; break; case CTF_K_FORWARD: case CTF_K_UNKNOWN: vbytes = 0; break; case CTF_K_POINTER: case CTF_K_TYPEDEF: case CTF_K_VOLATILE: case CTF_K_CONST: case CTF_K_RESTRICT: vbytes = 0; break; default: printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind); return (EIO); } *xp = (uint32_t)((uintptr_t) tp - (uintptr_t) ctfdata); tp = (ctf_type_t *)((uintptr_t)tp + increment + vbytes); } return (0); } /* * CTF Declaration Stack * * In order to implement ctf_type_name(), we must convert a type graph back * into a C type declaration. Unfortunately, a type graph represents a storage * class ordering of the type whereas a type declaration must obey the C rules * for operator precedence, and the two orderings are frequently in conflict. * For example, consider these CTF type graphs and their C declarations: * * CTF_K_POINTER -> CTF_K_FUNCTION -> CTF_K_INTEGER : int (*)() * CTF_K_POINTER -> CTF_K_ARRAY -> CTF_K_INTEGER : int (*)[] * * In each case, parentheses are used to raise operator * to higher lexical * precedence, so the string form of the C declaration cannot be constructed by * walking the type graph links and forming the string from left to right. * * The functions in this file build a set of stacks from the type graph nodes * corresponding to the C operator precedence levels in the appropriate order. * The code in ctf_type_name() can then iterate over the levels and nodes in * lexical precedence order and construct the final C declaration string. */ typedef struct ctf_list { struct ctf_list *l_prev; /* previous pointer or tail pointer */ struct ctf_list *l_next; /* next pointer or head pointer */ } ctf_list_t; #define ctf_list_prev(elem) ((void *)(((ctf_list_t *)(elem))->l_prev)) #define ctf_list_next(elem) ((void *)(((ctf_list_t *)(elem))->l_next)) typedef enum { CTF_PREC_BASE, CTF_PREC_POINTER, CTF_PREC_ARRAY, CTF_PREC_FUNCTION, CTF_PREC_MAX } ctf_decl_prec_t; typedef struct ctf_decl_node { ctf_list_t cd_list; /* linked list pointers */ ctf_id_t cd_type; /* type identifier */ uint_t cd_kind; /* type kind */ uint_t cd_n; /* type dimension if array */ } ctf_decl_node_t; typedef struct ctf_decl { ctf_list_t cd_nodes[CTF_PREC_MAX]; /* declaration node stacks */ int cd_order[CTF_PREC_MAX]; /* storage order of decls */ ctf_decl_prec_t cd_qualp; /* qualifier precision */ ctf_decl_prec_t cd_ordp; /* ordered precision */ char *cd_buf; /* buffer for output */ char *cd_ptr; /* buffer location */ char *cd_end; /* buffer limit */ size_t cd_len; /* buffer space required */ int cd_err; /* saved error value */ } ctf_decl_t; /* * Simple doubly-linked list append routine. This implementation assumes that * each list element contains an embedded ctf_list_t as the first member. * An additional ctf_list_t is used to store the head (l_next) and tail * (l_prev) pointers. The current head and tail list elements have their * previous and next pointers set to NULL, respectively. */ static void ctf_list_append(ctf_list_t *lp, void *new) { ctf_list_t *p = lp->l_prev; /* p = tail list element */ ctf_list_t *q = new; /* q = new list element */ lp->l_prev = q; q->l_prev = p; q->l_next = NULL; if (p != NULL) p->l_next = q; else lp->l_next = q; } /* * Prepend the specified existing element to the given ctf_list_t. The * existing pointer should be pointing at a struct with embedded ctf_list_t. */ static void ctf_list_prepend(ctf_list_t *lp, void *new) { ctf_list_t *p = new; /* p = new list element */ ctf_list_t *q = lp->l_next; /* q = head list element */ lp->l_next = p; p->l_prev = NULL; p->l_next = q; if (q != NULL) q->l_prev = p; else lp->l_prev = p; } static void ctf_decl_init(ctf_decl_t *cd, char *buf, size_t len) { int i; bzero(cd, sizeof (ctf_decl_t)); for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) cd->cd_order[i] = CTF_PREC_BASE - 1; cd->cd_qualp = CTF_PREC_BASE; cd->cd_ordp = CTF_PREC_BASE; cd->cd_buf = buf; cd->cd_ptr = buf; cd->cd_end = buf + len; } static void ctf_decl_fini(ctf_decl_t *cd) { ctf_decl_node_t *cdp, *ndp; int i; for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) { for (cdp = ctf_list_next(&cd->cd_nodes[i]); cdp != NULL; cdp = ndp) { ndp = ctf_list_next(cdp); free(cdp, M_FBT); } } } static const ctf_type_t * ctf_lookup_by_id(linker_ctf_t *lc, ctf_id_t type) { const ctf_type_t *tp; uint32_t offset; uint32_t *typoff = *lc->typoffp; if (type >= *lc->typlenp) { printf("%s(%d): type %d exceeds max %ld\n",__func__,__LINE__,(int) type,*lc->typlenp); return(NULL); } /* Check if the type isn't cross-referenced. */ if ((offset = typoff[type]) == 0) { printf("%s(%d): type %d isn't cross referenced\n",__func__,__LINE__, (int) type); return(NULL); } tp = (const ctf_type_t *)(lc->ctftab + offset + sizeof(ctf_header_t)); return (tp); } static void fbt_array_info(linker_ctf_t *lc, ctf_id_t type, ctf_arinfo_t *arp) { const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; const ctf_type_t *tp; const ctf_array_t *ap; ssize_t increment; bzero(arp, sizeof(*arp)); if ((tp = ctf_lookup_by_id(lc, type)) == NULL) return; if (CTF_INFO_KIND(tp->ctt_info) != CTF_K_ARRAY) return; (void) fbt_get_ctt_size(hp->cth_version, tp, NULL, &increment); ap = (const ctf_array_t *)((uintptr_t)tp + increment); arp->ctr_contents = ap->cta_contents; arp->ctr_index = ap->cta_index; arp->ctr_nelems = ap->cta_nelems; } static const char * ctf_strptr(linker_ctf_t *lc, int name) { const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;; const char *strp = ""; if (name < 0 || name >= hp->cth_strlen) return(strp); strp = (const char *)(lc->ctftab + hp->cth_stroff + name + sizeof(ctf_header_t)); return (strp); } static void ctf_decl_push(ctf_decl_t *cd, linker_ctf_t *lc, ctf_id_t type) { ctf_decl_node_t *cdp; ctf_decl_prec_t prec; uint_t kind, n = 1; int is_qual = 0; const ctf_type_t *tp; ctf_arinfo_t ar; if ((tp = ctf_lookup_by_id(lc, type)) == NULL) { cd->cd_err = ENOENT; return; } switch (kind = CTF_INFO_KIND(tp->ctt_info)) { case CTF_K_ARRAY: fbt_array_info(lc, type, &ar); ctf_decl_push(cd, lc, ar.ctr_contents); n = ar.ctr_nelems; prec = CTF_PREC_ARRAY; break; case CTF_K_TYPEDEF: if (ctf_strptr(lc, tp->ctt_name)[0] == '\0') { ctf_decl_push(cd, lc, tp->ctt_type); return; } prec = CTF_PREC_BASE; break; case CTF_K_FUNCTION: ctf_decl_push(cd, lc, tp->ctt_type); prec = CTF_PREC_FUNCTION; break; case CTF_K_POINTER: ctf_decl_push(cd, lc, tp->ctt_type); prec = CTF_PREC_POINTER; break; case CTF_K_VOLATILE: case CTF_K_CONST: case CTF_K_RESTRICT: ctf_decl_push(cd, lc, tp->ctt_type); prec = cd->cd_qualp; is_qual++; break; default: prec = CTF_PREC_BASE; } cdp = malloc(sizeof(*cdp), M_FBT, M_WAITOK); cdp->cd_type = type; cdp->cd_kind = kind; cdp->cd_n = n; if (ctf_list_next(&cd->cd_nodes[prec]) == NULL) cd->cd_order[prec] = cd->cd_ordp++; /* * Reset cd_qualp to the highest precedence level that we've seen so * far that can be qualified (CTF_PREC_BASE or CTF_PREC_POINTER). */ if (prec > cd->cd_qualp && prec < CTF_PREC_ARRAY) cd->cd_qualp = prec; /* * C array declarators are ordered inside out so prepend them. Also by * convention qualifiers of base types precede the type specifier (e.g. * const int vs. int const) even though the two forms are equivalent. */ if (kind == CTF_K_ARRAY || (is_qual && prec == CTF_PREC_BASE)) ctf_list_prepend(&cd->cd_nodes[prec], cdp); else ctf_list_append(&cd->cd_nodes[prec], cdp); } static void ctf_decl_sprintf(ctf_decl_t *cd, const char *format, ...) { size_t len = (size_t)(cd->cd_end - cd->cd_ptr); va_list ap; size_t n; va_start(ap, format); n = vsnprintf(cd->cd_ptr, len, format, ap); va_end(ap); cd->cd_ptr += MIN(n, len); cd->cd_len += n; } static ssize_t fbt_type_name(linker_ctf_t *lc, ctf_id_t type, char *buf, size_t len) { ctf_decl_t cd; ctf_decl_node_t *cdp; ctf_decl_prec_t prec, lp, rp; int ptr, arr; uint_t k; if (lc == NULL && type == CTF_ERR) return (-1); /* simplify caller code by permitting CTF_ERR */ ctf_decl_init(&cd, buf, len); ctf_decl_push(&cd, lc, type); if (cd.cd_err != 0) { ctf_decl_fini(&cd); return (-1); } /* * If the type graph's order conflicts with lexical precedence order * for pointers or arrays, then we need to surround the declarations at * the corresponding lexical precedence with parentheses. This can * result in either a parenthesized pointer (*) as in int (*)() or * int (*)[], or in a parenthesized pointer and array as in int (*[])(). */ ptr = cd.cd_order[CTF_PREC_POINTER] > CTF_PREC_POINTER; arr = cd.cd_order[CTF_PREC_ARRAY] > CTF_PREC_ARRAY; rp = arr ? CTF_PREC_ARRAY : ptr ? CTF_PREC_POINTER : -1; lp = ptr ? CTF_PREC_POINTER : arr ? CTF_PREC_ARRAY : -1; k = CTF_K_POINTER; /* avoid leading whitespace (see below) */ for (prec = CTF_PREC_BASE; prec < CTF_PREC_MAX; prec++) { for (cdp = ctf_list_next(&cd.cd_nodes[prec]); cdp != NULL; cdp = ctf_list_next(cdp)) { const ctf_type_t *tp = ctf_lookup_by_id(lc, cdp->cd_type); const char *name = ctf_strptr(lc, tp->ctt_name); if (k != CTF_K_POINTER && k != CTF_K_ARRAY) ctf_decl_sprintf(&cd, " "); if (lp == prec) { ctf_decl_sprintf(&cd, "("); lp = -1; } switch (cdp->cd_kind) { case CTF_K_INTEGER: case CTF_K_FLOAT: case CTF_K_TYPEDEF: ctf_decl_sprintf(&cd, "%s", name); break; case CTF_K_POINTER: ctf_decl_sprintf(&cd, "*"); break; case CTF_K_ARRAY: ctf_decl_sprintf(&cd, "[%u]", cdp->cd_n); break; case CTF_K_FUNCTION: ctf_decl_sprintf(&cd, "()"); break; case CTF_K_STRUCT: case CTF_K_FORWARD: ctf_decl_sprintf(&cd, "struct %s", name); break; case CTF_K_UNION: ctf_decl_sprintf(&cd, "union %s", name); break; case CTF_K_ENUM: ctf_decl_sprintf(&cd, "enum %s", name); break; case CTF_K_VOLATILE: ctf_decl_sprintf(&cd, "volatile"); break; case CTF_K_CONST: ctf_decl_sprintf(&cd, "const"); break; case CTF_K_RESTRICT: ctf_decl_sprintf(&cd, "restrict"); break; } k = cdp->cd_kind; } if (rp == prec) ctf_decl_sprintf(&cd, ")"); } ctf_decl_fini(&cd); return (cd.cd_len); } static void fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_argdesc_t *desc) { const ushort_t *dp; fbt_probe_t *fbt = parg; linker_ctf_t lc; modctl_t *ctl = fbt->fbtp_ctl; int ndx = desc->dtargd_ndx; int symindx = fbt->fbtp_symindx; uint32_t *ctfoff; uint32_t offset; ushort_t info, kind, n; if (fbt->fbtp_roffset != 0 && desc->dtargd_ndx == 0) { (void) strcpy(desc->dtargd_native, "int"); return; } desc->dtargd_ndx = DTRACE_ARGNONE; /* Get a pointer to the CTF data and it's length. */ if (linker_ctf_get(ctl, &lc) != 0) /* No CTF data? Something wrong? *shrug* */ return; /* Check if this module hasn't been initialised yet. */ if (*lc.ctfoffp == NULL) { /* * Initialise the CTF object and function symindx to * byte offset array. */ if (fbt_ctfoff_init(ctl, &lc) != 0) return; /* Initialise the CTF type to byte offset array. */ if (fbt_typoff_init(&lc) != 0) return; } ctfoff = *lc.ctfoffp; if (ctfoff == NULL || *lc.typoffp == NULL) return; /* Check if the symbol index is out of range. */ if (symindx >= lc.nsym) return; /* Check if the symbol isn't cross-referenced. */ if ((offset = ctfoff[symindx]) == 0xffffffff) return; dp = (const ushort_t *)(lc.ctftab + offset + sizeof(ctf_header_t)); info = *dp++; kind = CTF_INFO_KIND(info); n = CTF_INFO_VLEN(info); if (kind == CTF_K_UNKNOWN && n == 0) { printf("%s(%d): Unknown function!\n",__func__,__LINE__); return; } if (kind != CTF_K_FUNCTION) { printf("%s(%d): Expected a function!\n",__func__,__LINE__); return; } if (fbt->fbtp_roffset != 0) { /* Only return type is available for args[1] in return probe. */ if (ndx > 1) return; ASSERT(ndx == 1); } else { /* Check if the requested argument doesn't exist. */ if (ndx >= n) return; /* Skip the return type and arguments up to the one requested. */ dp += ndx + 1; } if (fbt_type_name(&lc, *dp, desc->dtargd_native, sizeof(desc->dtargd_native)) > 0) desc->dtargd_ndx = ndx; return; } static int fbt_linker_file_cb(linker_file_t lf, void *arg) { fbt_provide_module(arg, lf); return (0); } static void fbt_load(void *dummy) { /* Create the /dev/dtrace/fbt entry. */ fbt_cdev = make_dev(&fbt_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "dtrace/fbt"); /* Default the probe table size if not specified. */ if (fbt_probetab_size == 0) fbt_probetab_size = FBT_PROBETAB_SIZE; /* Choose the hash mask for the probe table. */ fbt_probetab_mask = fbt_probetab_size - 1; /* Allocate memory for the probe table. */ fbt_probetab = malloc(fbt_probetab_size * sizeof (fbt_probe_t *), M_FBT, M_WAITOK | M_ZERO); dtrace_doubletrap_func = fbt_doubletrap; dtrace_invop_add(fbt_invop); if (dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_USER, NULL, &fbt_pops, NULL, &fbt_id) != 0) return; /* Create probes for the kernel and already-loaded modules. */ linker_file_foreach(fbt_linker_file_cb, NULL); } static int fbt_unload() { int error = 0; /* De-register the invalid opcode handler. */ dtrace_invop_remove(fbt_invop); dtrace_doubletrap_func = NULL; /* De-register this DTrace provider. */ if ((error = dtrace_unregister(fbt_id)) != 0) return (error); /* Free the probe table. */ free(fbt_probetab, M_FBT); fbt_probetab = NULL; fbt_probetab_mask = 0; destroy_dev(fbt_cdev); return (error); } static int fbt_modevent(module_t mod __unused, int type, void *data __unused) { int error = 0; switch (type) { case MOD_LOAD: break; case MOD_UNLOAD: break; case MOD_SHUTDOWN: break; default: error = EOPNOTSUPP; break; } return (error); } static int fbt_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused) { return (0); } SYSINIT(fbt_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_load, NULL); SYSUNINIT(fbt_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_unload, NULL); DEV_MODULE(fbt, fbt_modevent, NULL); MODULE_VERSION(fbt, 1); MODULE_DEPEND(fbt, dtrace, 1, 1, 1); MODULE_DEPEND(fbt, opensolaris, 1, 1, 1); Index: head/sys/cddl/dev/fbt/fbt.h =================================================================== --- head/sys/cddl/dev/fbt/fbt.h (revision 338358) +++ head/sys/cddl/dev/fbt/fbt.h (revision 338359) @@ -1,73 +1,81 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * * $FreeBSD$ * */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #ifndef _FBT_H_ #define _FBT_H_ #include "fbt_isa.h" +/* + * fbt_probe is a bit of a misnomer. One of these structures is created for + * each trace point of an FBT probe. A probe might have multiple trace points + * (e.g., a function with multiple return instructions), and different probes + * might have a trace point at the same address (e.g., GNU ifuncs). + */ typedef struct fbt_probe { - struct fbt_probe *fbtp_hashnext; - fbt_patchval_t *fbtp_patchpoint; + struct fbt_probe *fbtp_hashnext; /* global hash table linkage */ + struct fbt_probe *fbtp_tracenext; /* next probe for tracepoint */ + struct fbt_probe *fbtp_probenext; /* next tracepoint for probe */ + int fbtp_enabled; + fbt_patchval_t *fbtp_patchpoint; int8_t fbtp_rval; fbt_patchval_t fbtp_patchval; fbt_patchval_t fbtp_savedval; uintptr_t fbtp_roffset; dtrace_id_t fbtp_id; const char *fbtp_name; modctl_t *fbtp_ctl; int fbtp_loadcnt; int fbtp_symindx; - struct fbt_probe *fbtp_next; } fbt_probe_t; struct linker_file; struct linker_symval; struct trapframe; int fbt_invop(uintptr_t, struct trapframe *, uintptr_t); void fbt_patch_tracepoint(fbt_probe_t *, fbt_patchval_t); int fbt_provide_module_function(struct linker_file *, int, struct linker_symval *, void *); int fbt_excluded(const char *name); extern dtrace_provider_id_t fbt_id; extern fbt_probe_t **fbt_probetab; extern int fbt_probetab_mask; #define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask) #define FBT_PROBETAB_SIZE 0x8000 /* 32k entries -- 128K total */ #ifdef MALLOC_DECLARE MALLOC_DECLARE(M_FBT); #endif #endif Index: head/sys/cddl/dev/fbt/mips/fbt_isa.c =================================================================== --- head/sys/cddl/dev/fbt/mips/fbt_isa.c (revision 338358) +++ head/sys/cddl/dev/fbt/mips/fbt_isa.c (revision 338359) @@ -1,164 +1,164 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org * Portions Copyright 2013 Howard Su howardsu@freebsd.org * Portions Copyright 2015-2016 Ruslan Bukin * * $FreeBSD$ */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include "fbt.h" #define FBT_PATCHVAL (MIPS_BREAK_INSTR) #define FBT_ENTRY "entry" #define FBT_RETURN "return" int fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) { solaris_cpu_t *cpu; fbt_probe_t *fbt; cpu = &solaris_cpu[curcpu]; fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { if ((uintptr_t)fbt->fbtp_patchpoint == addr) { cpu->cpu_dtrace_caller = addr; dtrace_probe(fbt->fbtp_id, frame->a0, frame->a1, frame->a2, frame->a3, frame->a4); cpu->cpu_dtrace_caller = 0; return (fbt->fbtp_savedval); } } return (0); } void fbt_patch_tracepoint(fbt_probe_t *fbt, fbt_patchval_t val) { *fbt->fbtp_patchpoint = val; mips_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); } int fbt_provide_module_function(linker_file_t lf, int symindx, linker_symval_t *symval, void *opaque) { fbt_probe_t *fbt, *retfbt; uint32_t *instr, *limit; const char *name; char *modname; modname = opaque; name = symval->name; /* Check if function is excluded from instrumentation */ if (fbt_excluded(name)) return (0); instr = (uint32_t *)(symval->value); limit = (uint32_t *)(symval->value + symval->size); /* Look for store double to ra register */ for (; instr < limit; instr++) { if ((*instr & LDSD_RA_SP_MASK) == SD_RA_SP) break; } if (instr >= limit) return (0); fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_ENTRY, 3, fbt); fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_rval = DTRACE_INVOP_SD; fbt->fbtp_symindx = symindx; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; retfbt = NULL; again: for (; instr < limit; instr++) { if ((*instr & LDSD_RA_SP_MASK) == LD_RA_SP) { break; } } if (instr >= limit) return (0); /* * We have a winner! */ fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; if (retfbt == NULL) { fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_RETURN, 3, fbt); } else { - retfbt->fbtp_next = fbt; + retfbt->fbtp_probenext = fbt; fbt->fbtp_id = retfbt->fbtp_id; } retfbt = fbt; fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_symindx = symindx; fbt->fbtp_rval = DTRACE_INVOP_LD; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; instr++; goto again; } Index: head/sys/cddl/dev/fbt/powerpc/fbt_isa.c =================================================================== --- head/sys/cddl/dev/fbt/powerpc/fbt_isa.c (revision 338358) +++ head/sys/cddl/dev/fbt/powerpc/fbt_isa.c (revision 338359) @@ -1,240 +1,240 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org * * $FreeBSD$ * */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include "fbt.h" #define FBT_PATCHVAL 0x7ffff808 #define FBT_MFLR_R0 0x7c0802a6 #define FBT_MTLR_R0 0x7c0803a6 #define FBT_BLR 0x4e800020 #define FBT_BCTR 0x4e800030 #define FBT_BRANCH 0x48000000 #define FBT_BR_MASK 0x03fffffc #define FBT_IS_JUMP(instr) ((instr & ~FBT_BR_MASK) == FBT_BRANCH) #define FBT_ENTRY "entry" #define FBT_RETURN "return" #define FBT_AFRAMES 7 int fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) { solaris_cpu_t *cpu = &solaris_cpu[curcpu]; fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; uintptr_t tmp; for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { if ((uintptr_t)fbt->fbtp_patchpoint == addr) { if (fbt->fbtp_roffset == 0) { cpu->cpu_dtrace_caller = addr; dtrace_probe(fbt->fbtp_id, frame->fixreg[3], frame->fixreg[4], frame->fixreg[5], frame->fixreg[6], frame->fixreg[7]); cpu->cpu_dtrace_caller = 0; } else { dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0); /* * The caller doesn't have the fbt item, so * fixup tail calls here. */ if (fbt->fbtp_rval == DTRACE_INVOP_JUMP) { frame->srr0 = (uintptr_t)fbt->fbtp_patchpoint; tmp = fbt->fbtp_savedval & FBT_BR_MASK; /* Sign extend. */ if (tmp & 0x02000000) #ifdef __powerpc64__ tmp |= 0xfffffffffc000000ULL; #else tmp |= 0xfc000000UL; #endif frame->srr0 += tmp; } cpu->cpu_dtrace_caller = 0; } return (fbt->fbtp_rval); } } return (0); } void fbt_patch_tracepoint(fbt_probe_t *fbt, fbt_patchval_t val) { *fbt->fbtp_patchpoint = val; __syncicache(fbt->fbtp_patchpoint, 4); } int fbt_provide_module_function(linker_file_t lf, int symindx, linker_symval_t *symval, void *opaque) { char *modname = opaque; const char *name = symval->name; fbt_probe_t *fbt, *retfbt; int j; uint32_t *instr, *limit; #ifdef __powerpc64__ /* * PowerPC64 uses '.' prefixes on symbol names, ignore it, but only * allow symbols with the '.' prefix, so that we don't get the function * descriptor instead. */ if (name[0] == '.') name++; else return (0); #endif if (fbt_excluded(name)) return (0); instr = (uint32_t *) symval->value; limit = (uint32_t *) (symval->value + symval->size); for (; instr < limit; instr++) if (*instr == FBT_MFLR_R0) break; if (*instr != FBT_MFLR_R0) return (0); fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_ENTRY, FBT_AFRAMES, fbt); fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_rval = DTRACE_INVOP_MFLR_R0; fbt->fbtp_symindx = symindx; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; retfbt = NULL; again: if (instr >= limit) return (0); /* * We (desperately) want to avoid erroneously instrumenting a * jump table. To determine if we're looking at a true instruction * sequence or an inline jump table that happens to contain the same * byte sequences, we resort to some heuristic sleeze: we treat this * instruction as being contained within a pointer, and see if that * pointer points to within the body of the function. If it does, we * refuse to instrument it. */ { uint32_t *ptr; ptr = *(uint32_t **)instr; if (ptr >= (uint32_t *) symval->value && ptr < limit) { instr++; goto again; } } if (*instr != FBT_MTLR_R0) { instr++; goto again; } instr++; for (j = 0; j < 12 && instr < limit; j++, instr++) { if ((*instr == FBT_BCTR) || (*instr == FBT_BLR) || FBT_IS_JUMP(*instr)) break; } if (!(*instr == FBT_BCTR || *instr == FBT_BLR || FBT_IS_JUMP(*instr))) goto again; /* * We have a winner! */ fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; if (retfbt == NULL) { fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_RETURN, FBT_AFRAMES, fbt); } else { - retfbt->fbtp_next = fbt; + retfbt->fbtp_probenext = fbt; fbt->fbtp_id = retfbt->fbtp_id; } retfbt = fbt; fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_symindx = symindx; if (*instr == FBT_BCTR) fbt->fbtp_rval = DTRACE_INVOP_BCTR; else if (*instr == FBT_BLR) fbt->fbtp_rval = DTRACE_INVOP_RET; else fbt->fbtp_rval = DTRACE_INVOP_JUMP; fbt->fbtp_roffset = (uintptr_t)((uint8_t *)instr - (uint8_t *)symval->value); fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; instr += 4; goto again; } Index: head/sys/cddl/dev/fbt/riscv/fbt_isa.c =================================================================== --- head/sys/cddl/dev/fbt/riscv/fbt_isa.c (revision 338358) +++ head/sys/cddl/dev/fbt/riscv/fbt_isa.c (revision 338359) @@ -1,163 +1,163 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org * Portions Copyright 2013 Howard Su howardsu@freebsd.org * Portions Copyright 2016 Ruslan Bukin * * $FreeBSD$ */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include "fbt.h" #define FBT_PATCHVAL (RISCV_INSN_BREAK) #define FBT_ENTRY "entry" #define FBT_RETURN "return" int fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) { solaris_cpu_t *cpu; fbt_probe_t *fbt; cpu = &solaris_cpu[curcpu]; fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { if ((uintptr_t)fbt->fbtp_patchpoint == addr) { cpu->cpu_dtrace_caller = addr; dtrace_probe(fbt->fbtp_id, frame->tf_a[0], frame->tf_a[1], frame->tf_a[2], frame->tf_a[3], frame->tf_a[4]); cpu->cpu_dtrace_caller = 0; return (fbt->fbtp_savedval); } } return (0); } void fbt_patch_tracepoint(fbt_probe_t *fbt, fbt_patchval_t val) { *fbt->fbtp_patchpoint = val; cpu_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); } int fbt_provide_module_function(linker_file_t lf, int symindx, linker_symval_t *symval, void *opaque) { fbt_probe_t *fbt, *retfbt; uint32_t *instr, *limit; const char *name; char *modname; modname = opaque; name = symval->name; /* Check if function is excluded from instrumentation */ if (fbt_excluded(name)) return (0); instr = (uint32_t *)(symval->value); limit = (uint32_t *)(symval->value + symval->size); /* Look for sd operation */ for (; instr < limit; instr++) { if ((*instr & SD_RA_SP_MASK) == SD_RA_SP) break; } if (instr >= limit) return (0); fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_ENTRY, 3, fbt); fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_rval = DTRACE_INVOP_SD; fbt->fbtp_symindx = symindx; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; retfbt = NULL; again: for (; instr < limit; instr++) { if (*instr == RISCV_INSN_RET) break; } if (instr >= limit) return (0); /* * We have a winner! */ fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; if (retfbt == NULL) { fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_RETURN, 3, fbt); } else { - retfbt->fbtp_next = fbt; + retfbt->fbtp_probenext = fbt; fbt->fbtp_id = retfbt->fbtp_id; } retfbt = fbt; fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_symindx = symindx; fbt->fbtp_rval = DTRACE_INVOP_RET; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; instr++; goto again; } Index: head/sys/cddl/dev/fbt/x86/fbt_isa.c =================================================================== --- head/sys/cddl/dev/fbt/x86/fbt_isa.c (revision 338358) +++ head/sys/cddl/dev/fbt/x86/fbt_isa.c (revision 338359) @@ -1,340 +1,354 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * * $FreeBSD$ * */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include "fbt.h" #define FBT_PUSHL_EBP 0x55 #define FBT_MOVL_ESP_EBP0_V0 0x8b #define FBT_MOVL_ESP_EBP1_V0 0xec #define FBT_MOVL_ESP_EBP0_V1 0x89 #define FBT_MOVL_ESP_EBP1_V1 0xe5 #define FBT_REX_RSP_RBP 0x48 #define FBT_POPL_EBP 0x5d #define FBT_RET 0xc3 #define FBT_RET_IMM16 0xc2 #define FBT_LEAVE 0xc9 #ifdef __amd64__ #define FBT_PATCHVAL 0xcc #else #define FBT_PATCHVAL 0xf0 #endif #define FBT_ENTRY "entry" #define FBT_RETURN "return" int fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) { solaris_cpu_t *cpu; uintptr_t *stack; uintptr_t arg0, arg1, arg2, arg3, arg4; fbt_probe_t *fbt; + int8_t fbtrval; #ifdef __amd64__ stack = (uintptr_t *)frame->tf_rsp; #else /* Skip hardware-saved registers. */ stack = (uintptr_t *)frame->tf_isp + 3; #endif cpu = &solaris_cpu[curcpu]; fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { - if ((uintptr_t)fbt->fbtp_patchpoint == addr) { + if ((uintptr_t)fbt->fbtp_patchpoint != addr) + continue; + fbtrval = fbt->fbtp_rval; + for (; fbt != NULL; fbt = fbt->fbtp_tracenext) { + ASSERT(fbt->fbtp_rval == fbtrval); if (fbt->fbtp_roffset == 0) { #ifdef __amd64__ /* fbt->fbtp_rval == DTRACE_INVOP_PUSHQ_RBP */ DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); cpu->cpu_dtrace_caller = stack[0]; DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); arg0 = frame->tf_rdi; arg1 = frame->tf_rsi; arg2 = frame->tf_rdx; arg3 = frame->tf_rcx; arg4 = frame->tf_r8; #else int i = 0; /* * When accessing the arguments on the stack, * we must protect against accessing beyond * the stack. We can safely set NOFAULT here * -- we know that interrupts are already * disabled. */ DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); cpu->cpu_dtrace_caller = stack[i++]; arg0 = stack[i++]; arg1 = stack[i++]; arg2 = stack[i++]; arg3 = stack[i++]; arg4 = stack[i++]; DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); #endif dtrace_probe(fbt->fbtp_id, arg0, arg1, arg2, arg3, arg4); cpu->cpu_dtrace_caller = 0; } else { #ifdef __amd64__ /* * On amd64, we instrument the ret, not the * leave. We therefore need to set the caller * to ensure that the top frame of a stack() * action is correct. */ DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); cpu->cpu_dtrace_caller = stack[0]; DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); #endif dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0); cpu->cpu_dtrace_caller = 0; } - - return (fbt->fbtp_rval); } + return (fbtrval); } return (0); } void fbt_patch_tracepoint(fbt_probe_t *fbt, fbt_patchval_t val) { register_t intr; bool old_wp; intr = intr_disable(); old_wp = disable_wp(); *fbt->fbtp_patchpoint = val; restore_wp(old_wp); intr_restore(intr); } int fbt_provide_module_function(linker_file_t lf, int symindx, linker_symval_t *symval, void *opaque) { char *modname = opaque; const char *name = symval->name; - fbt_probe_t *fbt, *retfbt; + fbt_probe_t *fbt, *hash, *retfbt; int j; int size; uint8_t *instr, *limit; if (fbt_excluded(name)) return (0); /* * trap_check() is a wrapper for DTrace's fault handler, so we don't * want to be able to instrument it. */ if (strcmp(name, "trap_check") == 0) return (0); size = symval->size; instr = (uint8_t *) symval->value; limit = (uint8_t *) symval->value + symval->size; #ifdef __amd64__ while (instr < limit) { if (*instr == FBT_PUSHL_EBP) break; if ((size = dtrace_instr_size(instr)) <= 0) break; instr += size; } if (instr >= limit || *instr != FBT_PUSHL_EBP) { /* * We either don't save the frame pointer in this * function, or we ran into some disassembly * screw-up. Either way, we bail. */ return (0); } #else if (instr[0] != FBT_PUSHL_EBP) return (0); if (!(instr[1] == FBT_MOVL_ESP_EBP0_V0 && instr[2] == FBT_MOVL_ESP_EBP1_V0) && !(instr[1] == FBT_MOVL_ESP_EBP0_V1 && instr[2] == FBT_MOVL_ESP_EBP1_V1)) return (0); #endif fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_ENTRY, 3, fbt); fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_rval = DTRACE_INVOP_PUSHL_EBP; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_symindx = symindx; - fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; - fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; + for (hash = fbt_probetab[FBT_ADDR2NDX(instr)]; hash != NULL; + hash = hash->fbtp_hashnext) { + if (hash->fbtp_patchpoint == fbt->fbtp_patchpoint) { + fbt->fbtp_tracenext = hash->fbtp_tracenext; + hash->fbtp_tracenext = fbt; + break; + } + } + if (hash == NULL) { + fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; + fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; + } lf->fbt_nentries++; retfbt = NULL; again: if (instr >= limit) return (0); /* * If this disassembly fails, then we've likely walked off into * a jump table or some other unsuitable area. Bail out of the * disassembly now. */ if ((size = dtrace_instr_size(instr)) <= 0) return (0); #ifdef __amd64__ /* * We only instrument "ret" on amd64 -- we don't yet instrument * ret imm16, largely because the compiler doesn't seem to * (yet) emit them in the kernel... */ if (*instr != FBT_RET) { instr += size; goto again; } #else if (!(size == 1 && (*instr == FBT_POPL_EBP || *instr == FBT_LEAVE) && (*(instr + 1) == FBT_RET || *(instr + 1) == FBT_RET_IMM16))) { instr += size; goto again; } #endif /* * We (desperately) want to avoid erroneously instrumenting a * jump table, especially given that our markers are pretty * short: two bytes on x86, and just one byte on amd64. To * determine if we're looking at a true instruction sequence * or an inline jump table that happens to contain the same * byte sequences, we resort to some heuristic sleeze: we * treat this instruction as being contained within a pointer, * and see if that pointer points to within the body of the * function. If it does, we refuse to instrument it. */ for (j = 0; j < sizeof (uintptr_t); j++) { caddr_t check = (caddr_t) instr - j; uint8_t *ptr; if (check < symval->value) break; if (check + sizeof (caddr_t) > (caddr_t)limit) continue; ptr = *(uint8_t **)check; if (ptr >= (uint8_t *) symval->value && ptr < limit) { instr += size; goto again; } } /* * We have a winner! */ fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; if (retfbt == NULL) { fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_RETURN, 3, fbt); } else { - retfbt->fbtp_next = fbt; + retfbt->fbtp_probenext = fbt; fbt->fbtp_id = retfbt->fbtp_id; } retfbt = fbt; fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_symindx = symindx; #ifndef __amd64__ if (*instr == FBT_POPL_EBP) { fbt->fbtp_rval = DTRACE_INVOP_POPL_EBP; } else { ASSERT(*instr == FBT_LEAVE); fbt->fbtp_rval = DTRACE_INVOP_LEAVE; } fbt->fbtp_roffset = (uintptr_t)(instr - (uint8_t *) symval->value) + 1; #else ASSERT(*instr == FBT_RET); fbt->fbtp_rval = DTRACE_INVOP_RET; fbt->fbtp_roffset = (uintptr_t)(instr - (uint8_t *) symval->value); #endif fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; instr += size; goto again; }