Index: head/cddl/contrib/opensolaris/lib/libdtrace/arm/dt_isadep.c =================================================================== --- head/cddl/contrib/opensolaris/lib/libdtrace/arm/dt_isadep.c (nonexistent) +++ head/cddl/contrib/opensolaris/lib/libdtrace/arm/dt_isadep.c (revision 278529) @@ -0,0 +1,190 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + * Copyright 2014 Howard Su + * Copyright 2015 George V. Neville-Neil + * + */ + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include +#include +#include +#include +#include + +#include +#include + +#if !defined(sun) +#define PR_MODEL_ILP32 1 +#define PR_MODEL_LP64 2 +#include +#endif + +#define OP(x) ((x) >> 30) +#define OP2(x) (((x) >> 22) & 0x07) +#define COND(x) (((x) >> 25) & 0x0f) +#define A(x) (((x) >> 29) & 0x01) + +#define OP_BRANCH 0 + +#define OP2_BPcc 0x1 +#define OP2_Bicc 0x2 +#define OP2_BPr 0x3 +#define OP2_FBPfcc 0x5 +#define OP2_FBfcc 0x6 + +/*ARGSUSED*/ +int +dt_pid_create_entry_probe(struct ps_prochandle *P, dtrace_hdl_t *dtp, + fasttrap_probe_spec_t *ftp, const GElf_Sym *symp) +{ + ftp->ftps_type = DTFTP_ENTRY; + ftp->ftps_pc = (uintptr_t)symp->st_value; + ftp->ftps_size = (size_t)symp->st_size; + ftp->ftps_noffs = 1; + ftp->ftps_offs[0] = 0; + + if (ioctl(dtp->dt_ftfd, FASTTRAPIOC_MAKEPROBE, ftp) != 0) { + dt_dprintf("fasttrap probe creation ioctl failed: %s\n", + strerror(errno)); + return (dt_set_errno(dtp, errno)); + } + + return (1); +} + +int +dt_pid_create_return_probe(struct ps_prochandle *P, dtrace_hdl_t *dtp, + fasttrap_probe_spec_t *ftp, const GElf_Sym *symp, uint64_t *stret) +{ + + uint32_t *text; + int i; + int srdepth = 0; + + dt_dprintf("%s: unimplemented\n", __func__); + return (DT_PROC_ERR); + + if ((text = malloc(symp->st_size + 4)) == NULL) { + dt_dprintf("mr sparkle: malloc() failed\n"); + return (DT_PROC_ERR); + } + + if (Pread(P, text, symp->st_size, symp->st_value) != symp->st_size) { + dt_dprintf("mr sparkle: Pread() failed\n"); + free(text); + return (DT_PROC_ERR); + } + + /* + * Leave a dummy instruction in the last slot to simplify edge + * conditions. + */ + text[symp->st_size / 4] = 0; + + ftp->ftps_type = DTFTP_RETURN; + ftp->ftps_pc = symp->st_value; + ftp->ftps_size = symp->st_size; + ftp->ftps_noffs = 0; + + + free(text); + if (ftp->ftps_noffs > 0) { + if (ioctl(dtp->dt_ftfd, FASTTRAPIOC_MAKEPROBE, ftp) != 0) { + dt_dprintf("fasttrap probe creation ioctl failed: %s\n", + strerror(errno)); + return (dt_set_errno(dtp, errno)); + } + } + + + return (ftp->ftps_noffs); +} + +/*ARGSUSED*/ +int +dt_pid_create_offset_probe(struct ps_prochandle *P, dtrace_hdl_t *dtp, + fasttrap_probe_spec_t *ftp, const GElf_Sym *symp, ulong_t off) +{ + if (off & 0x3) + return (DT_PROC_ALIGN); + + ftp->ftps_type = DTFTP_OFFSETS; + ftp->ftps_pc = (uintptr_t)symp->st_value; + ftp->ftps_size = (size_t)symp->st_size; + ftp->ftps_noffs = 1; + ftp->ftps_offs[0] = off; + + if (ioctl(dtp->dt_ftfd, FASTTRAPIOC_MAKEPROBE, ftp) != 0) { + dt_dprintf("fasttrap probe creation ioctl failed: %s\n", + strerror(errno)); + return (dt_set_errno(dtp, errno)); + } + + return (1); +} + +/*ARGSUSED*/ +int +dt_pid_create_glob_offset_probes(struct ps_prochandle *P, dtrace_hdl_t *dtp, + fasttrap_probe_spec_t *ftp, const GElf_Sym *symp, const char *pattern) +{ + ulong_t i; + + ftp->ftps_type = DTFTP_OFFSETS; + ftp->ftps_pc = (uintptr_t)symp->st_value; + ftp->ftps_size = (size_t)symp->st_size; + ftp->ftps_noffs = 0; + + /* + * If we're matching against everything, just iterate through each + * instruction in the function, otherwise look for matching offset + * names by constructing the string and comparing it against the + * pattern. + */ + if (strcmp("*", pattern) == 0) { + for (i = 0; i < symp->st_size; i += 4) { + ftp->ftps_offs[ftp->ftps_noffs++] = i; + } + } else { + char name[sizeof (i) * 2 + 1]; + + for (i = 0; i < symp->st_size; i += 4) { + (void) sprintf(name, "%lx", i); + if (gmatch(name, pattern)) + ftp->ftps_offs[ftp->ftps_noffs++] = i; + } + } + + if (ioctl(dtp->dt_ftfd, FASTTRAPIOC_MAKEPROBE, ftp) != 0) { + dt_dprintf("fasttrap probe creation ioctl failed: %s\n", + strerror(errno)); + return (dt_set_errno(dtp, errno)); + } + + return (ftp->ftps_noffs); +} Property changes on: head/cddl/contrib/opensolaris/lib/libdtrace/arm/dt_isadep.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/cddl/contrib/opensolaris/tools/ctf/cvt/ctf.c =================================================================== --- head/cddl/contrib/opensolaris/tools/ctf/cvt/ctf.c (revision 278528) +++ head/cddl/contrib/opensolaris/tools/ctf/cvt/ctf.c (revision 278529) @@ -1,1395 +1,1395 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Create and parse buffers containing CTF data. */ #include #include #include #include #include #include #include #include "ctf_headers.h" #include "ctftools.h" #include "strtab.h" #include "memory.h" /* * Name of the file currently being read, used to print error messages. We * assume that only one file will be read at a time, and thus make no attempt * to allow curfile to be used simultaneously by multiple threads. * * The value is only valid during a call to ctf_load. */ static char *curfile; #define CTF_BUF_CHUNK_SIZE (64 * 1024) #define RES_BUF_CHUNK_SIZE (64 * 1024) static int ntypes = 0; /* The number of types. */ struct ctf_buf { strtab_t ctb_strtab; /* string table */ caddr_t ctb_base; /* pointer to base of buffer */ caddr_t ctb_end; /* pointer to end of buffer */ caddr_t ctb_ptr; /* pointer to empty buffer space */ size_t ctb_size; /* size of buffer */ int nptent; /* number of processed types */ int ntholes; /* number of type holes */ }; /* * Macros to reverse byte order */ #define BSWAP_8(x) ((x) & 0xff) #define BSWAP_16(x) ((BSWAP_8(x) << 8) | BSWAP_8((x) >> 8)) #define BSWAP_32(x) ((BSWAP_16(x) << 16) | BSWAP_16((x) >> 16)) #define SWAP_16(x) (x) = BSWAP_16(x) #define SWAP_32(x) (x) = BSWAP_32(x) static int target_requires_swap; /*PRINTFLIKE1*/ static void parseterminate(const char *fmt, ...) { static char msgbuf[1024]; /* sigh */ va_list ap; va_start(ap, fmt); vsnprintf(msgbuf, sizeof (msgbuf), fmt, ap); va_end(ap); terminate("%s: %s\n", curfile, msgbuf); } static void ctf_buf_grow(ctf_buf_t *b) { off_t ptroff = b->ctb_ptr - b->ctb_base; b->ctb_size += CTF_BUF_CHUNK_SIZE; b->ctb_base = xrealloc(b->ctb_base, b->ctb_size); b->ctb_end = b->ctb_base + b->ctb_size; b->ctb_ptr = b->ctb_base + ptroff; } static ctf_buf_t * ctf_buf_new(void) { ctf_buf_t *b = xcalloc(sizeof (ctf_buf_t)); strtab_create(&b->ctb_strtab); ctf_buf_grow(b); return (b); } static void ctf_buf_free(ctf_buf_t *b) { strtab_destroy(&b->ctb_strtab); free(b->ctb_base); free(b); } static uint_t ctf_buf_cur(ctf_buf_t *b) { return (b->ctb_ptr - b->ctb_base); } static void ctf_buf_write(ctf_buf_t *b, void const *p, size_t n) { size_t len; while (n != 0) { if (b->ctb_ptr == b->ctb_end) ctf_buf_grow(b); len = MIN((size_t)(b->ctb_end - b->ctb_ptr), n); bcopy(p, b->ctb_ptr, len); b->ctb_ptr += len; p = (char const *)p + len; n -= len; } } static int write_label(void *arg1, void *arg2) { labelent_t *le = arg1; ctf_buf_t *b = arg2; ctf_lblent_t ctl; ctl.ctl_label = strtab_insert(&b->ctb_strtab, le->le_name); ctl.ctl_typeidx = le->le_idx; if (target_requires_swap) { SWAP_32(ctl.ctl_label); SWAP_32(ctl.ctl_typeidx); } ctf_buf_write(b, &ctl, sizeof (ctl)); return (1); } static void write_objects(iidesc_t *idp, ctf_buf_t *b) { ushort_t id = (idp ? idp->ii_dtype->t_id : 0); - ctf_buf_write(b, &id, sizeof (id)); - if (target_requires_swap) { SWAP_16(id); } + + ctf_buf_write(b, &id, sizeof (id)); debug(3, "Wrote object %s (%d)\n", (idp ? idp->ii_name : "(null)"), id); } static void write_functions(iidesc_t *idp, ctf_buf_t *b) { ushort_t fdata[2]; ushort_t id; int nargs; int i; if (!idp) { fdata[0] = 0; ctf_buf_write(b, &fdata[0], sizeof (fdata[0])); debug(3, "Wrote function (null)\n"); return; } nargs = idp->ii_nargs + (idp->ii_vargs != 0); if (nargs > CTF_MAX_VLEN) { terminate("function %s has too many args: %d > %d\n", idp->ii_name, nargs, CTF_MAX_VLEN); } fdata[0] = CTF_TYPE_INFO(CTF_K_FUNCTION, 1, nargs); fdata[1] = idp->ii_dtype->t_id; if (target_requires_swap) { SWAP_16(fdata[0]); SWAP_16(fdata[1]); } ctf_buf_write(b, fdata, sizeof (fdata)); for (i = 0; i < idp->ii_nargs; i++) { id = idp->ii_args[i]->t_id; if (target_requires_swap) { SWAP_16(id); } ctf_buf_write(b, &id, sizeof (id)); } if (idp->ii_vargs) { id = 0; ctf_buf_write(b, &id, sizeof (id)); } debug(3, "Wrote function %s (%d args)\n", idp->ii_name, nargs); } /* * Depending on the size of the type being described, either a ctf_stype_t (for * types with size < CTF_LSTRUCT_THRESH) or a ctf_type_t (all others) will be * written. We isolate the determination here so the rest of the writer code * doesn't need to care. */ static void write_sized_type_rec(ctf_buf_t *b, ctf_type_t *ctt, size_t size) { if (size > CTF_MAX_SIZE) { ctt->ctt_size = CTF_LSIZE_SENT; ctt->ctt_lsizehi = CTF_SIZE_TO_LSIZE_HI(size); ctt->ctt_lsizelo = CTF_SIZE_TO_LSIZE_LO(size); if (target_requires_swap) { SWAP_32(ctt->ctt_name); SWAP_16(ctt->ctt_info); SWAP_16(ctt->ctt_size); SWAP_32(ctt->ctt_lsizehi); SWAP_32(ctt->ctt_lsizelo); } ctf_buf_write(b, ctt, sizeof (*ctt)); } else { ctf_stype_t *cts = (ctf_stype_t *)ctt; cts->ctt_size = (ushort_t)size; if (target_requires_swap) { SWAP_32(cts->ctt_name); SWAP_16(cts->ctt_info); SWAP_16(cts->ctt_size); } ctf_buf_write(b, cts, sizeof (*cts)); } } static void write_unsized_type_rec(ctf_buf_t *b, ctf_type_t *ctt) { ctf_stype_t *cts = (ctf_stype_t *)ctt; if (target_requires_swap) { SWAP_32(cts->ctt_name); SWAP_16(cts->ctt_info); SWAP_16(cts->ctt_size); } ctf_buf_write(b, cts, sizeof (*cts)); } static int write_type(void *arg1, void *arg2) { tdesc_t *tp = arg1; ctf_buf_t *b = arg2; elist_t *ep; mlist_t *mp; intr_t *ip; size_t offset; uint_t encoding; uint_t data; int isroot = tp->t_flags & TDESC_F_ISROOT; int i; ctf_type_t ctt; ctf_array_t cta; ctf_member_t ctm; ctf_lmember_t ctlm; ctf_enum_t cte; ushort_t id; ctlm.ctlm_pad = 0; /* * There shouldn't be any holes in the type list (where a hole is * defined as two consecutive tdescs without consecutive ids), but * check for them just in case. If we do find holes, we need to make * fake entries to fill the holes, or we won't be able to reconstruct * the tree from the written data. */ if (++b->nptent < CTF_TYPE_TO_INDEX(tp->t_id)) { debug(2, "genctf: type hole from %d < x < %d\n", b->nptent - 1, CTF_TYPE_TO_INDEX(tp->t_id)); ctt.ctt_name = CTF_TYPE_NAME(CTF_STRTAB_0, 0); ctt.ctt_info = CTF_TYPE_INFO(0, 0, 0); while (b->nptent < CTF_TYPE_TO_INDEX(tp->t_id)) { write_sized_type_rec(b, &ctt, 0); b->nptent++; } } offset = strtab_insert(&b->ctb_strtab, tp->t_name); ctt.ctt_name = CTF_TYPE_NAME(CTF_STRTAB_0, offset); switch (tp->t_type) { case INTRINSIC: ip = tp->t_intr; if (ip->intr_type == INTR_INT) ctt.ctt_info = CTF_TYPE_INFO(CTF_K_INTEGER, isroot, 1); else ctt.ctt_info = CTF_TYPE_INFO(CTF_K_FLOAT, isroot, 1); write_sized_type_rec(b, &ctt, tp->t_size); encoding = 0; if (ip->intr_type == INTR_INT) { if (ip->intr_signed) encoding |= CTF_INT_SIGNED; if (ip->intr_iformat == 'c') encoding |= CTF_INT_CHAR; else if (ip->intr_iformat == 'b') encoding |= CTF_INT_BOOL; else if (ip->intr_iformat == 'v') encoding |= CTF_INT_VARARGS; } else encoding = ip->intr_fformat; data = CTF_INT_DATA(encoding, ip->intr_offset, ip->intr_nbits); if (target_requires_swap) { SWAP_32(data); } ctf_buf_write(b, &data, sizeof (data)); break; case POINTER: ctt.ctt_info = CTF_TYPE_INFO(CTF_K_POINTER, isroot, 0); ctt.ctt_type = tp->t_tdesc->t_id; write_unsized_type_rec(b, &ctt); break; case ARRAY: ctt.ctt_info = CTF_TYPE_INFO(CTF_K_ARRAY, isroot, 1); write_sized_type_rec(b, &ctt, tp->t_size); cta.cta_contents = tp->t_ardef->ad_contents->t_id; cta.cta_index = tp->t_ardef->ad_idxtype->t_id; cta.cta_nelems = tp->t_ardef->ad_nelems; if (target_requires_swap) { SWAP_16(cta.cta_contents); SWAP_16(cta.cta_index); SWAP_32(cta.cta_nelems); } ctf_buf_write(b, &cta, sizeof (cta)); break; case STRUCT: case UNION: for (i = 0, mp = tp->t_members; mp != NULL; mp = mp->ml_next) i++; /* count up struct or union members */ if (i > CTF_MAX_VLEN) { terminate("sou %s has too many members: %d > %d\n", tdesc_name(tp), i, CTF_MAX_VLEN); } if (tp->t_type == STRUCT) ctt.ctt_info = CTF_TYPE_INFO(CTF_K_STRUCT, isroot, i); else ctt.ctt_info = CTF_TYPE_INFO(CTF_K_UNION, isroot, i); write_sized_type_rec(b, &ctt, tp->t_size); if (tp->t_size < CTF_LSTRUCT_THRESH) { for (mp = tp->t_members; mp != NULL; mp = mp->ml_next) { offset = strtab_insert(&b->ctb_strtab, mp->ml_name); ctm.ctm_name = CTF_TYPE_NAME(CTF_STRTAB_0, offset); ctm.ctm_type = mp->ml_type->t_id; ctm.ctm_offset = mp->ml_offset; if (target_requires_swap) { SWAP_32(ctm.ctm_name); SWAP_16(ctm.ctm_type); SWAP_16(ctm.ctm_offset); } ctf_buf_write(b, &ctm, sizeof (ctm)); } } else { for (mp = tp->t_members; mp != NULL; mp = mp->ml_next) { offset = strtab_insert(&b->ctb_strtab, mp->ml_name); ctlm.ctlm_name = CTF_TYPE_NAME(CTF_STRTAB_0, offset); ctlm.ctlm_type = mp->ml_type->t_id; ctlm.ctlm_offsethi = CTF_OFFSET_TO_LMEMHI(mp->ml_offset); ctlm.ctlm_offsetlo = CTF_OFFSET_TO_LMEMLO(mp->ml_offset); if (target_requires_swap) { SWAP_32(ctlm.ctlm_name); SWAP_16(ctlm.ctlm_type); SWAP_32(ctlm.ctlm_offsethi); SWAP_32(ctlm.ctlm_offsetlo); } ctf_buf_write(b, &ctlm, sizeof (ctlm)); } } break; case ENUM: for (i = 0, ep = tp->t_emem; ep != NULL; ep = ep->el_next) i++; /* count up enum members */ if (i > CTF_MAX_VLEN) { warning("enum %s has too many values: %d > %d\n", tdesc_name(tp), i, CTF_MAX_VLEN); i = CTF_MAX_VLEN; } ctt.ctt_info = CTF_TYPE_INFO(CTF_K_ENUM, isroot, i); write_sized_type_rec(b, &ctt, tp->t_size); for (ep = tp->t_emem; ep != NULL && i > 0; ep = ep->el_next) { offset = strtab_insert(&b->ctb_strtab, ep->el_name); cte.cte_name = CTF_TYPE_NAME(CTF_STRTAB_0, offset); cte.cte_value = ep->el_number; if (target_requires_swap) { SWAP_32(cte.cte_name); SWAP_32(cte.cte_value); } ctf_buf_write(b, &cte, sizeof (cte)); i--; } break; case FORWARD: ctt.ctt_info = CTF_TYPE_INFO(CTF_K_FORWARD, isroot, 0); ctt.ctt_type = 0; write_unsized_type_rec(b, &ctt); break; case TYPEDEF: ctt.ctt_info = CTF_TYPE_INFO(CTF_K_TYPEDEF, isroot, 0); ctt.ctt_type = tp->t_tdesc->t_id; write_unsized_type_rec(b, &ctt); break; case VOLATILE: ctt.ctt_info = CTF_TYPE_INFO(CTF_K_VOLATILE, isroot, 0); ctt.ctt_type = tp->t_tdesc->t_id; write_unsized_type_rec(b, &ctt); break; case CONST: ctt.ctt_info = CTF_TYPE_INFO(CTF_K_CONST, isroot, 0); ctt.ctt_type = tp->t_tdesc->t_id; write_unsized_type_rec(b, &ctt); break; case FUNCTION: i = tp->t_fndef->fn_nargs + tp->t_fndef->fn_vargs; if (i > CTF_MAX_VLEN) { terminate("function %s has too many args: %d > %d\n", i, CTF_MAX_VLEN); } ctt.ctt_info = CTF_TYPE_INFO(CTF_K_FUNCTION, isroot, i); ctt.ctt_type = tp->t_fndef->fn_ret->t_id; write_unsized_type_rec(b, &ctt); for (i = 0; i < (int) tp->t_fndef->fn_nargs; i++) { id = tp->t_fndef->fn_args[i]->t_id; if (target_requires_swap) { SWAP_16(id); } ctf_buf_write(b, &id, sizeof (id)); } if (tp->t_fndef->fn_vargs) { id = 0; ctf_buf_write(b, &id, sizeof (id)); i++; } if (i & 1) { id = 0; ctf_buf_write(b, &id, sizeof (id)); } break; case RESTRICT: ctt.ctt_info = CTF_TYPE_INFO(CTF_K_RESTRICT, isroot, 0); ctt.ctt_type = tp->t_tdesc->t_id; write_unsized_type_rec(b, &ctt); break; default: warning("Can't write unknown type %d\n", tp->t_type); } debug(3, "Wrote type %d %s\n", tp->t_id, tdesc_name(tp)); return (1); } typedef struct resbuf { caddr_t rb_base; caddr_t rb_ptr; size_t rb_size; z_stream rb_zstr; } resbuf_t; static void rbzs_grow(resbuf_t *rb) { off_t ptroff = (caddr_t)rb->rb_zstr.next_out - rb->rb_base; rb->rb_size += RES_BUF_CHUNK_SIZE; rb->rb_base = xrealloc(rb->rb_base, rb->rb_size); rb->rb_ptr = rb->rb_base + ptroff; rb->rb_zstr.next_out = (Bytef *)(rb->rb_ptr); rb->rb_zstr.avail_out += RES_BUF_CHUNK_SIZE; } static void compress_start(resbuf_t *rb) { int rc; rb->rb_zstr.zalloc = (alloc_func)0; rb->rb_zstr.zfree = (free_func)0; rb->rb_zstr.opaque = (voidpf)0; if ((rc = deflateInit(&rb->rb_zstr, Z_BEST_COMPRESSION)) != Z_OK) parseterminate("zlib start failed: %s", zError(rc)); } static ssize_t compress_buffer(void *buf, size_t n, void *data) { resbuf_t *rb = (resbuf_t *)data; int rc; rb->rb_zstr.next_out = (Bytef *)rb->rb_ptr; rb->rb_zstr.avail_out = rb->rb_size - (rb->rb_ptr - rb->rb_base); rb->rb_zstr.next_in = buf; rb->rb_zstr.avail_in = n; while (rb->rb_zstr.avail_in) { if (rb->rb_zstr.avail_out == 0) rbzs_grow(rb); if ((rc = deflate(&rb->rb_zstr, Z_NO_FLUSH)) != Z_OK) parseterminate("zlib deflate failed: %s", zError(rc)); } rb->rb_ptr = (caddr_t)rb->rb_zstr.next_out; return (n); } static void compress_flush(resbuf_t *rb, int type) { int rc; for (;;) { if (rb->rb_zstr.avail_out == 0) rbzs_grow(rb); rc = deflate(&rb->rb_zstr, type); if ((type == Z_FULL_FLUSH && rc == Z_BUF_ERROR) || (type == Z_FINISH && rc == Z_STREAM_END)) break; else if (rc != Z_OK) parseterminate("zlib finish failed: %s", zError(rc)); } rb->rb_ptr = (caddr_t)rb->rb_zstr.next_out; } static void compress_end(resbuf_t *rb) { int rc; compress_flush(rb, Z_FINISH); if ((rc = deflateEnd(&rb->rb_zstr)) != Z_OK) parseterminate("zlib end failed: %s", zError(rc)); } /* * Pad the buffer to a power-of-2 boundary */ static void pad_buffer(ctf_buf_t *buf, int align) { uint_t cur = ctf_buf_cur(buf); ssize_t topad = (align - (cur % align)) % align; static const char pad[8] = { 0 }; while (topad > 0) { ctf_buf_write(buf, pad, (topad > 8 ? 8 : topad)); topad -= 8; } } static ssize_t bcopy_data(void *buf, size_t n, void *data) { caddr_t *posp = (caddr_t *)data; bcopy(buf, *posp, n); *posp += n; return (n); } static caddr_t write_buffer(ctf_header_t *h, ctf_buf_t *buf, size_t *resszp) { caddr_t outbuf; caddr_t bufpos; outbuf = xmalloc(sizeof (ctf_header_t) + (buf->ctb_ptr - buf->ctb_base) + buf->ctb_strtab.str_size); bufpos = outbuf; (void) bcopy_data(h, sizeof (ctf_header_t), &bufpos); (void) bcopy_data(buf->ctb_base, buf->ctb_ptr - buf->ctb_base, &bufpos); (void) strtab_write(&buf->ctb_strtab, bcopy_data, &bufpos); *resszp = bufpos - outbuf; return (outbuf); } /* * Create the compression buffer, and fill it with the CTF and string * table data. We flush the compression state between the two so the * dictionary used for the string tables won't be polluted with values * that made sense for the CTF data. */ static caddr_t write_compressed_buffer(ctf_header_t *h, ctf_buf_t *buf, size_t *resszp) { resbuf_t resbuf; resbuf.rb_size = RES_BUF_CHUNK_SIZE; resbuf.rb_base = xmalloc(resbuf.rb_size); bcopy(h, resbuf.rb_base, sizeof (ctf_header_t)); resbuf.rb_ptr = resbuf.rb_base + sizeof (ctf_header_t); compress_start(&resbuf); (void) compress_buffer(buf->ctb_base, buf->ctb_ptr - buf->ctb_base, &resbuf); compress_flush(&resbuf, Z_FULL_FLUSH); (void) strtab_write(&buf->ctb_strtab, compress_buffer, &resbuf); compress_end(&resbuf); *resszp = (resbuf.rb_ptr - resbuf.rb_base); return (resbuf.rb_base); } caddr_t ctf_gen(iiburst_t *iiburst, size_t *resszp, int do_compress) { ctf_buf_t *buf = ctf_buf_new(); ctf_header_t h; caddr_t outbuf; int i; target_requires_swap = do_compress & CTF_SWAP_BYTES; do_compress &= ~CTF_SWAP_BYTES; /* * Prepare the header, and create the CTF output buffers. The data * object section and function section are both lists of 2-byte * integers; we pad these out to the next 4-byte boundary if needed. */ h.cth_magic = CTF_MAGIC; h.cth_version = CTF_VERSION; h.cth_flags = do_compress ? CTF_F_COMPRESS : 0; h.cth_parlabel = strtab_insert(&buf->ctb_strtab, iiburst->iib_td->td_parlabel); h.cth_parname = strtab_insert(&buf->ctb_strtab, iiburst->iib_td->td_parname); h.cth_lbloff = 0; (void) list_iter(iiburst->iib_td->td_labels, write_label, buf); pad_buffer(buf, 2); h.cth_objtoff = ctf_buf_cur(buf); for (i = 0; i < iiburst->iib_nobjts; i++) write_objects(iiburst->iib_objts[i], buf); pad_buffer(buf, 2); h.cth_funcoff = ctf_buf_cur(buf); for (i = 0; i < iiburst->iib_nfuncs; i++) write_functions(iiburst->iib_funcs[i], buf); pad_buffer(buf, 4); h.cth_typeoff = ctf_buf_cur(buf); (void) list_iter(iiburst->iib_types, write_type, buf); debug(2, "CTF wrote %d types\n", list_count(iiburst->iib_types)); h.cth_stroff = ctf_buf_cur(buf); h.cth_strlen = strtab_size(&buf->ctb_strtab); if (target_requires_swap) { SWAP_16(h.cth_preamble.ctp_magic); SWAP_32(h.cth_parlabel); SWAP_32(h.cth_parname); SWAP_32(h.cth_lbloff); SWAP_32(h.cth_objtoff); SWAP_32(h.cth_funcoff); SWAP_32(h.cth_typeoff); SWAP_32(h.cth_stroff); SWAP_32(h.cth_strlen); } /* * We only do compression for ctfmerge, as ctfconvert is only * supposed to be used on intermediary build objects. This is * significantly faster. */ if (do_compress) outbuf = write_compressed_buffer(&h, buf, resszp); else outbuf = write_buffer(&h, buf, resszp); ctf_buf_free(buf); return (outbuf); } static void get_ctt_size(ctf_type_t *ctt, size_t *sizep, size_t *incrementp) { if (ctt->ctt_size == CTF_LSIZE_SENT) { *sizep = (size_t)CTF_TYPE_LSIZE(ctt); *incrementp = sizeof (ctf_type_t); } else { *sizep = ctt->ctt_size; *incrementp = sizeof (ctf_stype_t); } } static int count_types(ctf_header_t *h, caddr_t data) { caddr_t dptr = data + h->cth_typeoff; int count = 0; dptr = data + h->cth_typeoff; while (dptr < data + h->cth_stroff) { void *v = (void *) dptr; ctf_type_t *ctt = v; size_t vlen = CTF_INFO_VLEN(ctt->ctt_info); size_t size, increment; get_ctt_size(ctt, &size, &increment); switch (CTF_INFO_KIND(ctt->ctt_info)) { case CTF_K_INTEGER: case CTF_K_FLOAT: dptr += 4; break; case CTF_K_POINTER: case CTF_K_FORWARD: case CTF_K_TYPEDEF: case CTF_K_VOLATILE: case CTF_K_CONST: case CTF_K_RESTRICT: case CTF_K_FUNCTION: dptr += sizeof (ushort_t) * (vlen + (vlen & 1)); break; case CTF_K_ARRAY: dptr += sizeof (ctf_array_t); break; case CTF_K_STRUCT: case CTF_K_UNION: if (size < CTF_LSTRUCT_THRESH) dptr += sizeof (ctf_member_t) * vlen; else dptr += sizeof (ctf_lmember_t) * vlen; break; case CTF_K_ENUM: dptr += sizeof (ctf_enum_t) * vlen; break; case CTF_K_UNKNOWN: break; default: parseterminate("Unknown CTF type %d (#%d) at %#x", CTF_INFO_KIND(ctt->ctt_info), count, dptr - data); } dptr += increment; count++; } debug(3, "CTF read %d types\n", count); return (count); } /* * Resurrect the labels stored in the CTF data, returning the index associated * with a label provided by the caller. There are several cases, outlined * below. Note that, given two labels, the one associated with the lesser type * index is considered to be older than the other. * * 1. matchlbl == NULL - return the index of the most recent label. * 2. matchlbl == "BASE" - return the index of the oldest label. * 3. matchlbl != NULL, but doesn't match any labels in the section - warn * the user, and proceed as if matchlbl == "BASE" (for safety). * 4. matchlbl != NULL, and matches one of the labels in the section - return * the type index associated with the label. */ static int resurrect_labels(ctf_header_t *h, tdata_t *td, caddr_t ctfdata, char *matchlbl) { caddr_t buf = ctfdata + h->cth_lbloff; caddr_t sbuf = ctfdata + h->cth_stroff; size_t bufsz = h->cth_objtoff - h->cth_lbloff; int lastidx = 0, baseidx = -1; char *baselabel = NULL; ctf_lblent_t *ctl; void *v = (void *) buf; for (ctl = v; (caddr_t)ctl < buf + bufsz; ctl++) { char *label = sbuf + ctl->ctl_label; lastidx = ctl->ctl_typeidx; debug(3, "Resurrected label %s type idx %d\n", label, lastidx); tdata_label_add(td, label, lastidx); if (baseidx == -1) { baseidx = lastidx; baselabel = label; if (matchlbl != NULL && streq(matchlbl, "BASE")) return (lastidx); } if (matchlbl != NULL && streq(label, matchlbl)) return (lastidx); } if (matchlbl != NULL) { /* User provided a label that didn't match */ warning("%s: Cannot find label `%s' - using base (%s)\n", curfile, matchlbl, (baselabel ? baselabel : "NONE")); tdata_label_free(td); tdata_label_add(td, baselabel, baseidx); return (baseidx); } return (lastidx); } static void resurrect_objects(ctf_header_t *h, tdata_t *td, tdesc_t **tdarr, int tdsize, caddr_t ctfdata, symit_data_t *si) { caddr_t buf = ctfdata + h->cth_objtoff; size_t bufsz = h->cth_funcoff - h->cth_objtoff; caddr_t dptr; symit_reset(si); for (dptr = buf; dptr < buf + bufsz; dptr += 2) { void *v = (void *) dptr; ushort_t id = *((ushort_t *)v); iidesc_t *ii; GElf_Sym *sym; if (!(sym = symit_next(si, STT_OBJECT)) && id != 0) { parseterminate( "Unexpected end of object symbols at %x of %x", dptr - buf, bufsz); } if (id == 0) { debug(3, "Skipping null object\n"); continue; } else if (id >= tdsize) { parseterminate("Reference to invalid type %d", id); } ii = iidesc_new(symit_name(si)); ii->ii_dtype = tdarr[id]; if (GELF_ST_BIND(sym->st_info) == STB_LOCAL) { ii->ii_type = II_SVAR; ii->ii_owner = xstrdup(symit_curfile(si)); } else ii->ii_type = II_GVAR; hash_add(td->td_iihash, ii); debug(3, "Resurrected %s object %s (%d) from %s\n", (ii->ii_type == II_GVAR ? "global" : "static"), ii->ii_name, id, (ii->ii_owner ? ii->ii_owner : "(none)")); } } static void resurrect_functions(ctf_header_t *h, tdata_t *td, tdesc_t **tdarr, int tdsize, caddr_t ctfdata, symit_data_t *si) { caddr_t buf = ctfdata + h->cth_funcoff; size_t bufsz = h->cth_typeoff - h->cth_funcoff; caddr_t dptr = buf; iidesc_t *ii; ushort_t info; ushort_t retid; GElf_Sym *sym; int i; symit_reset(si); while (dptr < buf + bufsz) { void *v = (void *) dptr; info = *((ushort_t *)v); dptr += 2; if (!(sym = symit_next(si, STT_FUNC)) && info != 0) parseterminate("Unexpected end of function symbols"); if (info == 0) { debug(3, "Skipping null function (%s)\n", symit_name(si)); continue; } v = (void *) dptr; retid = *((ushort_t *)v); dptr += 2; if (retid >= tdsize) parseterminate("Reference to invalid type %d", retid); ii = iidesc_new(symit_name(si)); ii->ii_dtype = tdarr[retid]; if (GELF_ST_BIND(sym->st_info) == STB_LOCAL) { ii->ii_type = II_SFUN; ii->ii_owner = xstrdup(symit_curfile(si)); } else ii->ii_type = II_GFUN; ii->ii_nargs = CTF_INFO_VLEN(info); if (ii->ii_nargs) ii->ii_args = xmalloc(sizeof (tdesc_t *) * ii->ii_nargs); for (i = 0; i < ii->ii_nargs; i++, dptr += 2) { v = (void *) dptr; ushort_t id = *((ushort_t *)v); if (id >= tdsize) parseterminate("Reference to invalid type %d", id); ii->ii_args[i] = tdarr[id]; } if (ii->ii_nargs && ii->ii_args[ii->ii_nargs - 1] == NULL) { ii->ii_nargs--; ii->ii_vargs = 1; } hash_add(td->td_iihash, ii); debug(3, "Resurrected %s function %s (%d, %d args)\n", (ii->ii_type == II_GFUN ? "global" : "static"), ii->ii_name, retid, ii->ii_nargs); } } static void resurrect_types(ctf_header_t *h, tdata_t *td, tdesc_t **tdarr, int tdsize, caddr_t ctfdata, int maxid) { caddr_t buf = ctfdata + h->cth_typeoff; size_t bufsz = h->cth_stroff - h->cth_typeoff; caddr_t sbuf = ctfdata + h->cth_stroff; caddr_t dptr = buf; tdesc_t *tdp; uint_t data; uint_t encoding; size_t size, increment; int tcnt; int iicnt = 0; tid_t tid, argid; int kind, vlen; int i; elist_t **epp; mlist_t **mpp; intr_t *ip; ctf_type_t *ctt; ctf_array_t *cta; ctf_enum_t *cte; /* * A maxid of zero indicates a request to resurrect all types, so reset * maxid to the maximum type id. */ if (maxid == 0) maxid = CTF_MAX_TYPE; for (dptr = buf, tcnt = 0, tid = 1; dptr < buf + bufsz; tcnt++, tid++) { if (tid > maxid) break; if (tid >= tdsize) parseterminate("Reference to invalid type %d", tid); void *v = (void *) dptr; ctt = v; get_ctt_size(ctt, &size, &increment); dptr += increment; tdp = tdarr[tid]; if (CTF_NAME_STID(ctt->ctt_name) != CTF_STRTAB_0) parseterminate( "Unable to cope with non-zero strtab id"); if (CTF_NAME_OFFSET(ctt->ctt_name) != 0) { tdp->t_name = xstrdup(sbuf + CTF_NAME_OFFSET(ctt->ctt_name)); } else tdp->t_name = NULL; kind = CTF_INFO_KIND(ctt->ctt_info); vlen = CTF_INFO_VLEN(ctt->ctt_info); switch (kind) { case CTF_K_INTEGER: tdp->t_type = INTRINSIC; tdp->t_size = size; v = (void *) dptr; data = *((uint_t *)v); dptr += sizeof (uint_t); encoding = CTF_INT_ENCODING(data); ip = xmalloc(sizeof (intr_t)); ip->intr_type = INTR_INT; ip->intr_signed = (encoding & CTF_INT_SIGNED) ? 1 : 0; if (encoding & CTF_INT_CHAR) ip->intr_iformat = 'c'; else if (encoding & CTF_INT_BOOL) ip->intr_iformat = 'b'; else if (encoding & CTF_INT_VARARGS) ip->intr_iformat = 'v'; else ip->intr_iformat = '\0'; ip->intr_offset = CTF_INT_OFFSET(data); ip->intr_nbits = CTF_INT_BITS(data); tdp->t_intr = ip; break; case CTF_K_FLOAT: tdp->t_type = INTRINSIC; tdp->t_size = size; v = (void *) dptr; data = *((uint_t *)v); dptr += sizeof (uint_t); ip = xcalloc(sizeof (intr_t)); ip->intr_type = INTR_REAL; ip->intr_fformat = CTF_FP_ENCODING(data); ip->intr_offset = CTF_FP_OFFSET(data); ip->intr_nbits = CTF_FP_BITS(data); tdp->t_intr = ip; break; case CTF_K_POINTER: tdp->t_type = POINTER; tdp->t_tdesc = tdarr[ctt->ctt_type]; break; case CTF_K_ARRAY: tdp->t_type = ARRAY; tdp->t_size = size; v = (void *) dptr; cta = v; dptr += sizeof (ctf_array_t); tdp->t_ardef = xmalloc(sizeof (ardef_t)); tdp->t_ardef->ad_contents = tdarr[cta->cta_contents]; tdp->t_ardef->ad_idxtype = tdarr[cta->cta_index]; tdp->t_ardef->ad_nelems = cta->cta_nelems; break; case CTF_K_STRUCT: case CTF_K_UNION: tdp->t_type = (kind == CTF_K_STRUCT ? STRUCT : UNION); tdp->t_size = size; if (size < CTF_LSTRUCT_THRESH) { for (i = 0, mpp = &tdp->t_members; i < vlen; i++, mpp = &((*mpp)->ml_next)) { v = (void *) dptr; ctf_member_t *ctm = v; dptr += sizeof (ctf_member_t); *mpp = xmalloc(sizeof (mlist_t)); (*mpp)->ml_name = xstrdup(sbuf + ctm->ctm_name); (*mpp)->ml_type = tdarr[ctm->ctm_type]; (*mpp)->ml_offset = ctm->ctm_offset; (*mpp)->ml_size = 0; if (ctm->ctm_type > ntypes) { parseterminate("Invalid member type ctm_type=%d", ctm->ctm_type); } } } else { for (i = 0, mpp = &tdp->t_members; i < vlen; i++, mpp = &((*mpp)->ml_next)) { v = (void *) dptr; ctf_lmember_t *ctlm = v; dptr += sizeof (ctf_lmember_t); *mpp = xmalloc(sizeof (mlist_t)); (*mpp)->ml_name = xstrdup(sbuf + ctlm->ctlm_name); (*mpp)->ml_type = tdarr[ctlm->ctlm_type]; (*mpp)->ml_offset = (int)CTF_LMEM_OFFSET(ctlm); (*mpp)->ml_size = 0; if (ctlm->ctlm_type > ntypes) { parseterminate("Invalid lmember type ctlm_type=%d", ctlm->ctlm_type); } } } *mpp = NULL; break; case CTF_K_ENUM: tdp->t_type = ENUM; tdp->t_size = size; for (i = 0, epp = &tdp->t_emem; i < vlen; i++, epp = &((*epp)->el_next)) { v = (void *) dptr; cte = v; dptr += sizeof (ctf_enum_t); *epp = xmalloc(sizeof (elist_t)); (*epp)->el_name = xstrdup(sbuf + cte->cte_name); (*epp)->el_number = cte->cte_value; } *epp = NULL; break; case CTF_K_FORWARD: tdp->t_type = FORWARD; list_add(&td->td_fwdlist, tdp); break; case CTF_K_TYPEDEF: tdp->t_type = TYPEDEF; tdp->t_tdesc = tdarr[ctt->ctt_type]; break; case CTF_K_VOLATILE: tdp->t_type = VOLATILE; tdp->t_tdesc = tdarr[ctt->ctt_type]; break; case CTF_K_CONST: tdp->t_type = CONST; tdp->t_tdesc = tdarr[ctt->ctt_type]; break; case CTF_K_FUNCTION: tdp->t_type = FUNCTION; tdp->t_fndef = xcalloc(sizeof (fndef_t)); tdp->t_fndef->fn_ret = tdarr[ctt->ctt_type]; v = (void *) (dptr + (sizeof (ushort_t) * (vlen - 1))); if (vlen > 0 && *(ushort_t *)v == 0) tdp->t_fndef->fn_vargs = 1; tdp->t_fndef->fn_nargs = vlen - tdp->t_fndef->fn_vargs; tdp->t_fndef->fn_args = xcalloc(sizeof (tdesc_t) * vlen - tdp->t_fndef->fn_vargs); for (i = 0; i < vlen; i++) { v = (void *) dptr; argid = *(ushort_t *)v; dptr += sizeof (ushort_t); if (argid != 0) tdp->t_fndef->fn_args[i] = tdarr[argid]; } if (vlen & 1) dptr += sizeof (ushort_t); break; case CTF_K_RESTRICT: tdp->t_type = RESTRICT; tdp->t_tdesc = tdarr[ctt->ctt_type]; break; case CTF_K_UNKNOWN: break; default: warning("Can't parse unknown CTF type %d\n", kind); } if (CTF_INFO_ISROOT(ctt->ctt_info)) { iidesc_t *ii = iidesc_new(tdp->t_name); if (tdp->t_type == STRUCT || tdp->t_type == UNION || tdp->t_type == ENUM) ii->ii_type = II_SOU; else ii->ii_type = II_TYPE; ii->ii_dtype = tdp; hash_add(td->td_iihash, ii); iicnt++; } debug(3, "Resurrected %d %stype %s (%d)\n", tdp->t_type, (CTF_INFO_ISROOT(ctt->ctt_info) ? "root " : ""), tdesc_name(tdp), tdp->t_id); } debug(3, "Resurrected %d types (%d were roots)\n", tcnt, iicnt); } /* * For lack of other inspiration, we're going to take the boring route. We * count the number of types. This lets us malloc that many tdesc structs * before we start filling them in. This has the advantage of allowing us to * avoid a merge-esque remap step. */ static tdata_t * ctf_parse(ctf_header_t *h, caddr_t buf, symit_data_t *si, char *label) { tdata_t *td = tdata_new(); tdesc_t **tdarr; int idx, i; ntypes = count_types(h, buf); /* shudder */ tdarr = xcalloc(sizeof (tdesc_t *) * (ntypes + 1)); tdarr[0] = NULL; for (i = 1; i <= ntypes; i++) { tdarr[i] = xcalloc(sizeof (tdesc_t)); tdarr[i]->t_id = i; } td->td_parlabel = xstrdup(buf + h->cth_stroff + h->cth_parlabel); /* we have the technology - we can rebuild them */ idx = resurrect_labels(h, td, buf, label); resurrect_objects(h, td, tdarr, ntypes + 1, buf, si); resurrect_functions(h, td, tdarr, ntypes + 1, buf, si); resurrect_types(h, td, tdarr, ntypes + 1, buf, idx); free(tdarr); td->td_nextid = ntypes + 1; return (td); } static size_t decompress_ctf(caddr_t cbuf, size_t cbufsz, caddr_t dbuf, size_t dbufsz) { z_stream zstr; int rc; zstr.zalloc = (alloc_func)0; zstr.zfree = (free_func)0; zstr.opaque = (voidpf)0; zstr.next_in = (Bytef *)cbuf; zstr.avail_in = cbufsz; zstr.next_out = (Bytef *)dbuf; zstr.avail_out = dbufsz; if ((rc = inflateInit(&zstr)) != Z_OK || (rc = inflate(&zstr, Z_NO_FLUSH)) != Z_STREAM_END || (rc = inflateEnd(&zstr)) != Z_OK) { warning("CTF decompress zlib error %s\n", zError(rc)); return (0); } debug(3, "reflated %lu bytes to %lu, pointer at %d\n", zstr.total_in, zstr.total_out, (caddr_t)zstr.next_in - cbuf); return (zstr.total_out); } /* * Reconstruct the type tree from a given buffer of CTF data. Only the types * up to the type associated with the provided label, inclusive, will be * reconstructed. If a NULL label is provided, all types will be reconstructed. * * This function won't work on files that have been uniquified. */ tdata_t * ctf_load(char *file, caddr_t buf, size_t bufsz, symit_data_t *si, char *label) { ctf_header_t *h; caddr_t ctfdata; size_t ctfdatasz; tdata_t *td; curfile = file; if (bufsz < sizeof (ctf_header_t)) parseterminate("Corrupt CTF - short header"); void *v = (void *) buf; h = v; buf += sizeof (ctf_header_t); bufsz -= sizeof (ctf_header_t); if (h->cth_magic != CTF_MAGIC) parseterminate("Corrupt CTF - bad magic 0x%x", h->cth_magic); if (h->cth_version != CTF_VERSION) parseterminate("Unknown CTF version %d", h->cth_version); ctfdatasz = h->cth_stroff + h->cth_strlen; if (h->cth_flags & CTF_F_COMPRESS) { size_t actual; ctfdata = xmalloc(ctfdatasz); if ((actual = decompress_ctf(buf, bufsz, ctfdata, ctfdatasz)) != ctfdatasz) { parseterminate("Corrupt CTF - short decompression " "(was %d, expecting %d)", actual, ctfdatasz); } } else { ctfdata = buf; ctfdatasz = bufsz; } td = ctf_parse(h, ctfdata, si, label); if (h->cth_flags & CTF_F_COMPRESS) free(ctfdata); curfile = NULL; return (td); } Index: head/cddl/lib/Makefile =================================================================== --- head/cddl/lib/Makefile (revision 278528) +++ head/cddl/lib/Makefile (revision 278529) @@ -1,35 +1,36 @@ # $FreeBSD$ .include SUBDIR= ${_drti} \ libavl \ libctf \ ${_libdtrace} \ libnvpair \ libumem \ libuutil \ ${_libzfs_core} \ ${_libzfs} \ ${_libzpool} \ ${_tests} .if ${MK_TESTS} != "no" _tests= tests .endif .if ${MK_ZFS} != "no" _libzfs_core= libzfs_core _libzfs= libzfs .if ${MK_LIBTHR} != "no" _libzpool= libzpool .endif .endif .if ${MACHINE_ARCH} == "amd64" || ${MACHINE_ARCH} == "i386" || \ - ${MACHINE_CPUARCH} == "mips" || ${MACHINE_CPUARCH} == "powerpc" + ${MACHINE_CPUARCH} == "mips" || ${MACHINE_CPUARCH} == "powerpc" || \ + ${MACHINE_CPUARCH} == "arm" _drti= drti _libdtrace= libdtrace .endif .include Index: head/cddl/lib/libdtrace/Makefile =================================================================== --- head/cddl/lib/libdtrace/Makefile (revision 278528) +++ head/cddl/lib/libdtrace/Makefile (revision 278529) @@ -1,119 +1,123 @@ # $FreeBSD$ .PATH: ${.CURDIR}/../../../cddl/contrib/opensolaris/lib/libdtrace/common .PATH: ${.CURDIR}/../../../cddl/contrib/opensolaris/lib/libgen/common LIB= dtrace SRCS= dt_aggregate.c \ dt_as.c \ dt_buf.c \ dt_cc.c \ dt_cg.c \ dt_consume.c \ dt_decl.c \ dt_dis.c \ dt_dof.c \ dt_error.c \ dt_errtags.c \ dt_grammar.y \ dt_handle.c \ dt_ident.c \ dt_isadep.c \ dt_inttab.c \ dt_lex.l \ dt_link.c \ dt_list.c \ dt_map.c \ dt_module.c \ dt_names.c \ dt_open.c \ dt_options.c \ dt_parser.c \ dt_pcb.c \ dt_pid.c \ dt_pq.c \ dt_pragma.c \ dt_print.c \ dt_printf.c \ dt_proc.c \ dt_program.c \ dt_provider.c \ dt_regset.c \ dt_string.c \ dt_strtab.c \ dt_subr.c \ dt_work.c \ dt_xlator.c \ gmatch.c DSRCS= errno.d \ io.d \ ip.d \ psinfo.d \ signal.d \ tcp.d \ udp.d \ unistd.d WARNS?= 1 CFLAGS+= -I${.OBJDIR} -I${.CURDIR} \ -I${.CURDIR}/../../../sys/cddl/dev/dtrace/${MACHINE_ARCH} \ -I${.CURDIR}/../../../sys/cddl/compat/opensolaris \ -I${.CURDIR}/../../../cddl/compat/opensolaris/include \ -I${OPENSOLARIS_USR_DISTDIR}/head \ -I${OPENSOLARIS_USR_DISTDIR}/lib/libctf/common \ -I${OPENSOLARIS_USR_DISTDIR}/lib/libdtrace/common \ -I${OPENSOLARIS_SYS_DISTDIR}/uts/common #CFLAGS+= -DYYDEBUG .if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64" CFLAGS+= -I${.CURDIR}/../../../sys/cddl/dev/dtrace/x86 CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/intel -DDIS_MEM .PATH: ${.CURDIR}/../../../cddl/contrib/opensolaris/lib/libdtrace/i386 .PATH: ${.CURDIR}/../../../sys/cddl/dev/dtrace/${MACHINE_ARCH} .PATH: ${.CURDIR}/../../../sys/cddl/dev/dtrace/x86 .elif ${MACHINE_CPUARCH} == "sparc64" CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/sparc .PATH: ${.CURDIR}/../../../cddl/contrib/opensolaris/lib/libdtrace/sparc .elif ${MACHINE_CPUARCH} == "mips" CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/mips .PATH: ${.CURDIR}/../../../cddl/contrib/opensolaris/lib/libdtrace/mips .PATH: ${.CURDIR}/../../../sys/cddl/dev/dtrace/mips +.elif ${MACHINE_CPUARCH} == "arm" +CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/arm +.PATH: ${.CURDIR}/../../../cddl/contrib/opensolaris/lib/libdtrace/arm +.PATH: ${.CURDIR}/../../../sys/cddl/dev/dtrace/arm .elif ${MACHINE_CPUARCH} == "powerpc" CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/powerpc .PATH: ${.CURDIR}/../../../cddl/contrib/opensolaris/lib/libdtrace/powerpc .PATH: ${.CURDIR}/../../../sys/cddl/dev/dtrace/powerpc .else # temporary hack CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/intel .endif .if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "amd64" SRCS+= dis_tables.c DSRCS+= regs_x86.d .endif LFLAGS+=-l YFLAGS+=-d CLEANFILES= dt_errtags.c dt_names.c dt_errtags.c: sh ${OPENSOLARIS_USR_DISTDIR}/lib/libdtrace/common/mkerrtags.sh < ${OPENSOLARIS_USR_DISTDIR}/lib/libdtrace/common/dt_errtags.h > dt_errtags.c dt_names.c: sh ${OPENSOLARIS_USR_DISTDIR}/lib/libdtrace/common/mknames.sh < ${OPENSOLARIS_SYS_DISTDIR}/uts/common/sys/dtrace.h > dt_names.c beforedepend: dt_errtags.c dt_names.c beforeinstall: .if !defined(LIBRARIES_ONLY) && exists(${DESTDIR}/usr/lib/dtrace) .for file in ${DSRCS} ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m ${NOBINMODE} ${.CURDIR}/${file} ${DESTDIR}/usr/lib/dtrace .endfor .endif .include Index: head/cddl/usr.sbin/Makefile =================================================================== --- head/cddl/usr.sbin/Makefile (revision 278528) +++ head/cddl/usr.sbin/Makefile (revision 278529) @@ -1,43 +1,49 @@ # $FreeBSD$ .include SUBDIR= ${_dtrace} \ ${_dtruss} \ ${_lockstat} \ ${_plockstat} \ ${_tests} \ ${_zdb} \ ${_zhack} .if ${MK_TESTS} != "no" _tests= tests .endif .if ${MK_ZFS} != "no" .if ${MK_LIBTHR} != "no" _zdb= zdb _zhack= zhack .endif .endif .if ${MACHINE_ARCH} == "amd64" || ${MACHINE_ARCH} == "i386" _dtrace= dtrace _dtruss= dtruss _lockstat= lockstat .if defined(WITH_PLOCKSTAT) _plockstat= plockstat .endif .endif +.if ${MACHINE_CPUARCH} == "arm" +_dtrace= dtrace +_dtruss= dtruss +_lockstat= lockstat +.endif + .if ${MACHINE_CPUARCH} == "mips" _dtrace= dtrace .endif .if ${MACHINE_CPUARCH} == "powerpc" _dtrace= dtrace _dtruss= dtruss _lockstat= lockstat .endif .include Index: head/lib/Makefile =================================================================== --- head/lib/Makefile (revision 278528) +++ head/lib/Makefile (revision 278529) @@ -1,335 +1,341 @@ # @(#)Makefile 8.1 (Berkeley) 6/4/93 # $FreeBSD$ .include # The SUBDIR_ORDERED list is a small set of libraries which are used by many # of the other libraries. These are built first with a .WAIT between them # and the main list to avoid needing a SUBDIR_DEPEND line on every library # naming just these few items. SUBDIR_ORDERED= ${_csu} \ .WAIT \ libc \ libc_nonshared \ libcompiler_rt \ ${_libclang_rt} \ ${_libcplusplus} \ ${_libcxxrt} \ libelf \ msun # The main list; please keep these sorted alphabetically. SUBDIR= ${SUBDIR_ORDERED} \ .WAIT \ libalias \ libarchive \ ${_libatm} \ libauditd \ libbegemot \ libblocksruntime \ ${_libbluetooth} \ ${_libbsnmp} \ libbsdstat \ libbsm \ libbz2 \ libcalendar \ libcam \ ${_libcapsicum} \ ${_libcasper} \ ${_libcom_err} \ libcompat \ libcrypt \ libdevctl \ libdevinfo \ libdevstat \ libdpv \ libdwarf \ libedit \ ${_libelftc} \ ${_libevent} \ libexecinfo \ libexpat \ libfetch \ libfigpar \ libgeom \ ${_libgpio} \ ${_libgssapi} \ ${_librpcsec_gss} \ ${_libiconv_modules} \ libipsec \ libjail \ libkiconv \ libkvm \ ${_libldns} \ liblzma \ ${_libmagic} \ libmandoc \ libmemstat \ libmd \ ${_libmilter} \ ${_libmp} \ ${_libnandfs} \ libnetbsd \ ${_libnetgraph} \ ${_libngatm} \ libnv \ libohash \ libopie \ libpam \ libpcap \ libpjdlog \ ${_libpmc} \ ${_libproc} \ libprocstat \ ${_libradius} \ librpcsvc \ librt \ ${_librtld_db} \ libsbuf \ ${_libsdp} \ ${_libsm} \ ${_libsmb} \ ${_libsmdb} \ ${_libsmutil} \ libsqlite3 \ libstand \ libstdbuf \ libstdthreads \ libtacplus \ ${_libtelnet} \ ${_libthr} \ libthread_db \ libucl \ libufs \ libugidfw \ libulog \ ${_libunbound} \ ${_libusbhid} \ ${_libusb} \ libutil \ ${_libvgl} \ ${_libvmmapi} \ libwrap \ libxo \ liby \ ${_libypclnt} \ libz \ ncurses \ ${_atf} \ ${_clang} \ ${_cuse} \ ${_tests} # Inter-library dependencies. When the makefile for a library contains LDADD # libraries, those libraries should be listed as build order dependencies here. SUBDIR_DEPEND_libarchive= libz libbz2 libexpat liblzma libmd SUBDIR_DEPEND_libatm= libmd SUBDIR_DEPEND_libauditdm= libbsm SUBDIR_DEPEND_libbsnmp= ${_libnetgraph} SUBDIR_DEPEND_libc++= libcxxrt SUBDIR_DEPEND_libc= libcompiler_rt SUBDIR_DEPEND_libcam= libsbuf SUBDIR_DEPEND_libcapsicum= libnv SUBDIR_DEPEND_libcasper= libcapsicum libnv libpjdlog SUBDIR_DEPEND_libdevstat= libkvm SUBDIR_DEPEND_libdpv= libfigpar ncurses libutil SUBDIR_DEPEND_libedit= ncurses SUBDIR_DEPEND_libg++= msun SUBDIR_DEPEND_libgeom= libexpat libsbuf SUBDIR_DEPEND_liblibrpcsec_gss= libgssapi SUBDIR_DEPEND_libmagic= libz SUBDIR_DEPEND_libmemstat= libkvm SUBDIR_DEPEND_libopie= libmd SUBDIR_DEPEND_libpam= libcrypt libopie ${_libradius} librpcsvc libtacplus libutil ${_libypclnt} ${_libcom_err} SUBDIR_DEPEND_libpjdlog= libutil SUBDIR_DEPEND_libprocstat= libkvm libutil SUBDIR_DEPEND_libradius= libmd SUBDIR_DEPEND_libreadline= ncurses SUBDIR_DEPEND_libsmb= libkiconv SUBDIR_DEPEND_libstdc++= msun SUBDIR_DEPEND_libtacplus= libmd SUBDIR_DEPEND_libulog= libmd SUBDIR_DEPEND_libunbound= ${_libldns} SUBDIR_DEPEND_liblzma= ${_libthr} .if exists(${.CURDIR}/csu/${MACHINE_ARCH}-elf) _csu=csu/${MACHINE_ARCH}-elf .elif exists(${.CURDIR}/csu/${MACHINE_ARCH}) _csu=csu/${MACHINE_ARCH} .elif exists(${.CURDIR}/csu/${MACHINE_CPUARCH}/Makefile) _csu=csu/${MACHINE_CPUARCH} .else _csu=csu .endif # NB: keep these sorted by MK_* knobs .if ${MK_ATM} != "no" _libngatm= libngatm .endif .if ${MK_BLUETOOTH} != "no" _libbluetooth= libbluetooth _libsdp= libsdp .endif .if ${MK_BSNMP} != "no" _libbsnmp= libbsnmp .endif .if ${MK_CASPER} != "no" _libcapsicum= libcapsicum _libcasper= libcasper .endif .if ${MK_CLANG} != "no" && !defined(COMPAT_32BIT) _clang= clang .endif .if ${MK_CUSE} != "no" _cuse= libcuse .endif .if ${MK_ELFTOOLCHAIN_TOOLS} != "no" _libelftc= libelftc .endif .if ${MK_FILE} != "no" _libmagic= libmagic .endif .if ${MK_GPIO} != "no" _libgpio= libgpio .endif .if ${MK_GSSAPI} != "no" _libgssapi= libgssapi _librpcsec_gss= librpcsec_gss .endif .if ${MK_ICONV} != "no" _libiconv_modules= libiconv_modules .endif .if ${MK_KERBEROS_SUPPORT} != "no" _libcom_err= libcom_err .endif .if ${MK_LDNS} != "no" _libldns= libldns .endif # The libraries under libclang_rt can only be built by clang, and only make # sense to build when clang is enabled at all. Furthermore, they can only be # built for certain architectures. .if ${MK_CLANG} != "no" && ${COMPILER_TYPE} == "clang" && \ (${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64" || \ (${MACHINE_CPUARCH} == "arm" && ${MACHINE_ARCH} != "armeb")) _libclang_rt= libclang_rt .endif .if ${MK_LIBCPLUSPLUS} != "no" _libcxxrt= libcxxrt _libcplusplus= libc++ .endif .if ${MK_LIBTHR} != "no" _libthr= libthr .endif .if ${MK_NAND} != "no" _libnandfs= libnandfs .endif .if ${MK_NETGRAPH} != "no" _libnetgraph= libnetgraph .endif .if ${MK_NIS} != "no" _libypclnt= libypclnt .endif .if ${MK_PF} != "no" _libevent= libevent .endif .if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64" _libsmb= libsmb _libvgl= libvgl _libproc= libproc _librtld_db= librtld_db .endif .if ${MACHINE_CPUARCH} == "amd64" .if ${MK_BHYVE} != "no" _libvmmapi= libvmmapi .endif .endif .if ${MACHINE_CPUARCH} == "mips" _libproc= libproc _librtld_db= librtld_db .endif .if ${MACHINE_CPUARCH} == "powerpc" _libproc= libproc _librtld_db= librtld_db _libsmb= libsmb .endif .if ${MACHINE_CPUARCH} == "sparc64" _libsmb= libsmb .endif +.if ${MACHINE_CPUARCH} == "arm" +_libsmb= libsmb +_libproc= libproc +_librtld_db= librtld_db +.endif + .if ${MK_OPENSSL} != "no" _libmp= libmp .endif .if ${MK_PMC} != "no" _libpmc= libpmc .endif .if ${MK_RADIUS_SUPPORT} != "no" _libradius= libradius .endif .if ${MK_SENDMAIL} != "no" _libmilter= libmilter _libsm= libsm _libsmdb= libsmdb _libsmutil= libsmutil .endif .if ${MK_TELNET} != "no" _libtelnet= libtelnet .endif .if ${MK_TESTS_SUPPORT} != "no" _atf= atf .endif .if ${MK_TESTS} != "no" _tests= tests .endif .if ${MK_UNBOUND} != "no" _libunbound= libunbound .endif .if ${MK_USB} != "no" _libusbhid= libusbhid _libusb= libusb .endif .if !defined(LIBRARIES_ONLY) afterinstall: ${INSTALL_SYMLINK} ../include ${DESTDIR}/usr/lib/include .endif .if !make(install) SUBDIR_PARALLEL= .endif .include Index: head/lib/libproc/proc_bkpt.c =================================================================== --- head/lib/libproc/proc_bkpt.c (revision 278528) +++ head/lib/libproc/proc_bkpt.c (revision 278529) @@ -1,241 +1,244 @@ /* * Copyright (c) 2010 The FreeBSD Foundation * All rights reserved. * * This software was developed by Rui Paulo under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include "_libproc.h" #if defined(__i386__) || defined(__amd64__) #define BREAKPOINT_INSTR 0xcc /* int 0x3 */ #define BREAKPOINT_INSTR_SZ 1 #elif defined(__mips__) #define BREAKPOINT_INSTR 0xd /* break */ #define BREAKPOINT_INSTR_SZ 4 #elif defined(__powerpc__) #define BREAKPOINT_INSTR 0x7fe00008 /* trap */ #define BREAKPOINT_INSTR_SZ 4 +#elif defined(__arm__) +#define BREAKPOINT_INSTR 0xe7ffffff /* bkpt */ +#define BREAKPOINT_INSTR_SZ 4 #else #error "Add support for your architecture" #endif static int proc_stop(struct proc_handle *phdl) { int status; if (kill(proc_getpid(phdl), SIGSTOP) == -1) { DPRINTF("kill %d", proc_getpid(phdl)); return (-1); } else if (waitpid(proc_getpid(phdl), &status, WSTOPPED) == -1) { DPRINTF("waitpid %d", proc_getpid(phdl)); return (-1); } else if (!WIFSTOPPED(status)) { DPRINTFX("waitpid: unexpected status 0x%x", status); return (-1); } return (0); } int proc_bkptset(struct proc_handle *phdl, uintptr_t address, unsigned long *saved) { struct ptrace_io_desc piod; unsigned long paddr, caddr; int ret = 0, stopped; *saved = 0; if (phdl->status == PS_DEAD || phdl->status == PS_UNDEAD || phdl->status == PS_IDLE) { errno = ENOENT; return (-1); } DPRINTFX("adding breakpoint at 0x%lx", address); stopped = 0; if (phdl->status != PS_STOP) { if (proc_stop(phdl) != 0) return (-1); stopped = 1; } /* * Read the original instruction. */ caddr = address; paddr = 0; piod.piod_op = PIOD_READ_I; piod.piod_offs = (void *)caddr; piod.piod_addr = &paddr; piod.piod_len = BREAKPOINT_INSTR_SZ; if (ptrace(PT_IO, proc_getpid(phdl), (caddr_t)&piod, 0) < 0) { DPRINTF("ERROR: couldn't read instruction at address 0x%" PRIuPTR, address); ret = -1; goto done; } *saved = paddr; /* * Write a breakpoint instruction to that address. */ caddr = address; paddr = BREAKPOINT_INSTR; piod.piod_op = PIOD_WRITE_I; piod.piod_offs = (void *)caddr; piod.piod_addr = &paddr; piod.piod_len = BREAKPOINT_INSTR_SZ; if (ptrace(PT_IO, proc_getpid(phdl), (caddr_t)&piod, 0) < 0) { DPRINTF("ERROR: couldn't write instruction at address 0x%" PRIuPTR, address); ret = -1; goto done; } done: if (stopped) /* Restart the process if we had to stop it. */ proc_continue(phdl); return (ret); } int proc_bkptdel(struct proc_handle *phdl, uintptr_t address, unsigned long saved) { struct ptrace_io_desc piod; unsigned long paddr, caddr; int ret = 0, stopped; if (phdl->status == PS_DEAD || phdl->status == PS_UNDEAD || phdl->status == PS_IDLE) { errno = ENOENT; return (-1); } DPRINTFX("removing breakpoint at 0x%lx", address); stopped = 0; if (phdl->status != PS_STOP) { if (proc_stop(phdl) != 0) return (-1); stopped = 1; } /* * Overwrite the breakpoint instruction that we setup previously. */ caddr = address; paddr = saved; piod.piod_op = PIOD_WRITE_I; piod.piod_offs = (void *)caddr; piod.piod_addr = &paddr; piod.piod_len = BREAKPOINT_INSTR_SZ; if (ptrace(PT_IO, proc_getpid(phdl), (caddr_t)&piod, 0) < 0) { DPRINTF("ERROR: couldn't write instruction at address 0x%" PRIuPTR, address); ret = -1; } if (stopped) /* Restart the process if we had to stop it. */ proc_continue(phdl); return (ret); } /* * Decrement pc so that we delete the breakpoint at the correct * address, i.e. at the BREAKPOINT_INSTR address. */ void proc_bkptregadj(unsigned long *pc) { *pc = *pc - BREAKPOINT_INSTR_SZ; } /* * Step over the breakpoint. */ int proc_bkptexec(struct proc_handle *phdl, unsigned long saved) { unsigned long pc; unsigned long samesaved; int status; if (proc_regget(phdl, REG_PC, &pc) < 0) { DPRINTFX("ERROR: couldn't get PC register"); return (-1); } proc_bkptregadj(&pc); if (proc_bkptdel(phdl, pc, saved) < 0) { DPRINTFX("ERROR: couldn't delete breakpoint"); return (-1); } /* * Go back in time and step over the new instruction just * set up by proc_bkptdel(). */ proc_regset(phdl, REG_PC, pc); if (ptrace(PT_STEP, proc_getpid(phdl), (caddr_t)1, 0) < 0) { DPRINTFX("ERROR: ptrace step failed"); return (-1); } proc_wstatus(phdl); status = proc_getwstat(phdl); if (!WIFSTOPPED(status)) { DPRINTFX("ERROR: don't know why process stopped"); return (-1); } /* * Restore the breakpoint. The saved instruction should be * the same as the one that we were passed in. */ if (proc_bkptset(phdl, pc, &samesaved) < 0) { DPRINTFX("ERROR: couldn't restore breakpoint"); return (-1); } assert(samesaved == saved); return (0); } Index: head/lib/libproc/proc_regs.c =================================================================== --- head/lib/libproc/proc_regs.c (revision 278528) +++ head/lib/libproc/proc_regs.c (revision 278529) @@ -1,129 +1,137 @@ /* * Copyright (c) 2010 The FreeBSD Foundation * All rights reserved. * * This software was developed by Rui Paulo under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include "_libproc.h" int proc_regget(struct proc_handle *phdl, proc_reg_t reg, unsigned long *regvalue) { struct reg regs; if (phdl->status == PS_DEAD || phdl->status == PS_UNDEAD || phdl->status == PS_IDLE) { errno = ENOENT; return (-1); } memset(®s, 0, sizeof(regs)); if (ptrace(PT_GETREGS, proc_getpid(phdl), (caddr_t)®s, 0) < 0) return (-1); switch (reg) { case REG_PC: #if defined(__amd64__) *regvalue = regs.r_rip; +#elif defined(__arm__) + *regvalue = regs.r_pc; #elif defined(__i386__) *regvalue = regs.r_eip; #elif defined(__mips__) *regvalue = regs.r_regs[PC]; #elif defined(__powerpc__) *regvalue = regs.pc; #endif break; case REG_SP: #if defined(__amd64__) *regvalue = regs.r_rsp; +#elif defined(__arm__) + *regvalue = regs.r_sp; #elif defined(__i386__) *regvalue = regs.r_esp; #elif defined(__mips__) *regvalue = regs.r_regs[SP]; #elif defined(__powerpc__) *regvalue = regs.fixreg[1]; #endif break; default: DPRINTFX("ERROR: no support for reg number %d", reg); return (-1); } return (0); } int proc_regset(struct proc_handle *phdl, proc_reg_t reg, unsigned long regvalue) { struct reg regs; if (phdl->status == PS_DEAD || phdl->status == PS_UNDEAD || phdl->status == PS_IDLE) { errno = ENOENT; return (-1); } if (ptrace(PT_GETREGS, proc_getpid(phdl), (caddr_t)®s, 0) < 0) return (-1); switch (reg) { case REG_PC: #if defined(__amd64__) regs.r_rip = regvalue; +#elif defined(__arm__) + regs.r_pc = regvalue; #elif defined(__i386__) regs.r_eip = regvalue; #elif defined(__mips__) regs.r_regs[PC] = regvalue; #elif defined(__powerpc__) regs.pc = regvalue; #endif break; case REG_SP: #if defined(__amd64__) regs.r_rsp = regvalue; +#elif defined(__arm__) + regs.r_sp = regvalue; #elif defined(__i386__) regs.r_esp = regvalue; #elif defined(__mips__) regs.r_regs[PC] = regvalue; #elif defined(__powerpc__) regs.fixreg[1] = regvalue; #endif break; default: DPRINTFX("ERROR: no support for reg number %d", reg); return (-1); } if (ptrace(PT_SETREGS, proc_getpid(phdl), (caddr_t)®s, 0) < 0) return (-1); return (0); } Index: head/sys/arm/arm/exception.S =================================================================== --- head/sys/arm/arm/exception.S (revision 278528) +++ head/sys/arm/arm/exception.S (revision 278529) @@ -1,465 +1,480 @@ /* $NetBSD: exception.S,v 1.13 2003/10/31 16:30:15 scw Exp $ */ /*- * Copyright (c) 1994-1997 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * exception.S * * Low level handlers for exception vectors * * Created : 24/09/94 * * Based on kate/display/abort.s * */ #include "assym.s" +#include "opt_kdtrace.h" #include #include #include __FBSDID("$FreeBSD$"); + +#ifdef KDTRACE_HOOKS + .bss + .align 4 + .global _C_LABEL(dtrace_invop_jump_addr) +_C_LABEL(dtrace_invop_jump_addr): + .word 0 + .word 0 + + .global _C_LABEL(dtrace_invop_calltrap_addr) +_C_LABEL(dtrace_invop_calltrap_addr): + .word 0 + .word 0 +#endif .text .align 2 /* * ASM macros for pushing and pulling trapframes from the stack * * These macros are used to handle the irqframe and trapframe structures * defined above. */ /* * PUSHFRAME - macro to push a trap frame on the stack in the current mode * Since the current mode is used, the SVC lr field is not defined. * * NOTE: r13 and r14 are stored separately as a work around for the * SA110 rev 2 STM^ bug */ #ifdef ARM_TP_ADDRESS #define PUSHFRAME \ sub sp, sp, #4; /* Align the stack */ \ str lr, [sp, #-4]!; /* Push the return address */ \ sub sp, sp, #(4*17); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ mrs r0, spsr; /* Put the SPSR on the stack */ \ str r0, [sp, #-4]!; \ ldr r0, =ARM_RAS_START; \ mov r1, #0; \ str r1, [r0]; \ mov r1, #0xffffffff; \ str r1, [r0, #4]; #else #define PUSHFRAME \ sub sp, sp, #4; /* Align the stack */ \ str lr, [sp, #-4]!; /* Push the return address */ \ sub sp, sp, #(4*17); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ mrs r0, spsr; /* Put the SPSR on the stack */ \ str r0, [sp, #-4]!; #endif /* * PULLFRAME - macro to pull a trap frame from the stack in the current mode * Since the current mode is used, the SVC lr field is ignored. */ #ifdef ARM_TP_ADDRESS #define PULLFRAME \ ldr r0, [sp], #4; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*17); /* Adjust the stack pointer */ \ ldr lr, [sp], #4; /* Pull the return address */ \ add sp, sp, #4 /* Align the stack */ #else #define PULLFRAME \ ldr r0, [sp], #4 ; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; \ clrex; \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*17); /* Adjust the stack pointer */ \ ldr lr, [sp], #4; /* Pull the return address */ \ add sp, sp, #4 /* Align the stack */ #endif /* * PUSHFRAMEINSVC - macro to push a trap frame on the stack in SVC32 mode * This should only be used if the processor is not currently in SVC32 * mode. The processor mode is switched to SVC mode and the trap frame is * stored. The SVC lr field is used to store the previous value of * lr in SVC mode. * * NOTE: r13 and r14 are stored separately as a work around for the * SA110 rev 2 STM^ bug */ #ifdef ARM_TP_ADDRESS #define PUSHFRAMEINSVC \ stmdb sp, {r0-r3}; /* Save 4 registers */ \ mov r0, lr; /* Save xxx32 r14 */ \ mov r1, sp; /* Save xxx32 sp */ \ mrs r3, spsr; /* Save xxx32 spsr */ \ mrs r2, cpsr; /* Get the CPSR */ \ bic r2, r2, #(PSR_MODE); /* Fix for SVC mode */ \ orr r2, r2, #(PSR_SVC32_MODE); \ msr cpsr_c, r2; /* Punch into SVC mode */ \ mov r2, sp; /* Save SVC sp */ \ bic sp, sp, #7; /* Align sp to an 8-byte addrress */ \ sub sp, sp, #4; /* Pad trapframe to keep alignment */ \ str r0, [sp, #-4]!; /* Push return address */ \ str lr, [sp, #-4]!; /* Push SVC lr */ \ str r2, [sp, #-4]!; /* Push SVC sp */ \ msr spsr_fsxc, r3; /* Restore correct spsr */ \ ldmdb r1, {r0-r3}; /* Restore 4 regs from xxx mode */ \ sub sp, sp, #(4*15); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ ldr r5, =ARM_RAS_START; /* Check if there's any RAS */ \ ldr r4, [r5, #4]; /* reset it to point at the */ \ cmp r4, #0xffffffff; /* end of memory if necessary; */ \ movne r1, #0xffffffff; /* leave value in r4 for later */ \ strne r1, [r5, #4]; /* comparision against PC. */ \ ldr r3, [r5]; /* Retrieve global RAS_START */ \ cmp r3, #0; /* and reset it if non-zero. */ \ movne r1, #0; /* If non-zero RAS_START and */ \ strne r1, [r5]; /* PC was lower than RAS_END, */ \ ldrne r1, [r0, #16]; /* adjust the saved PC so that */ \ cmpne r4, r1; /* execution later resumes at */ \ strhi r3, [r0, #16]; /* the RAS_START location. */ \ mrs r0, spsr; \ str r0, [sp, #-4]! #else #define PUSHFRAMEINSVC \ stmdb sp, {r0-r3}; /* Save 4 registers */ \ mov r0, lr; /* Save xxx32 r14 */ \ mov r1, sp; /* Save xxx32 sp */ \ mrs r3, spsr; /* Save xxx32 spsr */ \ mrs r2, cpsr; /* Get the CPSR */ \ bic r2, r2, #(PSR_MODE); /* Fix for SVC mode */ \ orr r2, r2, #(PSR_SVC32_MODE); \ msr cpsr_c, r2; /* Punch into SVC mode */ \ mov r2, sp; /* Save SVC sp */ \ bic sp, sp, #7; /* Align sp to an 8-byte addrress */ \ sub sp, sp, #4; /* Pad trapframe to keep alignment */ \ str r0, [sp, #-4]!; /* Push return address */ \ str lr, [sp, #-4]!; /* Push SVC lr */ \ str r2, [sp, #-4]!; /* Push SVC sp */ \ msr spsr_fsxc, r3; /* Restore correct spsr */ \ ldmdb r1, {r0-r3}; /* Restore 4 regs from xxx mode */ \ sub sp, sp, #(4*15); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ mrs r0, spsr; /* Put the SPSR on the stack */ \ str r0, [sp, #-4]! #endif /* * PULLFRAMEFROMSVCANDEXIT - macro to pull a trap frame from the stack * in SVC32 mode and restore the saved processor mode and PC. * This should be used when the SVC lr register needs to be restored on * exit. */ #ifdef ARM_TP_ADDRESS #define PULLFRAMEFROMSVCANDEXIT \ ldr r0, [sp], #4; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; /* restore SPSR */ \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*15); /* Adjust the stack pointer */ \ ldmia sp, {sp, lr, pc}^ /* Restore lr and exit */ #else #define PULLFRAMEFROMSVCANDEXIT \ ldr r0, [sp], #4; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; /* restore SPSR */ \ clrex; \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*15); /* Adjust the stack pointer */ \ ldmia sp, {sp, lr, pc}^ /* Restore lr and exit */ #endif #if defined(__ARM_EABI__) /* * Unwind hints so we can unwind past functions that use * PULLFRAMEFROMSVCANDEXIT. They are run in reverse order. * As the last thing we do is restore the stack pointer * we can ignore the padding at the end of struct trapframe. */ #define UNWINDSVCFRAME \ .save {r13-r15}; /* Restore sp, lr, pc */ \ .pad #(2*4); /* Skip user sp and lr */ \ .save {r0-r12}; /* Restore r0-r12 */ \ .pad #(4) /* Skip spsr */ #else #define UNWINDSVCFRAME #endif #define DO_AST \ ldr r0, [sp]; /* Get the SPSR from stack */ \ mrs r4, cpsr; /* save CPSR */ \ orr r1, r4, #(PSR_I|PSR_F); \ msr cpsr_c, r1; /* Disable interrupts */ \ and r0, r0, #(PSR_MODE); /* Returning to USR mode? */ \ teq r0, #(PSR_USR32_MODE); \ bne 2f; /* Nope, get out now */ \ bic r4, r4, #(PSR_I|PSR_F); \ 1: GET_CURTHREAD_PTR(r5); \ ldr r1, [r5, #(TD_FLAGS)]; \ and r1, r1, #(TDF_ASTPENDING|TDF_NEEDRESCHED); \ teq r1, #0; \ beq 2f; /* Nope. Just bail */ \ msr cpsr_c, r4; /* Restore interrupts */ \ mov r0, sp; \ bl _C_LABEL(ast); /* ast(frame) */ \ orr r0, r4, #(PSR_I|PSR_F); \ msr cpsr_c, r0; \ b 1b; \ 2: /* * Entry point for a Software Interrupt (SWI). * * The hardware switches to svc32 mode on a swi, so we're already on the * right stack; just build a trapframe and call the handler. */ ASENTRY_NP(swi_entry) PUSHFRAME /* Build the trapframe on the */ mov r0, sp /* scv32 stack, pass it to the */ bl _C_LABEL(swi_handler) /* swi handler. */ /* * The fork_trampoline() code in swtch.S aranges for the MI fork_exit() * to return to swi_exit here, to return to userland. The net effect is * that a newly created thread appears to return from a SWI just like * the parent thread that created it. */ ASEENTRY_NP(swi_exit) DO_AST /* Handle pending signals. */ PULLFRAME /* Deallocate trapframe. */ movs pc, lr /* Return to userland. */ STOP_UNWINDING /* Don't unwind into user mode. */ EEND(swi_exit) END(swi_entry) /* * Standard exception exit handler. * * This is used to return from all exceptions except SWI. It uses DO_AST and * PULLFRAMEFROMSVCANDEXIT and can only be called if the exception entry code * used PUSHFRAMEINSVC. * * If the return is to user mode, this uses DO_AST to deliver any pending * signals and/or handle TDF_NEEDRESCHED first. */ ASENTRY_NP(exception_exit) DO_AST /* Handle pending signals. */ PULLFRAMEFROMSVCANDEXIT /* Return. */ UNWINDSVCFRAME /* Special unwinding for exceptions. */ END(exception_exit) /* * Entry point for a Prefetch Abort exception. * * The hardware switches to the abort mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the abort mode stack). */ ASENTRY_NP(prefetch_abort_entry) #ifdef __XSCALE__ nop /* Make absolutely sure any pending */ nop /* imprecise aborts have occurred. */ #endif sub lr, lr, #4 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ mov r1, #1 /* Type flag */ b _C_LABEL(abort_handler) END(prefetch_abort_entry) /* * Entry point for a Data Abort exception. * * The hardware switches to the abort mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the abort mode stack). */ ASENTRY_NP(data_abort_entry) #ifdef __XSCALE__ nop /* Make absolutely sure any pending */ nop /* imprecise aborts have occurred. */ #endif sub lr, lr, #8 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Exception exit routine */ mov r0, sp /* Trapframe to the handler */ mov r1, #0 /* Type flag */ b _C_LABEL(abort_handler) END(data_abort_entry) /* * Entry point for an Undefined Instruction exception. * * The hardware switches to the undefined mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the undefined mode stack). */ ASENTRY_NP(undefined_entry) sub lr, lr, #4 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ b undefinedinstruction /* trapframe to the handler. */ END(undefined_entry) /* * Entry point for a normal IRQ. * * The hardware switches to the IRQ mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the IRQ mode stack). */ ASENTRY_NP(irq_entry) sub lr, lr, #4 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ b _C_LABEL(arm_irq_handler)/* trapframe to the handler. */ END(irq_entry) /* * Entry point for an FIQ interrupt. * * We don't currently support FIQ handlers very much. Something can * install itself in the FIQ vector using code (that may or may not work * these days) in fiq.c. If nobody does that and an FIQ happens, this * default handler just disables FIQs and otherwise ignores it. */ ASENTRY_NP(fiq_entry) mrs r8, cpsr /* FIQ handling isn't supported, */ bic r8, #(PSR_F) /* just disable FIQ and return. */ msr cpsr_c, r8 /* The r8 we trash here is the */ subs pc, lr, #4 /* banked FIQ-mode r8. */ END(fiq_entry) /* * Entry point for an Address Exception exception. * This is an arm26 exception that should never happen. */ ASENTRY_NP(addr_exception_entry) mov r3, lr mrs r2, spsr mrs r1, cpsr adr r0, Laddr_exception_msg b _C_LABEL(panic) Laddr_exception_msg: .asciz "Address Exception CPSR=0x%08x SPSR=0x%08x LR=0x%08x\n" .balign 4 END(addr_exception_entry) /* * Entry point for the system Reset vector. * This should never happen, so panic. */ ASENTRY_NP(reset_entry) mov r1, lr adr r0, Lreset_panicmsg b _C_LABEL(panic) /* NOTREACHED */ Lreset_panicmsg: .asciz "Reset vector called, LR = 0x%08x" .balign 4 END(reset_entry) /* * page0 and page0_data -- An image of the ARM vectors which is copied to * the ARM vectors page (high or low) as part of CPU initialization. The * code that does the copy assumes that page0_data holds one 32-bit word * of data for each of the predefined ARM vectors. It also assumes that * page0_data follows the vectors in page0, but other stuff can appear * between the two. We currently leave room between the two for some fiq * handler code to be copied in. */ .global _C_LABEL(page0), _C_LABEL(page0_data) _C_LABEL(page0): ldr pc, .Lreset_entry ldr pc, .Lundefined_entry ldr pc, .Lswi_entry ldr pc, .Lprefetch_abort_entry ldr pc, .Ldata_abort_entry ldr pc, .Laddr_exception_entry ldr pc, .Lirq_entry .fiqv: ldr pc, .Lfiq_entry .space 256 /* room for some fiq handler code */ _C_LABEL(page0_data): .Lreset_entry: .word reset_entry .Lundefined_entry: .word undefined_entry .Lswi_entry: .word swi_entry .Lprefetch_abort_entry: .word prefetch_abort_entry .Ldata_abort_entry: .word data_abort_entry .Laddr_exception_entry: .word addr_exception_entry .Lirq_entry: .word irq_entry .Lfiq_entry: .word fiq_entry /* * These items are used by the code in fiq.c to install what it calls the * "null" handler. It's actually our default vector entry that just jumps * to the default handler which just disables FIQs and returns. */ .global _C_LABEL(fiq_nullhandler_code), _C_LABEL(fiq_nullhandler_size) _C_LABEL(fiq_nullhandler_code): .word .fiqv _C_LABEL(fiq_nullhandler_size): .word 4 Index: head/sys/arm/arm/identcpu.c =================================================================== --- head/sys/arm/arm/identcpu.c (revision 278528) +++ head/sys/arm/arm/identcpu.c (revision 278529) @@ -1,530 +1,530 @@ /* $NetBSD: cpu.c,v 1.55 2004/02/13 11:36:10 wiz Exp $ */ /*- * Copyright (c) 1995 Mark Brinicombe. * Copyright (c) 1995 Brini. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * cpu.c * * Probing and configuration for the master CPU * * Created : 10/10/95 */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include char machine[] = "arm"; SYSCTL_STRING(_hw, HW_MACHINE, machine, CTLFLAG_RD, machine, 0, "Machine class"); static const char * const generic_steppings[16] = { "rev 0", "rev 1", "rev 2", "rev 3", "rev 4", "rev 5", "rev 6", "rev 7", "rev 8", "rev 9", "rev 10", "rev 11", "rev 12", "rev 13", "rev 14", "rev 15", }; static const char * const xscale_steppings[16] = { "step A-0", "step A-1", "step B-0", "step C-0", "step D-0", "rev 5", "rev 6", "rev 7", "rev 8", "rev 9", "rev 10", "rev 11", "rev 12", "rev 13", "rev 14", "rev 15", }; static const char * const i80219_steppings[16] = { "step A-0", "rev 1", "rev 2", "rev 3", "rev 4", "rev 5", "rev 6", "rev 7", "rev 8", "rev 9", "rev 10", "rev 11", "rev 12", "rev 13", "rev 14", "rev 15", }; static const char * const i80321_steppings[16] = { "step A-0", "step B-0", "rev 2", "rev 3", "rev 4", "rev 5", "rev 6", "rev 7", "rev 8", "rev 9", "rev 10", "rev 11", "rev 12", "rev 13", "rev 14", "rev 15", }; static const char * const i81342_steppings[16] = { "step A-0", "rev 1", "rev 2", "rev 3", "rev 4", "rev 5", "rev 6", "rev 7", "rev 8", "rev 9", "rev 10", "rev 11", "rev 12", "rev 13", "rev 14", "rev 15", }; /* Steppings for PXA2[15]0 */ static const char * const pxa2x0_steppings[16] = { "step A-0", "step A-1", "step B-0", "step B-1", "step B-2", "step C-0", "rev 6", "rev 7", "rev 8", "rev 9", "rev 10", "rev 11", "rev 12", "rev 13", "rev 14", "rev 15", }; /* Steppings for PXA255/26x. * rev 5: PXA26x B0, rev 6: PXA255 A0 */ static const char * const pxa255_steppings[16] = { "rev 0", "rev 1", "rev 2", "step A-0", "rev 4", "step B-0", "step A-0", "rev 7", "rev 8", "rev 9", "rev 10", "rev 11", "rev 12", "rev 13", "rev 14", "rev 15", }; /* Stepping for PXA27x */ static const char * const pxa27x_steppings[16] = { "step A-0", "step A-1", "step B-0", "step B-1", "step C-0", "rev 5", "rev 6", "rev 7", "rev 8", "rev 9", "rev 10", "rev 11", "rev 12", "rev 13", "rev 14", "rev 15", }; static const char * const ixp425_steppings[16] = { "step 0 (A0)", "rev 1 (ARMv5TE)", "rev 2", "rev 3", "rev 4", "rev 5", "rev 6", "rev 7", "rev 8", "rev 9", "rev 10", "rev 11", "rev 12", "rev 13", "rev 14", "rev 15", }; struct cpuidtab { u_int32_t cpuid; enum cpu_class cpu_class; const char *cpu_name; const char * const *cpu_steppings; }; const struct cpuidtab cpuids[] = { { CPU_ID_ARM920T, CPU_CLASS_ARM9TDMI, "ARM920T", generic_steppings }, { CPU_ID_ARM920T_ALT, CPU_CLASS_ARM9TDMI, "ARM920T", generic_steppings }, { CPU_ID_ARM922T, CPU_CLASS_ARM9TDMI, "ARM922T", generic_steppings }, { CPU_ID_ARM926EJS, CPU_CLASS_ARM9EJS, "ARM926EJ-S", generic_steppings }, { CPU_ID_ARM940T, CPU_CLASS_ARM9TDMI, "ARM940T", generic_steppings }, { CPU_ID_ARM946ES, CPU_CLASS_ARM9ES, "ARM946E-S", generic_steppings }, { CPU_ID_ARM966ES, CPU_CLASS_ARM9ES, "ARM966E-S", generic_steppings }, { CPU_ID_ARM966ESR1, CPU_CLASS_ARM9ES, "ARM966E-S", generic_steppings }, { CPU_ID_FA526, CPU_CLASS_ARM9TDMI, "FA526", generic_steppings }, { CPU_ID_FA626TE, CPU_CLASS_ARM9ES, "FA626TE", generic_steppings }, { CPU_ID_TI925T, CPU_CLASS_ARM9TDMI, "TI ARM925T", generic_steppings }, { CPU_ID_ARM1020E, CPU_CLASS_ARM10E, "ARM1020E", generic_steppings }, { CPU_ID_ARM1022ES, CPU_CLASS_ARM10E, "ARM1022E-S", generic_steppings }, { CPU_ID_ARM1026EJS, CPU_CLASS_ARM10EJ, "ARM1026EJ-S", generic_steppings }, { CPU_ID_CORTEXA5, CPU_CLASS_CORTEXA, "Cortex A5", generic_steppings }, { CPU_ID_CORTEXA7, CPU_CLASS_CORTEXA, "Cortex A7", generic_steppings }, { CPU_ID_CORTEXA8R1, CPU_CLASS_CORTEXA, "Cortex A8-r1", generic_steppings }, { CPU_ID_CORTEXA8R2, CPU_CLASS_CORTEXA, "Cortex A8-r2", generic_steppings }, { CPU_ID_CORTEXA8R3, CPU_CLASS_CORTEXA, "Cortex A8-r3", generic_steppings }, { CPU_ID_CORTEXA9R1, CPU_CLASS_CORTEXA, "Cortex A9-r1", generic_steppings }, { CPU_ID_CORTEXA9R2, CPU_CLASS_CORTEXA, "Cortex A9-r2", generic_steppings }, { CPU_ID_CORTEXA9R3, CPU_CLASS_CORTEXA, "Cortex A9-r3", generic_steppings }, { CPU_ID_CORTEXA12R0, CPU_CLASS_CORTEXA, "Cortex A12-r0", generic_steppings }, { CPU_ID_CORTEXA15R0, CPU_CLASS_CORTEXA, "Cortex A15-r0", generic_steppings }, { CPU_ID_CORTEXA15R1, CPU_CLASS_CORTEXA, "Cortex A15-r1", generic_steppings }, { CPU_ID_CORTEXA15R2, CPU_CLASS_CORTEXA, "Cortex A15-r2", generic_steppings }, { CPU_ID_CORTEXA15R3, CPU_CLASS_CORTEXA, "Cortex A15-r3", generic_steppings }, { CPU_ID_KRAIT, CPU_CLASS_KRAIT, "Krait", generic_steppings }, { CPU_ID_80200, CPU_CLASS_XSCALE, "i80200", xscale_steppings }, { CPU_ID_80321_400, CPU_CLASS_XSCALE, "i80321 400MHz", i80321_steppings }, { CPU_ID_80321_600, CPU_CLASS_XSCALE, "i80321 600MHz", i80321_steppings }, { CPU_ID_80321_400_B0, CPU_CLASS_XSCALE, "i80321 400MHz", i80321_steppings }, { CPU_ID_80321_600_B0, CPU_CLASS_XSCALE, "i80321 600MHz", i80321_steppings }, { CPU_ID_81342, CPU_CLASS_XSCALE, "i81342", i81342_steppings }, { CPU_ID_80219_400, CPU_CLASS_XSCALE, "i80219 400MHz", i80219_steppings }, { CPU_ID_80219_600, CPU_CLASS_XSCALE, "i80219 600MHz", i80219_steppings }, { CPU_ID_PXA27X, CPU_CLASS_XSCALE, "PXA27x", pxa27x_steppings }, { CPU_ID_PXA250A, CPU_CLASS_XSCALE, "PXA250", pxa2x0_steppings }, { CPU_ID_PXA210A, CPU_CLASS_XSCALE, "PXA210", pxa2x0_steppings }, { CPU_ID_PXA250B, CPU_CLASS_XSCALE, "PXA250", pxa2x0_steppings }, { CPU_ID_PXA210B, CPU_CLASS_XSCALE, "PXA210", pxa2x0_steppings }, { CPU_ID_PXA250C, CPU_CLASS_XSCALE, "PXA255", pxa255_steppings }, { CPU_ID_PXA210C, CPU_CLASS_XSCALE, "PXA210", pxa2x0_steppings }, { CPU_ID_IXP425_533, CPU_CLASS_XSCALE, "IXP425 533MHz", ixp425_steppings }, { CPU_ID_IXP425_400, CPU_CLASS_XSCALE, "IXP425 400MHz", ixp425_steppings }, { CPU_ID_IXP425_266, CPU_CLASS_XSCALE, "IXP425 266MHz", ixp425_steppings }, /* XXX ixp435 steppings? */ { CPU_ID_IXP435, CPU_CLASS_XSCALE, "IXP435", ixp425_steppings }, { CPU_ID_ARM1136JS, CPU_CLASS_ARM11J, "ARM1136J-S", generic_steppings }, { CPU_ID_ARM1136JSR1, CPU_CLASS_ARM11J, "ARM1136J-S R1", generic_steppings }, { CPU_ID_ARM1176JZS, CPU_CLASS_ARM11J, "ARM1176JZ-S", generic_steppings }, { CPU_ID_MV88FR131, CPU_CLASS_MARVELL, "Feroceon 88FR131", generic_steppings }, { CPU_ID_MV88FR571_VD, CPU_CLASS_MARVELL, "Feroceon 88FR571-VD", generic_steppings }, { CPU_ID_MV88SV581X_V7, CPU_CLASS_MARVELL, "Sheeva 88SV581x", generic_steppings }, { CPU_ID_ARM_88SV581X_V7, CPU_CLASS_MARVELL, "Sheeva 88SV581x", generic_steppings }, { CPU_ID_MV88SV584X_V7, CPU_CLASS_MARVELL, "Sheeva 88SV584x", generic_steppings }, { 0, CPU_CLASS_NONE, NULL, NULL } }; struct cpu_classtab { const char *class_name; const char *class_option; }; const struct cpu_classtab cpu_classes[] = { { "unknown", NULL }, /* CPU_CLASS_NONE */ { "ARM9TDMI", "CPU_ARM9TDMI" }, /* CPU_CLASS_ARM9TDMI */ { "ARM9E-S", "CPU_ARM9E" }, /* CPU_CLASS_ARM9ES */ { "ARM9EJ-S", "CPU_ARM9E" }, /* CPU_CLASS_ARM9EJS */ { "ARM10E", "CPU_ARM10" }, /* CPU_CLASS_ARM10E */ { "ARM10EJ", "CPU_ARM10" }, /* CPU_CLASS_ARM10EJ */ { "Cortex-A", "CPU_CORTEXA" }, /* CPU_CLASS_CORTEXA */ { "Krait", "CPU_KRAIT" }, /* CPU_CLASS_KRAIT */ { "XScale", "CPU_XSCALE_..." }, /* CPU_CLASS_XSCALE */ { "ARM11J", "CPU_ARM11" }, /* CPU_CLASS_ARM11J */ { "Marvell", "CPU_MARVELL" }, /* CPU_CLASS_MARVELL */ }; /* * Report the type of the specified arm processor. This uses the generic and * arm specific information in the cpu structure to identify the processor. * The remaining fields in the cpu structure are filled in appropriately. */ static const char * const wtnames[] = { "write-through", "write-back", "write-back", "**unknown 3**", "**unknown 4**", "write-back-locking", /* XXX XScale-specific? */ "write-back-locking-A", "write-back-locking-B", "**unknown 8**", "**unknown 9**", "**unknown 10**", "**unknown 11**", "**unknown 12**", "**unknown 13**", "write-back-locking-C", "**unknown 15**", }; static void print_enadis(int enadis, char *s) { printf(" %s %sabled", s, (enadis == 0) ? "dis" : "en"); } extern int ctrl; enum cpu_class cpu_class = CPU_CLASS_NONE; u_int cpu_pfr(int num) { u_int feat; switch (num) { case 0: __asm __volatile("mrc p15, 0, %0, c0, c1, 0" : "=r" (feat)); break; case 1: __asm __volatile("mrc p15, 0, %0, c0, c1, 1" : "=r" (feat)); break; default: panic("Processor Feature Register %d not implemented", num); break; } return (feat); } static void identify_armv7(void) { u_int feature; printf("Supported features:"); /* Get Processor Feature Register 0 */ feature = cpu_pfr(0); if (feature & ARM_PFR0_ARM_ISA_MASK) printf(" ARM_ISA"); if (feature & ARM_PFR0_THUMB2) printf(" THUMB2"); else if (feature & ARM_PFR0_THUMB) printf(" THUMB"); if (feature & ARM_PFR0_JAZELLE_MASK) printf(" JAZELLE"); if (feature & ARM_PFR0_THUMBEE_MASK) printf(" THUMBEE"); /* Get Processor Feature Register 1 */ feature = cpu_pfr(1); if (feature & ARM_PFR1_ARMV4_MASK) printf(" ARMv4"); if (feature & ARM_PFR1_SEC_EXT_MASK) printf(" Security_Ext"); if (feature & ARM_PFR1_MICROCTRL_MASK) printf(" M_profile"); printf("\n"); } void identify_arm_cpu(void) { u_int cpuid, reg, size, sets, ways; u_int8_t type, linesize; int i; - cpuid = cpu_id(); + cpuid = cpu_ident(); if (cpuid == 0) { printf("Processor failed probe - no CPU ID\n"); return; } for (i = 0; cpuids[i].cpuid != 0; i++) if (cpuids[i].cpuid == (cpuid & CPU_ID_CPU_MASK)) { cpu_class = cpuids[i].cpu_class; printf("CPU: %s %s (%s core)\n", cpuids[i].cpu_name, cpuids[i].cpu_steppings[cpuid & CPU_ID_REVISION_MASK], cpu_classes[cpu_class].class_name); break; } if (cpuids[i].cpuid == 0) printf("unknown CPU (ID = 0x%x)\n", cpuid); printf(" "); if ((cpuid & CPU_ID_ARCH_MASK) == CPU_ID_CPUID_SCHEME) { identify_armv7(); } else { if (ctrl & CPU_CONTROL_BEND_ENABLE) printf(" Big-endian"); else printf(" Little-endian"); switch (cpu_class) { case CPU_CLASS_ARM9TDMI: case CPU_CLASS_ARM9ES: case CPU_CLASS_ARM9EJS: case CPU_CLASS_ARM10E: case CPU_CLASS_ARM10EJ: case CPU_CLASS_XSCALE: case CPU_CLASS_ARM11J: case CPU_CLASS_MARVELL: print_enadis(ctrl & CPU_CONTROL_DC_ENABLE, "DC"); print_enadis(ctrl & CPU_CONTROL_IC_ENABLE, "IC"); #ifdef CPU_XSCALE_81342 print_enadis(ctrl & CPU_CONTROL_L2_ENABLE, "L2"); #endif #if defined(SOC_MV_KIRKWOOD) || defined(SOC_MV_DISCOVERY) i = sheeva_control_ext(0, 0); print_enadis(i & MV_WA_ENABLE, "WA"); print_enadis(i & MV_DC_STREAM_ENABLE, "DC streaming"); printf("\n "); print_enadis((i & MV_BTB_DISABLE) == 0, "BTB"); print_enadis(i & MV_L2_ENABLE, "L2"); print_enadis((i & MV_L2_PREFETCH_DISABLE) == 0, "L2 prefetch"); printf("\n "); #endif break; default: break; } } print_enadis(ctrl & CPU_CONTROL_WBUF_ENABLE, "WB"); if (ctrl & CPU_CONTROL_LABT_ENABLE) printf(" LABT"); else printf(" EABT"); print_enadis(ctrl & CPU_CONTROL_BPRD_ENABLE, "branch prediction"); printf("\n"); if (arm_cache_level) { printf("LoUU:%d LoC:%d LoUIS:%d \n", CPU_CLIDR_LOUU(arm_cache_level) + 1, arm_cache_loc + 1, CPU_CLIDR_LOUIS(arm_cache_level) + 1); i = 0; while (((type = CPU_CLIDR_CTYPE(arm_cache_level, i)) != 0) && i < 7) { printf("Cache level %d: \n", i + 1); if (type == CACHE_DCACHE || type == CACHE_UNI_CACHE || type == CACHE_SEP_CACHE) { reg = arm_cache_type[2 * i]; ways = CPUV7_CT_xSIZE_ASSOC(reg) + 1; sets = CPUV7_CT_xSIZE_SET(reg) + 1; linesize = 1 << (CPUV7_CT_xSIZE_LEN(reg) + 4); size = (ways * sets * linesize) / 1024; if (type == CACHE_UNI_CACHE) printf(" %dKB/%dB %d-way unified cache", size, linesize,ways); else printf(" %dKB/%dB %d-way data cache", size, linesize, ways); if (reg & CPUV7_CT_CTYPE_WT) printf(" WT"); if (reg & CPUV7_CT_CTYPE_WB) printf(" WB"); if (reg & CPUV7_CT_CTYPE_RA) printf(" Read-Alloc"); if (reg & CPUV7_CT_CTYPE_WA) printf(" Write-Alloc"); printf("\n"); } if (type == CACHE_ICACHE || type == CACHE_SEP_CACHE) { reg = arm_cache_type[(2 * i) + 1]; ways = CPUV7_CT_xSIZE_ASSOC(reg) + 1; sets = CPUV7_CT_xSIZE_SET(reg) + 1; linesize = 1 << (CPUV7_CT_xSIZE_LEN(reg) + 4); size = (ways * sets * linesize) / 1024; printf(" %dKB/%dB %d-way instruction cache", size, linesize, ways); if (reg & CPUV7_CT_CTYPE_WT) printf(" WT"); if (reg & CPUV7_CT_CTYPE_WB) printf(" WB"); if (reg & CPUV7_CT_CTYPE_RA) printf(" Read-Alloc"); if (reg & CPUV7_CT_CTYPE_WA) printf(" Write-Alloc"); printf("\n"); } i++; } } else { /* Print cache info. */ if (arm_picache_line_size == 0 && arm_pdcache_line_size == 0) return; if (arm_pcache_unified) { printf(" %dKB/%dB %d-way %s unified cache\n", arm_pdcache_size / 1024, arm_pdcache_line_size, arm_pdcache_ways, wtnames[arm_pcache_type]); } else { printf(" %dKB/%dB %d-way instruction cache\n", arm_picache_size / 1024, arm_picache_line_size, arm_picache_ways); printf(" %dKB/%dB %d-way %s data cache\n", arm_pdcache_size / 1024, arm_pdcache_line_size, arm_pdcache_ways, wtnames[arm_pcache_type]); } } } Index: head/sys/arm/arm/trap.c =================================================================== --- head/sys/arm/arm/trap.c (revision 278528) +++ head/sys/arm/arm/trap.c (revision 278529) @@ -1,756 +1,766 @@ /* $NetBSD: fault.c,v 1.45 2003/11/20 14:44:36 scw Exp $ */ /*- * Copyright 2004 Olivier Houchard * Copyright 2003 Wasabi Systems, Inc. * All rights reserved. * * Written by Steve C. Woodford for Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed for the NetBSD Project by * Wasabi Systems, Inc. * 4. The name of Wasabi Systems, Inc. may not be used to endorse * or promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1994-1997 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * fault.c * * Fault handlers * * Created : 28/11/94 */ +#ifdef KDTRACE_HOOKS +#include +#endif #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef KDB #include #endif extern char fusubailout[]; #ifdef DEBUG int last_fault_code; /* For the benefit of pmap_fault_fixup() */ #endif struct ksig { int signb; u_long code; }; struct data_abort { int (*func)(struct trapframe *, u_int, u_int, struct thread *, struct ksig *); const char *desc; }; static int dab_fatal(struct trapframe *, u_int, u_int, struct thread *, struct ksig *); static int dab_align(struct trapframe *, u_int, u_int, struct thread *, struct ksig *); static int dab_buserr(struct trapframe *, u_int, u_int, struct thread *, struct ksig *); static void prefetch_abort_handler(struct trapframe *); static const struct data_abort data_aborts[] = { {dab_fatal, "Vector Exception"}, {dab_align, "Alignment Fault 1"}, {dab_fatal, "Terminal Exception"}, {dab_align, "Alignment Fault 3"}, {dab_buserr, "External Linefetch Abort (S)"}, {NULL, "Translation Fault (S)"}, #if (ARM_MMU_V6 + ARM_MMU_V7) != 0 {NULL, "Translation Flag Fault"}, #else {dab_buserr, "External Linefetch Abort (P)"}, #endif {NULL, "Translation Fault (P)"}, {dab_buserr, "External Non-Linefetch Abort (S)"}, {NULL, "Domain Fault (S)"}, {dab_buserr, "External Non-Linefetch Abort (P)"}, {NULL, "Domain Fault (P)"}, {dab_buserr, "External Translation Abort (L1)"}, {NULL, "Permission Fault (S)"}, {dab_buserr, "External Translation Abort (L2)"}, {NULL, "Permission Fault (P)"} }; /* Determine if a fault came from user mode */ #define TRAP_USERMODE(tf) ((tf->tf_spsr & PSR_MODE) == PSR_USR32_MODE) /* Determine if 'x' is a permission fault */ #define IS_PERMISSION_FAULT(x) \ (((1 << ((x) & FAULT_TYPE_MASK)) & \ ((1 << FAULT_PERM_P) | (1 << FAULT_PERM_S))) != 0) static __inline void call_trapsignal(struct thread *td, int sig, u_long code) { ksiginfo_t ksi; ksiginfo_init_trap(&ksi); ksi.ksi_signo = sig; ksi.ksi_code = (int)code; trapsignal(td, &ksi); } void abort_handler(struct trapframe *tf, int type) { struct vm_map *map; struct pcb *pcb; struct thread *td; u_int user, far, fsr; vm_prot_t ftype; void *onfault; vm_offset_t va; int error = 0; struct ksig ksig; struct proc *p; if (type == 1) return (prefetch_abort_handler(tf)); /* Grab FAR/FSR before enabling interrupts */ far = cpu_faultaddress(); fsr = cpu_faultstatus(); #if 0 printf("data abort: fault address=%p (from pc=%p lr=%p)\n", (void*)far, (void*)tf->tf_pc, (void*)tf->tf_svc_lr); #endif /* Update vmmeter statistics */ #if 0 vmexp.traps++; #endif td = curthread; p = td->td_proc; PCPU_INC(cnt.v_trap); /* Data abort came from user mode? */ user = TRAP_USERMODE(tf); if (user) { td->td_pticks = 0; td->td_frame = tf; if (td->td_ucred != td->td_proc->p_ucred) cred_update_thread(td); } /* Grab the current pcb */ pcb = td->td_pcb; /* Re-enable interrupts if they were enabled previously */ if (td->td_md.md_spinlock_count == 0) { if (__predict_true(tf->tf_spsr & PSR_I) == 0) enable_interrupts(PSR_I); if (__predict_true(tf->tf_spsr & PSR_F) == 0) enable_interrupts(PSR_F); } /* Invoke the appropriate handler, if necessary */ if (__predict_false(data_aborts[fsr & FAULT_TYPE_MASK].func != NULL)) { if ((data_aborts[fsr & FAULT_TYPE_MASK].func)(tf, fsr, far, td, &ksig)) { goto do_trapsignal; } goto out; } /* * At this point, we're dealing with one of the following data aborts: * * FAULT_TRANS_S - Translation -- Section * FAULT_TRANS_P - Translation -- Page * FAULT_DOMAIN_S - Domain -- Section * FAULT_DOMAIN_P - Domain -- Page * FAULT_PERM_S - Permission -- Section * FAULT_PERM_P - Permission -- Page * * These are the main virtual memory-related faults signalled by * the MMU. */ /* fusubailout is used by [fs]uswintr to avoid page faulting */ if (__predict_false(pcb->pcb_onfault == fusubailout)) { tf->tf_r0 = EFAULT; tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault; return; } /* * Make sure the Program Counter is sane. We could fall foul of * someone executing Thumb code, in which case the PC might not * be word-aligned. This would cause a kernel alignment fault * further down if we have to decode the current instruction. * XXX: It would be nice to be able to support Thumb at some point. */ if (__predict_false((tf->tf_pc & 3) != 0)) { if (user) { /* * Give the user an illegal instruction signal. */ /* Deliver a SIGILL to the process */ ksig.signb = SIGILL; ksig.code = 0; goto do_trapsignal; } /* * The kernel never executes Thumb code. */ printf("\ndata_abort_fault: Misaligned Kernel-mode " "Program Counter\n"); dab_fatal(tf, fsr, far, td, &ksig); } va = trunc_page((vm_offset_t)far); /* * It is only a kernel address space fault iff: * 1. user == 0 and * 2. pcb_onfault not set or * 3. pcb_onfault set and not LDRT/LDRBT/STRT/STRBT instruction. */ if (user == 0 && (va >= VM_MIN_KERNEL_ADDRESS || (va < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW)) && __predict_true((pcb->pcb_onfault == NULL || (ReadWord(tf->tf_pc) & 0x05200000) != 0x04200000))) { map = kernel_map; /* Was the fault due to the FPE/IPKDB ? */ if (__predict_false((tf->tf_spsr & PSR_MODE)==PSR_UND32_MODE)) { /* * Force exit via userret() * This is necessary as the FPE is an extension to * userland that actually runs in a priveledged mode * but uses USR mode permissions for its accesses. */ user = 1; ksig.signb = SIGSEGV; ksig.code = 0; goto do_trapsignal; } } else { map = &td->td_proc->p_vmspace->vm_map; } /* * We need to know whether the page should be mapped as R or R/W. On * armv6 and later the fault status register indicates whether the * access was a read or write. Prior to armv6, we know that a * permission fault can only be the result of a write to a read-only * location, so we can deal with those quickly. Otherwise we need to * disassemble the faulting instruction to determine if it was a write. */ #if ARM_ARCH_6 || ARM_ARCH_7A ftype = (fsr & FAULT_WNR) ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; #else if (IS_PERMISSION_FAULT(fsr)) ftype = VM_PROT_WRITE; else { u_int insn = ReadWord(tf->tf_pc); if (((insn & 0x0c100000) == 0x04000000) || /* STR/STRB */ ((insn & 0x0e1000b0) == 0x000000b0) || /* STRH/STRD */ ((insn & 0x0a100000) == 0x08000000)) { /* STM/CDT */ ftype = VM_PROT_WRITE; } else { if ((insn & 0x0fb00ff0) == 0x01000090) /* SWP */ ftype = VM_PROT_READ | VM_PROT_WRITE; else ftype = VM_PROT_READ; } } #endif /* * See if the fault is as a result of ref/mod emulation, * or domain mismatch. */ #ifdef DEBUG last_fault_code = fsr; #endif if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, "Kernel page fault") != 0) goto fatal_pagefault; if (pmap_fault_fixup(vmspace_pmap(td->td_proc->p_vmspace), va, ftype, user)) { goto out; } onfault = pcb->pcb_onfault; pcb->pcb_onfault = NULL; if (map != kernel_map) { PROC_LOCK(p); p->p_lock++; PROC_UNLOCK(p); } error = vm_fault(map, va, ftype, VM_FAULT_NORMAL); pcb->pcb_onfault = onfault; if (map != kernel_map) { PROC_LOCK(p); p->p_lock--; PROC_UNLOCK(p); } if (__predict_true(error == 0)) goto out; fatal_pagefault: if (user == 0) { if (pcb->pcb_onfault) { tf->tf_r0 = error; tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault; return; } printf("\nvm_fault(%p, %x, %x, 0) -> %x\n", map, va, ftype, error); dab_fatal(tf, fsr, far, td, &ksig); } if (error == ENOMEM) { printf("VM: pid %d (%s), uid %d killed: " "out of swap\n", td->td_proc->p_pid, td->td_name, (td->td_proc->p_ucred) ? td->td_proc->p_ucred->cr_uid : -1); ksig.signb = SIGKILL; } else { ksig.signb = SIGSEGV; } ksig.code = 0; do_trapsignal: call_trapsignal(td, ksig.signb, ksig.code); out: /* If returning to user mode, make sure to invoke userret() */ if (user) userret(td, tf); } /* * dab_fatal() handles the following data aborts: * * FAULT_WRTBUF_0 - Vector Exception * FAULT_WRTBUF_1 - Terminal Exception * * We should never see these on a properly functioning system. * * This function is also called by the other handlers if they * detect a fatal problem. * * Note: If 'l' is NULL, we assume we're dealing with a prefetch abort. */ static int dab_fatal(struct trapframe *tf, u_int fsr, u_int far, struct thread *td, struct ksig *ksig) { const char *mode; + +#ifdef KDTRACE_HOOKS + if (!TRAP_USERMODE(tf)) { + if (dtrace_trap_func != NULL && (*dtrace_trap_func)(tf, far & FAULT_TYPE_MASK)) + return (0); + } +#endif mode = TRAP_USERMODE(tf) ? "user" : "kernel"; disable_interrupts(PSR_I|PSR_F); if (td != NULL) { printf("Fatal %s mode data abort: '%s'\n", mode, data_aborts[fsr & FAULT_TYPE_MASK].desc); printf("trapframe: %p\nFSR=%08x, FAR=", tf, fsr); if ((fsr & FAULT_IMPRECISE) == 0) printf("%08x, ", far); else printf("Invalid, "); printf("spsr=%08x\n", tf->tf_spsr); } else { printf("Fatal %s mode prefetch abort at 0x%08x\n", mode, tf->tf_pc); printf("trapframe: %p, spsr=%08x\n", tf, tf->tf_spsr); } printf("r0 =%08x, r1 =%08x, r2 =%08x, r3 =%08x\n", tf->tf_r0, tf->tf_r1, tf->tf_r2, tf->tf_r3); printf("r4 =%08x, r5 =%08x, r6 =%08x, r7 =%08x\n", tf->tf_r4, tf->tf_r5, tf->tf_r6, tf->tf_r7); printf("r8 =%08x, r9 =%08x, r10=%08x, r11=%08x\n", tf->tf_r8, tf->tf_r9, tf->tf_r10, tf->tf_r11); printf("r12=%08x, ", tf->tf_r12); if (TRAP_USERMODE(tf)) printf("usp=%08x, ulr=%08x", tf->tf_usr_sp, tf->tf_usr_lr); else printf("ssp=%08x, slr=%08x", tf->tf_svc_sp, tf->tf_svc_lr); printf(", pc =%08x\n\n", tf->tf_pc); #ifdef KDB if (debugger_on_panic || kdb_active) if (kdb_trap(fsr, 0, tf)) return (0); #endif panic("Fatal abort"); /*NOTREACHED*/ } /* * dab_align() handles the following data aborts: * * FAULT_ALIGN_0 - Alignment fault * FAULT_ALIGN_1 - Alignment fault * * These faults are fatal if they happen in kernel mode. Otherwise, we * deliver a bus error to the process. */ static int dab_align(struct trapframe *tf, u_int fsr, u_int far, struct thread *td, struct ksig *ksig) { /* Alignment faults are always fatal if they occur in kernel mode */ if (!TRAP_USERMODE(tf)) { if (!td || !td->td_pcb->pcb_onfault) dab_fatal(tf, fsr, far, td, ksig); tf->tf_r0 = EFAULT; tf->tf_pc = (int)td->td_pcb->pcb_onfault; return (0); } /* pcb_onfault *must* be NULL at this point */ /* Deliver a bus error signal to the process */ ksig->code = 0; ksig->signb = SIGBUS; td->td_frame = tf; return (1); } /* * dab_buserr() handles the following data aborts: * * FAULT_BUSERR_0 - External Abort on Linefetch -- Section * FAULT_BUSERR_1 - External Abort on Linefetch -- Page * FAULT_BUSERR_2 - External Abort on Non-linefetch -- Section * FAULT_BUSERR_3 - External Abort on Non-linefetch -- Page * FAULT_BUSTRNL1 - External abort on Translation -- Level 1 * FAULT_BUSTRNL2 - External abort on Translation -- Level 2 * * If pcb_onfault is set, flag the fault and return to the handler. * If the fault occurred in user mode, give the process a SIGBUS. * * Note: On XScale, FAULT_BUSERR_0, FAULT_BUSERR_1, and FAULT_BUSERR_2 * can be flagged as imprecise in the FSR. This causes a real headache * since some of the machine state is lost. In this case, tf->tf_pc * may not actually point to the offending instruction. In fact, if * we've taken a double abort fault, it generally points somewhere near * the top of "data_abort_entry" in exception.S. * * In all other cases, these data aborts are considered fatal. */ static int dab_buserr(struct trapframe *tf, u_int fsr, u_int far, struct thread *td, struct ksig *ksig) { struct pcb *pcb = td->td_pcb; #ifdef __XSCALE__ if ((fsr & FAULT_IMPRECISE) != 0 && (tf->tf_spsr & PSR_MODE) == PSR_ABT32_MODE) { /* * Oops, an imprecise, double abort fault. We've lost the * r14_abt/spsr_abt values corresponding to the original * abort, and the spsr saved in the trapframe indicates * ABT mode. */ tf->tf_spsr &= ~PSR_MODE; /* * We use a simple heuristic to determine if the double abort * happened as a result of a kernel or user mode access. * If the current trapframe is at the top of the kernel stack, * the fault _must_ have come from user mode. */ if (tf != ((struct trapframe *)pcb->pcb_regs.sf_sp) - 1) { /* * Kernel mode. We're either about to die a * spectacular death, or pcb_onfault will come * to our rescue. Either way, the current value * of tf->tf_pc is irrelevant. */ tf->tf_spsr |= PSR_SVC32_MODE; if (pcb->pcb_onfault == NULL) printf("\nKernel mode double abort!\n"); } else { /* * User mode. We've lost the program counter at the * time of the fault (not that it was accurate anyway; * it's not called an imprecise fault for nothing). * About all we can do is copy r14_usr to tf_pc and * hope for the best. The process is about to get a * SIGBUS, so it's probably history anyway. */ tf->tf_spsr |= PSR_USR32_MODE; tf->tf_pc = tf->tf_usr_lr; } } /* FAR is invalid for imprecise exceptions */ if ((fsr & FAULT_IMPRECISE) != 0) far = 0; #endif /* __XSCALE__ */ if (pcb->pcb_onfault) { tf->tf_r0 = EFAULT; tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault; return (0); } /* * At this point, if the fault happened in kernel mode, we're toast */ if (!TRAP_USERMODE(tf)) dab_fatal(tf, fsr, far, td, ksig); /* Deliver a bus error signal to the process */ ksig->signb = SIGBUS; ksig->code = 0; td->td_frame = tf; return (1); } /* * void prefetch_abort_handler(struct trapframe *tf) * * Abort handler called when instruction execution occurs at * a non existent or restricted (access permissions) memory page. * If the address is invalid and we were in SVC mode then panic as * the kernel should never prefetch abort. * If the address is invalid and the page is mapped then the user process * does no have read permission so send it a signal. * Otherwise fault the page in and try again. */ static void prefetch_abort_handler(struct trapframe *tf) { struct thread *td; struct proc * p; struct vm_map *map; vm_offset_t fault_pc, va; int error = 0; struct ksig ksig; #if 0 /* Update vmmeter statistics */ uvmexp.traps++; #endif #if 0 printf("prefetch abort handler: %p %p\n", (void*)tf->tf_pc, (void*)tf->tf_usr_lr); #endif td = curthread; p = td->td_proc; PCPU_INC(cnt.v_trap); if (TRAP_USERMODE(tf)) { td->td_frame = tf; if (td->td_ucred != td->td_proc->p_ucred) cred_update_thread(td); } fault_pc = tf->tf_pc; if (td->td_md.md_spinlock_count == 0) { if (__predict_true(tf->tf_spsr & PSR_I) == 0) enable_interrupts(PSR_I); if (__predict_true(tf->tf_spsr & PSR_F) == 0) enable_interrupts(PSR_F); } /* Prefetch aborts cannot happen in kernel mode */ if (__predict_false(!TRAP_USERMODE(tf))) dab_fatal(tf, 0, tf->tf_pc, NULL, &ksig); td->td_pticks = 0; /* Ok validate the address, can only execute in USER space */ if (__predict_false(fault_pc >= VM_MAXUSER_ADDRESS || (fault_pc < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW))) { ksig.signb = SIGSEGV; ksig.code = 0; goto do_trapsignal; } map = &td->td_proc->p_vmspace->vm_map; va = trunc_page(fault_pc); /* * See if the pmap can handle this fault on its own... */ #ifdef DEBUG last_fault_code = -1; #endif if (pmap_fault_fixup(map->pmap, va, VM_PROT_READ, 1)) goto out; if (map != kernel_map) { PROC_LOCK(p); p->p_lock++; PROC_UNLOCK(p); } error = vm_fault(map, va, VM_PROT_READ | VM_PROT_EXECUTE, VM_FAULT_NORMAL); if (map != kernel_map) { PROC_LOCK(p); p->p_lock--; PROC_UNLOCK(p); } if (__predict_true(error == 0)) goto out; if (error == ENOMEM) { printf("VM: pid %d (%s), uid %d killed: " "out of swap\n", td->td_proc->p_pid, td->td_name, (td->td_proc->p_ucred) ? td->td_proc->p_ucred->cr_uid : -1); ksig.signb = SIGKILL; } else { ksig.signb = SIGSEGV; } ksig.code = 0; do_trapsignal: call_trapsignal(td, ksig.signb, ksig.code); out: userret(td, tf); } extern int badaddr_read_1(const uint8_t *, uint8_t *); extern int badaddr_read_2(const uint16_t *, uint16_t *); extern int badaddr_read_4(const uint32_t *, uint32_t *); /* * Tentatively read an 8, 16, or 32-bit value from 'addr'. * If the read succeeds, the value is written to 'rptr' and zero is returned. * Else, return EFAULT. */ int badaddr_read(void *addr, size_t size, void *rptr) { union { uint8_t v1; uint16_t v2; uint32_t v4; } u; int rv; cpu_drain_writebuf(); /* Read from the test address. */ switch (size) { case sizeof(uint8_t): rv = badaddr_read_1(addr, &u.v1); if (rv == 0 && rptr) *(uint8_t *) rptr = u.v1; break; case sizeof(uint16_t): rv = badaddr_read_2(addr, &u.v2); if (rv == 0 && rptr) *(uint16_t *) rptr = u.v2; break; case sizeof(uint32_t): rv = badaddr_read_4(addr, &u.v4); if (rv == 0 && rptr) *(uint32_t *) rptr = u.v4; break; default: panic("badaddr: invalid size (%lu)", (u_long) size); } /* Return EFAULT if the address was invalid, else zero */ return (rv); } \ No newline at end of file Index: head/sys/arm/conf/BEAGLEBONE =================================================================== --- head/sys/arm/conf/BEAGLEBONE (revision 278528) +++ head/sys/arm/conf/BEAGLEBONE (revision 278529) @@ -1,163 +1,169 @@ # # BEAGLEBONE -- Custom configuration for the BeagleBone ARM development # platforms, check out http://www.beagleboard.org/bone and # http://www.beagleboard.org/black. This kernel config file is used for the # original BeagleBone and the BeagleBone Black. # # For more information on this file, please read the config(5) manual page, # and/or the handbook section on Kernel Configuration Files: # # http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html # # The handbook is also available locally in /usr/share/doc/handbook # if you've installed the doc distribution, otherwise always see the # FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the # latest information. # # An exhaustive list of options and more detailed explanations of the # device lines is also present in the ../../conf/NOTES and NOTES files. # If you are in doubt as to the purpose or necessity of a line, check first # in NOTES. # # $FreeBSD$ ident BEAGLEBONE include "../ti/am335x/std.am335x" makeoptions WITHOUT_MODULES="ahc" +# DTrace support +options KDTRACE_HOOKS # Kernel DTrace hooks +options DDB_CTF # all architectures - kernel ELF linker loads CTF data +makeoptions WITH_CTF=1 +makeoptions MODULES_OVERRIDE="opensolaris dtrace dtrace/lockstat dtrace/profile dtrace/fbt" + options HZ=100 options SCHED_4BSD # 4BSD scheduler options PREEMPTION # Enable kernel thread preemption options INET # InterNETworking options INET6 # IPv6 communications protocols options SCTP # Stream Control Transmission Protocol options FFS # Berkeley Fast Filesystem options SOFTUPDATES # Enable FFS soft updates support options UFS_ACL # Support for access control lists options UFS_DIRHASH # Improve performance on big directories options UFS_GJOURNAL # Enable gjournal-based UFS journaling options QUOTA # Enable disk quotas for UFS options NFSCL # Network Filesystem Client options NFSLOCKD # Network Lock Manager options NFS_ROOT # NFS usable as /, requires NFSCL options MSDOSFS # MSDOS Filesystem options CD9660 # ISO 9660 Filesystem options PROCFS # Process filesystem (requires PSEUDOFS) options PSEUDOFS # Pseudo-filesystem framework options TMPFS # Efficient memory filesystem options GEOM_PART_GPT # GUID Partition Tables options GEOM_PART_BSD # BSD partition scheme options GEOM_PART_MBR # MBR partition scheme options COMPAT_43 # Compatible with BSD 4.3 [KEEP THIS!] options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI options KTRACE # ktrace(1) support options SYSVSHM # SYSV-style shared memory options SYSVMSG # SYSV-style message queues options SYSVSEM # SYSV-style semaphores options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions options KBD_INSTALL_CDEV # install a CDEV entry in /dev options PLATFORM options FREEBSD_BOOT_LOADER # Process metadata passed from loader(8) options VFP # Enable floating point hardware support # Debugging for use in -current makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols options BREAK_TO_DEBUGGER #options VERBOSE_SYSINIT # Enable verbose sysinit messages options KDB # Enable kernel debugger support # For minimum debugger support (stable branch) use: #options KDB_TRACE # Print a stack trace for a panic # For full debugger support use this instead: options DDB # Enable the kernel debugger options INVARIANTS # Enable calls of extra sanity checking options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS options WITNESS # Enable checks to detect deadlocks and cycles options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed #options DIAGNOSTIC # NFS server support #options NFSD # NFS root from boopt/dhcp #options BOOTP #options BOOTP_NFSROOT #options BOOTP_COMPAT #options BOOTP_NFSV3 #options BOOTP_WIRED_TO=cpsw0 # Boot device is 2nd slice on MMC/SD card options ROOTDEVNAME=\"ufs:mmcsd0s2\" # MMC/SD/SDIO Card slot support device mmc # mmc/sd bus device mmcsd # mmc/sd flash cards device sdhci # mmc/sd host controller # I2C support device iicbus device iic device ti_i2c device am335x_pmic # AM335x Power Management IC (TPC65217) device am335x_rtc # RTC support (power management only) # Console and misc device uart device uart_ns8250 device pty device snp device md device random # Entropy device # GPIO device gpio device gpioled # ADC support device ti_adc # Watchdog support # If we don't enable the watchdog driver, the system could potentially # reboot automatically because the boot loader might have enabled the # watchdog. device ti_wdt # TI Programmable Realtime Unit support device ti_pruss # Mailbox support device ti_mbox # USB support device usb options USB_HOST_ALIGN=64 # Align usb buffers to cache line size. options USB_DEBUG #options USB_REQ_DEBUG #options USB_VERBOSE device musb device umass device scbus # SCSI bus (required for ATA/SCSI) device da # Direct Access (disks) # Ethernet device loop device ether device mii device smscphy device cpsw device bpf # USB Ethernet support, requires miibus device miibus device axe # ASIX Electronics USB Ethernet # Device mode support and USFS template device usb_template # Control of the gadget device usfs # Flattened Device Tree options FDT # Configure using FDT/DTB data options FDT_DTB_STATIC makeoptions FDT_DTS_FILE=beaglebone.dts Index: head/sys/arm/include/cpufunc.h =================================================================== --- head/sys/arm/include/cpufunc.h (revision 278528) +++ head/sys/arm/include/cpufunc.h (revision 278529) @@ -1,670 +1,670 @@ /* $NetBSD: cpufunc.h,v 1.29 2003/09/06 09:08:35 rearnsha Exp $ */ /*- * Copyright (c) 1997 Mark Brinicombe. * Copyright (c) 1997 Causality Limited * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Causality Limited. * 4. The name of Causality Limited may not be used to endorse or promote * products derived from this software without specific prior written * permission. * * THIS SOFTWARE IS PROVIDED BY CAUSALITY LIMITED ``AS IS'' AND ANY EXPRESS * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL CAUSALITY LIMITED BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * cpufunc.h * * Prototypes for cpu, mmu and tlb related functions. * * $FreeBSD$ */ #ifndef _MACHINE_CPUFUNC_H_ #define _MACHINE_CPUFUNC_H_ #ifdef _KERNEL #include #include #include /* For in[bwl] and out[bwl] */ static __inline void breakpoint(void) { __asm(".word 0xe7ffffff"); } struct cpu_functions { /* CPU functions */ u_int (*cf_id) (void); void (*cf_cpwait) (void); /* MMU functions */ u_int (*cf_control) (u_int bic, u_int eor); void (*cf_domains) (u_int domains); void (*cf_setttb) (u_int ttb); u_int (*cf_faultstatus) (void); u_int (*cf_faultaddress) (void); /* TLB functions */ void (*cf_tlb_flushID) (void); void (*cf_tlb_flushID_SE) (u_int va); void (*cf_tlb_flushI) (void); void (*cf_tlb_flushI_SE) (u_int va); void (*cf_tlb_flushD) (void); void (*cf_tlb_flushD_SE) (u_int va); /* * Cache operations: * * We define the following primitives: * * icache_sync_all Synchronize I-cache * icache_sync_range Synchronize I-cache range * * dcache_wbinv_all Write-back and Invalidate D-cache * dcache_wbinv_range Write-back and Invalidate D-cache range * dcache_inv_range Invalidate D-cache range * dcache_wb_range Write-back D-cache range * * idcache_wbinv_all Write-back and Invalidate D-cache, * Invalidate I-cache * idcache_wbinv_range Write-back and Invalidate D-cache, * Invalidate I-cache range * * Note that the ARM term for "write-back" is "clean". We use * the term "write-back" since it's a more common way to describe * the operation. * * There are some rules that must be followed: * * ID-cache Invalidate All: * Unlike other functions, this one must never write back. * It is used to intialize the MMU when it is in an unknown * state (such as when it may have lines tagged as valid * that belong to a previous set of mappings). * * I-cache Synch (all or range): * The goal is to synchronize the instruction stream, * so you may beed to write-back dirty D-cache blocks * first. If a range is requested, and you can't * synchronize just a range, you have to hit the whole * thing. * * D-cache Write-Back and Invalidate range: * If you can't WB-Inv a range, you must WB-Inv the * entire D-cache. * * D-cache Invalidate: * If you can't Inv the D-cache, you must Write-Back * and Invalidate. Code that uses this operation * MUST NOT assume that the D-cache will not be written * back to memory. * * D-cache Write-Back: * If you can't Write-back without doing an Inv, * that's fine. Then treat this as a WB-Inv. * Skipping the invalidate is merely an optimization. * * All operations: * Valid virtual addresses must be passed to each * cache operation. */ void (*cf_icache_sync_all) (void); void (*cf_icache_sync_range) (vm_offset_t, vm_size_t); void (*cf_dcache_wbinv_all) (void); void (*cf_dcache_wbinv_range) (vm_offset_t, vm_size_t); void (*cf_dcache_inv_range) (vm_offset_t, vm_size_t); void (*cf_dcache_wb_range) (vm_offset_t, vm_size_t); void (*cf_idcache_inv_all) (void); void (*cf_idcache_wbinv_all) (void); void (*cf_idcache_wbinv_range) (vm_offset_t, vm_size_t); void (*cf_l2cache_wbinv_all) (void); void (*cf_l2cache_wbinv_range) (vm_offset_t, vm_size_t); void (*cf_l2cache_inv_range) (vm_offset_t, vm_size_t); void (*cf_l2cache_wb_range) (vm_offset_t, vm_size_t); void (*cf_l2cache_drain_writebuf) (void); /* Other functions */ void (*cf_flush_prefetchbuf) (void); void (*cf_drain_writebuf) (void); void (*cf_flush_brnchtgt_C) (void); void (*cf_flush_brnchtgt_E) (u_int va); void (*cf_sleep) (int mode); /* Soft functions */ int (*cf_dataabt_fixup) (void *arg); int (*cf_prefetchabt_fixup) (void *arg); void (*cf_context_switch) (void); void (*cf_setup) (char *string); }; extern struct cpu_functions cpufuncs; extern u_int cputype; -#define cpu_id() cpufuncs.cf_id() +#define cpu_ident() cpufuncs.cf_id() #define cpu_cpwait() cpufuncs.cf_cpwait() #define cpu_control(c, e) cpufuncs.cf_control(c, e) #define cpu_domains(d) cpufuncs.cf_domains(d) #define cpu_setttb(t) cpufuncs.cf_setttb(t) #define cpu_faultstatus() cpufuncs.cf_faultstatus() #define cpu_faultaddress() cpufuncs.cf_faultaddress() #ifndef SMP #define cpu_tlb_flushID() cpufuncs.cf_tlb_flushID() #define cpu_tlb_flushID_SE(e) cpufuncs.cf_tlb_flushID_SE(e) #define cpu_tlb_flushI() cpufuncs.cf_tlb_flushI() #define cpu_tlb_flushI_SE(e) cpufuncs.cf_tlb_flushI_SE(e) #define cpu_tlb_flushD() cpufuncs.cf_tlb_flushD() #define cpu_tlb_flushD_SE(e) cpufuncs.cf_tlb_flushD_SE(e) #else void tlb_broadcast(int); #if defined(CPU_CORTEXA) || defined(CPU_MV_PJ4B) || defined(CPU_KRAIT) #define TLB_BROADCAST /* No need to explicitely send an IPI */ #else #define TLB_BROADCAST tlb_broadcast(7) #endif #define cpu_tlb_flushID() do { \ cpufuncs.cf_tlb_flushID(); \ TLB_BROADCAST; \ } while(0) #define cpu_tlb_flushID_SE(e) do { \ cpufuncs.cf_tlb_flushID_SE(e); \ TLB_BROADCAST; \ } while(0) #define cpu_tlb_flushI() do { \ cpufuncs.cf_tlb_flushI(); \ TLB_BROADCAST; \ } while(0) #define cpu_tlb_flushI_SE(e) do { \ cpufuncs.cf_tlb_flushI_SE(e); \ TLB_BROADCAST; \ } while(0) #define cpu_tlb_flushD() do { \ cpufuncs.cf_tlb_flushD(); \ TLB_BROADCAST; \ } while(0) #define cpu_tlb_flushD_SE(e) do { \ cpufuncs.cf_tlb_flushD_SE(e); \ TLB_BROADCAST; \ } while(0) #endif #define cpu_icache_sync_all() cpufuncs.cf_icache_sync_all() #define cpu_icache_sync_range(a, s) cpufuncs.cf_icache_sync_range((a), (s)) #define cpu_dcache_wbinv_all() cpufuncs.cf_dcache_wbinv_all() #define cpu_dcache_wbinv_range(a, s) cpufuncs.cf_dcache_wbinv_range((a), (s)) #define cpu_dcache_inv_range(a, s) cpufuncs.cf_dcache_inv_range((a), (s)) #define cpu_dcache_wb_range(a, s) cpufuncs.cf_dcache_wb_range((a), (s)) #define cpu_idcache_inv_all() cpufuncs.cf_idcache_inv_all() #define cpu_idcache_wbinv_all() cpufuncs.cf_idcache_wbinv_all() #define cpu_idcache_wbinv_range(a, s) cpufuncs.cf_idcache_wbinv_range((a), (s)) #define cpu_l2cache_wbinv_all() cpufuncs.cf_l2cache_wbinv_all() #define cpu_l2cache_wb_range(a, s) cpufuncs.cf_l2cache_wb_range((a), (s)) #define cpu_l2cache_inv_range(a, s) cpufuncs.cf_l2cache_inv_range((a), (s)) #define cpu_l2cache_wbinv_range(a, s) cpufuncs.cf_l2cache_wbinv_range((a), (s)) #define cpu_l2cache_drain_writebuf() cpufuncs.cf_l2cache_drain_writebuf() #define cpu_flush_prefetchbuf() cpufuncs.cf_flush_prefetchbuf() #define cpu_drain_writebuf() cpufuncs.cf_drain_writebuf() #define cpu_flush_brnchtgt_C() cpufuncs.cf_flush_brnchtgt_C() #define cpu_flush_brnchtgt_E(e) cpufuncs.cf_flush_brnchtgt_E(e) #define cpu_sleep(m) cpufuncs.cf_sleep(m) #define cpu_dataabt_fixup(a) cpufuncs.cf_dataabt_fixup(a) #define cpu_prefetchabt_fixup(a) cpufuncs.cf_prefetchabt_fixup(a) #define ABORT_FIXUP_OK 0 /* fixup succeeded */ #define ABORT_FIXUP_FAILED 1 /* fixup failed */ #define ABORT_FIXUP_RETURN 2 /* abort handler should return */ #define cpu_setup(a) cpufuncs.cf_setup(a) int set_cpufuncs (void); #define ARCHITECTURE_NOT_PRESENT 1 /* known but not configured */ #define ARCHITECTURE_NOT_SUPPORTED 2 /* not known */ void cpufunc_nullop (void); int cpufunc_null_fixup (void *); int early_abort_fixup (void *); int late_abort_fixup (void *); u_int cpufunc_id (void); u_int cpufunc_cpuid (void); u_int cpufunc_control (u_int clear, u_int bic); void cpufunc_domains (u_int domains); u_int cpufunc_faultstatus (void); u_int cpufunc_faultaddress (void); u_int cpu_pfr (int); #if defined(CPU_FA526) || defined(CPU_FA626TE) void fa526_setup (char *arg); void fa526_setttb (u_int ttb); void fa526_context_switch (void); void fa526_cpu_sleep (int); void fa526_tlb_flushI_SE (u_int); void fa526_tlb_flushID_SE (u_int); void fa526_flush_prefetchbuf (void); void fa526_flush_brnchtgt_E (u_int); void fa526_icache_sync_all (void); void fa526_icache_sync_range(vm_offset_t start, vm_size_t end); void fa526_dcache_wbinv_all (void); void fa526_dcache_wbinv_range(vm_offset_t start, vm_size_t end); void fa526_dcache_inv_range (vm_offset_t start, vm_size_t end); void fa526_dcache_wb_range (vm_offset_t start, vm_size_t end); void fa526_idcache_wbinv_all(void); void fa526_idcache_wbinv_range(vm_offset_t start, vm_size_t end); #endif #ifdef CPU_ARM9 void arm9_setttb (u_int); void arm9_tlb_flushID_SE (u_int va); void arm9_icache_sync_all (void); void arm9_icache_sync_range (vm_offset_t, vm_size_t); void arm9_dcache_wbinv_all (void); void arm9_dcache_wbinv_range (vm_offset_t, vm_size_t); void arm9_dcache_inv_range (vm_offset_t, vm_size_t); void arm9_dcache_wb_range (vm_offset_t, vm_size_t); void arm9_idcache_wbinv_all (void); void arm9_idcache_wbinv_range (vm_offset_t, vm_size_t); void arm9_context_switch (void); void arm9_setup (char *string); extern unsigned arm9_dcache_sets_max; extern unsigned arm9_dcache_sets_inc; extern unsigned arm9_dcache_index_max; extern unsigned arm9_dcache_index_inc; #endif #if defined(CPU_ARM9E) || defined(CPU_ARM10) void arm10_setttb (u_int); void arm10_tlb_flushID_SE (u_int); void arm10_tlb_flushI_SE (u_int); void arm10_icache_sync_all (void); void arm10_icache_sync_range (vm_offset_t, vm_size_t); void arm10_dcache_wbinv_all (void); void arm10_dcache_wbinv_range (vm_offset_t, vm_size_t); void arm10_dcache_inv_range (vm_offset_t, vm_size_t); void arm10_dcache_wb_range (vm_offset_t, vm_size_t); void arm10_idcache_wbinv_all (void); void arm10_idcache_wbinv_range (vm_offset_t, vm_size_t); void arm10_context_switch (void); void arm10_setup (char *string); extern unsigned arm10_dcache_sets_max; extern unsigned arm10_dcache_sets_inc; extern unsigned arm10_dcache_index_max; extern unsigned arm10_dcache_index_inc; u_int sheeva_control_ext (u_int, u_int); void sheeva_cpu_sleep (int); void sheeva_setttb (u_int); void sheeva_dcache_wbinv_range (vm_offset_t, vm_size_t); void sheeva_dcache_inv_range (vm_offset_t, vm_size_t); void sheeva_dcache_wb_range (vm_offset_t, vm_size_t); void sheeva_idcache_wbinv_range (vm_offset_t, vm_size_t); void sheeva_l2cache_wbinv_range (vm_offset_t, vm_size_t); void sheeva_l2cache_inv_range (vm_offset_t, vm_size_t); void sheeva_l2cache_wb_range (vm_offset_t, vm_size_t); void sheeva_l2cache_wbinv_all (void); #endif #if defined(CPU_ARM1136) || defined(CPU_ARM1176) || \ defined(CPU_MV_PJ4B) || defined(CPU_CORTEXA) || defined(CPU_KRAIT) void arm11_setttb (u_int); void arm11_sleep (int); void arm11_tlb_flushID_SE (u_int); void arm11_tlb_flushI_SE (u_int); void arm11_context_switch (void); void arm11_setup (char *string); void arm11_tlb_flushID (void); void arm11_tlb_flushI (void); void arm11_tlb_flushD (void); void arm11_tlb_flushD_SE (u_int va); void arm11_drain_writebuf (void); void armv6_icache_sync_all (void); void armv6_icache_sync_range (vm_offset_t, vm_size_t); void armv6_dcache_wbinv_all (void); void armv6_dcache_wbinv_range (vm_offset_t, vm_size_t); void armv6_dcache_inv_range (vm_offset_t, vm_size_t); void armv6_dcache_wb_range (vm_offset_t, vm_size_t); void armv6_idcache_inv_all (void); void armv6_idcache_wbinv_all (void); void armv6_idcache_wbinv_range (vm_offset_t, vm_size_t); void armv7_setttb (u_int); void armv7_tlb_flushID (void); void armv7_tlb_flushID_SE (u_int); void armv7_icache_sync_all (void); void armv7_icache_sync_range (vm_offset_t, vm_size_t); void armv7_idcache_wbinv_range (vm_offset_t, vm_size_t); void armv7_idcache_inv_all (void); void armv7_dcache_wbinv_all (void); void armv7_idcache_wbinv_all (void); void armv7_dcache_wbinv_range (vm_offset_t, vm_size_t); void armv7_dcache_inv_range (vm_offset_t, vm_size_t); void armv7_dcache_wb_range (vm_offset_t, vm_size_t); void armv7_cpu_sleep (int); void armv7_setup (char *string); void armv7_context_switch (void); void armv7_drain_writebuf (void); void armv7_sev (void); void armv7_sleep (int unused); u_int armv7_auxctrl (u_int, u_int); void pj4bv7_setup (char *string); void pj4b_config (void); void armadaxp_idcache_wbinv_all (void); void cortexa_setup (char *); #endif #if defined(CPU_ARM1136) || defined(CPU_ARM1176) void arm11x6_setttb (u_int); void arm11x6_idcache_wbinv_all (void); void arm11x6_dcache_wbinv_all (void); void arm11x6_icache_sync_all (void); void arm11x6_flush_prefetchbuf (void); void arm11x6_icache_sync_range (vm_offset_t, vm_size_t); void arm11x6_idcache_wbinv_range (vm_offset_t, vm_size_t); void arm11x6_setup (char *string); void arm11x6_sleep (int); /* no ref. for errata */ #endif #if defined(CPU_ARM1136) void arm1136_sleep_rev0 (int); /* for errata 336501 */ #endif #if defined(CPU_ARM9E) || defined (CPU_ARM10) void armv5_ec_setttb(u_int); void armv5_ec_icache_sync_all(void); void armv5_ec_icache_sync_range(vm_offset_t, vm_size_t); void armv5_ec_dcache_wbinv_all(void); void armv5_ec_dcache_wbinv_range(vm_offset_t, vm_size_t); void armv5_ec_dcache_inv_range(vm_offset_t, vm_size_t); void armv5_ec_dcache_wb_range(vm_offset_t, vm_size_t); void armv5_ec_idcache_wbinv_all(void); void armv5_ec_idcache_wbinv_range(vm_offset_t, vm_size_t); #endif #if defined (CPU_ARM10) void armv5_setttb(u_int); void armv5_icache_sync_all(void); void armv5_icache_sync_range(vm_offset_t, vm_size_t); void armv5_dcache_wbinv_all(void); void armv5_dcache_wbinv_range(vm_offset_t, vm_size_t); void armv5_dcache_inv_range(vm_offset_t, vm_size_t); void armv5_dcache_wb_range(vm_offset_t, vm_size_t); void armv5_idcache_wbinv_all(void); void armv5_idcache_wbinv_range(vm_offset_t, vm_size_t); extern unsigned armv5_dcache_sets_max; extern unsigned armv5_dcache_sets_inc; extern unsigned armv5_dcache_index_max; extern unsigned armv5_dcache_index_inc; #endif #if defined(CPU_ARM9) || defined(CPU_ARM9E) || defined(CPU_ARM10) || \ defined(CPU_XSCALE_80200) || defined(CPU_XSCALE_80321) || \ defined(CPU_FA526) || defined(CPU_FA626TE) || \ defined(CPU_XSCALE_PXA2X0) || defined(CPU_XSCALE_IXP425) || \ defined(CPU_XSCALE_80219) || defined(CPU_XSCALE_81342) void armv4_tlb_flushID (void); void armv4_tlb_flushI (void); void armv4_tlb_flushD (void); void armv4_tlb_flushD_SE (u_int va); void armv4_drain_writebuf (void); void armv4_idcache_inv_all (void); #endif #if defined(CPU_XSCALE_80200) || defined(CPU_XSCALE_80321) || \ defined(CPU_XSCALE_PXA2X0) || defined(CPU_XSCALE_IXP425) || \ defined(CPU_XSCALE_80219) || defined(CPU_XSCALE_81342) void xscale_cpwait (void); void xscale_cpu_sleep (int mode); u_int xscale_control (u_int clear, u_int bic); void xscale_setttb (u_int ttb); void xscale_tlb_flushID_SE (u_int va); void xscale_cache_flushID (void); void xscale_cache_flushI (void); void xscale_cache_flushD (void); void xscale_cache_flushD_SE (u_int entry); void xscale_cache_cleanID (void); void xscale_cache_cleanD (void); void xscale_cache_cleanD_E (u_int entry); void xscale_cache_clean_minidata (void); void xscale_cache_purgeID (void); void xscale_cache_purgeID_E (u_int entry); void xscale_cache_purgeD (void); void xscale_cache_purgeD_E (u_int entry); void xscale_cache_syncI (void); void xscale_cache_cleanID_rng (vm_offset_t start, vm_size_t end); void xscale_cache_cleanD_rng (vm_offset_t start, vm_size_t end); void xscale_cache_purgeID_rng (vm_offset_t start, vm_size_t end); void xscale_cache_purgeD_rng (vm_offset_t start, vm_size_t end); void xscale_cache_syncI_rng (vm_offset_t start, vm_size_t end); void xscale_cache_flushD_rng (vm_offset_t start, vm_size_t end); void xscale_context_switch (void); void xscale_setup (char *string); #endif /* CPU_XSCALE_80200 || CPU_XSCALE_80321 || CPU_XSCALE_PXA2X0 || CPU_XSCALE_IXP425 CPU_XSCALE_80219 */ #ifdef CPU_XSCALE_81342 void xscalec3_l2cache_purge (void); void xscalec3_cache_purgeID (void); void xscalec3_cache_purgeD (void); void xscalec3_cache_cleanID (void); void xscalec3_cache_cleanD (void); void xscalec3_cache_syncI (void); void xscalec3_cache_purgeID_rng (vm_offset_t start, vm_size_t end); void xscalec3_cache_purgeD_rng (vm_offset_t start, vm_size_t end); void xscalec3_cache_cleanID_rng (vm_offset_t start, vm_size_t end); void xscalec3_cache_cleanD_rng (vm_offset_t start, vm_size_t end); void xscalec3_cache_syncI_rng (vm_offset_t start, vm_size_t end); void xscalec3_l2cache_flush_rng (vm_offset_t, vm_size_t); void xscalec3_l2cache_clean_rng (vm_offset_t start, vm_size_t end); void xscalec3_l2cache_purge_rng (vm_offset_t start, vm_size_t end); void xscalec3_setttb (u_int ttb); void xscalec3_context_switch (void); #endif /* CPU_XSCALE_81342 */ #define setttb cpu_setttb #define drain_writebuf cpu_drain_writebuf /* * Macros for manipulating CPU interrupts */ static __inline u_int32_t __set_cpsr_c(u_int bic, u_int eor) __attribute__((__unused__)); static __inline u_int32_t __set_cpsr_c(u_int bic, u_int eor) { u_int32_t tmp, ret; __asm __volatile( "mrs %0, cpsr\n" /* Get the CPSR */ "bic %1, %0, %2\n" /* Clear bits */ "eor %1, %1, %3\n" /* XOR bits */ "msr cpsr_c, %1\n" /* Set the control field of CPSR */ : "=&r" (ret), "=&r" (tmp) : "r" (bic), "r" (eor) : "memory"); return ret; } #define ARM_CPSR_F32 (1 << 6) /* FIQ disable */ #define ARM_CPSR_I32 (1 << 7) /* IRQ disable */ #define disable_interrupts(mask) \ (__set_cpsr_c((mask) & (ARM_CPSR_I32 | ARM_CPSR_F32), \ (mask) & (ARM_CPSR_I32 | ARM_CPSR_F32))) #define enable_interrupts(mask) \ (__set_cpsr_c((mask) & (ARM_CPSR_I32 | ARM_CPSR_F32), 0)) #define restore_interrupts(old_cpsr) \ (__set_cpsr_c((ARM_CPSR_I32 | ARM_CPSR_F32), \ (old_cpsr) & (ARM_CPSR_I32 | ARM_CPSR_F32))) static __inline register_t intr_disable(void) { register_t s; s = disable_interrupts(ARM_CPSR_I32 | ARM_CPSR_F32); return (s); } static __inline void intr_restore(register_t s) { restore_interrupts(s); } /* Functions to manipulate the CPSR. */ u_int SetCPSR(u_int bic, u_int eor); u_int GetCPSR(void); /* * Functions to manipulate cpu r13 * (in arm/arm32/setstack.S) */ void set_stackptr (u_int mode, u_int address); u_int get_stackptr (u_int mode); /* * Miscellany */ int get_pc_str_offset (void); /* * CPU functions from locore.S */ void cpu_reset (void) __attribute__((__noreturn__)); /* * Cache info variables. */ /* PRIMARY CACHE VARIABLES */ extern int arm_picache_size; extern int arm_picache_line_size; extern int arm_picache_ways; extern int arm_pdcache_size; /* and unified */ extern int arm_pdcache_line_size; extern int arm_pdcache_ways; extern int arm_pcache_type; extern int arm_pcache_unified; extern int arm_dcache_align; extern int arm_dcache_align_mask; extern u_int arm_cache_level; extern u_int arm_cache_loc; extern u_int arm_cache_type[14]; #endif /* _KERNEL */ #endif /* _MACHINE_CPUFUNC_H_ */ /* End of cpufunc.h */ Index: head/sys/cddl/contrib/opensolaris/uts/arm/dtrace/fasttrap_isa.c =================================================================== --- head/sys/cddl/contrib/opensolaris/uts/arm/dtrace/fasttrap_isa.c (nonexistent) +++ head/sys/cddl/contrib/opensolaris/uts/arm/dtrace/fasttrap_isa.c (revision 278529) @@ -0,0 +1,30 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + + +/* + * XXX: Placeholder for ARM fasttrap code + */ Property changes on: head/sys/cddl/contrib/opensolaris/uts/arm/dtrace/fasttrap_isa.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/cddl/contrib/opensolaris/uts/arm/sys/fasttrap_isa.h =================================================================== --- head/sys/cddl/contrib/opensolaris/uts/arm/sys/fasttrap_isa.h (nonexistent) +++ head/sys/cddl/contrib/opensolaris/uts/arm/sys/fasttrap_isa.h (revision 278529) @@ -0,0 +1,94 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _FASTTRAP_ISA_H +#define _FASTTRAP_ISA_H + +#pragma ident "%Z%%M% %I% %E% SMI" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * This is our reserved trap instruction: ta 0x38 + */ +#define FASTTRAP_INSTR 0x91d02038 + +#define FASTTRAP_SUNWDTRACE_SIZE 128 + +typedef uint32_t fasttrap_instr_t; + +typedef struct fasttrap_machtp { + fasttrap_instr_t ftmt_instr; /* original instruction */ + uintptr_t ftmt_dest; /* destination of DCTI */ + uint8_t ftmt_type; /* emulation type */ + uint8_t ftmt_flags; /* emulation flags */ + uint8_t ftmt_cc; /* which cc to look at */ + uint8_t ftmt_code; /* branch condition */ +} fasttrap_machtp_t; + +#define ftt_instr ftt_mtp.ftmt_instr +#define ftt_dest ftt_mtp.ftmt_dest +#define ftt_type ftt_mtp.ftmt_type +#define ftt_flags ftt_mtp.ftmt_flags +#define ftt_cc ftt_mtp.ftmt_cc +#define ftt_code ftt_mtp.ftmt_code + +#define FASTTRAP_T_COMMON 0x00 /* common case -- no emulation */ +#define FASTTRAP_T_CCR 0x01 /* integer condition code branch */ +#define FASTTRAP_T_FCC 0x02 /* floating-point branch */ +#define FASTTRAP_T_REG 0x03 /* register predicated branch */ +#define FASTTRAP_T_ALWAYS 0x04 /* branch always */ +#define FASTTRAP_T_CALL 0x05 /* call instruction */ +#define FASTTRAP_T_JMPL 0x06 /* jmpl instruction */ +#define FASTTRAP_T_RDPC 0x07 /* rdpc instruction */ +#define FASTTRAP_T_RETURN 0x08 /* return instruction */ + +/* + * For performance rather than correctness. + */ +#define FASTTRAP_T_SAVE 0x10 /* save instruction (func entry only) */ +#define FASTTRAP_T_RESTORE 0x11 /* restore instruction */ +#define FASTTRAP_T_OR 0x12 /* mov instruction */ +#define FASTTRAP_T_SETHI 0x13 /* sethi instruction (includes nop) */ + +#define FASTTRAP_F_ANNUL 0x01 /* branch is annulled */ +#define FASTTRAP_F_RETMAYBE 0x02 /* not definitely a return site */ + +#define FASTTRAP_AFRAMES 3 +#define FASTTRAP_RETURN_AFRAMES 4 +#define FASTTRAP_ENTRY_AFRAMES 3 +#define FASTTRAP_OFFSET_AFRAMES 3 + + +#ifdef __cplusplus +} +#endif + +#endif /* _FASTTRAP_ISA_H */ Property changes on: head/sys/cddl/contrib/opensolaris/uts/arm/sys/fasttrap_isa.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c =================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c (revision 278528) +++ head/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c (revision 278529) @@ -1,17981 +1,17981 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * $FreeBSD$ */ /* * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright (c) 2012, 2014 by Delphix. All rights reserved. */ /* * DTrace - Dynamic Tracing for Solaris * * This is the implementation of the Solaris Dynamic Tracing framework * (DTrace). The user-visible interface to DTrace is described at length in * the "Solaris Dynamic Tracing Guide". The interfaces between the libdtrace * library, the in-kernel DTrace framework, and the DTrace providers are * described in the block comments in the header file. The * internal architecture of DTrace is described in the block comments in the * header file. The comments contained within the DTrace * implementation very much assume mastery of all of these sources; if one has * an unanswered question about the implementation, one should consult them * first. * * The functions here are ordered roughly as follows: * * - Probe context functions * - Probe hashing functions * - Non-probe context utility functions * - Matching functions * - Provider-to-Framework API functions * - Probe management functions * - DIF object functions * - Format functions * - Predicate functions * - ECB functions * - Buffer functions * - Enabling functions * - DOF functions * - Anonymous enabling functions * - Consumer state functions * - Helper functions * - Hook functions * - Driver cookbook functions * * Each group of functions begins with a block comment labelled the "DTrace * [Group] Functions", allowing one to find each block by searching forward * on capital-f functions. */ #include #ifndef illumos #include #endif #include #include #include #include #ifdef illumos #include #include #endif #include #include #ifdef illumos #include #endif #include #include #include #include #ifdef illumos #include #include #endif #include #ifdef illumos #include #include #endif #include #ifdef illumos #include #include #endif #include #ifdef illumos #include #include #endif #include #include #include #include "strtolctype.h" /* FreeBSD includes: */ #ifndef illumos #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "dtrace_cddl.h" #include "dtrace_debug.c" #endif /* * DTrace Tunable Variables * * The following variables may be tuned by adding a line to /etc/system that * includes both the name of the DTrace module ("dtrace") and the name of the * variable. For example: * * set dtrace:dtrace_destructive_disallow = 1 * * In general, the only variables that one should be tuning this way are those * that affect system-wide DTrace behavior, and for which the default behavior * is undesirable. Most of these variables are tunable on a per-consumer * basis using DTrace options, and need not be tuned on a system-wide basis. * When tuning these variables, avoid pathological values; while some attempt * is made to verify the integrity of these variables, they are not considered * part of the supported interface to DTrace, and they are therefore not * checked comprehensively. Further, these variables should not be tuned * dynamically via "mdb -kw" or other means; they should only be tuned via * /etc/system. */ int dtrace_destructive_disallow = 0; dtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024); size_t dtrace_difo_maxsize = (256 * 1024); dtrace_optval_t dtrace_dof_maxsize = (8 * 1024 * 1024); size_t dtrace_global_maxsize = (16 * 1024); size_t dtrace_actions_max = (16 * 1024); size_t dtrace_retain_max = 1024; dtrace_optval_t dtrace_helper_actions_max = 128; dtrace_optval_t dtrace_helper_providers_max = 32; dtrace_optval_t dtrace_dstate_defsize = (1 * 1024 * 1024); size_t dtrace_strsize_default = 256; dtrace_optval_t dtrace_cleanrate_default = 9900990; /* 101 hz */ dtrace_optval_t dtrace_cleanrate_min = 200000; /* 5000 hz */ dtrace_optval_t dtrace_cleanrate_max = (uint64_t)60 * NANOSEC; /* 1/minute */ dtrace_optval_t dtrace_aggrate_default = NANOSEC; /* 1 hz */ dtrace_optval_t dtrace_statusrate_default = NANOSEC; /* 1 hz */ dtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC; /* 6/minute */ dtrace_optval_t dtrace_switchrate_default = NANOSEC; /* 1 hz */ dtrace_optval_t dtrace_nspec_default = 1; dtrace_optval_t dtrace_specsize_default = 32 * 1024; dtrace_optval_t dtrace_stackframes_default = 20; dtrace_optval_t dtrace_ustackframes_default = 20; dtrace_optval_t dtrace_jstackframes_default = 50; dtrace_optval_t dtrace_jstackstrsize_default = 512; int dtrace_msgdsize_max = 128; hrtime_t dtrace_chill_max = MSEC2NSEC(500); /* 500 ms */ hrtime_t dtrace_chill_interval = NANOSEC; /* 1000 ms */ int dtrace_devdepth_max = 32; int dtrace_err_verbose; hrtime_t dtrace_deadman_interval = NANOSEC; hrtime_t dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC; hrtime_t dtrace_deadman_user = (hrtime_t)30 * NANOSEC; hrtime_t dtrace_unregister_defunct_reap = (hrtime_t)60 * NANOSEC; #ifndef illumos int dtrace_memstr_max = 4096; #endif /* * DTrace External Variables * * As dtrace(7D) is a kernel module, any DTrace variables are obviously * available to DTrace consumers via the backtick (`) syntax. One of these, * dtrace_zero, is made deliberately so: it is provided as a source of * well-known, zero-filled memory. While this variable is not documented, * it is used by some translators as an implementation detail. */ const char dtrace_zero[256] = { 0 }; /* zero-filled memory */ /* * DTrace Internal Variables */ #ifdef illumos static dev_info_t *dtrace_devi; /* device info */ #endif #ifdef illumos static vmem_t *dtrace_arena; /* probe ID arena */ static vmem_t *dtrace_minor; /* minor number arena */ #else static taskq_t *dtrace_taskq; /* task queue */ static struct unrhdr *dtrace_arena; /* Probe ID number. */ #endif static dtrace_probe_t **dtrace_probes; /* array of all probes */ static int dtrace_nprobes; /* number of probes */ static dtrace_provider_t *dtrace_provider; /* provider list */ static dtrace_meta_t *dtrace_meta_pid; /* user-land meta provider */ static int dtrace_opens; /* number of opens */ static int dtrace_helpers; /* number of helpers */ static int dtrace_getf; /* number of unpriv getf()s */ #ifdef illumos static void *dtrace_softstate; /* softstate pointer */ #endif static dtrace_hash_t *dtrace_bymod; /* probes hashed by module */ static dtrace_hash_t *dtrace_byfunc; /* probes hashed by function */ static dtrace_hash_t *dtrace_byname; /* probes hashed by name */ static dtrace_toxrange_t *dtrace_toxrange; /* toxic range array */ static int dtrace_toxranges; /* number of toxic ranges */ static int dtrace_toxranges_max; /* size of toxic range array */ static dtrace_anon_t dtrace_anon; /* anonymous enabling */ static kmem_cache_t *dtrace_state_cache; /* cache for dynamic state */ static uint64_t dtrace_vtime_references; /* number of vtimestamp refs */ static kthread_t *dtrace_panicked; /* panicking thread */ static dtrace_ecb_t *dtrace_ecb_create_cache; /* cached created ECB */ static dtrace_genid_t dtrace_probegen; /* current probe generation */ static dtrace_helpers_t *dtrace_deferred_pid; /* deferred helper list */ static dtrace_enabling_t *dtrace_retained; /* list of retained enablings */ static dtrace_genid_t dtrace_retained_gen; /* current retained enab gen */ static dtrace_dynvar_t dtrace_dynhash_sink; /* end of dynamic hash chains */ static int dtrace_dynvar_failclean; /* dynvars failed to clean */ #ifndef illumos static struct mtx dtrace_unr_mtx; MTX_SYSINIT(dtrace_unr_mtx, &dtrace_unr_mtx, "Unique resource identifier", MTX_DEF); int dtrace_in_probe; /* non-zero if executing a probe */ #if defined(__i386__) || defined(__amd64__) || defined(__mips__) || defined(__powerpc__) uintptr_t dtrace_in_probe_addr; /* Address of invop when already in probe */ #endif static eventhandler_tag dtrace_kld_load_tag; static eventhandler_tag dtrace_kld_unload_try_tag; #endif /* * DTrace Locking * DTrace is protected by three (relatively coarse-grained) locks: * * (1) dtrace_lock is required to manipulate essentially any DTrace state, * including enabling state, probes, ECBs, consumer state, helper state, * etc. Importantly, dtrace_lock is _not_ required when in probe context; * probe context is lock-free -- synchronization is handled via the * dtrace_sync() cross call mechanism. * * (2) dtrace_provider_lock is required when manipulating provider state, or * when provider state must be held constant. * * (3) dtrace_meta_lock is required when manipulating meta provider state, or * when meta provider state must be held constant. * * The lock ordering between these three locks is dtrace_meta_lock before * dtrace_provider_lock before dtrace_lock. (In particular, there are * several places where dtrace_provider_lock is held by the framework as it * calls into the providers -- which then call back into the framework, * grabbing dtrace_lock.) * * There are two other locks in the mix: mod_lock and cpu_lock. With respect * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical * role as a coarse-grained lock; it is acquired before both of these locks. * With respect to dtrace_meta_lock, its behavior is stranger: cpu_lock must * be acquired _between_ dtrace_meta_lock and any other DTrace locks. * mod_lock is similar with respect to dtrace_provider_lock in that it must be * acquired _between_ dtrace_provider_lock and dtrace_lock. */ static kmutex_t dtrace_lock; /* probe state lock */ static kmutex_t dtrace_provider_lock; /* provider state lock */ static kmutex_t dtrace_meta_lock; /* meta-provider state lock */ #ifndef illumos /* XXX FreeBSD hacks. */ #define cr_suid cr_svuid #define cr_sgid cr_svgid #define ipaddr_t in_addr_t #define mod_modname pathname #define vuprintf vprintf #define ttoproc(_a) ((_a)->td_proc) #define crgetzoneid(_a) 0 #define NCPU MAXCPU #define SNOCD 0 #define CPU_ON_INTR(_a) 0 #define PRIV_EFFECTIVE (1 << 0) #define PRIV_DTRACE_KERNEL (1 << 1) #define PRIV_DTRACE_PROC (1 << 2) #define PRIV_DTRACE_USER (1 << 3) #define PRIV_PROC_OWNER (1 << 4) #define PRIV_PROC_ZONE (1 << 5) #define PRIV_ALL ~0 SYSCTL_DECL(_debug_dtrace); SYSCTL_DECL(_kern_dtrace); #endif #ifdef illumos #define curcpu CPU->cpu_id #endif /* * DTrace Provider Variables * * These are the variables relating to DTrace as a provider (that is, the * provider of the BEGIN, END, and ERROR probes). */ static dtrace_pattr_t dtrace_provider_attr = { { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON }, { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON }, }; static void dtrace_nullop(void) {} static dtrace_pops_t dtrace_provider_ops = { (void (*)(void *, dtrace_probedesc_t *))dtrace_nullop, (void (*)(void *, modctl_t *))dtrace_nullop, (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, NULL, NULL, NULL, (void (*)(void *, dtrace_id_t, void *))dtrace_nullop }; static dtrace_id_t dtrace_probeid_begin; /* special BEGIN probe */ static dtrace_id_t dtrace_probeid_end; /* special END probe */ dtrace_id_t dtrace_probeid_error; /* special ERROR probe */ /* * DTrace Helper Tracing Variables * * These variables should be set dynamically to enable helper tracing. The * only variables that should be set are dtrace_helptrace_enable (which should * be set to a non-zero value to allocate helper tracing buffers on the next * open of /dev/dtrace) and dtrace_helptrace_disable (which should be set to a * non-zero value to deallocate helper tracing buffers on the next close of * /dev/dtrace). When (and only when) helper tracing is disabled, the * buffer size may also be set via dtrace_helptrace_bufsize. */ int dtrace_helptrace_enable = 0; int dtrace_helptrace_disable = 0; int dtrace_helptrace_bufsize = 16 * 1024 * 1024; uint32_t dtrace_helptrace_nlocals; static dtrace_helptrace_t *dtrace_helptrace_buffer; static uint32_t dtrace_helptrace_next = 0; static int dtrace_helptrace_wrapped = 0; /* * DTrace Error Hashing * * On DEBUG kernels, DTrace will track the errors that has seen in a hash * table. This is very useful for checking coverage of tests that are * expected to induce DIF or DOF processing errors, and may be useful for * debugging problems in the DIF code generator or in DOF generation . The * error hash may be examined with the ::dtrace_errhash MDB dcmd. */ #ifdef DEBUG static dtrace_errhash_t dtrace_errhash[DTRACE_ERRHASHSZ]; static const char *dtrace_errlast; static kthread_t *dtrace_errthread; static kmutex_t dtrace_errlock; #endif /* * DTrace Macros and Constants * * These are various macros that are useful in various spots in the * implementation, along with a few random constants that have no meaning * outside of the implementation. There is no real structure to this cpp * mishmash -- but is there ever? */ #define DTRACE_HASHSTR(hash, probe) \ dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs))) #define DTRACE_HASHNEXT(hash, probe) \ (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs) #define DTRACE_HASHPREV(hash, probe) \ (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs) #define DTRACE_HASHEQ(hash, lhs, rhs) \ (strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \ *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0) #define DTRACE_AGGHASHSIZE_SLEW 17 #define DTRACE_V4MAPPED_OFFSET (sizeof (uint32_t) * 3) /* * The key for a thread-local variable consists of the lower 61 bits of the * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL. * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never * equal to a variable identifier. This is necessary (but not sufficient) to * assure that global associative arrays never collide with thread-local * variables. To guarantee that they cannot collide, we must also define the * order for keying dynamic variables. That order is: * * [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ] * * Because the variable-key and the tls-key are in orthogonal spaces, there is * no way for a global variable key signature to match a thread-local key * signature. */ #ifdef illumos #define DTRACE_TLS_THRKEY(where) { \ uint_t intr = 0; \ uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \ for (; actv; actv >>= 1) \ intr++; \ ASSERT(intr < (1 << 3)); \ (where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \ (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \ } #else #define DTRACE_TLS_THRKEY(where) { \ solaris_cpu_t *_c = &solaris_cpu[curcpu]; \ uint_t intr = 0; \ uint_t actv = _c->cpu_intr_actv; \ for (; actv; actv >>= 1) \ intr++; \ ASSERT(intr < (1 << 3)); \ (where) = ((curthread->td_tid + DIF_VARIABLE_MAX) & \ (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \ } #endif #define DT_BSWAP_8(x) ((x) & 0xff) #define DT_BSWAP_16(x) ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8)) #define DT_BSWAP_32(x) ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16)) #define DT_BSWAP_64(x) ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32)) #define DT_MASK_LO 0x00000000FFFFFFFFULL #define DTRACE_STORE(type, tomax, offset, what) \ *((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what); #ifndef __x86 #define DTRACE_ALIGNCHECK(addr, size, flags) \ if (addr & (size - 1)) { \ *flags |= CPU_DTRACE_BADALIGN; \ cpu_core[curcpu].cpuc_dtrace_illval = addr; \ return (0); \ } #else #define DTRACE_ALIGNCHECK(addr, size, flags) #endif /* * Test whether a range of memory starting at testaddr of size testsz falls * within the range of memory described by addr, sz. We take care to avoid * problems with overflow and underflow of the unsigned quantities, and * disallow all negative sizes. Ranges of size 0 are allowed. */ #define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \ ((testaddr) - (uintptr_t)(baseaddr) < (basesz) && \ (testaddr) + (testsz) - (uintptr_t)(baseaddr) <= (basesz) && \ (testaddr) + (testsz) >= (testaddr)) /* * Test whether alloc_sz bytes will fit in the scratch region. We isolate * alloc_sz on the righthand side of the comparison in order to avoid overflow * or underflow in the comparison with it. This is simpler than the INRANGE * check above, because we know that the dtms_scratch_ptr is valid in the * range. Allocations of size zero are allowed. */ #define DTRACE_INSCRATCH(mstate, alloc_sz) \ ((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \ (mstate)->dtms_scratch_ptr >= (alloc_sz)) #define DTRACE_LOADFUNC(bits) \ /*CSTYLED*/ \ uint##bits##_t \ dtrace_load##bits(uintptr_t addr) \ { \ size_t size = bits / NBBY; \ /*CSTYLED*/ \ uint##bits##_t rval; \ int i; \ volatile uint16_t *flags = (volatile uint16_t *) \ &cpu_core[curcpu].cpuc_dtrace_flags; \ \ DTRACE_ALIGNCHECK(addr, size, flags); \ \ for (i = 0; i < dtrace_toxranges; i++) { \ if (addr >= dtrace_toxrange[i].dtt_limit) \ continue; \ \ if (addr + size <= dtrace_toxrange[i].dtt_base) \ continue; \ \ /* \ * This address falls within a toxic region; return 0. \ */ \ *flags |= CPU_DTRACE_BADADDR; \ cpu_core[curcpu].cpuc_dtrace_illval = addr; \ return (0); \ } \ \ *flags |= CPU_DTRACE_NOFAULT; \ /*CSTYLED*/ \ rval = *((volatile uint##bits##_t *)addr); \ *flags &= ~CPU_DTRACE_NOFAULT; \ \ return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0); \ } #ifdef _LP64 #define dtrace_loadptr dtrace_load64 #else #define dtrace_loadptr dtrace_load32 #endif #define DTRACE_DYNHASH_FREE 0 #define DTRACE_DYNHASH_SINK 1 #define DTRACE_DYNHASH_VALID 2 #define DTRACE_MATCH_NEXT 0 #define DTRACE_MATCH_DONE 1 #define DTRACE_ANCHORED(probe) ((probe)->dtpr_func[0] != '\0') #define DTRACE_STATE_ALIGN 64 #define DTRACE_FLAGS2FLT(flags) \ (((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR : \ ((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP : \ ((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO : \ ((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV : \ ((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV : \ ((flags) & CPU_DTRACE_TUPOFLOW) ? DTRACEFLT_TUPOFLOW : \ ((flags) & CPU_DTRACE_BADALIGN) ? DTRACEFLT_BADALIGN : \ ((flags) & CPU_DTRACE_NOSCRATCH) ? DTRACEFLT_NOSCRATCH : \ ((flags) & CPU_DTRACE_BADSTACK) ? DTRACEFLT_BADSTACK : \ DTRACEFLT_UNKNOWN) #define DTRACEACT_ISSTRING(act) \ ((act)->dta_kind == DTRACEACT_DIFEXPR && \ (act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) /* Function prototype definitions: */ static size_t dtrace_strlen(const char *, size_t); static dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id); static void dtrace_enabling_provide(dtrace_provider_t *); static int dtrace_enabling_match(dtrace_enabling_t *, int *); static void dtrace_enabling_matchall(void); static void dtrace_enabling_reap(void); static dtrace_state_t *dtrace_anon_grab(void); static uint64_t dtrace_helper(int, dtrace_mstate_t *, dtrace_state_t *, uint64_t, uint64_t); static dtrace_helpers_t *dtrace_helpers_create(proc_t *); static void dtrace_buffer_drop(dtrace_buffer_t *); static int dtrace_buffer_consumed(dtrace_buffer_t *, hrtime_t when); static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t, dtrace_state_t *, dtrace_mstate_t *); static int dtrace_state_option(dtrace_state_t *, dtrace_optid_t, dtrace_optval_t); static int dtrace_ecb_create_enable(dtrace_probe_t *, void *); static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *); uint16_t dtrace_load16(uintptr_t); uint32_t dtrace_load32(uintptr_t); uint64_t dtrace_load64(uintptr_t); uint8_t dtrace_load8(uintptr_t); void dtrace_dynvar_clean(dtrace_dstate_t *); dtrace_dynvar_t *dtrace_dynvar(dtrace_dstate_t *, uint_t, dtrace_key_t *, size_t, dtrace_dynvar_op_t, dtrace_mstate_t *, dtrace_vstate_t *); uintptr_t dtrace_dif_varstr(uintptr_t, dtrace_state_t *, dtrace_mstate_t *); static int dtrace_priv_proc(dtrace_state_t *); static void dtrace_getf_barrier(void); /* * DTrace Probe Context Functions * * These functions are called from probe context. Because probe context is * any context in which C may be called, arbitrarily locks may be held, * interrupts may be disabled, we may be in arbitrary dispatched state, etc. * As a result, functions called from probe context may only call other DTrace * support functions -- they may not interact at all with the system at large. * (Note that the ASSERT macro is made probe-context safe by redefining it in * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary * loads are to be performed from probe context, they _must_ be in terms of * the safe dtrace_load*() variants. * * Some functions in this block are not actually called from probe context; * for these functions, there will be a comment above the function reading * "Note: not called from probe context." */ void dtrace_panic(const char *format, ...) { va_list alist; va_start(alist, format); dtrace_vpanic(format, alist); va_end(alist); } int dtrace_assfail(const char *a, const char *f, int l) { dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l); /* * We just need something here that even the most clever compiler * cannot optimize away. */ return (a[(uintptr_t)f]); } /* * Atomically increment a specified error counter from probe context. */ static void dtrace_error(uint32_t *counter) { /* * Most counters stored to in probe context are per-CPU counters. * However, there are some error conditions that are sufficiently * arcane that they don't merit per-CPU storage. If these counters * are incremented concurrently on different CPUs, scalability will be * adversely affected -- but we don't expect them to be white-hot in a * correctly constructed enabling... */ uint32_t oval, nval; do { oval = *counter; if ((nval = oval + 1) == 0) { /* * If the counter would wrap, set it to 1 -- assuring * that the counter is never zero when we have seen * errors. (The counter must be 32-bits because we * aren't guaranteed a 64-bit compare&swap operation.) * To save this code both the infamy of being fingered * by a priggish news story and the indignity of being * the target of a neo-puritan witch trial, we're * carefully avoiding any colorful description of the * likelihood of this condition -- but suffice it to * say that it is only slightly more likely than the * overflow of predicate cache IDs, as discussed in * dtrace_predicate_create(). */ nval = 1; } } while (dtrace_cas32(counter, oval, nval) != oval); } /* * Use the DTRACE_LOADFUNC macro to define functions for each of loading a * uint8_t, a uint16_t, a uint32_t and a uint64_t. */ DTRACE_LOADFUNC(8) DTRACE_LOADFUNC(16) DTRACE_LOADFUNC(32) DTRACE_LOADFUNC(64) static int dtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate) { if (dest < mstate->dtms_scratch_base) return (0); if (dest + size < dest) return (0); if (dest + size > mstate->dtms_scratch_ptr) return (0); return (1); } static int dtrace_canstore_statvar(uint64_t addr, size_t sz, dtrace_statvar_t **svars, int nsvars) { int i; for (i = 0; i < nsvars; i++) { dtrace_statvar_t *svar = svars[i]; if (svar == NULL || svar->dtsv_size == 0) continue; if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, svar->dtsv_size)) return (1); } return (0); } /* * Check to see if the address is within a memory region to which a store may * be issued. This includes the DTrace scratch areas, and any DTrace variable * region. The caller of dtrace_canstore() is responsible for performing any * alignment checks that are needed before stores are actually executed. */ static int dtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) { /* * First, check to see if the address is in scratch space... */ if (DTRACE_INRANGE(addr, sz, mstate->dtms_scratch_base, mstate->dtms_scratch_size)) return (1); /* * Now check to see if it's a dynamic variable. This check will pick * up both thread-local variables and any global dynamically-allocated * variables. */ if (DTRACE_INRANGE(addr, sz, vstate->dtvs_dynvars.dtds_base, vstate->dtvs_dynvars.dtds_size)) { dtrace_dstate_t *dstate = &vstate->dtvs_dynvars; uintptr_t base = (uintptr_t)dstate->dtds_base + (dstate->dtds_hashsize * sizeof (dtrace_dynhash_t)); uintptr_t chunkoffs; /* * Before we assume that we can store here, we need to make * sure that it isn't in our metadata -- storing to our * dynamic variable metadata would corrupt our state. For * the range to not include any dynamic variable metadata, * it must: * * (1) Start above the hash table that is at the base of * the dynamic variable space * * (2) Have a starting chunk offset that is beyond the * dtrace_dynvar_t that is at the base of every chunk * * (3) Not span a chunk boundary * */ if (addr < base) return (0); chunkoffs = (addr - base) % dstate->dtds_chunksize; if (chunkoffs < sizeof (dtrace_dynvar_t)) return (0); if (chunkoffs + sz > dstate->dtds_chunksize) return (0); return (1); } /* * Finally, check the static local and global variables. These checks * take the longest, so we perform them last. */ if (dtrace_canstore_statvar(addr, sz, vstate->dtvs_locals, vstate->dtvs_nlocals)) return (1); if (dtrace_canstore_statvar(addr, sz, vstate->dtvs_globals, vstate->dtvs_nglobals)) return (1); return (0); } /* * Convenience routine to check to see if the address is within a memory * region in which a load may be issued given the user's privilege level; * if not, it sets the appropriate error flags and loads 'addr' into the * illegal value slot. * * DTrace subroutines (DIF_SUBR_*) should use this helper to implement * appropriate memory access protection. */ static int dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) { volatile uintptr_t *illval = &cpu_core[curcpu].cpuc_dtrace_illval; file_t *fp; /* * If we hold the privilege to read from kernel memory, then * everything is readable. */ if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) return (1); /* * You can obviously read that which you can store. */ if (dtrace_canstore(addr, sz, mstate, vstate)) return (1); /* * We're allowed to read from our own string table. */ if (DTRACE_INRANGE(addr, sz, mstate->dtms_difo->dtdo_strtab, mstate->dtms_difo->dtdo_strlen)) return (1); if (vstate->dtvs_state != NULL && dtrace_priv_proc(vstate->dtvs_state)) { proc_t *p; /* * When we have privileges to the current process, there are * several context-related kernel structures that are safe to * read, even absent the privilege to read from kernel memory. * These reads are safe because these structures contain only * state that (1) we're permitted to read, (2) is harmless or * (3) contains pointers to additional kernel state that we're * not permitted to read (and as such, do not present an * opportunity for privilege escalation). Finally (and * critically), because of the nature of their relation with * the current thread context, the memory associated with these * structures cannot change over the duration of probe context, * and it is therefore impossible for this memory to be * deallocated and reallocated as something else while it's * being operated upon. */ if (DTRACE_INRANGE(addr, sz, curthread, sizeof (kthread_t))) return (1); if ((p = curthread->t_procp) != NULL && DTRACE_INRANGE(addr, sz, curthread->t_procp, sizeof (proc_t))) { return (1); } if (curthread->t_cred != NULL && DTRACE_INRANGE(addr, sz, curthread->t_cred, sizeof (cred_t))) { return (1); } #ifdef illumos if (p != NULL && p->p_pidp != NULL && DTRACE_INRANGE(addr, sz, &(p->p_pidp->pid_id), sizeof (pid_t))) { return (1); } if (curthread->t_cpu != NULL && DTRACE_INRANGE(addr, sz, curthread->t_cpu, offsetof(cpu_t, cpu_pause_thread))) { return (1); } #endif } if ((fp = mstate->dtms_getf) != NULL) { uintptr_t psz = sizeof (void *); vnode_t *vp; vnodeops_t *op; /* * When getf() returns a file_t, the enabling is implicitly * granted the (transient) right to read the returned file_t * as well as the v_path and v_op->vnop_name of the underlying * vnode. These accesses are allowed after a successful * getf() because the members that they refer to cannot change * once set -- and the barrier logic in the kernel's closef() * path assures that the file_t and its referenced vode_t * cannot themselves be stale (that is, it impossible for * either dtms_getf itself or its f_vnode member to reference * freed memory). */ if (DTRACE_INRANGE(addr, sz, fp, sizeof (file_t))) return (1); if ((vp = fp->f_vnode) != NULL) { #ifdef illumos if (DTRACE_INRANGE(addr, sz, &vp->v_path, psz)) return (1); if (vp->v_path != NULL && DTRACE_INRANGE(addr, sz, vp->v_path, strlen(vp->v_path) + 1)) { return (1); } #endif if (DTRACE_INRANGE(addr, sz, &vp->v_op, psz)) return (1); #ifdef illumos if ((op = vp->v_op) != NULL && DTRACE_INRANGE(addr, sz, &op->vnop_name, psz)) { return (1); } if (op != NULL && op->vnop_name != NULL && DTRACE_INRANGE(addr, sz, op->vnop_name, strlen(op->vnop_name) + 1)) { return (1); } #endif } } DTRACE_CPUFLAG_SET(CPU_DTRACE_KPRIV); *illval = addr; return (0); } /* * Convenience routine to check to see if a given string is within a memory * region in which a load may be issued given the user's privilege level; * this exists so that we don't need to issue unnecessary dtrace_strlen() * calls in the event that the user has all privileges. */ static int dtrace_strcanload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) { size_t strsz; /* * If we hold the privilege to read from kernel memory, then * everything is readable. */ if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) return (1); strsz = 1 + dtrace_strlen((char *)(uintptr_t)addr, sz); if (dtrace_canload(addr, strsz, mstate, vstate)) return (1); return (0); } /* * Convenience routine to check to see if a given variable is within a memory * region in which a load may be issued given the user's privilege level. */ static int dtrace_vcanload(void *src, dtrace_diftype_t *type, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) { size_t sz; ASSERT(type->dtdt_flags & DIF_TF_BYREF); /* * If we hold the privilege to read from kernel memory, then * everything is readable. */ if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) return (1); if (type->dtdt_kind == DIF_TYPE_STRING) sz = dtrace_strlen(src, vstate->dtvs_state->dts_options[DTRACEOPT_STRSIZE]) + 1; else sz = type->dtdt_size; return (dtrace_canload((uintptr_t)src, sz, mstate, vstate)); } /* * Convert a string to a signed integer using safe loads. * * NOTE: This function uses various macros from strtolctype.h to manipulate * digit values, etc -- these have all been checked to ensure they make * no additional function calls. */ static int64_t dtrace_strtoll(char *input, int base, size_t limit) { uintptr_t pos = (uintptr_t)input; int64_t val = 0; int x; boolean_t neg = B_FALSE; char c, cc, ccc; uintptr_t end = pos + limit; /* * Consume any whitespace preceding digits. */ while ((c = dtrace_load8(pos)) == ' ' || c == '\t') pos++; /* * Handle an explicit sign if one is present. */ if (c == '-' || c == '+') { if (c == '-') neg = B_TRUE; c = dtrace_load8(++pos); } /* * Check for an explicit hexadecimal prefix ("0x" or "0X") and skip it * if present. */ if (base == 16 && c == '0' && ((cc = dtrace_load8(pos + 1)) == 'x' || cc == 'X') && isxdigit(ccc = dtrace_load8(pos + 2))) { pos += 2; c = ccc; } /* * Read in contiguous digits until the first non-digit character. */ for (; pos < end && c != '\0' && lisalnum(c) && (x = DIGIT(c)) < base; c = dtrace_load8(++pos)) val = val * base + x; return (neg ? -val : val); } /* * Compare two strings using safe loads. */ static int dtrace_strncmp(char *s1, char *s2, size_t limit) { uint8_t c1, c2; volatile uint16_t *flags; if (s1 == s2 || limit == 0) return (0); flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; do { if (s1 == NULL) { c1 = '\0'; } else { c1 = dtrace_load8((uintptr_t)s1++); } if (s2 == NULL) { c2 = '\0'; } else { c2 = dtrace_load8((uintptr_t)s2++); } if (c1 != c2) return (c1 - c2); } while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT)); return (0); } /* * Compute strlen(s) for a string using safe memory accesses. The additional * len parameter is used to specify a maximum length to ensure completion. */ static size_t dtrace_strlen(const char *s, size_t lim) { uint_t len; for (len = 0; len != lim; len++) { if (dtrace_load8((uintptr_t)s++) == '\0') break; } return (len); } /* * Check if an address falls within a toxic region. */ static int dtrace_istoxic(uintptr_t kaddr, size_t size) { uintptr_t taddr, tsize; int i; for (i = 0; i < dtrace_toxranges; i++) { taddr = dtrace_toxrange[i].dtt_base; tsize = dtrace_toxrange[i].dtt_limit - taddr; if (kaddr - taddr < tsize) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = kaddr; return (1); } if (taddr - kaddr < size) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = taddr; return (1); } } return (0); } /* * Copy src to dst using safe memory accesses. The src is assumed to be unsafe * memory specified by the DIF program. The dst is assumed to be safe memory * that we can store to directly because it is managed by DTrace. As with * standard bcopy, overlapping copies are handled properly. */ static void dtrace_bcopy(const void *src, void *dst, size_t len) { if (len != 0) { uint8_t *s1 = dst; const uint8_t *s2 = src; if (s1 <= s2) { do { *s1++ = dtrace_load8((uintptr_t)s2++); } while (--len != 0); } else { s2 += len; s1 += len; do { *--s1 = dtrace_load8((uintptr_t)--s2); } while (--len != 0); } } } /* * Copy src to dst using safe memory accesses, up to either the specified * length, or the point that a nul byte is encountered. The src is assumed to * be unsafe memory specified by the DIF program. The dst is assumed to be * safe memory that we can store to directly because it is managed by DTrace. * Unlike dtrace_bcopy(), overlapping regions are not handled. */ static void dtrace_strcpy(const void *src, void *dst, size_t len) { if (len != 0) { uint8_t *s1 = dst, c; const uint8_t *s2 = src; do { *s1++ = c = dtrace_load8((uintptr_t)s2++); } while (--len != 0 && c != '\0'); } } /* * Copy src to dst, deriving the size and type from the specified (BYREF) * variable type. The src is assumed to be unsafe memory specified by the DIF * program. The dst is assumed to be DTrace variable memory that is of the * specified type; we assume that we can store to directly. */ static void dtrace_vcopy(void *src, void *dst, dtrace_diftype_t *type) { ASSERT(type->dtdt_flags & DIF_TF_BYREF); if (type->dtdt_kind == DIF_TYPE_STRING) { dtrace_strcpy(src, dst, type->dtdt_size); } else { dtrace_bcopy(src, dst, type->dtdt_size); } } /* * Compare s1 to s2 using safe memory accesses. The s1 data is assumed to be * unsafe memory specified by the DIF program. The s2 data is assumed to be * safe memory that we can access directly because it is managed by DTrace. */ static int dtrace_bcmp(const void *s1, const void *s2, size_t len) { volatile uint16_t *flags; flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; if (s1 == s2) return (0); if (s1 == NULL || s2 == NULL) return (1); if (s1 != s2 && len != 0) { const uint8_t *ps1 = s1; const uint8_t *ps2 = s2; do { if (dtrace_load8((uintptr_t)ps1++) != *ps2++) return (1); } while (--len != 0 && !(*flags & CPU_DTRACE_FAULT)); } return (0); } /* * Zero the specified region using a simple byte-by-byte loop. Note that this * is for safe DTrace-managed memory only. */ static void dtrace_bzero(void *dst, size_t len) { uchar_t *cp; for (cp = dst; len != 0; len--) *cp++ = 0; } static void dtrace_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum) { uint64_t result[2]; result[0] = addend1[0] + addend2[0]; result[1] = addend1[1] + addend2[1] + (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0); sum[0] = result[0]; sum[1] = result[1]; } /* * Shift the 128-bit value in a by b. If b is positive, shift left. * If b is negative, shift right. */ static void dtrace_shift_128(uint64_t *a, int b) { uint64_t mask; if (b == 0) return; if (b < 0) { b = -b; if (b >= 64) { a[0] = a[1] >> (b - 64); a[1] = 0; } else { a[0] >>= b; mask = 1LL << (64 - b); mask -= 1; a[0] |= ((a[1] & mask) << (64 - b)); a[1] >>= b; } } else { if (b >= 64) { a[1] = a[0] << (b - 64); a[0] = 0; } else { a[1] <<= b; mask = a[0] >> (64 - b); a[1] |= mask; a[0] <<= b; } } } /* * The basic idea is to break the 2 64-bit values into 4 32-bit values, * use native multiplication on those, and then re-combine into the * resulting 128-bit value. * * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) = * hi1 * hi2 << 64 + * hi1 * lo2 << 32 + * hi2 * lo1 << 32 + * lo1 * lo2 */ static void dtrace_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product) { uint64_t hi1, hi2, lo1, lo2; uint64_t tmp[2]; hi1 = factor1 >> 32; hi2 = factor2 >> 32; lo1 = factor1 & DT_MASK_LO; lo2 = factor2 & DT_MASK_LO; product[0] = lo1 * lo2; product[1] = hi1 * hi2; tmp[0] = hi1 * lo2; tmp[1] = 0; dtrace_shift_128(tmp, 32); dtrace_add_128(product, tmp, product); tmp[0] = hi2 * lo1; tmp[1] = 0; dtrace_shift_128(tmp, 32); dtrace_add_128(product, tmp, product); } /* * This privilege check should be used by actions and subroutines to * verify that the user credentials of the process that enabled the * invoking ECB match the target credentials */ static int dtrace_priv_proc_common_user(dtrace_state_t *state) { cred_t *cr, *s_cr = state->dts_cred.dcr_cred; /* * We should always have a non-NULL state cred here, since if cred * is null (anonymous tracing), we fast-path bypass this routine. */ ASSERT(s_cr != NULL); if ((cr = CRED()) != NULL && s_cr->cr_uid == cr->cr_uid && s_cr->cr_uid == cr->cr_ruid && s_cr->cr_uid == cr->cr_suid && s_cr->cr_gid == cr->cr_gid && s_cr->cr_gid == cr->cr_rgid && s_cr->cr_gid == cr->cr_sgid) return (1); return (0); } /* * This privilege check should be used by actions and subroutines to * verify that the zone of the process that enabled the invoking ECB * matches the target credentials */ static int dtrace_priv_proc_common_zone(dtrace_state_t *state) { #ifdef illumos cred_t *cr, *s_cr = state->dts_cred.dcr_cred; /* * We should always have a non-NULL state cred here, since if cred * is null (anonymous tracing), we fast-path bypass this routine. */ ASSERT(s_cr != NULL); if ((cr = CRED()) != NULL && s_cr->cr_zone == cr->cr_zone) return (1); return (0); #else return (1); #endif } /* * This privilege check should be used by actions and subroutines to * verify that the process has not setuid or changed credentials. */ static int dtrace_priv_proc_common_nocd(void) { proc_t *proc; if ((proc = ttoproc(curthread)) != NULL && !(proc->p_flag & SNOCD)) return (1); return (0); } static int dtrace_priv_proc_destructive(dtrace_state_t *state) { int action = state->dts_cred.dcr_action; if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE) == 0) && dtrace_priv_proc_common_zone(state) == 0) goto bad; if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER) == 0) && dtrace_priv_proc_common_user(state) == 0) goto bad; if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG) == 0) && dtrace_priv_proc_common_nocd() == 0) goto bad; return (1); bad: cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV; return (0); } static int dtrace_priv_proc_control(dtrace_state_t *state) { if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_CONTROL) return (1); if (dtrace_priv_proc_common_zone(state) && dtrace_priv_proc_common_user(state) && dtrace_priv_proc_common_nocd()) return (1); cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV; return (0); } static int dtrace_priv_proc(dtrace_state_t *state) { if (state->dts_cred.dcr_action & DTRACE_CRA_PROC) return (1); cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV; return (0); } static int dtrace_priv_kernel(dtrace_state_t *state) { if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL) return (1); cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV; return (0); } static int dtrace_priv_kernel_destructive(dtrace_state_t *state) { if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL_DESTRUCTIVE) return (1); cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV; return (0); } /* * Determine if the dte_cond of the specified ECB allows for processing of * the current probe to continue. Note that this routine may allow continued * processing, but with access(es) stripped from the mstate's dtms_access * field. */ static int dtrace_priv_probe(dtrace_state_t *state, dtrace_mstate_t *mstate, dtrace_ecb_t *ecb) { dtrace_probe_t *probe = ecb->dte_probe; dtrace_provider_t *prov = probe->dtpr_provider; dtrace_pops_t *pops = &prov->dtpv_pops; int mode = DTRACE_MODE_NOPRIV_DROP; ASSERT(ecb->dte_cond); #ifdef illumos if (pops->dtps_mode != NULL) { mode = pops->dtps_mode(prov->dtpv_arg, probe->dtpr_id, probe->dtpr_arg); ASSERT((mode & DTRACE_MODE_USER) || (mode & DTRACE_MODE_KERNEL)); ASSERT((mode & DTRACE_MODE_NOPRIV_RESTRICT) || (mode & DTRACE_MODE_NOPRIV_DROP)); } /* * If the dte_cond bits indicate that this consumer is only allowed to * see user-mode firings of this probe, call the provider's dtps_mode() * entry point to check that the probe was fired while in a user * context. If that's not the case, use the policy specified by the * provider to determine if we drop the probe or merely restrict * operation. */ if (ecb->dte_cond & DTRACE_COND_USERMODE) { ASSERT(mode != DTRACE_MODE_NOPRIV_DROP); if (!(mode & DTRACE_MODE_USER)) { if (mode & DTRACE_MODE_NOPRIV_DROP) return (0); mstate->dtms_access &= ~DTRACE_ACCESS_ARGS; } } #endif /* * This is more subtle than it looks. We have to be absolutely certain * that CRED() isn't going to change out from under us so it's only * legit to examine that structure if we're in constrained situations. * Currently, the only times we'll this check is if a non-super-user * has enabled the profile or syscall providers -- providers that * allow visibility of all processes. For the profile case, the check * above will ensure that we're examining a user context. */ if (ecb->dte_cond & DTRACE_COND_OWNER) { cred_t *cr; cred_t *s_cr = state->dts_cred.dcr_cred; proc_t *proc; ASSERT(s_cr != NULL); if ((cr = CRED()) == NULL || s_cr->cr_uid != cr->cr_uid || s_cr->cr_uid != cr->cr_ruid || s_cr->cr_uid != cr->cr_suid || s_cr->cr_gid != cr->cr_gid || s_cr->cr_gid != cr->cr_rgid || s_cr->cr_gid != cr->cr_sgid || (proc = ttoproc(curthread)) == NULL || (proc->p_flag & SNOCD)) { if (mode & DTRACE_MODE_NOPRIV_DROP) return (0); #ifdef illumos mstate->dtms_access &= ~DTRACE_ACCESS_PROC; #endif } } #ifdef illumos /* * If our dte_cond is set to DTRACE_COND_ZONEOWNER and we are not * in our zone, check to see if our mode policy is to restrict rather * than to drop; if to restrict, strip away both DTRACE_ACCESS_PROC * and DTRACE_ACCESS_ARGS */ if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) { cred_t *cr; cred_t *s_cr = state->dts_cred.dcr_cred; ASSERT(s_cr != NULL); if ((cr = CRED()) == NULL || s_cr->cr_zone->zone_id != cr->cr_zone->zone_id) { if (mode & DTRACE_MODE_NOPRIV_DROP) return (0); mstate->dtms_access &= ~(DTRACE_ACCESS_PROC | DTRACE_ACCESS_ARGS); } } #endif return (1); } /* * Note: not called from probe context. This function is called * asynchronously (and at a regular interval) from outside of probe context to * clean the dirty dynamic variable lists on all CPUs. Dynamic variable * cleaning is explained in detail in . */ void dtrace_dynvar_clean(dtrace_dstate_t *dstate) { dtrace_dynvar_t *dirty; dtrace_dstate_percpu_t *dcpu; dtrace_dynvar_t **rinsep; int i, j, work = 0; for (i = 0; i < NCPU; i++) { dcpu = &dstate->dtds_percpu[i]; rinsep = &dcpu->dtdsc_rinsing; /* * If the dirty list is NULL, there is no dirty work to do. */ if (dcpu->dtdsc_dirty == NULL) continue; if (dcpu->dtdsc_rinsing != NULL) { /* * If the rinsing list is non-NULL, then it is because * this CPU was selected to accept another CPU's * dirty list -- and since that time, dirty buffers * have accumulated. This is a highly unlikely * condition, but we choose to ignore the dirty * buffers -- they'll be picked up a future cleanse. */ continue; } if (dcpu->dtdsc_clean != NULL) { /* * If the clean list is non-NULL, then we're in a * situation where a CPU has done deallocations (we * have a non-NULL dirty list) but no allocations (we * also have a non-NULL clean list). We can't simply * move the dirty list into the clean list on this * CPU, yet we also don't want to allow this condition * to persist, lest a short clean list prevent a * massive dirty list from being cleaned (which in * turn could lead to otherwise avoidable dynamic * drops). To deal with this, we look for some CPU * with a NULL clean list, NULL dirty list, and NULL * rinsing list -- and then we borrow this CPU to * rinse our dirty list. */ for (j = 0; j < NCPU; j++) { dtrace_dstate_percpu_t *rinser; rinser = &dstate->dtds_percpu[j]; if (rinser->dtdsc_rinsing != NULL) continue; if (rinser->dtdsc_dirty != NULL) continue; if (rinser->dtdsc_clean != NULL) continue; rinsep = &rinser->dtdsc_rinsing; break; } if (j == NCPU) { /* * We were unable to find another CPU that * could accept this dirty list -- we are * therefore unable to clean it now. */ dtrace_dynvar_failclean++; continue; } } work = 1; /* * Atomically move the dirty list aside. */ do { dirty = dcpu->dtdsc_dirty; /* * Before we zap the dirty list, set the rinsing list. * (This allows for a potential assertion in * dtrace_dynvar(): if a free dynamic variable appears * on a hash chain, either the dirty list or the * rinsing list for some CPU must be non-NULL.) */ *rinsep = dirty; dtrace_membar_producer(); } while (dtrace_casptr(&dcpu->dtdsc_dirty, dirty, NULL) != dirty); } if (!work) { /* * We have no work to do; we can simply return. */ return; } dtrace_sync(); for (i = 0; i < NCPU; i++) { dcpu = &dstate->dtds_percpu[i]; if (dcpu->dtdsc_rinsing == NULL) continue; /* * We are now guaranteed that no hash chain contains a pointer * into this dirty list; we can make it clean. */ ASSERT(dcpu->dtdsc_clean == NULL); dcpu->dtdsc_clean = dcpu->dtdsc_rinsing; dcpu->dtdsc_rinsing = NULL; } /* * Before we actually set the state to be DTRACE_DSTATE_CLEAN, make * sure that all CPUs have seen all of the dtdsc_clean pointers. * This prevents a race whereby a CPU incorrectly decides that * the state should be something other than DTRACE_DSTATE_CLEAN * after dtrace_dynvar_clean() has completed. */ dtrace_sync(); dstate->dtds_state = DTRACE_DSTATE_CLEAN; } /* * Depending on the value of the op parameter, this function looks-up, * allocates or deallocates an arbitrarily-keyed dynamic variable. If an * allocation is requested, this function will return a pointer to a * dtrace_dynvar_t corresponding to the allocated variable -- or NULL if no * variable can be allocated. If NULL is returned, the appropriate counter * will be incremented. */ dtrace_dynvar_t * dtrace_dynvar(dtrace_dstate_t *dstate, uint_t nkeys, dtrace_key_t *key, size_t dsize, dtrace_dynvar_op_t op, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) { uint64_t hashval = DTRACE_DYNHASH_VALID; dtrace_dynhash_t *hash = dstate->dtds_hash; dtrace_dynvar_t *free, *new_free, *next, *dvar, *start, *prev = NULL; processorid_t me = curcpu, cpu = me; dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[me]; size_t bucket, ksize; size_t chunksize = dstate->dtds_chunksize; uintptr_t kdata, lock, nstate; uint_t i; ASSERT(nkeys != 0); /* * Hash the key. As with aggregations, we use Jenkins' "One-at-a-time" * algorithm. For the by-value portions, we perform the algorithm in * 16-bit chunks (as opposed to 8-bit chunks). This speeds things up a * bit, and seems to have only a minute effect on distribution. For * the by-reference data, we perform "One-at-a-time" iterating (safely) * over each referenced byte. It's painful to do this, but it's much * better than pathological hash distribution. The efficacy of the * hashing algorithm (and a comparison with other algorithms) may be * found by running the ::dtrace_dynstat MDB dcmd. */ for (i = 0; i < nkeys; i++) { if (key[i].dttk_size == 0) { uint64_t val = key[i].dttk_value; hashval += (val >> 48) & 0xffff; hashval += (hashval << 10); hashval ^= (hashval >> 6); hashval += (val >> 32) & 0xffff; hashval += (hashval << 10); hashval ^= (hashval >> 6); hashval += (val >> 16) & 0xffff; hashval += (hashval << 10); hashval ^= (hashval >> 6); hashval += val & 0xffff; hashval += (hashval << 10); hashval ^= (hashval >> 6); } else { /* * This is incredibly painful, but it beats the hell * out of the alternative. */ uint64_t j, size = key[i].dttk_size; uintptr_t base = (uintptr_t)key[i].dttk_value; if (!dtrace_canload(base, size, mstate, vstate)) break; for (j = 0; j < size; j++) { hashval += dtrace_load8(base + j); hashval += (hashval << 10); hashval ^= (hashval >> 6); } } } if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) return (NULL); hashval += (hashval << 3); hashval ^= (hashval >> 11); hashval += (hashval << 15); /* * There is a remote chance (ideally, 1 in 2^31) that our hashval * comes out to be one of our two sentinel hash values. If this * actually happens, we set the hashval to be a value known to be a * non-sentinel value. */ if (hashval == DTRACE_DYNHASH_FREE || hashval == DTRACE_DYNHASH_SINK) hashval = DTRACE_DYNHASH_VALID; /* * Yes, it's painful to do a divide here. If the cycle count becomes * important here, tricks can be pulled to reduce it. (However, it's * critical that hash collisions be kept to an absolute minimum; * they're much more painful than a divide.) It's better to have a * solution that generates few collisions and still keeps things * relatively simple. */ bucket = hashval % dstate->dtds_hashsize; if (op == DTRACE_DYNVAR_DEALLOC) { volatile uintptr_t *lockp = &hash[bucket].dtdh_lock; for (;;) { while ((lock = *lockp) & 1) continue; if (dtrace_casptr((volatile void *)lockp, (volatile void *)lock, (volatile void *)(lock + 1)) == (void *)lock) break; } dtrace_membar_producer(); } top: prev = NULL; lock = hash[bucket].dtdh_lock; dtrace_membar_consumer(); start = hash[bucket].dtdh_chain; ASSERT(start != NULL && (start->dtdv_hashval == DTRACE_DYNHASH_SINK || start->dtdv_hashval != DTRACE_DYNHASH_FREE || op != DTRACE_DYNVAR_DEALLOC)); for (dvar = start; dvar != NULL; dvar = dvar->dtdv_next) { dtrace_tuple_t *dtuple = &dvar->dtdv_tuple; dtrace_key_t *dkey = &dtuple->dtt_key[0]; if (dvar->dtdv_hashval != hashval) { if (dvar->dtdv_hashval == DTRACE_DYNHASH_SINK) { /* * We've reached the sink, and therefore the * end of the hash chain; we can kick out of * the loop knowing that we have seen a valid * snapshot of state. */ ASSERT(dvar->dtdv_next == NULL); ASSERT(dvar == &dtrace_dynhash_sink); break; } if (dvar->dtdv_hashval == DTRACE_DYNHASH_FREE) { /* * We've gone off the rails: somewhere along * the line, one of the members of this hash * chain was deleted. Note that we could also * detect this by simply letting this loop run * to completion, as we would eventually hit * the end of the dirty list. However, we * want to avoid running the length of the * dirty list unnecessarily (it might be quite * long), so we catch this as early as * possible by detecting the hash marker. In * this case, we simply set dvar to NULL and * break; the conditional after the loop will * send us back to top. */ dvar = NULL; break; } goto next; } if (dtuple->dtt_nkeys != nkeys) goto next; for (i = 0; i < nkeys; i++, dkey++) { if (dkey->dttk_size != key[i].dttk_size) goto next; /* size or type mismatch */ if (dkey->dttk_size != 0) { if (dtrace_bcmp( (void *)(uintptr_t)key[i].dttk_value, (void *)(uintptr_t)dkey->dttk_value, dkey->dttk_size)) goto next; } else { if (dkey->dttk_value != key[i].dttk_value) goto next; } } if (op != DTRACE_DYNVAR_DEALLOC) return (dvar); ASSERT(dvar->dtdv_next == NULL || dvar->dtdv_next->dtdv_hashval != DTRACE_DYNHASH_FREE); if (prev != NULL) { ASSERT(hash[bucket].dtdh_chain != dvar); ASSERT(start != dvar); ASSERT(prev->dtdv_next == dvar); prev->dtdv_next = dvar->dtdv_next; } else { if (dtrace_casptr(&hash[bucket].dtdh_chain, start, dvar->dtdv_next) != start) { /* * We have failed to atomically swing the * hash table head pointer, presumably because * of a conflicting allocation on another CPU. * We need to reread the hash chain and try * again. */ goto top; } } dtrace_membar_producer(); /* * Now set the hash value to indicate that it's free. */ ASSERT(hash[bucket].dtdh_chain != dvar); dvar->dtdv_hashval = DTRACE_DYNHASH_FREE; dtrace_membar_producer(); /* * Set the next pointer to point at the dirty list, and * atomically swing the dirty pointer to the newly freed dvar. */ do { next = dcpu->dtdsc_dirty; dvar->dtdv_next = next; } while (dtrace_casptr(&dcpu->dtdsc_dirty, next, dvar) != next); /* * Finally, unlock this hash bucket. */ ASSERT(hash[bucket].dtdh_lock == lock); ASSERT(lock & 1); hash[bucket].dtdh_lock++; return (NULL); next: prev = dvar; continue; } if (dvar == NULL) { /* * If dvar is NULL, it is because we went off the rails: * one of the elements that we traversed in the hash chain * was deleted while we were traversing it. In this case, * we assert that we aren't doing a dealloc (deallocs lock * the hash bucket to prevent themselves from racing with * one another), and retry the hash chain traversal. */ ASSERT(op != DTRACE_DYNVAR_DEALLOC); goto top; } if (op != DTRACE_DYNVAR_ALLOC) { /* * If we are not to allocate a new variable, we want to * return NULL now. Before we return, check that the value * of the lock word hasn't changed. If it has, we may have * seen an inconsistent snapshot. */ if (op == DTRACE_DYNVAR_NOALLOC) { if (hash[bucket].dtdh_lock != lock) goto top; } else { ASSERT(op == DTRACE_DYNVAR_DEALLOC); ASSERT(hash[bucket].dtdh_lock == lock); ASSERT(lock & 1); hash[bucket].dtdh_lock++; } return (NULL); } /* * We need to allocate a new dynamic variable. The size we need is the * size of dtrace_dynvar plus the size of nkeys dtrace_key_t's plus the * size of any auxiliary key data (rounded up to 8-byte alignment) plus * the size of any referred-to data (dsize). We then round the final * size up to the chunksize for allocation. */ for (ksize = 0, i = 0; i < nkeys; i++) ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t)); /* * This should be pretty much impossible, but could happen if, say, * strange DIF specified the tuple. Ideally, this should be an * assertion and not an error condition -- but that requires that the * chunksize calculation in dtrace_difo_chunksize() be absolutely * bullet-proof. (That is, it must not be able to be fooled by * malicious DIF.) Given the lack of backwards branches in DIF, * solving this would presumably not amount to solving the Halting * Problem -- but it still seems awfully hard. */ if (sizeof (dtrace_dynvar_t) + sizeof (dtrace_key_t) * (nkeys - 1) + ksize + dsize > chunksize) { dcpu->dtdsc_drops++; return (NULL); } nstate = DTRACE_DSTATE_EMPTY; do { retry: free = dcpu->dtdsc_free; if (free == NULL) { dtrace_dynvar_t *clean = dcpu->dtdsc_clean; void *rval; if (clean == NULL) { /* * We're out of dynamic variable space on * this CPU. Unless we have tried all CPUs, * we'll try to allocate from a different * CPU. */ switch (dstate->dtds_state) { case DTRACE_DSTATE_CLEAN: { void *sp = &dstate->dtds_state; if (++cpu >= NCPU) cpu = 0; if (dcpu->dtdsc_dirty != NULL && nstate == DTRACE_DSTATE_EMPTY) nstate = DTRACE_DSTATE_DIRTY; if (dcpu->dtdsc_rinsing != NULL) nstate = DTRACE_DSTATE_RINSING; dcpu = &dstate->dtds_percpu[cpu]; if (cpu != me) goto retry; (void) dtrace_cas32(sp, DTRACE_DSTATE_CLEAN, nstate); /* * To increment the correct bean * counter, take another lap. */ goto retry; } case DTRACE_DSTATE_DIRTY: dcpu->dtdsc_dirty_drops++; break; case DTRACE_DSTATE_RINSING: dcpu->dtdsc_rinsing_drops++; break; case DTRACE_DSTATE_EMPTY: dcpu->dtdsc_drops++; break; } DTRACE_CPUFLAG_SET(CPU_DTRACE_DROP); return (NULL); } /* * The clean list appears to be non-empty. We want to * move the clean list to the free list; we start by * moving the clean pointer aside. */ if (dtrace_casptr(&dcpu->dtdsc_clean, clean, NULL) != clean) { /* * We are in one of two situations: * * (a) The clean list was switched to the * free list by another CPU. * * (b) The clean list was added to by the * cleansing cyclic. * * In either of these situations, we can * just reattempt the free list allocation. */ goto retry; } ASSERT(clean->dtdv_hashval == DTRACE_DYNHASH_FREE); /* * Now we'll move the clean list to our free list. * It's impossible for this to fail: the only way * the free list can be updated is through this * code path, and only one CPU can own the clean list. * Thus, it would only be possible for this to fail if * this code were racing with dtrace_dynvar_clean(). * (That is, if dtrace_dynvar_clean() updated the clean * list, and we ended up racing to update the free * list.) This race is prevented by the dtrace_sync() * in dtrace_dynvar_clean() -- which flushes the * owners of the clean lists out before resetting * the clean lists. */ dcpu = &dstate->dtds_percpu[me]; rval = dtrace_casptr(&dcpu->dtdsc_free, NULL, clean); ASSERT(rval == NULL); goto retry; } dvar = free; new_free = dvar->dtdv_next; } while (dtrace_casptr(&dcpu->dtdsc_free, free, new_free) != free); /* * We have now allocated a new chunk. We copy the tuple keys into the * tuple array and copy any referenced key data into the data space * following the tuple array. As we do this, we relocate dttk_value * in the final tuple to point to the key data address in the chunk. */ kdata = (uintptr_t)&dvar->dtdv_tuple.dtt_key[nkeys]; dvar->dtdv_data = (void *)(kdata + ksize); dvar->dtdv_tuple.dtt_nkeys = nkeys; for (i = 0; i < nkeys; i++) { dtrace_key_t *dkey = &dvar->dtdv_tuple.dtt_key[i]; size_t kesize = key[i].dttk_size; if (kesize != 0) { dtrace_bcopy( (const void *)(uintptr_t)key[i].dttk_value, (void *)kdata, kesize); dkey->dttk_value = kdata; kdata += P2ROUNDUP(kesize, sizeof (uint64_t)); } else { dkey->dttk_value = key[i].dttk_value; } dkey->dttk_size = kesize; } ASSERT(dvar->dtdv_hashval == DTRACE_DYNHASH_FREE); dvar->dtdv_hashval = hashval; dvar->dtdv_next = start; if (dtrace_casptr(&hash[bucket].dtdh_chain, start, dvar) == start) return (dvar); /* * The cas has failed. Either another CPU is adding an element to * this hash chain, or another CPU is deleting an element from this * hash chain. The simplest way to deal with both of these cases * (though not necessarily the most efficient) is to free our * allocated block and tail-call ourselves. Note that the free is * to the dirty list and _not_ to the free list. This is to prevent * races with allocators, above. */ dvar->dtdv_hashval = DTRACE_DYNHASH_FREE; dtrace_membar_producer(); do { free = dcpu->dtdsc_dirty; dvar->dtdv_next = free; } while (dtrace_casptr(&dcpu->dtdsc_dirty, free, dvar) != free); return (dtrace_dynvar(dstate, nkeys, key, dsize, op, mstate, vstate)); } /*ARGSUSED*/ static void dtrace_aggregate_min(uint64_t *oval, uint64_t nval, uint64_t arg) { if ((int64_t)nval < (int64_t)*oval) *oval = nval; } /*ARGSUSED*/ static void dtrace_aggregate_max(uint64_t *oval, uint64_t nval, uint64_t arg) { if ((int64_t)nval > (int64_t)*oval) *oval = nval; } static void dtrace_aggregate_quantize(uint64_t *quanta, uint64_t nval, uint64_t incr) { int i, zero = DTRACE_QUANTIZE_ZEROBUCKET; int64_t val = (int64_t)nval; if (val < 0) { for (i = 0; i < zero; i++) { if (val <= DTRACE_QUANTIZE_BUCKETVAL(i)) { quanta[i] += incr; return; } } } else { for (i = zero + 1; i < DTRACE_QUANTIZE_NBUCKETS; i++) { if (val < DTRACE_QUANTIZE_BUCKETVAL(i)) { quanta[i - 1] += incr; return; } } quanta[DTRACE_QUANTIZE_NBUCKETS - 1] += incr; return; } ASSERT(0); } static void dtrace_aggregate_lquantize(uint64_t *lquanta, uint64_t nval, uint64_t incr) { uint64_t arg = *lquanta++; int32_t base = DTRACE_LQUANTIZE_BASE(arg); uint16_t step = DTRACE_LQUANTIZE_STEP(arg); uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg); int32_t val = (int32_t)nval, level; ASSERT(step != 0); ASSERT(levels != 0); if (val < base) { /* * This is an underflow. */ lquanta[0] += incr; return; } level = (val - base) / step; if (level < levels) { lquanta[level + 1] += incr; return; } /* * This is an overflow. */ lquanta[levels + 1] += incr; } static int dtrace_aggregate_llquantize_bucket(uint16_t factor, uint16_t low, uint16_t high, uint16_t nsteps, int64_t value) { int64_t this = 1, last, next; int base = 1, order; ASSERT(factor <= nsteps); ASSERT(nsteps % factor == 0); for (order = 0; order < low; order++) this *= factor; /* * If our value is less than our factor taken to the power of the * low order of magnitude, it goes into the zeroth bucket. */ if (value < (last = this)) return (0); for (this *= factor; order <= high; order++) { int nbuckets = this > nsteps ? nsteps : this; if ((next = this * factor) < this) { /* * We should not generally get log/linear quantizations * with a high magnitude that allows 64-bits to * overflow, but we nonetheless protect against this * by explicitly checking for overflow, and clamping * our value accordingly. */ value = this - 1; } if (value < this) { /* * If our value lies within this order of magnitude, * determine its position by taking the offset within * the order of magnitude, dividing by the bucket * width, and adding to our (accumulated) base. */ return (base + (value - last) / (this / nbuckets)); } base += nbuckets - (nbuckets / factor); last = this; this = next; } /* * Our value is greater than or equal to our factor taken to the * power of one plus the high magnitude -- return the top bucket. */ return (base); } static void dtrace_aggregate_llquantize(uint64_t *llquanta, uint64_t nval, uint64_t incr) { uint64_t arg = *llquanta++; uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg); uint16_t low = DTRACE_LLQUANTIZE_LOW(arg); uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg); uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg); llquanta[dtrace_aggregate_llquantize_bucket(factor, low, high, nsteps, nval)] += incr; } /*ARGSUSED*/ static void dtrace_aggregate_avg(uint64_t *data, uint64_t nval, uint64_t arg) { data[0]++; data[1] += nval; } /*ARGSUSED*/ static void dtrace_aggregate_stddev(uint64_t *data, uint64_t nval, uint64_t arg) { int64_t snval = (int64_t)nval; uint64_t tmp[2]; data[0]++; data[1] += nval; /* * What we want to say here is: * * data[2] += nval * nval; * * But given that nval is 64-bit, we could easily overflow, so * we do this as 128-bit arithmetic. */ if (snval < 0) snval = -snval; dtrace_multiply_128((uint64_t)snval, (uint64_t)snval, tmp); dtrace_add_128(data + 2, tmp, data + 2); } /*ARGSUSED*/ static void dtrace_aggregate_count(uint64_t *oval, uint64_t nval, uint64_t arg) { *oval = *oval + 1; } /*ARGSUSED*/ static void dtrace_aggregate_sum(uint64_t *oval, uint64_t nval, uint64_t arg) { *oval += nval; } /* * Aggregate given the tuple in the principal data buffer, and the aggregating * action denoted by the specified dtrace_aggregation_t. The aggregation * buffer is specified as the buf parameter. This routine does not return * failure; if there is no space in the aggregation buffer, the data will be * dropped, and a corresponding counter incremented. */ static void dtrace_aggregate(dtrace_aggregation_t *agg, dtrace_buffer_t *dbuf, intptr_t offset, dtrace_buffer_t *buf, uint64_t expr, uint64_t arg) { dtrace_recdesc_t *rec = &agg->dtag_action.dta_rec; uint32_t i, ndx, size, fsize; uint32_t align = sizeof (uint64_t) - 1; dtrace_aggbuffer_t *agb; dtrace_aggkey_t *key; uint32_t hashval = 0, limit, isstr; caddr_t tomax, data, kdata; dtrace_actkind_t action; dtrace_action_t *act; uintptr_t offs; if (buf == NULL) return; if (!agg->dtag_hasarg) { /* * Currently, only quantize() and lquantize() take additional * arguments, and they have the same semantics: an increment * value that defaults to 1 when not present. If additional * aggregating actions take arguments, the setting of the * default argument value will presumably have to become more * sophisticated... */ arg = 1; } action = agg->dtag_action.dta_kind - DTRACEACT_AGGREGATION; size = rec->dtrd_offset - agg->dtag_base; fsize = size + rec->dtrd_size; ASSERT(dbuf->dtb_tomax != NULL); data = dbuf->dtb_tomax + offset + agg->dtag_base; if ((tomax = buf->dtb_tomax) == NULL) { dtrace_buffer_drop(buf); return; } /* * The metastructure is always at the bottom of the buffer. */ agb = (dtrace_aggbuffer_t *)(tomax + buf->dtb_size - sizeof (dtrace_aggbuffer_t)); if (buf->dtb_offset == 0) { /* * We just kludge up approximately 1/8th of the size to be * buckets. If this guess ends up being routinely * off-the-mark, we may need to dynamically readjust this * based on past performance. */ uintptr_t hashsize = (buf->dtb_size >> 3) / sizeof (uintptr_t); if ((uintptr_t)agb - hashsize * sizeof (dtrace_aggkey_t *) < (uintptr_t)tomax || hashsize == 0) { /* * We've been given a ludicrously small buffer; * increment our drop count and leave. */ dtrace_buffer_drop(buf); return; } /* * And now, a pathetic attempt to try to get a an odd (or * perchance, a prime) hash size for better hash distribution. */ if (hashsize > (DTRACE_AGGHASHSIZE_SLEW << 3)) hashsize -= DTRACE_AGGHASHSIZE_SLEW; agb->dtagb_hashsize = hashsize; agb->dtagb_hash = (dtrace_aggkey_t **)((uintptr_t)agb - agb->dtagb_hashsize * sizeof (dtrace_aggkey_t *)); agb->dtagb_free = (uintptr_t)agb->dtagb_hash; for (i = 0; i < agb->dtagb_hashsize; i++) agb->dtagb_hash[i] = NULL; } ASSERT(agg->dtag_first != NULL); ASSERT(agg->dtag_first->dta_intuple); /* * Calculate the hash value based on the key. Note that we _don't_ * include the aggid in the hashing (but we will store it as part of * the key). The hashing algorithm is Bob Jenkins' "One-at-a-time" * algorithm: a simple, quick algorithm that has no known funnels, and * gets good distribution in practice. The efficacy of the hashing * algorithm (and a comparison with other algorithms) may be found by * running the ::dtrace_aggstat MDB dcmd. */ for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) { i = act->dta_rec.dtrd_offset - agg->dtag_base; limit = i + act->dta_rec.dtrd_size; ASSERT(limit <= size); isstr = DTRACEACT_ISSTRING(act); for (; i < limit; i++) { hashval += data[i]; hashval += (hashval << 10); hashval ^= (hashval >> 6); if (isstr && data[i] == '\0') break; } } hashval += (hashval << 3); hashval ^= (hashval >> 11); hashval += (hashval << 15); /* * Yes, the divide here is expensive -- but it's generally the least * of the performance issues given the amount of data that we iterate * over to compute hash values, compare data, etc. */ ndx = hashval % agb->dtagb_hashsize; for (key = agb->dtagb_hash[ndx]; key != NULL; key = key->dtak_next) { ASSERT((caddr_t)key >= tomax); ASSERT((caddr_t)key < tomax + buf->dtb_size); if (hashval != key->dtak_hashval || key->dtak_size != size) continue; kdata = key->dtak_data; ASSERT(kdata >= tomax && kdata < tomax + buf->dtb_size); for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) { i = act->dta_rec.dtrd_offset - agg->dtag_base; limit = i + act->dta_rec.dtrd_size; ASSERT(limit <= size); isstr = DTRACEACT_ISSTRING(act); for (; i < limit; i++) { if (kdata[i] != data[i]) goto next; if (isstr && data[i] == '\0') break; } } if (action != key->dtak_action) { /* * We are aggregating on the same value in the same * aggregation with two different aggregating actions. * (This should have been picked up in the compiler, * so we may be dealing with errant or devious DIF.) * This is an error condition; we indicate as much, * and return. */ DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); return; } /* * This is a hit: we need to apply the aggregator to * the value at this key. */ agg->dtag_aggregate((uint64_t *)(kdata + size), expr, arg); return; next: continue; } /* * We didn't find it. We need to allocate some zero-filled space, * link it into the hash table appropriately, and apply the aggregator * to the (zero-filled) value. */ offs = buf->dtb_offset; while (offs & (align - 1)) offs += sizeof (uint32_t); /* * If we don't have enough room to both allocate a new key _and_ * its associated data, increment the drop count and return. */ if ((uintptr_t)tomax + offs + fsize > agb->dtagb_free - sizeof (dtrace_aggkey_t)) { dtrace_buffer_drop(buf); return; } /*CONSTCOND*/ ASSERT(!(sizeof (dtrace_aggkey_t) & (sizeof (uintptr_t) - 1))); key = (dtrace_aggkey_t *)(agb->dtagb_free - sizeof (dtrace_aggkey_t)); agb->dtagb_free -= sizeof (dtrace_aggkey_t); key->dtak_data = kdata = tomax + offs; buf->dtb_offset = offs + fsize; /* * Now copy the data across. */ *((dtrace_aggid_t *)kdata) = agg->dtag_id; for (i = sizeof (dtrace_aggid_t); i < size; i++) kdata[i] = data[i]; /* * Because strings are not zeroed out by default, we need to iterate * looking for actions that store strings, and we need to explicitly * pad these strings out with zeroes. */ for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) { int nul; if (!DTRACEACT_ISSTRING(act)) continue; i = act->dta_rec.dtrd_offset - agg->dtag_base; limit = i + act->dta_rec.dtrd_size; ASSERT(limit <= size); for (nul = 0; i < limit; i++) { if (nul) { kdata[i] = '\0'; continue; } if (data[i] != '\0') continue; nul = 1; } } for (i = size; i < fsize; i++) kdata[i] = 0; key->dtak_hashval = hashval; key->dtak_size = size; key->dtak_action = action; key->dtak_next = agb->dtagb_hash[ndx]; agb->dtagb_hash[ndx] = key; /* * Finally, apply the aggregator. */ *((uint64_t *)(key->dtak_data + size)) = agg->dtag_initial; agg->dtag_aggregate((uint64_t *)(key->dtak_data + size), expr, arg); } /* * Given consumer state, this routine finds a speculation in the INACTIVE * state and transitions it into the ACTIVE state. If there is no speculation * in the INACTIVE state, 0 is returned. In this case, no error counter is * incremented -- it is up to the caller to take appropriate action. */ static int dtrace_speculation(dtrace_state_t *state) { int i = 0; dtrace_speculation_state_t current; uint32_t *stat = &state->dts_speculations_unavail, count; while (i < state->dts_nspeculations) { dtrace_speculation_t *spec = &state->dts_speculations[i]; current = spec->dtsp_state; if (current != DTRACESPEC_INACTIVE) { if (current == DTRACESPEC_COMMITTINGMANY || current == DTRACESPEC_COMMITTING || current == DTRACESPEC_DISCARDING) stat = &state->dts_speculations_busy; i++; continue; } if (dtrace_cas32((uint32_t *)&spec->dtsp_state, current, DTRACESPEC_ACTIVE) == current) return (i + 1); } /* * We couldn't find a speculation. If we found as much as a single * busy speculation buffer, we'll attribute this failure as "busy" * instead of "unavail". */ do { count = *stat; } while (dtrace_cas32(stat, count, count + 1) != count); return (0); } /* * This routine commits an active speculation. If the specified speculation * is not in a valid state to perform a commit(), this routine will silently do * nothing. The state of the specified speculation is transitioned according * to the state transition diagram outlined in */ static void dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu, dtrace_specid_t which) { dtrace_speculation_t *spec; dtrace_buffer_t *src, *dest; uintptr_t daddr, saddr, dlimit, slimit; dtrace_speculation_state_t current, new = 0; intptr_t offs; uint64_t timestamp; if (which == 0) return; if (which > state->dts_nspeculations) { cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; return; } spec = &state->dts_speculations[which - 1]; src = &spec->dtsp_buffer[cpu]; dest = &state->dts_buffer[cpu]; do { current = spec->dtsp_state; if (current == DTRACESPEC_COMMITTINGMANY) break; switch (current) { case DTRACESPEC_INACTIVE: case DTRACESPEC_DISCARDING: return; case DTRACESPEC_COMMITTING: /* * This is only possible if we are (a) commit()'ing * without having done a prior speculate() on this CPU * and (b) racing with another commit() on a different * CPU. There's nothing to do -- we just assert that * our offset is 0. */ ASSERT(src->dtb_offset == 0); return; case DTRACESPEC_ACTIVE: new = DTRACESPEC_COMMITTING; break; case DTRACESPEC_ACTIVEONE: /* * This speculation is active on one CPU. If our * buffer offset is non-zero, we know that the one CPU * must be us. Otherwise, we are committing on a * different CPU from the speculate(), and we must * rely on being asynchronously cleaned. */ if (src->dtb_offset != 0) { new = DTRACESPEC_COMMITTING; break; } /*FALLTHROUGH*/ case DTRACESPEC_ACTIVEMANY: new = DTRACESPEC_COMMITTINGMANY; break; default: ASSERT(0); } } while (dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new) != current); /* * We have set the state to indicate that we are committing this * speculation. Now reserve the necessary space in the destination * buffer. */ if ((offs = dtrace_buffer_reserve(dest, src->dtb_offset, sizeof (uint64_t), state, NULL)) < 0) { dtrace_buffer_drop(dest); goto out; } /* * We have sufficient space to copy the speculative buffer into the * primary buffer. First, modify the speculative buffer, filling * in the timestamp of all entries with the current time. The data * must have the commit() time rather than the time it was traced, * so that all entries in the primary buffer are in timestamp order. */ timestamp = dtrace_gethrtime(); saddr = (uintptr_t)src->dtb_tomax; slimit = saddr + src->dtb_offset; while (saddr < slimit) { size_t size; dtrace_rechdr_t *dtrh = (dtrace_rechdr_t *)saddr; if (dtrh->dtrh_epid == DTRACE_EPIDNONE) { saddr += sizeof (dtrace_epid_t); continue; } ASSERT3U(dtrh->dtrh_epid, <=, state->dts_necbs); size = state->dts_ecbs[dtrh->dtrh_epid - 1]->dte_size; ASSERT3U(saddr + size, <=, slimit); ASSERT3U(size, >=, sizeof (dtrace_rechdr_t)); ASSERT3U(DTRACE_RECORD_LOAD_TIMESTAMP(dtrh), ==, UINT64_MAX); DTRACE_RECORD_STORE_TIMESTAMP(dtrh, timestamp); saddr += size; } /* * Copy the buffer across. (Note that this is a * highly subobtimal bcopy(); in the unlikely event that this becomes * a serious performance issue, a high-performance DTrace-specific * bcopy() should obviously be invented.) */ daddr = (uintptr_t)dest->dtb_tomax + offs; dlimit = daddr + src->dtb_offset; saddr = (uintptr_t)src->dtb_tomax; /* * First, the aligned portion. */ while (dlimit - daddr >= sizeof (uint64_t)) { *((uint64_t *)daddr) = *((uint64_t *)saddr); daddr += sizeof (uint64_t); saddr += sizeof (uint64_t); } /* * Now any left-over bit... */ while (dlimit - daddr) *((uint8_t *)daddr++) = *((uint8_t *)saddr++); /* * Finally, commit the reserved space in the destination buffer. */ dest->dtb_offset = offs + src->dtb_offset; out: /* * If we're lucky enough to be the only active CPU on this speculation * buffer, we can just set the state back to DTRACESPEC_INACTIVE. */ if (current == DTRACESPEC_ACTIVE || (current == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) { uint32_t rval = dtrace_cas32((uint32_t *)&spec->dtsp_state, DTRACESPEC_COMMITTING, DTRACESPEC_INACTIVE); ASSERT(rval == DTRACESPEC_COMMITTING); } src->dtb_offset = 0; src->dtb_xamot_drops += src->dtb_drops; src->dtb_drops = 0; } /* * This routine discards an active speculation. If the specified speculation * is not in a valid state to perform a discard(), this routine will silently * do nothing. The state of the specified speculation is transitioned * according to the state transition diagram outlined in */ static void dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu, dtrace_specid_t which) { dtrace_speculation_t *spec; dtrace_speculation_state_t current, new = 0; dtrace_buffer_t *buf; if (which == 0) return; if (which > state->dts_nspeculations) { cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; return; } spec = &state->dts_speculations[which - 1]; buf = &spec->dtsp_buffer[cpu]; do { current = spec->dtsp_state; switch (current) { case DTRACESPEC_INACTIVE: case DTRACESPEC_COMMITTINGMANY: case DTRACESPEC_COMMITTING: case DTRACESPEC_DISCARDING: return; case DTRACESPEC_ACTIVE: case DTRACESPEC_ACTIVEMANY: new = DTRACESPEC_DISCARDING; break; case DTRACESPEC_ACTIVEONE: if (buf->dtb_offset != 0) { new = DTRACESPEC_INACTIVE; } else { new = DTRACESPEC_DISCARDING; } break; default: ASSERT(0); } } while (dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new) != current); buf->dtb_offset = 0; buf->dtb_drops = 0; } /* * Note: not called from probe context. This function is called * asynchronously from cross call context to clean any speculations that are * in the COMMITTINGMANY or DISCARDING states. These speculations may not be * transitioned back to the INACTIVE state until all CPUs have cleaned the * speculation. */ static void dtrace_speculation_clean_here(dtrace_state_t *state) { dtrace_icookie_t cookie; processorid_t cpu = curcpu; dtrace_buffer_t *dest = &state->dts_buffer[cpu]; dtrace_specid_t i; cookie = dtrace_interrupt_disable(); if (dest->dtb_tomax == NULL) { dtrace_interrupt_enable(cookie); return; } for (i = 0; i < state->dts_nspeculations; i++) { dtrace_speculation_t *spec = &state->dts_speculations[i]; dtrace_buffer_t *src = &spec->dtsp_buffer[cpu]; if (src->dtb_tomax == NULL) continue; if (spec->dtsp_state == DTRACESPEC_DISCARDING) { src->dtb_offset = 0; continue; } if (spec->dtsp_state != DTRACESPEC_COMMITTINGMANY) continue; if (src->dtb_offset == 0) continue; dtrace_speculation_commit(state, cpu, i + 1); } dtrace_interrupt_enable(cookie); } /* * Note: not called from probe context. This function is called * asynchronously (and at a regular interval) to clean any speculations that * are in the COMMITTINGMANY or DISCARDING states. If it discovers that there * is work to be done, it cross calls all CPUs to perform that work; * COMMITMANY and DISCARDING speculations may not be transitioned back to the * INACTIVE state until they have been cleaned by all CPUs. */ static void dtrace_speculation_clean(dtrace_state_t *state) { int work = 0, rv; dtrace_specid_t i; for (i = 0; i < state->dts_nspeculations; i++) { dtrace_speculation_t *spec = &state->dts_speculations[i]; ASSERT(!spec->dtsp_cleaning); if (spec->dtsp_state != DTRACESPEC_DISCARDING && spec->dtsp_state != DTRACESPEC_COMMITTINGMANY) continue; work++; spec->dtsp_cleaning = 1; } if (!work) return; dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_speculation_clean_here, state); /* * We now know that all CPUs have committed or discarded their * speculation buffers, as appropriate. We can now set the state * to inactive. */ for (i = 0; i < state->dts_nspeculations; i++) { dtrace_speculation_t *spec = &state->dts_speculations[i]; dtrace_speculation_state_t current, new; if (!spec->dtsp_cleaning) continue; current = spec->dtsp_state; ASSERT(current == DTRACESPEC_DISCARDING || current == DTRACESPEC_COMMITTINGMANY); new = DTRACESPEC_INACTIVE; rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new); ASSERT(rv == current); spec->dtsp_cleaning = 0; } } /* * Called as part of a speculate() to get the speculative buffer associated * with a given speculation. Returns NULL if the specified speculation is not * in an ACTIVE state. If the speculation is in the ACTIVEONE state -- and * the active CPU is not the specified CPU -- the speculation will be * atomically transitioned into the ACTIVEMANY state. */ static dtrace_buffer_t * dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid, dtrace_specid_t which) { dtrace_speculation_t *spec; dtrace_speculation_state_t current, new = 0; dtrace_buffer_t *buf; if (which == 0) return (NULL); if (which > state->dts_nspeculations) { cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; return (NULL); } spec = &state->dts_speculations[which - 1]; buf = &spec->dtsp_buffer[cpuid]; do { current = spec->dtsp_state; switch (current) { case DTRACESPEC_INACTIVE: case DTRACESPEC_COMMITTINGMANY: case DTRACESPEC_DISCARDING: return (NULL); case DTRACESPEC_COMMITTING: ASSERT(buf->dtb_offset == 0); return (NULL); case DTRACESPEC_ACTIVEONE: /* * This speculation is currently active on one CPU. * Check the offset in the buffer; if it's non-zero, * that CPU must be us (and we leave the state alone). * If it's zero, assume that we're starting on a new * CPU -- and change the state to indicate that the * speculation is active on more than one CPU. */ if (buf->dtb_offset != 0) return (buf); new = DTRACESPEC_ACTIVEMANY; break; case DTRACESPEC_ACTIVEMANY: return (buf); case DTRACESPEC_ACTIVE: new = DTRACESPEC_ACTIVEONE; break; default: ASSERT(0); } } while (dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new) != current); ASSERT(new == DTRACESPEC_ACTIVEONE || new == DTRACESPEC_ACTIVEMANY); return (buf); } /* * Return a string. In the event that the user lacks the privilege to access * arbitrary kernel memory, we copy the string out to scratch memory so that we * don't fail access checking. * * dtrace_dif_variable() uses this routine as a helper for various * builtin values such as 'execname' and 'probefunc.' */ uintptr_t dtrace_dif_varstr(uintptr_t addr, dtrace_state_t *state, dtrace_mstate_t *mstate) { uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; uintptr_t ret; size_t strsz; /* * The easy case: this probe is allowed to read all of memory, so * we can just return this as a vanilla pointer. */ if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) return (addr); /* * This is the tougher case: we copy the string in question from * kernel memory into scratch memory and return it that way: this * ensures that we won't trip up when access checking tests the * BYREF return value. */ strsz = dtrace_strlen((char *)addr, size) + 1; if (mstate->dtms_scratch_ptr + strsz > mstate->dtms_scratch_base + mstate->dtms_scratch_size) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); return (0); } dtrace_strcpy((const void *)addr, (void *)mstate->dtms_scratch_ptr, strsz); ret = mstate->dtms_scratch_ptr; mstate->dtms_scratch_ptr += strsz; return (ret); } /* * Return a string from a memoy address which is known to have one or * more concatenated, individually zero terminated, sub-strings. * In the event that the user lacks the privilege to access * arbitrary kernel memory, we copy the string out to scratch memory so that we * don't fail access checking. * * dtrace_dif_variable() uses this routine as a helper for various * builtin values such as 'execargs'. */ static uintptr_t dtrace_dif_varstrz(uintptr_t addr, size_t strsz, dtrace_state_t *state, dtrace_mstate_t *mstate) { char *p; size_t i; uintptr_t ret; if (mstate->dtms_scratch_ptr + strsz > mstate->dtms_scratch_base + mstate->dtms_scratch_size) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); return (0); } dtrace_bcopy((const void *)addr, (void *)mstate->dtms_scratch_ptr, strsz); /* Replace sub-string termination characters with a space. */ for (p = (char *) mstate->dtms_scratch_ptr, i = 0; i < strsz - 1; p++, i++) if (*p == '\0') *p = ' '; ret = mstate->dtms_scratch_ptr; mstate->dtms_scratch_ptr += strsz; return (ret); } /* * This function implements the DIF emulator's variable lookups. The emulator * passes a reserved variable identifier and optional built-in array index. */ static uint64_t dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v, uint64_t ndx) { /* * If we're accessing one of the uncached arguments, we'll turn this * into a reference in the args array. */ if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) { ndx = v - DIF_VAR_ARG0; v = DIF_VAR_ARGS; } switch (v) { case DIF_VAR_ARGS: ASSERT(mstate->dtms_present & DTRACE_MSTATE_ARGS); if (ndx >= sizeof (mstate->dtms_arg) / sizeof (mstate->dtms_arg[0])) { int aframes = mstate->dtms_probe->dtpr_aframes + 2; dtrace_provider_t *pv; uint64_t val; pv = mstate->dtms_probe->dtpr_provider; if (pv->dtpv_pops.dtps_getargval != NULL) val = pv->dtpv_pops.dtps_getargval(pv->dtpv_arg, mstate->dtms_probe->dtpr_id, mstate->dtms_probe->dtpr_arg, ndx, aframes); else val = dtrace_getarg(ndx, aframes); /* * This is regrettably required to keep the compiler * from tail-optimizing the call to dtrace_getarg(). * The condition always evaluates to true, but the * compiler has no way of figuring that out a priori. * (None of this would be necessary if the compiler * could be relied upon to _always_ tail-optimize * the call to dtrace_getarg() -- but it can't.) */ if (mstate->dtms_probe != NULL) return (val); ASSERT(0); } return (mstate->dtms_arg[ndx]); #ifdef illumos case DIF_VAR_UREGS: { klwp_t *lwp; if (!dtrace_priv_proc(state)) return (0); if ((lwp = curthread->t_lwp) == NULL) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = NULL; return (0); } return (dtrace_getreg(lwp->lwp_regs, ndx)); return (0); } #else case DIF_VAR_UREGS: { struct trapframe *tframe; if (!dtrace_priv_proc(state)) return (0); if ((tframe = curthread->td_frame) == NULL) { DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); cpu_core[curcpu].cpuc_dtrace_illval = 0; return (0); } return (dtrace_getreg(tframe, ndx)); } #endif case DIF_VAR_CURTHREAD: if (!dtrace_priv_proc(state)) return (0); return ((uint64_t)(uintptr_t)curthread); case DIF_VAR_TIMESTAMP: if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) { mstate->dtms_timestamp = dtrace_gethrtime(); mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP; } return (mstate->dtms_timestamp); case DIF_VAR_VTIMESTAMP: ASSERT(dtrace_vtime_references != 0); return (curthread->t_dtrace_vtime); case DIF_VAR_WALLTIMESTAMP: if (!(mstate->dtms_present & DTRACE_MSTATE_WALLTIMESTAMP)) { mstate->dtms_walltimestamp = dtrace_gethrestime(); mstate->dtms_present |= DTRACE_MSTATE_WALLTIMESTAMP; } return (mstate->dtms_walltimestamp); #ifdef illumos case DIF_VAR_IPL: if (!dtrace_priv_kernel(state)) return (0); if (!(mstate->dtms_present & DTRACE_MSTATE_IPL)) { mstate->dtms_ipl = dtrace_getipl(); mstate->dtms_present |= DTRACE_MSTATE_IPL; } return (mstate->dtms_ipl); #endif case DIF_VAR_EPID: ASSERT(mstate->dtms_present & DTRACE_MSTATE_EPID); return (mstate->dtms_epid); case DIF_VAR_ID: ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); return (mstate->dtms_probe->dtpr_id); case DIF_VAR_STACKDEPTH: if (!dtrace_priv_kernel(state)) return (0); if (!(mstate->dtms_present & DTRACE_MSTATE_STACKDEPTH)) { int aframes = mstate->dtms_probe->dtpr_aframes + 2; mstate->dtms_stackdepth = dtrace_getstackdepth(aframes); mstate->dtms_present |= DTRACE_MSTATE_STACKDEPTH; } return (mstate->dtms_stackdepth); case DIF_VAR_USTACKDEPTH: if (!dtrace_priv_proc(state)) return (0); if (!(mstate->dtms_present & DTRACE_MSTATE_USTACKDEPTH)) { /* * See comment in DIF_VAR_PID. */ if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) { mstate->dtms_ustackdepth = 0; } else { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); mstate->dtms_ustackdepth = dtrace_getustackdepth(); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); } mstate->dtms_present |= DTRACE_MSTATE_USTACKDEPTH; } return (mstate->dtms_ustackdepth); case DIF_VAR_CALLER: if (!dtrace_priv_kernel(state)) return (0); if (!(mstate->dtms_present & DTRACE_MSTATE_CALLER)) { int aframes = mstate->dtms_probe->dtpr_aframes + 2; if (!DTRACE_ANCHORED(mstate->dtms_probe)) { /* * If this is an unanchored probe, we are * required to go through the slow path: * dtrace_caller() only guarantees correct * results for anchored probes. */ pc_t caller[2] = {0, 0}; dtrace_getpcstack(caller, 2, aframes, (uint32_t *)(uintptr_t)mstate->dtms_arg[0]); mstate->dtms_caller = caller[1]; } else if ((mstate->dtms_caller = dtrace_caller(aframes)) == -1) { /* * We have failed to do this the quick way; * we must resort to the slower approach of * calling dtrace_getpcstack(). */ pc_t caller = 0; dtrace_getpcstack(&caller, 1, aframes, NULL); mstate->dtms_caller = caller; } mstate->dtms_present |= DTRACE_MSTATE_CALLER; } return (mstate->dtms_caller); case DIF_VAR_UCALLER: if (!dtrace_priv_proc(state)) return (0); if (!(mstate->dtms_present & DTRACE_MSTATE_UCALLER)) { uint64_t ustack[3]; /* * dtrace_getupcstack() fills in the first uint64_t * with the current PID. The second uint64_t will * be the program counter at user-level. The third * uint64_t will contain the caller, which is what * we're after. */ ustack[2] = 0; DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); dtrace_getupcstack(ustack, 3); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); mstate->dtms_ucaller = ustack[2]; mstate->dtms_present |= DTRACE_MSTATE_UCALLER; } return (mstate->dtms_ucaller); case DIF_VAR_PROBEPROV: ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); return (dtrace_dif_varstr( (uintptr_t)mstate->dtms_probe->dtpr_provider->dtpv_name, state, mstate)); case DIF_VAR_PROBEMOD: ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); return (dtrace_dif_varstr( (uintptr_t)mstate->dtms_probe->dtpr_mod, state, mstate)); case DIF_VAR_PROBEFUNC: ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); return (dtrace_dif_varstr( (uintptr_t)mstate->dtms_probe->dtpr_func, state, mstate)); case DIF_VAR_PROBENAME: ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); return (dtrace_dif_varstr( (uintptr_t)mstate->dtms_probe->dtpr_name, state, mstate)); case DIF_VAR_PID: if (!dtrace_priv_proc(state)) return (0); #ifdef illumos /* * Note that we are assuming that an unanchored probe is * always due to a high-level interrupt. (And we're assuming * that there is only a single high level interrupt.) */ if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) return (pid0.pid_id); /* * It is always safe to dereference one's own t_procp pointer: * it always points to a valid, allocated proc structure. * Further, it is always safe to dereference the p_pidp member * of one's own proc structure. (These are truisms becuase * threads and processes don't clean up their own state -- * they leave that task to whomever reaps them.) */ return ((uint64_t)curthread->t_procp->p_pidp->pid_id); #else return ((uint64_t)curproc->p_pid); #endif case DIF_VAR_PPID: if (!dtrace_priv_proc(state)) return (0); #ifdef illumos /* * See comment in DIF_VAR_PID. */ if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) return (pid0.pid_id); /* * It is always safe to dereference one's own t_procp pointer: * it always points to a valid, allocated proc structure. * (This is true because threads don't clean up their own * state -- they leave that task to whomever reaps them.) */ return ((uint64_t)curthread->t_procp->p_ppid); #else if (curproc->p_pid == proc0.p_pid) return (curproc->p_pid); else return (curproc->p_pptr->p_pid); #endif case DIF_VAR_TID: #ifdef illumos /* * See comment in DIF_VAR_PID. */ if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) return (0); #endif return ((uint64_t)curthread->t_tid); case DIF_VAR_EXECARGS: { struct pargs *p_args = curthread->td_proc->p_args; if (p_args == NULL) return(0); return (dtrace_dif_varstrz( (uintptr_t) p_args->ar_args, p_args->ar_length, state, mstate)); } case DIF_VAR_EXECNAME: #ifdef illumos if (!dtrace_priv_proc(state)) return (0); /* * See comment in DIF_VAR_PID. */ if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) return ((uint64_t)(uintptr_t)p0.p_user.u_comm); /* * It is always safe to dereference one's own t_procp pointer: * it always points to a valid, allocated proc structure. * (This is true because threads don't clean up their own * state -- they leave that task to whomever reaps them.) */ return (dtrace_dif_varstr( (uintptr_t)curthread->t_procp->p_user.u_comm, state, mstate)); #else return (dtrace_dif_varstr( (uintptr_t) curthread->td_proc->p_comm, state, mstate)); #endif case DIF_VAR_ZONENAME: #ifdef illumos if (!dtrace_priv_proc(state)) return (0); /* * See comment in DIF_VAR_PID. */ if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) return ((uint64_t)(uintptr_t)p0.p_zone->zone_name); /* * It is always safe to dereference one's own t_procp pointer: * it always points to a valid, allocated proc structure. * (This is true because threads don't clean up their own * state -- they leave that task to whomever reaps them.) */ return (dtrace_dif_varstr( (uintptr_t)curthread->t_procp->p_zone->zone_name, state, mstate)); #else return (0); #endif case DIF_VAR_UID: if (!dtrace_priv_proc(state)) return (0); #ifdef illumos /* * See comment in DIF_VAR_PID. */ if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) return ((uint64_t)p0.p_cred->cr_uid); #endif /* * It is always safe to dereference one's own t_procp pointer: * it always points to a valid, allocated proc structure. * (This is true because threads don't clean up their own * state -- they leave that task to whomever reaps them.) * * Additionally, it is safe to dereference one's own process * credential, since this is never NULL after process birth. */ return ((uint64_t)curthread->t_procp->p_cred->cr_uid); case DIF_VAR_GID: if (!dtrace_priv_proc(state)) return (0); #ifdef illumos /* * See comment in DIF_VAR_PID. */ if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) return ((uint64_t)p0.p_cred->cr_gid); #endif /* * It is always safe to dereference one's own t_procp pointer: * it always points to a valid, allocated proc structure. * (This is true because threads don't clean up their own * state -- they leave that task to whomever reaps them.) * * Additionally, it is safe to dereference one's own process * credential, since this is never NULL after process birth. */ return ((uint64_t)curthread->t_procp->p_cred->cr_gid); case DIF_VAR_ERRNO: { #ifdef illumos klwp_t *lwp; if (!dtrace_priv_proc(state)) return (0); /* * See comment in DIF_VAR_PID. */ if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) return (0); /* * It is always safe to dereference one's own t_lwp pointer in * the event that this pointer is non-NULL. (This is true * because threads and lwps don't clean up their own state -- * they leave that task to whomever reaps them.) */ if ((lwp = curthread->t_lwp) == NULL) return (0); return ((uint64_t)lwp->lwp_errno); #else return (curthread->td_errno); #endif } #ifndef illumos case DIF_VAR_CPU: { return curcpu; } #endif default: DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); return (0); } } typedef enum dtrace_json_state { DTRACE_JSON_REST = 1, DTRACE_JSON_OBJECT, DTRACE_JSON_STRING, DTRACE_JSON_STRING_ESCAPE, DTRACE_JSON_STRING_ESCAPE_UNICODE, DTRACE_JSON_COLON, DTRACE_JSON_COMMA, DTRACE_JSON_VALUE, DTRACE_JSON_IDENTIFIER, DTRACE_JSON_NUMBER, DTRACE_JSON_NUMBER_FRAC, DTRACE_JSON_NUMBER_EXP, DTRACE_JSON_COLLECT_OBJECT } dtrace_json_state_t; /* * This function possesses just enough knowledge about JSON to extract a single * value from a JSON string and store it in the scratch buffer. It is able * to extract nested object values, and members of arrays by index. * * elemlist is a list of JSON keys, stored as packed NUL-terminated strings, to * be looked up as we descend into the object tree. e.g. * * foo[0].bar.baz[32] --> "foo" NUL "0" NUL "bar" NUL "baz" NUL "32" NUL * with nelems = 5. * * The run time of this function must be bounded above by strsize to limit the * amount of work done in probe context. As such, it is implemented as a * simple state machine, reading one character at a time using safe loads * until we find the requested element, hit a parsing error or run off the * end of the object or string. * * As there is no way for a subroutine to return an error without interrupting * clause execution, we simply return NULL in the event of a missing key or any * other error condition. Each NULL return in this function is commented with * the error condition it represents -- parsing or otherwise. * * The set of states for the state machine closely matches the JSON * specification (http://json.org/). Briefly: * * DTRACE_JSON_REST: * Skip whitespace until we find either a top-level Object, moving * to DTRACE_JSON_OBJECT; or an Array, moving to DTRACE_JSON_VALUE. * * DTRACE_JSON_OBJECT: * Locate the next key String in an Object. Sets a flag to denote * the next String as a key string and moves to DTRACE_JSON_STRING. * * DTRACE_JSON_COLON: * Skip whitespace until we find the colon that separates key Strings * from their values. Once found, move to DTRACE_JSON_VALUE. * * DTRACE_JSON_VALUE: * Detects the type of the next value (String, Number, Identifier, Object * or Array) and routes to the states that process that type. Here we also * deal with the element selector list if we are requested to traverse down * into the object tree. * * DTRACE_JSON_COMMA: * Skip whitespace until we find the comma that separates key-value pairs * in Objects (returning to DTRACE_JSON_OBJECT) or values in Arrays * (similarly DTRACE_JSON_VALUE). All following literal value processing * states return to this state at the end of their value, unless otherwise * noted. * * DTRACE_JSON_NUMBER, DTRACE_JSON_NUMBER_FRAC, DTRACE_JSON_NUMBER_EXP: * Processes a Number literal from the JSON, including any exponent * component that may be present. Numbers are returned as strings, which * may be passed to strtoll() if an integer is required. * * DTRACE_JSON_IDENTIFIER: * Processes a "true", "false" or "null" literal in the JSON. * * DTRACE_JSON_STRING, DTRACE_JSON_STRING_ESCAPE, * DTRACE_JSON_STRING_ESCAPE_UNICODE: * Processes a String literal from the JSON, whether the String denotes * a key, a value or part of a larger Object. Handles all escape sequences * present in the specification, including four-digit unicode characters, * but merely includes the escape sequence without converting it to the * actual escaped character. If the String is flagged as a key, we * move to DTRACE_JSON_COLON rather than DTRACE_JSON_COMMA. * * DTRACE_JSON_COLLECT_OBJECT: * This state collects an entire Object (or Array), correctly handling * embedded strings. If the full element selector list matches this nested * object, we return the Object in full as a string. If not, we use this * state to skip to the next value at this level and continue processing. * * NOTE: This function uses various macros from strtolctype.h to manipulate * digit values, etc -- these have all been checked to ensure they make * no additional function calls. */ static char * dtrace_json(uint64_t size, uintptr_t json, char *elemlist, int nelems, char *dest) { dtrace_json_state_t state = DTRACE_JSON_REST; int64_t array_elem = INT64_MIN; int64_t array_pos = 0; uint8_t escape_unicount = 0; boolean_t string_is_key = B_FALSE; boolean_t collect_object = B_FALSE; boolean_t found_key = B_FALSE; boolean_t in_array = B_FALSE; uint32_t braces = 0, brackets = 0; char *elem = elemlist; char *dd = dest; uintptr_t cur; for (cur = json; cur < json + size; cur++) { char cc = dtrace_load8(cur); if (cc == '\0') return (NULL); switch (state) { case DTRACE_JSON_REST: if (isspace(cc)) break; if (cc == '{') { state = DTRACE_JSON_OBJECT; break; } if (cc == '[') { in_array = B_TRUE; array_pos = 0; array_elem = dtrace_strtoll(elem, 10, size); found_key = array_elem == 0 ? B_TRUE : B_FALSE; state = DTRACE_JSON_VALUE; break; } /* * ERROR: expected to find a top-level object or array. */ return (NULL); case DTRACE_JSON_OBJECT: if (isspace(cc)) break; if (cc == '"') { state = DTRACE_JSON_STRING; string_is_key = B_TRUE; break; } /* * ERROR: either the object did not start with a key * string, or we've run off the end of the object * without finding the requested key. */ return (NULL); case DTRACE_JSON_STRING: if (cc == '\\') { *dd++ = '\\'; state = DTRACE_JSON_STRING_ESCAPE; break; } if (cc == '"') { if (collect_object) { /* * We don't reset the dest here, as * the string is part of a larger * object being collected. */ *dd++ = cc; collect_object = B_FALSE; state = DTRACE_JSON_COLLECT_OBJECT; break; } *dd = '\0'; dd = dest; /* reset string buffer */ if (string_is_key) { if (dtrace_strncmp(dest, elem, size) == 0) found_key = B_TRUE; } else if (found_key) { if (nelems > 1) { /* * We expected an object, not * this string. */ return (NULL); } return (dest); } state = string_is_key ? DTRACE_JSON_COLON : DTRACE_JSON_COMMA; string_is_key = B_FALSE; break; } *dd++ = cc; break; case DTRACE_JSON_STRING_ESCAPE: *dd++ = cc; if (cc == 'u') { escape_unicount = 0; state = DTRACE_JSON_STRING_ESCAPE_UNICODE; } else { state = DTRACE_JSON_STRING; } break; case DTRACE_JSON_STRING_ESCAPE_UNICODE: if (!isxdigit(cc)) { /* * ERROR: invalid unicode escape, expected * four valid hexidecimal digits. */ return (NULL); } *dd++ = cc; if (++escape_unicount == 4) state = DTRACE_JSON_STRING; break; case DTRACE_JSON_COLON: if (isspace(cc)) break; if (cc == ':') { state = DTRACE_JSON_VALUE; break; } /* * ERROR: expected a colon. */ return (NULL); case DTRACE_JSON_COMMA: if (isspace(cc)) break; if (cc == ',') { if (in_array) { state = DTRACE_JSON_VALUE; if (++array_pos == array_elem) found_key = B_TRUE; } else { state = DTRACE_JSON_OBJECT; } break; } /* * ERROR: either we hit an unexpected character, or * we reached the end of the object or array without * finding the requested key. */ return (NULL); case DTRACE_JSON_IDENTIFIER: if (islower(cc)) { *dd++ = cc; break; } *dd = '\0'; dd = dest; /* reset string buffer */ if (dtrace_strncmp(dest, "true", 5) == 0 || dtrace_strncmp(dest, "false", 6) == 0 || dtrace_strncmp(dest, "null", 5) == 0) { if (found_key) { if (nelems > 1) { /* * ERROR: We expected an object, * not this identifier. */ return (NULL); } return (dest); } else { cur--; state = DTRACE_JSON_COMMA; break; } } /* * ERROR: we did not recognise the identifier as one * of those in the JSON specification. */ return (NULL); case DTRACE_JSON_NUMBER: if (cc == '.') { *dd++ = cc; state = DTRACE_JSON_NUMBER_FRAC; break; } if (cc == 'x' || cc == 'X') { /* * ERROR: specification explicitly excludes * hexidecimal or octal numbers. */ return (NULL); } /* FALLTHRU */ case DTRACE_JSON_NUMBER_FRAC: if (cc == 'e' || cc == 'E') { *dd++ = cc; state = DTRACE_JSON_NUMBER_EXP; break; } if (cc == '+' || cc == '-') { /* * ERROR: expect sign as part of exponent only. */ return (NULL); } /* FALLTHRU */ case DTRACE_JSON_NUMBER_EXP: if (isdigit(cc) || cc == '+' || cc == '-') { *dd++ = cc; break; } *dd = '\0'; dd = dest; /* reset string buffer */ if (found_key) { if (nelems > 1) { /* * ERROR: We expected an object, not * this number. */ return (NULL); } return (dest); } cur--; state = DTRACE_JSON_COMMA; break; case DTRACE_JSON_VALUE: if (isspace(cc)) break; if (cc == '{' || cc == '[') { if (nelems > 1 && found_key) { in_array = cc == '[' ? B_TRUE : B_FALSE; /* * If our element selector directs us * to descend into this nested object, * then move to the next selector * element in the list and restart the * state machine. */ while (*elem != '\0') elem++; elem++; /* skip the inter-element NUL */ nelems--; dd = dest; if (in_array) { state = DTRACE_JSON_VALUE; array_pos = 0; array_elem = dtrace_strtoll( elem, 10, size); found_key = array_elem == 0 ? B_TRUE : B_FALSE; } else { found_key = B_FALSE; state = DTRACE_JSON_OBJECT; } break; } /* * Otherwise, we wish to either skip this * nested object or return it in full. */ if (cc == '[') brackets = 1; else braces = 1; *dd++ = cc; state = DTRACE_JSON_COLLECT_OBJECT; break; } if (cc == '"') { state = DTRACE_JSON_STRING; break; } if (islower(cc)) { /* * Here we deal with true, false and null. */ *dd++ = cc; state = DTRACE_JSON_IDENTIFIER; break; } if (cc == '-' || isdigit(cc)) { *dd++ = cc; state = DTRACE_JSON_NUMBER; break; } /* * ERROR: unexpected character at start of value. */ return (NULL); case DTRACE_JSON_COLLECT_OBJECT: if (cc == '\0') /* * ERROR: unexpected end of input. */ return (NULL); *dd++ = cc; if (cc == '"') { collect_object = B_TRUE; state = DTRACE_JSON_STRING; break; } if (cc == ']') { if (brackets-- == 0) { /* * ERROR: unbalanced brackets. */ return (NULL); } } else if (cc == '}') { if (braces-- == 0) { /* * ERROR: unbalanced braces. */ return (NULL); } } else if (cc == '{') { braces++; } else if (cc == '[') { brackets++; } if (brackets == 0 && braces == 0) { if (found_key) { *dd = '\0'; return (dest); } dd = dest; /* reset string buffer */ state = DTRACE_JSON_COMMA; } break; } } return (NULL); } /* * Emulate the execution of DTrace ID subroutines invoked by the call opcode. * Notice that we don't bother validating the proper number of arguments or * their types in the tuple stack. This isn't needed because all argument * interpretation is safe because of our load safety -- the worst that can * happen is that a bogus program can obtain bogus results. */ static void dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, dtrace_key_t *tupregs, int nargs, dtrace_mstate_t *mstate, dtrace_state_t *state) { volatile uint16_t *flags = &cpu_core[curcpu].cpuc_dtrace_flags; volatile uintptr_t *illval = &cpu_core[curcpu].cpuc_dtrace_illval; dtrace_vstate_t *vstate = &state->dts_vstate; #ifdef illumos union { mutex_impl_t mi; uint64_t mx; } m; union { krwlock_t ri; uintptr_t rw; } r; #else struct thread *lowner; union { struct lock_object *li; uintptr_t lx; } l; #endif switch (subr) { case DIF_SUBR_RAND: regs[rd] = (dtrace_gethrtime() * 2416 + 374441) % 1771875; break; #ifdef illumos case DIF_SUBR_MUTEX_OWNED: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t), mstate, vstate)) { regs[rd] = 0; break; } m.mx = dtrace_load64(tupregs[0].dttk_value); if (MUTEX_TYPE_ADAPTIVE(&m.mi)) regs[rd] = MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER; else regs[rd] = LOCK_HELD(&m.mi.m_spin.m_spinlock); break; case DIF_SUBR_MUTEX_OWNER: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t), mstate, vstate)) { regs[rd] = 0; break; } m.mx = dtrace_load64(tupregs[0].dttk_value); if (MUTEX_TYPE_ADAPTIVE(&m.mi) && MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER) regs[rd] = (uintptr_t)MUTEX_OWNER(&m.mi); else regs[rd] = 0; break; case DIF_SUBR_MUTEX_TYPE_ADAPTIVE: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t), mstate, vstate)) { regs[rd] = 0; break; } m.mx = dtrace_load64(tupregs[0].dttk_value); regs[rd] = MUTEX_TYPE_ADAPTIVE(&m.mi); break; case DIF_SUBR_MUTEX_TYPE_SPIN: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t), mstate, vstate)) { regs[rd] = 0; break; } m.mx = dtrace_load64(tupregs[0].dttk_value); regs[rd] = MUTEX_TYPE_SPIN(&m.mi); break; case DIF_SUBR_RW_READ_HELD: { uintptr_t tmp; if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t), mstate, vstate)) { regs[rd] = 0; break; } r.rw = dtrace_loadptr(tupregs[0].dttk_value); regs[rd] = _RW_READ_HELD(&r.ri, tmp); break; } case DIF_SUBR_RW_WRITE_HELD: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t), mstate, vstate)) { regs[rd] = 0; break; } r.rw = dtrace_loadptr(tupregs[0].dttk_value); regs[rd] = _RW_WRITE_HELD(&r.ri); break; case DIF_SUBR_RW_ISWRITER: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t), mstate, vstate)) { regs[rd] = 0; break; } r.rw = dtrace_loadptr(tupregs[0].dttk_value); regs[rd] = _RW_ISWRITER(&r.ri); break; #else /* !illumos */ case DIF_SUBR_MUTEX_OWNED: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (struct lock_object), mstate, vstate)) { regs[rd] = 0; break; } l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); regs[rd] = LOCK_CLASS(l.li)->lc_owner(l.li, &lowner); break; case DIF_SUBR_MUTEX_OWNER: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (struct lock_object), mstate, vstate)) { regs[rd] = 0; break; } l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); LOCK_CLASS(l.li)->lc_owner(l.li, &lowner); regs[rd] = (uintptr_t)lowner; break; case DIF_SUBR_MUTEX_TYPE_ADAPTIVE: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (struct mtx), mstate, vstate)) { regs[rd] = 0; break; } l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); /* XXX - should be only LC_SLEEPABLE? */ regs[rd] = (LOCK_CLASS(l.li)->lc_flags & (LC_SLEEPLOCK | LC_SLEEPABLE)) != 0; break; case DIF_SUBR_MUTEX_TYPE_SPIN: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (struct mtx), mstate, vstate)) { regs[rd] = 0; break; } l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); regs[rd] = (LOCK_CLASS(l.li)->lc_flags & LC_SPINLOCK) != 0; break; case DIF_SUBR_RW_READ_HELD: case DIF_SUBR_SX_SHARED_HELD: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t), mstate, vstate)) { regs[rd] = 0; break; } l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); regs[rd] = LOCK_CLASS(l.li)->lc_owner(l.li, &lowner) && lowner == NULL; break; case DIF_SUBR_RW_WRITE_HELD: case DIF_SUBR_SX_EXCLUSIVE_HELD: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t), mstate, vstate)) { regs[rd] = 0; break; } l.lx = dtrace_loadptr(tupregs[0].dttk_value); LOCK_CLASS(l.li)->lc_owner(l.li, &lowner); regs[rd] = (lowner == curthread); break; case DIF_SUBR_RW_ISWRITER: case DIF_SUBR_SX_ISEXCLUSIVE: if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t), mstate, vstate)) { regs[rd] = 0; break; } l.lx = dtrace_loadptr(tupregs[0].dttk_value); regs[rd] = LOCK_CLASS(l.li)->lc_owner(l.li, &lowner) && lowner != NULL; break; #endif /* illumos */ case DIF_SUBR_BCOPY: { /* * We need to be sure that the destination is in the scratch * region -- no other region is allowed. */ uintptr_t src = tupregs[0].dttk_value; uintptr_t dest = tupregs[1].dttk_value; size_t size = tupregs[2].dttk_value; if (!dtrace_inscratch(dest, size, mstate)) { *flags |= CPU_DTRACE_BADADDR; *illval = regs[rd]; break; } if (!dtrace_canload(src, size, mstate, vstate)) { regs[rd] = 0; break; } dtrace_bcopy((void *)src, (void *)dest, size); break; } case DIF_SUBR_ALLOCA: case DIF_SUBR_COPYIN: { uintptr_t dest = P2ROUNDUP(mstate->dtms_scratch_ptr, 8); uint64_t size = tupregs[subr == DIF_SUBR_ALLOCA ? 0 : 1].dttk_value; size_t scratch_size = (dest - mstate->dtms_scratch_ptr) + size; /* * This action doesn't require any credential checks since * probes will not activate in user contexts to which the * enabling user does not have permissions. */ /* * Rounding up the user allocation size could have overflowed * a large, bogus allocation (like -1ULL) to 0. */ if (scratch_size < size || !DTRACE_INSCRATCH(mstate, scratch_size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } if (subr == DIF_SUBR_COPYIN) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); dtrace_copyin(tupregs[0].dttk_value, dest, size, flags); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); } mstate->dtms_scratch_ptr += scratch_size; regs[rd] = dest; break; } case DIF_SUBR_COPYINTO: { uint64_t size = tupregs[1].dttk_value; uintptr_t dest = tupregs[2].dttk_value; /* * This action doesn't require any credential checks since * probes will not activate in user contexts to which the * enabling user does not have permissions. */ if (!dtrace_inscratch(dest, size, mstate)) { *flags |= CPU_DTRACE_BADADDR; *illval = regs[rd]; break; } DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); dtrace_copyin(tupregs[0].dttk_value, dest, size, flags); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; } case DIF_SUBR_COPYINSTR: { uintptr_t dest = mstate->dtms_scratch_ptr; uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; if (nargs > 1 && tupregs[1].dttk_value < size) size = tupregs[1].dttk_value + 1; /* * This action doesn't require any credential checks since * probes will not activate in user contexts to which the * enabling user does not have permissions. */ if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); dtrace_copyinstr(tupregs[0].dttk_value, dest, size, flags); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); ((char *)dest)[size - 1] = '\0'; mstate->dtms_scratch_ptr += size; regs[rd] = dest; break; } #ifdef illumos case DIF_SUBR_MSGSIZE: case DIF_SUBR_MSGDSIZE: { uintptr_t baddr = tupregs[0].dttk_value, daddr; uintptr_t wptr, rptr; size_t count = 0; int cont = 0; while (baddr != 0 && !(*flags & CPU_DTRACE_FAULT)) { if (!dtrace_canload(baddr, sizeof (mblk_t), mstate, vstate)) { regs[rd] = 0; break; } wptr = dtrace_loadptr(baddr + offsetof(mblk_t, b_wptr)); rptr = dtrace_loadptr(baddr + offsetof(mblk_t, b_rptr)); if (wptr < rptr) { *flags |= CPU_DTRACE_BADADDR; *illval = tupregs[0].dttk_value; break; } daddr = dtrace_loadptr(baddr + offsetof(mblk_t, b_datap)); baddr = dtrace_loadptr(baddr + offsetof(mblk_t, b_cont)); /* * We want to prevent against denial-of-service here, * so we're only going to search the list for * dtrace_msgdsize_max mblks. */ if (cont++ > dtrace_msgdsize_max) { *flags |= CPU_DTRACE_ILLOP; break; } if (subr == DIF_SUBR_MSGDSIZE) { if (dtrace_load8(daddr + offsetof(dblk_t, db_type)) != M_DATA) continue; } count += wptr - rptr; } if (!(*flags & CPU_DTRACE_FAULT)) regs[rd] = count; break; } #endif case DIF_SUBR_PROGENYOF: { pid_t pid = tupregs[0].dttk_value; proc_t *p; int rval = 0; DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); for (p = curthread->t_procp; p != NULL; p = p->p_parent) { #ifdef illumos if (p->p_pidp->pid_id == pid) { #else if (p->p_pid == pid) { #endif rval = 1; break; } } DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); regs[rd] = rval; break; } case DIF_SUBR_SPECULATION: regs[rd] = dtrace_speculation(state); break; case DIF_SUBR_COPYOUT: { uintptr_t kaddr = tupregs[0].dttk_value; uintptr_t uaddr = tupregs[1].dttk_value; uint64_t size = tupregs[2].dttk_value; if (!dtrace_destructive_disallow && dtrace_priv_proc_control(state) && !dtrace_istoxic(kaddr, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); dtrace_copyout(kaddr, uaddr, size, flags); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); } break; } case DIF_SUBR_COPYOUTSTR: { uintptr_t kaddr = tupregs[0].dttk_value; uintptr_t uaddr = tupregs[1].dttk_value; uint64_t size = tupregs[2].dttk_value; if (!dtrace_destructive_disallow && dtrace_priv_proc_control(state) && !dtrace_istoxic(kaddr, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); dtrace_copyoutstr(kaddr, uaddr, size, flags); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); } break; } case DIF_SUBR_STRLEN: { size_t sz; uintptr_t addr = (uintptr_t)tupregs[0].dttk_value; sz = dtrace_strlen((char *)addr, state->dts_options[DTRACEOPT_STRSIZE]); if (!dtrace_canload(addr, sz + 1, mstate, vstate)) { regs[rd] = 0; break; } regs[rd] = sz; break; } case DIF_SUBR_STRCHR: case DIF_SUBR_STRRCHR: { /* * We're going to iterate over the string looking for the * specified character. We will iterate until we have reached * the string length or we have found the character. If this * is DIF_SUBR_STRRCHR, we will look for the last occurrence * of the specified character instead of the first. */ uintptr_t saddr = tupregs[0].dttk_value; uintptr_t addr = tupregs[0].dttk_value; uintptr_t limit = addr + state->dts_options[DTRACEOPT_STRSIZE]; char c, target = (char)tupregs[1].dttk_value; for (regs[rd] = 0; addr < limit; addr++) { if ((c = dtrace_load8(addr)) == target) { regs[rd] = addr; if (subr == DIF_SUBR_STRCHR) break; } if (c == '\0') break; } if (!dtrace_canload(saddr, addr - saddr, mstate, vstate)) { regs[rd] = 0; break; } break; } case DIF_SUBR_STRSTR: case DIF_SUBR_INDEX: case DIF_SUBR_RINDEX: { /* * We're going to iterate over the string looking for the * specified string. We will iterate until we have reached * the string length or we have found the string. (Yes, this * is done in the most naive way possible -- but considering * that the string we're searching for is likely to be * relatively short, the complexity of Rabin-Karp or similar * hardly seems merited.) */ char *addr = (char *)(uintptr_t)tupregs[0].dttk_value; char *substr = (char *)(uintptr_t)tupregs[1].dttk_value; uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; size_t len = dtrace_strlen(addr, size); size_t sublen = dtrace_strlen(substr, size); char *limit = addr + len, *orig = addr; int notfound = subr == DIF_SUBR_STRSTR ? 0 : -1; int inc = 1; regs[rd] = notfound; if (!dtrace_canload((uintptr_t)addr, len + 1, mstate, vstate)) { regs[rd] = 0; break; } if (!dtrace_canload((uintptr_t)substr, sublen + 1, mstate, vstate)) { regs[rd] = 0; break; } /* * strstr() and index()/rindex() have similar semantics if * both strings are the empty string: strstr() returns a * pointer to the (empty) string, and index() and rindex() * both return index 0 (regardless of any position argument). */ if (sublen == 0 && len == 0) { if (subr == DIF_SUBR_STRSTR) regs[rd] = (uintptr_t)addr; else regs[rd] = 0; break; } if (subr != DIF_SUBR_STRSTR) { if (subr == DIF_SUBR_RINDEX) { limit = orig - 1; addr += len; inc = -1; } /* * Both index() and rindex() take an optional position * argument that denotes the starting position. */ if (nargs == 3) { int64_t pos = (int64_t)tupregs[2].dttk_value; /* * If the position argument to index() is * negative, Perl implicitly clamps it at * zero. This semantic is a little surprising * given the special meaning of negative * positions to similar Perl functions like * substr(), but it appears to reflect a * notion that index() can start from a * negative index and increment its way up to * the string. Given this notion, Perl's * rindex() is at least self-consistent in * that it implicitly clamps positions greater * than the string length to be the string * length. Where Perl completely loses * coherence, however, is when the specified * substring is the empty string (""). In * this case, even if the position is * negative, rindex() returns 0 -- and even if * the position is greater than the length, * index() returns the string length. These * semantics violate the notion that index() * should never return a value less than the * specified position and that rindex() should * never return a value greater than the * specified position. (One assumes that * these semantics are artifacts of Perl's * implementation and not the results of * deliberate design -- it beggars belief that * even Larry Wall could desire such oddness.) * While in the abstract one would wish for * consistent position semantics across * substr(), index() and rindex() -- or at the * very least self-consistent position * semantics for index() and rindex() -- we * instead opt to keep with the extant Perl * semantics, in all their broken glory. (Do * we have more desire to maintain Perl's * semantics than Perl does? Probably.) */ if (subr == DIF_SUBR_RINDEX) { if (pos < 0) { if (sublen == 0) regs[rd] = 0; break; } if (pos > len) pos = len; } else { if (pos < 0) pos = 0; if (pos >= len) { if (sublen == 0) regs[rd] = len; break; } } addr = orig + pos; } } for (regs[rd] = notfound; addr != limit; addr += inc) { if (dtrace_strncmp(addr, substr, sublen) == 0) { if (subr != DIF_SUBR_STRSTR) { /* * As D index() and rindex() are * modeled on Perl (and not on awk), * we return a zero-based (and not a * one-based) index. (For you Perl * weenies: no, we're not going to add * $[ -- and shouldn't you be at a con * or something?) */ regs[rd] = (uintptr_t)(addr - orig); break; } ASSERT(subr == DIF_SUBR_STRSTR); regs[rd] = (uintptr_t)addr; break; } } break; } case DIF_SUBR_STRTOK: { uintptr_t addr = tupregs[0].dttk_value; uintptr_t tokaddr = tupregs[1].dttk_value; uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; uintptr_t limit, toklimit = tokaddr + size; uint8_t c = 0, tokmap[32]; /* 256 / 8 */ char *dest = (char *)mstate->dtms_scratch_ptr; int i; /* * Check both the token buffer and (later) the input buffer, * since both could be non-scratch addresses. */ if (!dtrace_strcanload(tokaddr, size, mstate, vstate)) { regs[rd] = 0; break; } if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } if (addr == 0) { /* * If the address specified is NULL, we use our saved * strtok pointer from the mstate. Note that this * means that the saved strtok pointer is _only_ * valid within multiple enablings of the same probe -- * it behaves like an implicit clause-local variable. */ addr = mstate->dtms_strtok; } else { /* * If the user-specified address is non-NULL we must * access check it. This is the only time we have * a chance to do so, since this address may reside * in the string table of this clause-- future calls * (when we fetch addr from mstate->dtms_strtok) * would fail this access check. */ if (!dtrace_strcanload(addr, size, mstate, vstate)) { regs[rd] = 0; break; } } /* * First, zero the token map, and then process the token * string -- setting a bit in the map for every character * found in the token string. */ for (i = 0; i < sizeof (tokmap); i++) tokmap[i] = 0; for (; tokaddr < toklimit; tokaddr++) { if ((c = dtrace_load8(tokaddr)) == '\0') break; ASSERT((c >> 3) < sizeof (tokmap)); tokmap[c >> 3] |= (1 << (c & 0x7)); } for (limit = addr + size; addr < limit; addr++) { /* * We're looking for a character that is _not_ contained * in the token string. */ if ((c = dtrace_load8(addr)) == '\0') break; if (!(tokmap[c >> 3] & (1 << (c & 0x7)))) break; } if (c == '\0') { /* * We reached the end of the string without finding * any character that was not in the token string. * We return NULL in this case, and we set the saved * address to NULL as well. */ regs[rd] = 0; mstate->dtms_strtok = 0; break; } /* * From here on, we're copying into the destination string. */ for (i = 0; addr < limit && i < size - 1; addr++) { if ((c = dtrace_load8(addr)) == '\0') break; if (tokmap[c >> 3] & (1 << (c & 0x7))) break; ASSERT(i < size); dest[i++] = c; } ASSERT(i < size); dest[i] = '\0'; regs[rd] = (uintptr_t)dest; mstate->dtms_scratch_ptr += size; mstate->dtms_strtok = addr; break; } case DIF_SUBR_SUBSTR: { uintptr_t s = tupregs[0].dttk_value; uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; char *d = (char *)mstate->dtms_scratch_ptr; int64_t index = (int64_t)tupregs[1].dttk_value; int64_t remaining = (int64_t)tupregs[2].dttk_value; size_t len = dtrace_strlen((char *)s, size); int64_t i; if (!dtrace_canload(s, len + 1, mstate, vstate)) { regs[rd] = 0; break; } if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } if (nargs <= 2) remaining = (int64_t)size; if (index < 0) { index += len; if (index < 0 && index + remaining > 0) { remaining += index; index = 0; } } if (index >= len || index < 0) { remaining = 0; } else if (remaining < 0) { remaining += len - index; } else if (index + remaining > size) { remaining = size - index; } for (i = 0; i < remaining; i++) { if ((d[i] = dtrace_load8(s + index + i)) == '\0') break; } d[i] = '\0'; mstate->dtms_scratch_ptr += size; regs[rd] = (uintptr_t)d; break; } case DIF_SUBR_JSON: { uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; uintptr_t json = tupregs[0].dttk_value; size_t jsonlen = dtrace_strlen((char *)json, size); uintptr_t elem = tupregs[1].dttk_value; size_t elemlen = dtrace_strlen((char *)elem, size); char *dest = (char *)mstate->dtms_scratch_ptr; char *elemlist = (char *)mstate->dtms_scratch_ptr + jsonlen + 1; char *ee = elemlist; int nelems = 1; uintptr_t cur; if (!dtrace_canload(json, jsonlen + 1, mstate, vstate) || !dtrace_canload(elem, elemlen + 1, mstate, vstate)) { regs[rd] = 0; break; } if (!DTRACE_INSCRATCH(mstate, jsonlen + 1 + elemlen + 1)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } /* * Read the element selector and split it up into a packed list * of strings. */ for (cur = elem; cur < elem + elemlen; cur++) { char cc = dtrace_load8(cur); if (cur == elem && cc == '[') { /* * If the first element selector key is * actually an array index then ignore the * bracket. */ continue; } if (cc == ']') continue; if (cc == '.' || cc == '[') { nelems++; cc = '\0'; } *ee++ = cc; } *ee++ = '\0'; if ((regs[rd] = (uintptr_t)dtrace_json(size, json, elemlist, nelems, dest)) != 0) mstate->dtms_scratch_ptr += jsonlen + 1; break; } case DIF_SUBR_TOUPPER: case DIF_SUBR_TOLOWER: { uintptr_t s = tupregs[0].dttk_value; uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; char *dest = (char *)mstate->dtms_scratch_ptr, c; size_t len = dtrace_strlen((char *)s, size); char lower, upper, convert; int64_t i; if (subr == DIF_SUBR_TOUPPER) { lower = 'a'; upper = 'z'; convert = 'A'; } else { lower = 'A'; upper = 'Z'; convert = 'a'; } if (!dtrace_canload(s, len + 1, mstate, vstate)) { regs[rd] = 0; break; } if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } for (i = 0; i < size - 1; i++) { if ((c = dtrace_load8(s + i)) == '\0') break; if (c >= lower && c <= upper) c = convert + (c - lower); dest[i] = c; } ASSERT(i < size); dest[i] = '\0'; regs[rd] = (uintptr_t)dest; mstate->dtms_scratch_ptr += size; break; } #ifdef illumos case DIF_SUBR_GETMAJOR: #ifdef _LP64 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR64) & MAXMAJ64; #else regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR) & MAXMAJ; #endif break; case DIF_SUBR_GETMINOR: #ifdef _LP64 regs[rd] = tupregs[0].dttk_value & MAXMIN64; #else regs[rd] = tupregs[0].dttk_value & MAXMIN; #endif break; case DIF_SUBR_DDI_PATHNAME: { /* * This one is a galactic mess. We are going to roughly * emulate ddi_pathname(), but it's made more complicated * by the fact that we (a) want to include the minor name and * (b) must proceed iteratively instead of recursively. */ uintptr_t dest = mstate->dtms_scratch_ptr; uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; char *start = (char *)dest, *end = start + size - 1; uintptr_t daddr = tupregs[0].dttk_value; int64_t minor = (int64_t)tupregs[1].dttk_value; char *s; int i, len, depth = 0; /* * Due to all the pointer jumping we do and context we must * rely upon, we just mandate that the user must have kernel * read privileges to use this routine. */ if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) == 0) { *flags |= CPU_DTRACE_KPRIV; *illval = daddr; regs[rd] = 0; } if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } *end = '\0'; /* * We want to have a name for the minor. In order to do this, * we need to walk the minor list from the devinfo. We want * to be sure that we don't infinitely walk a circular list, * so we check for circularity by sending a scout pointer * ahead two elements for every element that we iterate over; * if the list is circular, these will ultimately point to the * same element. You may recognize this little trick as the * answer to a stupid interview question -- one that always * seems to be asked by those who had to have it laboriously * explained to them, and who can't even concisely describe * the conditions under which one would be forced to resort to * this technique. Needless to say, those conditions are * found here -- and probably only here. Is this the only use * of this infamous trick in shipping, production code? If it * isn't, it probably should be... */ if (minor != -1) { uintptr_t maddr = dtrace_loadptr(daddr + offsetof(struct dev_info, devi_minor)); uintptr_t next = offsetof(struct ddi_minor_data, next); uintptr_t name = offsetof(struct ddi_minor_data, d_minor) + offsetof(struct ddi_minor, name); uintptr_t dev = offsetof(struct ddi_minor_data, d_minor) + offsetof(struct ddi_minor, dev); uintptr_t scout; if (maddr != NULL) scout = dtrace_loadptr(maddr + next); while (maddr != NULL && !(*flags & CPU_DTRACE_FAULT)) { uint64_t m; #ifdef _LP64 m = dtrace_load64(maddr + dev) & MAXMIN64; #else m = dtrace_load32(maddr + dev) & MAXMIN; #endif if (m != minor) { maddr = dtrace_loadptr(maddr + next); if (scout == NULL) continue; scout = dtrace_loadptr(scout + next); if (scout == NULL) continue; scout = dtrace_loadptr(scout + next); if (scout == NULL) continue; if (scout == maddr) { *flags |= CPU_DTRACE_ILLOP; break; } continue; } /* * We have the minor data. Now we need to * copy the minor's name into the end of the * pathname. */ s = (char *)dtrace_loadptr(maddr + name); len = dtrace_strlen(s, size); if (*flags & CPU_DTRACE_FAULT) break; if (len != 0) { if ((end -= (len + 1)) < start) break; *end = ':'; } for (i = 1; i <= len; i++) end[i] = dtrace_load8((uintptr_t)s++); break; } } while (daddr != NULL && !(*flags & CPU_DTRACE_FAULT)) { ddi_node_state_t devi_state; devi_state = dtrace_load32(daddr + offsetof(struct dev_info, devi_node_state)); if (*flags & CPU_DTRACE_FAULT) break; if (devi_state >= DS_INITIALIZED) { s = (char *)dtrace_loadptr(daddr + offsetof(struct dev_info, devi_addr)); len = dtrace_strlen(s, size); if (*flags & CPU_DTRACE_FAULT) break; if (len != 0) { if ((end -= (len + 1)) < start) break; *end = '@'; } for (i = 1; i <= len; i++) end[i] = dtrace_load8((uintptr_t)s++); } /* * Now for the node name... */ s = (char *)dtrace_loadptr(daddr + offsetof(struct dev_info, devi_node_name)); daddr = dtrace_loadptr(daddr + offsetof(struct dev_info, devi_parent)); /* * If our parent is NULL (that is, if we're the root * node), we're going to use the special path * "devices". */ if (daddr == 0) s = "devices"; len = dtrace_strlen(s, size); if (*flags & CPU_DTRACE_FAULT) break; if ((end -= (len + 1)) < start) break; for (i = 1; i <= len; i++) end[i] = dtrace_load8((uintptr_t)s++); *end = '/'; if (depth++ > dtrace_devdepth_max) { *flags |= CPU_DTRACE_ILLOP; break; } } if (end < start) DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); if (daddr == 0) { regs[rd] = (uintptr_t)end; mstate->dtms_scratch_ptr += size; } break; } #endif case DIF_SUBR_STRJOIN: { char *d = (char *)mstate->dtms_scratch_ptr; uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; uintptr_t s1 = tupregs[0].dttk_value; uintptr_t s2 = tupregs[1].dttk_value; int i = 0; if (!dtrace_strcanload(s1, size, mstate, vstate) || !dtrace_strcanload(s2, size, mstate, vstate)) { regs[rd] = 0; break; } if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } for (;;) { if (i >= size) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } if ((d[i++] = dtrace_load8(s1++)) == '\0') { i--; break; } } for (;;) { if (i >= size) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } if ((d[i++] = dtrace_load8(s2++)) == '\0') break; } if (i < size) { mstate->dtms_scratch_ptr += i; regs[rd] = (uintptr_t)d; } break; } case DIF_SUBR_STRTOLL: { uintptr_t s = tupregs[0].dttk_value; uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; int base = 10; if (nargs > 1) { if ((base = tupregs[1].dttk_value) <= 1 || base > ('z' - 'a' + 1) + ('9' - '0' + 1)) { *flags |= CPU_DTRACE_ILLOP; break; } } if (!dtrace_strcanload(s, size, mstate, vstate)) { regs[rd] = INT64_MIN; break; } regs[rd] = dtrace_strtoll((char *)s, base, size); break; } case DIF_SUBR_LLTOSTR: { int64_t i = (int64_t)tupregs[0].dttk_value; uint64_t val, digit; uint64_t size = 65; /* enough room for 2^64 in binary */ char *end = (char *)mstate->dtms_scratch_ptr + size - 1; int base = 10; if (nargs > 1) { if ((base = tupregs[1].dttk_value) <= 1 || base > ('z' - 'a' + 1) + ('9' - '0' + 1)) { *flags |= CPU_DTRACE_ILLOP; break; } } val = (base == 10 && i < 0) ? i * -1 : i; if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } for (*end-- = '\0'; val; val /= base) { if ((digit = val % base) <= '9' - '0') { *end-- = '0' + digit; } else { *end-- = 'a' + (digit - ('9' - '0') - 1); } } if (i == 0 && base == 16) *end-- = '0'; if (base == 16) *end-- = 'x'; if (i == 0 || base == 8 || base == 16) *end-- = '0'; if (i < 0 && base == 10) *end-- = '-'; regs[rd] = (uintptr_t)end + 1; mstate->dtms_scratch_ptr += size; break; } case DIF_SUBR_HTONS: case DIF_SUBR_NTOHS: #if BYTE_ORDER == BIG_ENDIAN regs[rd] = (uint16_t)tupregs[0].dttk_value; #else regs[rd] = DT_BSWAP_16((uint16_t)tupregs[0].dttk_value); #endif break; case DIF_SUBR_HTONL: case DIF_SUBR_NTOHL: #if BYTE_ORDER == BIG_ENDIAN regs[rd] = (uint32_t)tupregs[0].dttk_value; #else regs[rd] = DT_BSWAP_32((uint32_t)tupregs[0].dttk_value); #endif break; case DIF_SUBR_HTONLL: case DIF_SUBR_NTOHLL: #if BYTE_ORDER == BIG_ENDIAN regs[rd] = (uint64_t)tupregs[0].dttk_value; #else regs[rd] = DT_BSWAP_64((uint64_t)tupregs[0].dttk_value); #endif break; case DIF_SUBR_DIRNAME: case DIF_SUBR_BASENAME: { char *dest = (char *)mstate->dtms_scratch_ptr; uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; uintptr_t src = tupregs[0].dttk_value; int i, j, len = dtrace_strlen((char *)src, size); int lastbase = -1, firstbase = -1, lastdir = -1; int start, end; if (!dtrace_canload(src, len + 1, mstate, vstate)) { regs[rd] = 0; break; } if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } /* * The basename and dirname for a zero-length string is * defined to be "." */ if (len == 0) { len = 1; src = (uintptr_t)"."; } /* * Start from the back of the string, moving back toward the * front until we see a character that isn't a slash. That * character is the last character in the basename. */ for (i = len - 1; i >= 0; i--) { if (dtrace_load8(src + i) != '/') break; } if (i >= 0) lastbase = i; /* * Starting from the last character in the basename, move * towards the front until we find a slash. The character * that we processed immediately before that is the first * character in the basename. */ for (; i >= 0; i--) { if (dtrace_load8(src + i) == '/') break; } if (i >= 0) firstbase = i + 1; /* * Now keep going until we find a non-slash character. That * character is the last character in the dirname. */ for (; i >= 0; i--) { if (dtrace_load8(src + i) != '/') break; } if (i >= 0) lastdir = i; ASSERT(!(lastbase == -1 && firstbase != -1)); ASSERT(!(firstbase == -1 && lastdir != -1)); if (lastbase == -1) { /* * We didn't find a non-slash character. We know that * the length is non-zero, so the whole string must be * slashes. In either the dirname or the basename * case, we return '/'. */ ASSERT(firstbase == -1); firstbase = lastbase = lastdir = 0; } if (firstbase == -1) { /* * The entire string consists only of a basename * component. If we're looking for dirname, we need * to change our string to be just "."; if we're * looking for a basename, we'll just set the first * character of the basename to be 0. */ if (subr == DIF_SUBR_DIRNAME) { ASSERT(lastdir == -1); src = (uintptr_t)"."; lastdir = 0; } else { firstbase = 0; } } if (subr == DIF_SUBR_DIRNAME) { if (lastdir == -1) { /* * We know that we have a slash in the name -- * or lastdir would be set to 0, above. And * because lastdir is -1, we know that this * slash must be the first character. (That * is, the full string must be of the form * "/basename".) In this case, the last * character of the directory name is 0. */ lastdir = 0; } start = 0; end = lastdir; } else { ASSERT(subr == DIF_SUBR_BASENAME); ASSERT(firstbase != -1 && lastbase != -1); start = firstbase; end = lastbase; } for (i = start, j = 0; i <= end && j < size - 1; i++, j++) dest[j] = dtrace_load8(src + i); dest[j] = '\0'; regs[rd] = (uintptr_t)dest; mstate->dtms_scratch_ptr += size; break; } case DIF_SUBR_GETF: { uintptr_t fd = tupregs[0].dttk_value; struct filedesc *fdp; file_t *fp; if (!dtrace_priv_proc(state)) { regs[rd] = 0; break; } fdp = curproc->p_fd; FILEDESC_SLOCK(fdp); fp = fget_locked(fdp, fd); mstate->dtms_getf = fp; regs[rd] = (uintptr_t)fp; FILEDESC_SUNLOCK(fdp); break; } case DIF_SUBR_CLEANPATH: { char *dest = (char *)mstate->dtms_scratch_ptr, c; uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; uintptr_t src = tupregs[0].dttk_value; int i = 0, j = 0; #ifdef illumos zone_t *z; #endif if (!dtrace_strcanload(src, size, mstate, vstate)) { regs[rd] = 0; break; } if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } /* * Move forward, loading each character. */ do { c = dtrace_load8(src + i++); next: if (j + 5 >= size) /* 5 = strlen("/..c\0") */ break; if (c != '/') { dest[j++] = c; continue; } c = dtrace_load8(src + i++); if (c == '/') { /* * We have two slashes -- we can just advance * to the next character. */ goto next; } if (c != '.') { /* * This is not "." and it's not ".." -- we can * just store the "/" and this character and * drive on. */ dest[j++] = '/'; dest[j++] = c; continue; } c = dtrace_load8(src + i++); if (c == '/') { /* * This is a "/./" component. We're not going * to store anything in the destination buffer; * we're just going to go to the next component. */ goto next; } if (c != '.') { /* * This is not ".." -- we can just store the * "/." and this character and continue * processing. */ dest[j++] = '/'; dest[j++] = '.'; dest[j++] = c; continue; } c = dtrace_load8(src + i++); if (c != '/' && c != '\0') { /* * This is not ".." -- it's "..[mumble]". * We'll store the "/.." and this character * and continue processing. */ dest[j++] = '/'; dest[j++] = '.'; dest[j++] = '.'; dest[j++] = c; continue; } /* * This is "/../" or "/..\0". We need to back up * our destination pointer until we find a "/". */ i--; while (j != 0 && dest[--j] != '/') continue; if (c == '\0') dest[++j] = '/'; } while (c != '\0'); dest[j] = '\0'; #ifdef illumos if (mstate->dtms_getf != NULL && !(mstate->dtms_access & DTRACE_ACCESS_KERNEL) && (z = state->dts_cred.dcr_cred->cr_zone) != kcred->cr_zone) { /* * If we've done a getf() as a part of this ECB and we * don't have kernel access (and we're not in the global * zone), check if the path we cleaned up begins with * the zone's root path, and trim it off if so. Note * that this is an output cleanliness issue, not a * security issue: knowing one's zone root path does * not enable privilege escalation. */ if (strstr(dest, z->zone_rootpath) == dest) dest += strlen(z->zone_rootpath) - 1; } #endif regs[rd] = (uintptr_t)dest; mstate->dtms_scratch_ptr += size; break; } case DIF_SUBR_INET_NTOA: case DIF_SUBR_INET_NTOA6: case DIF_SUBR_INET_NTOP: { size_t size; int af, argi, i; char *base, *end; if (subr == DIF_SUBR_INET_NTOP) { af = (int)tupregs[0].dttk_value; argi = 1; } else { af = subr == DIF_SUBR_INET_NTOA ? AF_INET: AF_INET6; argi = 0; } if (af == AF_INET) { ipaddr_t ip4; uint8_t *ptr8, val; /* * Safely load the IPv4 address. */ ip4 = dtrace_load32(tupregs[argi].dttk_value); /* * Check an IPv4 string will fit in scratch. */ size = INET_ADDRSTRLEN; if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } base = (char *)mstate->dtms_scratch_ptr; end = (char *)mstate->dtms_scratch_ptr + size - 1; /* * Stringify as a dotted decimal quad. */ *end-- = '\0'; ptr8 = (uint8_t *)&ip4; for (i = 3; i >= 0; i--) { val = ptr8[i]; if (val == 0) { *end-- = '0'; } else { for (; val; val /= 10) { *end-- = '0' + (val % 10); } } if (i > 0) *end-- = '.'; } ASSERT(end + 1 >= base); } else if (af == AF_INET6) { struct in6_addr ip6; int firstzero, tryzero, numzero, v6end; uint16_t val; const char digits[] = "0123456789abcdef"; /* * Stringify using RFC 1884 convention 2 - 16 bit * hexadecimal values with a zero-run compression. * Lower case hexadecimal digits are used. * eg, fe80::214:4fff:fe0b:76c8. * The IPv4 embedded form is returned for inet_ntop, * just the IPv4 string is returned for inet_ntoa6. */ /* * Safely load the IPv6 address. */ dtrace_bcopy( (void *)(uintptr_t)tupregs[argi].dttk_value, (void *)(uintptr_t)&ip6, sizeof (struct in6_addr)); /* * Check an IPv6 string will fit in scratch. */ size = INET6_ADDRSTRLEN; if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } base = (char *)mstate->dtms_scratch_ptr; end = (char *)mstate->dtms_scratch_ptr + size - 1; *end-- = '\0'; /* * Find the longest run of 16 bit zero values * for the single allowed zero compression - "::". */ firstzero = -1; tryzero = -1; numzero = 1; for (i = 0; i < sizeof (struct in6_addr); i++) { #ifdef illumos if (ip6._S6_un._S6_u8[i] == 0 && #else if (ip6.__u6_addr.__u6_addr8[i] == 0 && #endif tryzero == -1 && i % 2 == 0) { tryzero = i; continue; } if (tryzero != -1 && #ifdef illumos (ip6._S6_un._S6_u8[i] != 0 || #else (ip6.__u6_addr.__u6_addr8[i] != 0 || #endif i == sizeof (struct in6_addr) - 1)) { if (i - tryzero <= numzero) { tryzero = -1; continue; } firstzero = tryzero; numzero = i - i % 2 - tryzero; tryzero = -1; #ifdef illumos if (ip6._S6_un._S6_u8[i] == 0 && #else if (ip6.__u6_addr.__u6_addr8[i] == 0 && #endif i == sizeof (struct in6_addr) - 1) numzero += 2; } } ASSERT(firstzero + numzero <= sizeof (struct in6_addr)); /* * Check for an IPv4 embedded address. */ v6end = sizeof (struct in6_addr) - 2; if (IN6_IS_ADDR_V4MAPPED(&ip6) || IN6_IS_ADDR_V4COMPAT(&ip6)) { for (i = sizeof (struct in6_addr) - 1; i >= DTRACE_V4MAPPED_OFFSET; i--) { ASSERT(end >= base); #ifdef illumos val = ip6._S6_un._S6_u8[i]; #else val = ip6.__u6_addr.__u6_addr8[i]; #endif if (val == 0) { *end-- = '0'; } else { for (; val; val /= 10) { *end-- = '0' + val % 10; } } if (i > DTRACE_V4MAPPED_OFFSET) *end-- = '.'; } if (subr == DIF_SUBR_INET_NTOA6) goto inetout; /* * Set v6end to skip the IPv4 address that * we have already stringified. */ v6end = 10; } /* * Build the IPv6 string by working through the * address in reverse. */ for (i = v6end; i >= 0; i -= 2) { ASSERT(end >= base); if (i == firstzero + numzero - 2) { *end-- = ':'; *end-- = ':'; i -= numzero - 2; continue; } if (i < 14 && i != firstzero - 2) *end-- = ':'; #ifdef illumos val = (ip6._S6_un._S6_u8[i] << 8) + ip6._S6_un._S6_u8[i + 1]; #else val = (ip6.__u6_addr.__u6_addr8[i] << 8) + ip6.__u6_addr.__u6_addr8[i + 1]; #endif if (val == 0) { *end-- = '0'; } else { for (; val; val /= 16) { *end-- = digits[val % 16]; } } } ASSERT(end + 1 >= base); } else { /* * The user didn't use AH_INET or AH_INET6. */ DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); regs[rd] = 0; break; } inetout: regs[rd] = (uintptr_t)end + 1; mstate->dtms_scratch_ptr += size; break; } case DIF_SUBR_MEMREF: { uintptr_t size = 2 * sizeof(uintptr_t); uintptr_t *memref = (uintptr_t *) P2ROUNDUP(mstate->dtms_scratch_ptr, sizeof(uintptr_t)); size_t scratch_size = ((uintptr_t) memref - mstate->dtms_scratch_ptr) + size; /* address and length */ memref[0] = tupregs[0].dttk_value; memref[1] = tupregs[1].dttk_value; regs[rd] = (uintptr_t) memref; mstate->dtms_scratch_ptr += scratch_size; break; } #ifndef illumos case DIF_SUBR_MEMSTR: { char *str = (char *)mstate->dtms_scratch_ptr; uintptr_t mem = tupregs[0].dttk_value; char c = tupregs[1].dttk_value; size_t size = tupregs[2].dttk_value; uint8_t n; int i; regs[rd] = 0; if (size == 0) break; if (!dtrace_canload(mem, size - 1, mstate, vstate)) break; if (!DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); break; } if (dtrace_memstr_max != 0 && size > dtrace_memstr_max) { *flags |= CPU_DTRACE_ILLOP; break; } for (i = 0; i < size - 1; i++) { n = dtrace_load8(mem++); str[i] = (n == 0) ? c : n; } str[size - 1] = 0; regs[rd] = (uintptr_t)str; mstate->dtms_scratch_ptr += size; break; } #endif case DIF_SUBR_TYPEREF: { uintptr_t size = 4 * sizeof(uintptr_t); uintptr_t *typeref = (uintptr_t *) P2ROUNDUP(mstate->dtms_scratch_ptr, sizeof(uintptr_t)); size_t scratch_size = ((uintptr_t) typeref - mstate->dtms_scratch_ptr) + size; /* address, num_elements, type_str, type_len */ typeref[0] = tupregs[0].dttk_value; typeref[1] = tupregs[1].dttk_value; typeref[2] = tupregs[2].dttk_value; typeref[3] = tupregs[3].dttk_value; regs[rd] = (uintptr_t) typeref; mstate->dtms_scratch_ptr += scratch_size; break; } } } /* * Emulate the execution of DTrace IR instructions specified by the given * DIF object. This function is deliberately void of assertions as all of * the necessary checks are handled by a call to dtrace_difo_validate(). */ static uint64_t dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate, dtrace_state_t *state) { const dif_instr_t *text = difo->dtdo_buf; const uint_t textlen = difo->dtdo_len; const char *strtab = difo->dtdo_strtab; const uint64_t *inttab = difo->dtdo_inttab; uint64_t rval = 0; dtrace_statvar_t *svar; dtrace_dstate_t *dstate = &vstate->dtvs_dynvars; dtrace_difv_t *v; volatile uint16_t *flags = &cpu_core[curcpu].cpuc_dtrace_flags; volatile uintptr_t *illval = &cpu_core[curcpu].cpuc_dtrace_illval; dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */ uint64_t regs[DIF_DIR_NREGS]; uint64_t *tmp; uint8_t cc_n = 0, cc_z = 0, cc_v = 0, cc_c = 0; int64_t cc_r; uint_t pc = 0, id, opc = 0; uint8_t ttop = 0; dif_instr_t instr; uint_t r1, r2, rd; /* * We stash the current DIF object into the machine state: we need it * for subsequent access checking. */ mstate->dtms_difo = difo; regs[DIF_REG_R0] = 0; /* %r0 is fixed at zero */ while (pc < textlen && !(*flags & CPU_DTRACE_FAULT)) { opc = pc; instr = text[pc++]; r1 = DIF_INSTR_R1(instr); r2 = DIF_INSTR_R2(instr); rd = DIF_INSTR_RD(instr); switch (DIF_INSTR_OP(instr)) { case DIF_OP_OR: regs[rd] = regs[r1] | regs[r2]; break; case DIF_OP_XOR: regs[rd] = regs[r1] ^ regs[r2]; break; case DIF_OP_AND: regs[rd] = regs[r1] & regs[r2]; break; case DIF_OP_SLL: regs[rd] = regs[r1] << regs[r2]; break; case DIF_OP_SRL: regs[rd] = regs[r1] >> regs[r2]; break; case DIF_OP_SUB: regs[rd] = regs[r1] - regs[r2]; break; case DIF_OP_ADD: regs[rd] = regs[r1] + regs[r2]; break; case DIF_OP_MUL: regs[rd] = regs[r1] * regs[r2]; break; case DIF_OP_SDIV: if (regs[r2] == 0) { regs[rd] = 0; *flags |= CPU_DTRACE_DIVZERO; } else { regs[rd] = (int64_t)regs[r1] / (int64_t)regs[r2]; } break; case DIF_OP_UDIV: if (regs[r2] == 0) { regs[rd] = 0; *flags |= CPU_DTRACE_DIVZERO; } else { regs[rd] = regs[r1] / regs[r2]; } break; case DIF_OP_SREM: if (regs[r2] == 0) { regs[rd] = 0; *flags |= CPU_DTRACE_DIVZERO; } else { regs[rd] = (int64_t)regs[r1] % (int64_t)regs[r2]; } break; case DIF_OP_UREM: if (regs[r2] == 0) { regs[rd] = 0; *flags |= CPU_DTRACE_DIVZERO; } else { regs[rd] = regs[r1] % regs[r2]; } break; case DIF_OP_NOT: regs[rd] = ~regs[r1]; break; case DIF_OP_MOV: regs[rd] = regs[r1]; break; case DIF_OP_CMP: cc_r = regs[r1] - regs[r2]; cc_n = cc_r < 0; cc_z = cc_r == 0; cc_v = 0; cc_c = regs[r1] < regs[r2]; break; case DIF_OP_TST: cc_n = cc_v = cc_c = 0; cc_z = regs[r1] == 0; break; case DIF_OP_BA: pc = DIF_INSTR_LABEL(instr); break; case DIF_OP_BE: if (cc_z) pc = DIF_INSTR_LABEL(instr); break; case DIF_OP_BNE: if (cc_z == 0) pc = DIF_INSTR_LABEL(instr); break; case DIF_OP_BG: if ((cc_z | (cc_n ^ cc_v)) == 0) pc = DIF_INSTR_LABEL(instr); break; case DIF_OP_BGU: if ((cc_c | cc_z) == 0) pc = DIF_INSTR_LABEL(instr); break; case DIF_OP_BGE: if ((cc_n ^ cc_v) == 0) pc = DIF_INSTR_LABEL(instr); break; case DIF_OP_BGEU: if (cc_c == 0) pc = DIF_INSTR_LABEL(instr); break; case DIF_OP_BL: if (cc_n ^ cc_v) pc = DIF_INSTR_LABEL(instr); break; case DIF_OP_BLU: if (cc_c) pc = DIF_INSTR_LABEL(instr); break; case DIF_OP_BLE: if (cc_z | (cc_n ^ cc_v)) pc = DIF_INSTR_LABEL(instr); break; case DIF_OP_BLEU: if (cc_c | cc_z) pc = DIF_INSTR_LABEL(instr); break; case DIF_OP_RLDSB: if (!dtrace_canload(regs[r1], 1, mstate, vstate)) break; /*FALLTHROUGH*/ case DIF_OP_LDSB: regs[rd] = (int8_t)dtrace_load8(regs[r1]); break; case DIF_OP_RLDSH: if (!dtrace_canload(regs[r1], 2, mstate, vstate)) break; /*FALLTHROUGH*/ case DIF_OP_LDSH: regs[rd] = (int16_t)dtrace_load16(regs[r1]); break; case DIF_OP_RLDSW: if (!dtrace_canload(regs[r1], 4, mstate, vstate)) break; /*FALLTHROUGH*/ case DIF_OP_LDSW: regs[rd] = (int32_t)dtrace_load32(regs[r1]); break; case DIF_OP_RLDUB: if (!dtrace_canload(regs[r1], 1, mstate, vstate)) break; /*FALLTHROUGH*/ case DIF_OP_LDUB: regs[rd] = dtrace_load8(regs[r1]); break; case DIF_OP_RLDUH: if (!dtrace_canload(regs[r1], 2, mstate, vstate)) break; /*FALLTHROUGH*/ case DIF_OP_LDUH: regs[rd] = dtrace_load16(regs[r1]); break; case DIF_OP_RLDUW: if (!dtrace_canload(regs[r1], 4, mstate, vstate)) break; /*FALLTHROUGH*/ case DIF_OP_LDUW: regs[rd] = dtrace_load32(regs[r1]); break; case DIF_OP_RLDX: if (!dtrace_canload(regs[r1], 8, mstate, vstate)) break; /*FALLTHROUGH*/ case DIF_OP_LDX: regs[rd] = dtrace_load64(regs[r1]); break; case DIF_OP_ULDSB: DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); regs[rd] = (int8_t) dtrace_fuword8((void *)(uintptr_t)regs[r1]); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; case DIF_OP_ULDSH: DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); regs[rd] = (int16_t) dtrace_fuword16((void *)(uintptr_t)regs[r1]); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; case DIF_OP_ULDSW: DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); regs[rd] = (int32_t) dtrace_fuword32((void *)(uintptr_t)regs[r1]); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; case DIF_OP_ULDUB: DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); regs[rd] = dtrace_fuword8((void *)(uintptr_t)regs[r1]); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; case DIF_OP_ULDUH: DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); regs[rd] = dtrace_fuword16((void *)(uintptr_t)regs[r1]); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; case DIF_OP_ULDUW: DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); regs[rd] = dtrace_fuword32((void *)(uintptr_t)regs[r1]); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; case DIF_OP_ULDX: DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); regs[rd] = dtrace_fuword64((void *)(uintptr_t)regs[r1]); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; case DIF_OP_RET: rval = regs[rd]; pc = textlen; break; case DIF_OP_NOP: break; case DIF_OP_SETX: regs[rd] = inttab[DIF_INSTR_INTEGER(instr)]; break; case DIF_OP_SETS: regs[rd] = (uint64_t)(uintptr_t) (strtab + DIF_INSTR_STRING(instr)); break; case DIF_OP_SCMP: { size_t sz = state->dts_options[DTRACEOPT_STRSIZE]; uintptr_t s1 = regs[r1]; uintptr_t s2 = regs[r2]; if (s1 != 0 && !dtrace_strcanload(s1, sz, mstate, vstate)) break; if (s2 != 0 && !dtrace_strcanload(s2, sz, mstate, vstate)) break; cc_r = dtrace_strncmp((char *)s1, (char *)s2, sz); cc_n = cc_r < 0; cc_z = cc_r == 0; cc_v = cc_c = 0; break; } case DIF_OP_LDGA: regs[rd] = dtrace_dif_variable(mstate, state, r1, regs[r2]); break; case DIF_OP_LDGS: id = DIF_INSTR_VAR(instr); if (id >= DIF_VAR_OTHER_UBASE) { uintptr_t a; id -= DIF_VAR_OTHER_UBASE; svar = vstate->dtvs_globals[id]; ASSERT(svar != NULL); v = &svar->dtsv_var; if (!(v->dtdv_type.dtdt_flags & DIF_TF_BYREF)) { regs[rd] = svar->dtsv_data; break; } a = (uintptr_t)svar->dtsv_data; if (*(uint8_t *)a == UINT8_MAX) { /* * If the 0th byte is set to UINT8_MAX * then this is to be treated as a * reference to a NULL variable. */ regs[rd] = 0; } else { regs[rd] = a + sizeof (uint64_t); } break; } regs[rd] = dtrace_dif_variable(mstate, state, id, 0); break; case DIF_OP_STGS: id = DIF_INSTR_VAR(instr); ASSERT(id >= DIF_VAR_OTHER_UBASE); id -= DIF_VAR_OTHER_UBASE; svar = vstate->dtvs_globals[id]; ASSERT(svar != NULL); v = &svar->dtsv_var; if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { uintptr_t a = (uintptr_t)svar->dtsv_data; ASSERT(a != 0); ASSERT(svar->dtsv_size != 0); if (regs[rd] == 0) { *(uint8_t *)a = UINT8_MAX; break; } else { *(uint8_t *)a = 0; a += sizeof (uint64_t); } if (!dtrace_vcanload( (void *)(uintptr_t)regs[rd], &v->dtdv_type, mstate, vstate)) break; dtrace_vcopy((void *)(uintptr_t)regs[rd], (void *)a, &v->dtdv_type); break; } svar->dtsv_data = regs[rd]; break; case DIF_OP_LDTA: /* * There are no DTrace built-in thread-local arrays at * present. This opcode is saved for future work. */ *flags |= CPU_DTRACE_ILLOP; regs[rd] = 0; break; case DIF_OP_LDLS: id = DIF_INSTR_VAR(instr); if (id < DIF_VAR_OTHER_UBASE) { /* * For now, this has no meaning. */ regs[rd] = 0; break; } id -= DIF_VAR_OTHER_UBASE; ASSERT(id < vstate->dtvs_nlocals); ASSERT(vstate->dtvs_locals != NULL); svar = vstate->dtvs_locals[id]; ASSERT(svar != NULL); v = &svar->dtsv_var; if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { uintptr_t a = (uintptr_t)svar->dtsv_data; size_t sz = v->dtdv_type.dtdt_size; sz += sizeof (uint64_t); ASSERT(svar->dtsv_size == NCPU * sz); a += curcpu * sz; if (*(uint8_t *)a == UINT8_MAX) { /* * If the 0th byte is set to UINT8_MAX * then this is to be treated as a * reference to a NULL variable. */ regs[rd] = 0; } else { regs[rd] = a + sizeof (uint64_t); } break; } ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t)); tmp = (uint64_t *)(uintptr_t)svar->dtsv_data; regs[rd] = tmp[curcpu]; break; case DIF_OP_STLS: id = DIF_INSTR_VAR(instr); ASSERT(id >= DIF_VAR_OTHER_UBASE); id -= DIF_VAR_OTHER_UBASE; ASSERT(id < vstate->dtvs_nlocals); ASSERT(vstate->dtvs_locals != NULL); svar = vstate->dtvs_locals[id]; ASSERT(svar != NULL); v = &svar->dtsv_var; if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { uintptr_t a = (uintptr_t)svar->dtsv_data; size_t sz = v->dtdv_type.dtdt_size; sz += sizeof (uint64_t); ASSERT(svar->dtsv_size == NCPU * sz); a += curcpu * sz; if (regs[rd] == 0) { *(uint8_t *)a = UINT8_MAX; break; } else { *(uint8_t *)a = 0; a += sizeof (uint64_t); } if (!dtrace_vcanload( (void *)(uintptr_t)regs[rd], &v->dtdv_type, mstate, vstate)) break; dtrace_vcopy((void *)(uintptr_t)regs[rd], (void *)a, &v->dtdv_type); break; } ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t)); tmp = (uint64_t *)(uintptr_t)svar->dtsv_data; tmp[curcpu] = regs[rd]; break; case DIF_OP_LDTS: { dtrace_dynvar_t *dvar; dtrace_key_t *key; id = DIF_INSTR_VAR(instr); ASSERT(id >= DIF_VAR_OTHER_UBASE); id -= DIF_VAR_OTHER_UBASE; v = &vstate->dtvs_tlocals[id]; key = &tupregs[DIF_DTR_NREGS]; key[0].dttk_value = (uint64_t)id; key[0].dttk_size = 0; DTRACE_TLS_THRKEY(key[1].dttk_value); key[1].dttk_size = 0; dvar = dtrace_dynvar(dstate, 2, key, sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC, mstate, vstate); if (dvar == NULL) { regs[rd] = 0; break; } if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data; } else { regs[rd] = *((uint64_t *)dvar->dtdv_data); } break; } case DIF_OP_STTS: { dtrace_dynvar_t *dvar; dtrace_key_t *key; id = DIF_INSTR_VAR(instr); ASSERT(id >= DIF_VAR_OTHER_UBASE); id -= DIF_VAR_OTHER_UBASE; key = &tupregs[DIF_DTR_NREGS]; key[0].dttk_value = (uint64_t)id; key[0].dttk_size = 0; DTRACE_TLS_THRKEY(key[1].dttk_value); key[1].dttk_size = 0; v = &vstate->dtvs_tlocals[id]; dvar = dtrace_dynvar(dstate, 2, key, v->dtdv_type.dtdt_size > sizeof (uint64_t) ? v->dtdv_type.dtdt_size : sizeof (uint64_t), regs[rd] ? DTRACE_DYNVAR_ALLOC : DTRACE_DYNVAR_DEALLOC, mstate, vstate); /* * Given that we're storing to thread-local data, * we need to flush our predicate cache. */ curthread->t_predcache = 0; if (dvar == NULL) break; if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { if (!dtrace_vcanload( (void *)(uintptr_t)regs[rd], &v->dtdv_type, mstate, vstate)) break; dtrace_vcopy((void *)(uintptr_t)regs[rd], dvar->dtdv_data, &v->dtdv_type); } else { *((uint64_t *)dvar->dtdv_data) = regs[rd]; } break; } case DIF_OP_SRA: regs[rd] = (int64_t)regs[r1] >> regs[r2]; break; case DIF_OP_CALL: dtrace_dif_subr(DIF_INSTR_SUBR(instr), rd, regs, tupregs, ttop, mstate, state); break; case DIF_OP_PUSHTR: if (ttop == DIF_DTR_NREGS) { *flags |= CPU_DTRACE_TUPOFLOW; break; } if (r1 == DIF_TYPE_STRING) { /* * If this is a string type and the size is 0, * we'll use the system-wide default string * size. Note that we are _not_ looking at * the value of the DTRACEOPT_STRSIZE option; * had this been set, we would expect to have * a non-zero size value in the "pushtr". */ tupregs[ttop].dttk_size = dtrace_strlen((char *)(uintptr_t)regs[rd], regs[r2] ? regs[r2] : dtrace_strsize_default) + 1; } else { tupregs[ttop].dttk_size = regs[r2]; } tupregs[ttop++].dttk_value = regs[rd]; break; case DIF_OP_PUSHTV: if (ttop == DIF_DTR_NREGS) { *flags |= CPU_DTRACE_TUPOFLOW; break; } tupregs[ttop].dttk_value = regs[rd]; tupregs[ttop++].dttk_size = 0; break; case DIF_OP_POPTS: if (ttop != 0) ttop--; break; case DIF_OP_FLUSHTS: ttop = 0; break; case DIF_OP_LDGAA: case DIF_OP_LDTAA: { dtrace_dynvar_t *dvar; dtrace_key_t *key = tupregs; uint_t nkeys = ttop; id = DIF_INSTR_VAR(instr); ASSERT(id >= DIF_VAR_OTHER_UBASE); id -= DIF_VAR_OTHER_UBASE; key[nkeys].dttk_value = (uint64_t)id; key[nkeys++].dttk_size = 0; if (DIF_INSTR_OP(instr) == DIF_OP_LDTAA) { DTRACE_TLS_THRKEY(key[nkeys].dttk_value); key[nkeys++].dttk_size = 0; v = &vstate->dtvs_tlocals[id]; } else { v = &vstate->dtvs_globals[id]->dtsv_var; } dvar = dtrace_dynvar(dstate, nkeys, key, v->dtdv_type.dtdt_size > sizeof (uint64_t) ? v->dtdv_type.dtdt_size : sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC, mstate, vstate); if (dvar == NULL) { regs[rd] = 0; break; } if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data; } else { regs[rd] = *((uint64_t *)dvar->dtdv_data); } break; } case DIF_OP_STGAA: case DIF_OP_STTAA: { dtrace_dynvar_t *dvar; dtrace_key_t *key = tupregs; uint_t nkeys = ttop; id = DIF_INSTR_VAR(instr); ASSERT(id >= DIF_VAR_OTHER_UBASE); id -= DIF_VAR_OTHER_UBASE; key[nkeys].dttk_value = (uint64_t)id; key[nkeys++].dttk_size = 0; if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) { DTRACE_TLS_THRKEY(key[nkeys].dttk_value); key[nkeys++].dttk_size = 0; v = &vstate->dtvs_tlocals[id]; } else { v = &vstate->dtvs_globals[id]->dtsv_var; } dvar = dtrace_dynvar(dstate, nkeys, key, v->dtdv_type.dtdt_size > sizeof (uint64_t) ? v->dtdv_type.dtdt_size : sizeof (uint64_t), regs[rd] ? DTRACE_DYNVAR_ALLOC : DTRACE_DYNVAR_DEALLOC, mstate, vstate); if (dvar == NULL) break; if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { if (!dtrace_vcanload( (void *)(uintptr_t)regs[rd], &v->dtdv_type, mstate, vstate)) break; dtrace_vcopy((void *)(uintptr_t)regs[rd], dvar->dtdv_data, &v->dtdv_type); } else { *((uint64_t *)dvar->dtdv_data) = regs[rd]; } break; } case DIF_OP_ALLOCS: { uintptr_t ptr = P2ROUNDUP(mstate->dtms_scratch_ptr, 8); size_t size = ptr - mstate->dtms_scratch_ptr + regs[r1]; /* * Rounding up the user allocation size could have * overflowed large, bogus allocations (like -1ULL) to * 0. */ if (size < regs[r1] || !DTRACE_INSCRATCH(mstate, size)) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); regs[rd] = 0; break; } dtrace_bzero((void *) mstate->dtms_scratch_ptr, size); mstate->dtms_scratch_ptr += size; regs[rd] = ptr; break; } case DIF_OP_COPYS: if (!dtrace_canstore(regs[rd], regs[r2], mstate, vstate)) { *flags |= CPU_DTRACE_BADADDR; *illval = regs[rd]; break; } if (!dtrace_canload(regs[r1], regs[r2], mstate, vstate)) break; dtrace_bcopy((void *)(uintptr_t)regs[r1], (void *)(uintptr_t)regs[rd], (size_t)regs[r2]); break; case DIF_OP_STB: if (!dtrace_canstore(regs[rd], 1, mstate, vstate)) { *flags |= CPU_DTRACE_BADADDR; *illval = regs[rd]; break; } *((uint8_t *)(uintptr_t)regs[rd]) = (uint8_t)regs[r1]; break; case DIF_OP_STH: if (!dtrace_canstore(regs[rd], 2, mstate, vstate)) { *flags |= CPU_DTRACE_BADADDR; *illval = regs[rd]; break; } if (regs[rd] & 1) { *flags |= CPU_DTRACE_BADALIGN; *illval = regs[rd]; break; } *((uint16_t *)(uintptr_t)regs[rd]) = (uint16_t)regs[r1]; break; case DIF_OP_STW: if (!dtrace_canstore(regs[rd], 4, mstate, vstate)) { *flags |= CPU_DTRACE_BADADDR; *illval = regs[rd]; break; } if (regs[rd] & 3) { *flags |= CPU_DTRACE_BADALIGN; *illval = regs[rd]; break; } *((uint32_t *)(uintptr_t)regs[rd]) = (uint32_t)regs[r1]; break; case DIF_OP_STX: if (!dtrace_canstore(regs[rd], 8, mstate, vstate)) { *flags |= CPU_DTRACE_BADADDR; *illval = regs[rd]; break; } if (regs[rd] & 7) { *flags |= CPU_DTRACE_BADALIGN; *illval = regs[rd]; break; } *((uint64_t *)(uintptr_t)regs[rd]) = regs[r1]; break; } } if (!(*flags & CPU_DTRACE_FAULT)) return (rval); mstate->dtms_fltoffs = opc * sizeof (dif_instr_t); mstate->dtms_present |= DTRACE_MSTATE_FLTOFFS; return (0); } static void dtrace_action_breakpoint(dtrace_ecb_t *ecb) { dtrace_probe_t *probe = ecb->dte_probe; dtrace_provider_t *prov = probe->dtpr_provider; char c[DTRACE_FULLNAMELEN + 80], *str; char *msg = "dtrace: breakpoint action at probe "; char *ecbmsg = " (ecb "; uintptr_t mask = (0xf << (sizeof (uintptr_t) * NBBY / 4)); uintptr_t val = (uintptr_t)ecb; int shift = (sizeof (uintptr_t) * NBBY) - 4, i = 0; if (dtrace_destructive_disallow) return; /* * It's impossible to be taking action on the NULL probe. */ ASSERT(probe != NULL); /* * This is a poor man's (destitute man's?) sprintf(): we want to * print the provider name, module name, function name and name of * the probe, along with the hex address of the ECB with the breakpoint * action -- all of which we must place in the character buffer by * hand. */ while (*msg != '\0') c[i++] = *msg++; for (str = prov->dtpv_name; *str != '\0'; str++) c[i++] = *str; c[i++] = ':'; for (str = probe->dtpr_mod; *str != '\0'; str++) c[i++] = *str; c[i++] = ':'; for (str = probe->dtpr_func; *str != '\0'; str++) c[i++] = *str; c[i++] = ':'; for (str = probe->dtpr_name; *str != '\0'; str++) c[i++] = *str; while (*ecbmsg != '\0') c[i++] = *ecbmsg++; while (shift >= 0) { mask = (uintptr_t)0xf << shift; if (val >= ((uintptr_t)1 << shift)) c[i++] = "0123456789abcdef"[(val & mask) >> shift]; shift -= 4; } c[i++] = ')'; c[i] = '\0'; #ifdef illumos debug_enter(c); #else kdb_enter(KDB_WHY_DTRACE, "breakpoint action"); #endif } static void dtrace_action_panic(dtrace_ecb_t *ecb) { dtrace_probe_t *probe = ecb->dte_probe; /* * It's impossible to be taking action on the NULL probe. */ ASSERT(probe != NULL); if (dtrace_destructive_disallow) return; if (dtrace_panicked != NULL) return; if (dtrace_casptr(&dtrace_panicked, NULL, curthread) != NULL) return; /* * We won the right to panic. (We want to be sure that only one * thread calls panic() from dtrace_probe(), and that panic() is * called exactly once.) */ dtrace_panic("dtrace: panic action at probe %s:%s:%s:%s (ecb %p)", probe->dtpr_provider->dtpv_name, probe->dtpr_mod, probe->dtpr_func, probe->dtpr_name, (void *)ecb); } static void dtrace_action_raise(uint64_t sig) { if (dtrace_destructive_disallow) return; if (sig >= NSIG) { DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); return; } #ifdef illumos /* * raise() has a queue depth of 1 -- we ignore all subsequent * invocations of the raise() action. */ if (curthread->t_dtrace_sig == 0) curthread->t_dtrace_sig = (uint8_t)sig; curthread->t_sig_check = 1; aston(curthread); #else struct proc *p = curproc; PROC_LOCK(p); kern_psignal(p, sig); PROC_UNLOCK(p); #endif } static void dtrace_action_stop(void) { if (dtrace_destructive_disallow) return; #ifdef illumos if (!curthread->t_dtrace_stop) { curthread->t_dtrace_stop = 1; curthread->t_sig_check = 1; aston(curthread); } #else struct proc *p = curproc; PROC_LOCK(p); kern_psignal(p, SIGSTOP); PROC_UNLOCK(p); #endif } static void dtrace_action_chill(dtrace_mstate_t *mstate, hrtime_t val) { hrtime_t now; volatile uint16_t *flags; #ifdef illumos cpu_t *cpu = CPU; #else cpu_t *cpu = &solaris_cpu[curcpu]; #endif if (dtrace_destructive_disallow) return; flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; now = dtrace_gethrtime(); if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) { /* * We need to advance the mark to the current time. */ cpu->cpu_dtrace_chillmark = now; cpu->cpu_dtrace_chilled = 0; } /* * Now check to see if the requested chill time would take us over * the maximum amount of time allowed in the chill interval. (Or * worse, if the calculation itself induces overflow.) */ if (cpu->cpu_dtrace_chilled + val > dtrace_chill_max || cpu->cpu_dtrace_chilled + val < cpu->cpu_dtrace_chilled) { *flags |= CPU_DTRACE_ILLOP; return; } while (dtrace_gethrtime() - now < val) continue; /* * Normally, we assure that the value of the variable "timestamp" does * not change within an ECB. The presence of chill() represents an * exception to this rule, however. */ mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP; cpu->cpu_dtrace_chilled += val; } static void dtrace_action_ustack(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t *buf, uint64_t arg) { int nframes = DTRACE_USTACK_NFRAMES(arg); int strsize = DTRACE_USTACK_STRSIZE(arg); uint64_t *pcs = &buf[1], *fps; char *str = (char *)&pcs[nframes]; int size, offs = 0, i, j; uintptr_t old = mstate->dtms_scratch_ptr, saved; uint16_t *flags = &cpu_core[curcpu].cpuc_dtrace_flags; char *sym; /* * Should be taking a faster path if string space has not been * allocated. */ ASSERT(strsize != 0); /* * We will first allocate some temporary space for the frame pointers. */ fps = (uint64_t *)P2ROUNDUP(mstate->dtms_scratch_ptr, 8); size = (uintptr_t)fps - mstate->dtms_scratch_ptr + (nframes * sizeof (uint64_t)); if (!DTRACE_INSCRATCH(mstate, size)) { /* * Not enough room for our frame pointers -- need to indicate * that we ran out of scratch space. */ DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); return; } mstate->dtms_scratch_ptr += size; saved = mstate->dtms_scratch_ptr; /* * Now get a stack with both program counters and frame pointers. */ DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); dtrace_getufpstack(buf, fps, nframes + 1); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); /* * If that faulted, we're cooked. */ if (*flags & CPU_DTRACE_FAULT) goto out; /* * Now we want to walk up the stack, calling the USTACK helper. For * each iteration, we restore the scratch pointer. */ for (i = 0; i < nframes; i++) { mstate->dtms_scratch_ptr = saved; if (offs >= strsize) break; sym = (char *)(uintptr_t)dtrace_helper( DTRACE_HELPER_ACTION_USTACK, mstate, state, pcs[i], fps[i]); /* * If we faulted while running the helper, we're going to * clear the fault and null out the corresponding string. */ if (*flags & CPU_DTRACE_FAULT) { *flags &= ~CPU_DTRACE_FAULT; str[offs++] = '\0'; continue; } if (sym == NULL) { str[offs++] = '\0'; continue; } DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); /* * Now copy in the string that the helper returned to us. */ for (j = 0; offs + j < strsize; j++) { if ((str[offs + j] = sym[j]) == '\0') break; } DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); offs += j + 1; } if (offs >= strsize) { /* * If we didn't have room for all of the strings, we don't * abort processing -- this needn't be a fatal error -- but we * still want to increment a counter (dts_stkstroverflows) to * allow this condition to be warned about. (If this is from * a jstack() action, it is easily tuned via jstackstrsize.) */ dtrace_error(&state->dts_stkstroverflows); } while (offs < strsize) str[offs++] = '\0'; out: mstate->dtms_scratch_ptr = old; } static void dtrace_store_by_ref(dtrace_difo_t *dp, caddr_t tomax, size_t size, size_t *valoffsp, uint64_t *valp, uint64_t end, int intuple, int dtkind) { volatile uint16_t *flags; uint64_t val = *valp; size_t valoffs = *valoffsp; flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; ASSERT(dtkind == DIF_TF_BYREF || dtkind == DIF_TF_BYUREF); /* * If this is a string, we're going to only load until we find the zero * byte -- after which we'll store zero bytes. */ if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) { char c = '\0' + 1; size_t s; for (s = 0; s < size; s++) { if (c != '\0' && dtkind == DIF_TF_BYREF) { c = dtrace_load8(val++); } else if (c != '\0' && dtkind == DIF_TF_BYUREF) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); c = dtrace_fuword8((void *)(uintptr_t)val++); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); if (*flags & CPU_DTRACE_FAULT) break; } DTRACE_STORE(uint8_t, tomax, valoffs++, c); if (c == '\0' && intuple) break; } } else { uint8_t c; while (valoffs < end) { if (dtkind == DIF_TF_BYREF) { c = dtrace_load8(val++); } else if (dtkind == DIF_TF_BYUREF) { DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); c = dtrace_fuword8((void *)(uintptr_t)val++); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); if (*flags & CPU_DTRACE_FAULT) break; } DTRACE_STORE(uint8_t, tomax, valoffs++, c); } } *valp = val; *valoffsp = valoffs; } /* * If you're looking for the epicenter of DTrace, you just found it. This * is the function called by the provider to fire a probe -- from which all * subsequent probe-context DTrace activity emanates. */ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4) { processorid_t cpuid; dtrace_icookie_t cookie; dtrace_probe_t *probe; dtrace_mstate_t mstate; dtrace_ecb_t *ecb; dtrace_action_t *act; intptr_t offs; size_t size; int vtime, onintr; volatile uint16_t *flags; hrtime_t now; if (panicstr != NULL) return; #ifdef illumos /* * Kick out immediately if this CPU is still being born (in which case * curthread will be set to -1) or the current thread can't allow * probes in its current context. */ if (((uintptr_t)curthread & 1) || (curthread->t_flag & T_DONTDTRACE)) return; #endif cookie = dtrace_interrupt_disable(); probe = dtrace_probes[id - 1]; cpuid = curcpu; onintr = CPU_ON_INTR(CPU); if (!onintr && probe->dtpr_predcache != DTRACE_CACHEIDNONE && probe->dtpr_predcache == curthread->t_predcache) { /* * We have hit in the predicate cache; we know that * this predicate would evaluate to be false. */ dtrace_interrupt_enable(cookie); return; } #ifdef illumos if (panic_quiesce) { #else if (panicstr != NULL) { #endif /* * We don't trace anything if we're panicking. */ dtrace_interrupt_enable(cookie); return; } now = mstate.dtms_timestamp = dtrace_gethrtime(); mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP; vtime = dtrace_vtime_references != 0; if (vtime && curthread->t_dtrace_start) curthread->t_dtrace_vtime += now - curthread->t_dtrace_start; mstate.dtms_difo = NULL; mstate.dtms_probe = probe; mstate.dtms_strtok = 0; mstate.dtms_arg[0] = arg0; mstate.dtms_arg[1] = arg1; mstate.dtms_arg[2] = arg2; mstate.dtms_arg[3] = arg3; mstate.dtms_arg[4] = arg4; flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags; for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) { dtrace_predicate_t *pred = ecb->dte_predicate; dtrace_state_t *state = ecb->dte_state; dtrace_buffer_t *buf = &state->dts_buffer[cpuid]; dtrace_buffer_t *aggbuf = &state->dts_aggbuffer[cpuid]; dtrace_vstate_t *vstate = &state->dts_vstate; dtrace_provider_t *prov = probe->dtpr_provider; uint64_t tracememsize = 0; int committed = 0; caddr_t tomax; /* * A little subtlety with the following (seemingly innocuous) * declaration of the automatic 'val': by looking at the * code, you might think that it could be declared in the * action processing loop, below. (That is, it's only used in * the action processing loop.) However, it must be declared * out of that scope because in the case of DIF expression * arguments to aggregating actions, one iteration of the * action loop will use the last iteration's value. */ uint64_t val = 0; mstate.dtms_present = DTRACE_MSTATE_ARGS | DTRACE_MSTATE_PROBE; mstate.dtms_getf = NULL; *flags &= ~CPU_DTRACE_ERROR; if (prov == dtrace_provider) { /* * If dtrace itself is the provider of this probe, * we're only going to continue processing the ECB if * arg0 (the dtrace_state_t) is equal to the ECB's * creating state. (This prevents disjoint consumers * from seeing one another's metaprobes.) */ if (arg0 != (uint64_t)(uintptr_t)state) continue; } if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) { /* * We're not currently active. If our provider isn't * the dtrace pseudo provider, we're not interested. */ if (prov != dtrace_provider) continue; /* * Now we must further check if we are in the BEGIN * probe. If we are, we will only continue processing * if we're still in WARMUP -- if one BEGIN enabling * has invoked the exit() action, we don't want to * evaluate subsequent BEGIN enablings. */ if (probe->dtpr_id == dtrace_probeid_begin && state->dts_activity != DTRACE_ACTIVITY_WARMUP) { ASSERT(state->dts_activity == DTRACE_ACTIVITY_DRAINING); continue; } } if (ecb->dte_cond) { /* * If the dte_cond bits indicate that this * consumer is only allowed to see user-mode firings * of this probe, call the provider's dtps_usermode() * entry point to check that the probe was fired * while in a user context. Skip this ECB if that's * not the case. */ if ((ecb->dte_cond & DTRACE_COND_USERMODE) && prov->dtpv_pops.dtps_usermode(prov->dtpv_arg, probe->dtpr_id, probe->dtpr_arg) == 0) continue; #ifdef illumos /* * This is more subtle than it looks. We have to be * absolutely certain that CRED() isn't going to * change out from under us so it's only legit to * examine that structure if we're in constrained * situations. Currently, the only times we'll this * check is if a non-super-user has enabled the * profile or syscall providers -- providers that * allow visibility of all processes. For the * profile case, the check above will ensure that * we're examining a user context. */ if (ecb->dte_cond & DTRACE_COND_OWNER) { cred_t *cr; cred_t *s_cr = ecb->dte_state->dts_cred.dcr_cred; proc_t *proc; ASSERT(s_cr != NULL); if ((cr = CRED()) == NULL || s_cr->cr_uid != cr->cr_uid || s_cr->cr_uid != cr->cr_ruid || s_cr->cr_uid != cr->cr_suid || s_cr->cr_gid != cr->cr_gid || s_cr->cr_gid != cr->cr_rgid || s_cr->cr_gid != cr->cr_sgid || (proc = ttoproc(curthread)) == NULL || (proc->p_flag & SNOCD)) continue; } if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) { cred_t *cr; cred_t *s_cr = ecb->dte_state->dts_cred.dcr_cred; ASSERT(s_cr != NULL); if ((cr = CRED()) == NULL || s_cr->cr_zone->zone_id != cr->cr_zone->zone_id) continue; } #endif } if (now - state->dts_alive > dtrace_deadman_timeout) { /* * We seem to be dead. Unless we (a) have kernel * destructive permissions (b) have explicitly enabled * destructive actions and (c) destructive actions have * not been disabled, we're going to transition into * the KILLED state, from which no further processing * on this state will be performed. */ if (!dtrace_priv_kernel_destructive(state) || !state->dts_cred.dcr_destructive || dtrace_destructive_disallow) { void *activity = &state->dts_activity; dtrace_activity_t current; do { current = state->dts_activity; } while (dtrace_cas32(activity, current, DTRACE_ACTIVITY_KILLED) != current); continue; } } if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed, ecb->dte_alignment, state, &mstate)) < 0) continue; tomax = buf->dtb_tomax; ASSERT(tomax != NULL); if (ecb->dte_size != 0) { dtrace_rechdr_t dtrh; if (!(mstate.dtms_present & DTRACE_MSTATE_TIMESTAMP)) { mstate.dtms_timestamp = dtrace_gethrtime(); mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP; } ASSERT3U(ecb->dte_size, >=, sizeof (dtrace_rechdr_t)); dtrh.dtrh_epid = ecb->dte_epid; DTRACE_RECORD_STORE_TIMESTAMP(&dtrh, mstate.dtms_timestamp); *((dtrace_rechdr_t *)(tomax + offs)) = dtrh; } mstate.dtms_epid = ecb->dte_epid; mstate.dtms_present |= DTRACE_MSTATE_EPID; if (state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) mstate.dtms_access = DTRACE_ACCESS_KERNEL; else mstate.dtms_access = 0; if (pred != NULL) { dtrace_difo_t *dp = pred->dtp_difo; int rval; rval = dtrace_dif_emulate(dp, &mstate, vstate, state); if (!(*flags & CPU_DTRACE_ERROR) && !rval) { dtrace_cacheid_t cid = probe->dtpr_predcache; if (cid != DTRACE_CACHEIDNONE && !onintr) { /* * Update the predicate cache... */ ASSERT(cid == pred->dtp_cacheid); curthread->t_predcache = cid; } continue; } } for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) && act != NULL; act = act->dta_next) { size_t valoffs; dtrace_difo_t *dp; dtrace_recdesc_t *rec = &act->dta_rec; size = rec->dtrd_size; valoffs = offs + rec->dtrd_offset; if (DTRACEACT_ISAGG(act->dta_kind)) { uint64_t v = 0xbad; dtrace_aggregation_t *agg; agg = (dtrace_aggregation_t *)act; if ((dp = act->dta_difo) != NULL) v = dtrace_dif_emulate(dp, &mstate, vstate, state); if (*flags & CPU_DTRACE_ERROR) continue; /* * Note that we always pass the expression * value from the previous iteration of the * action loop. This value will only be used * if there is an expression argument to the * aggregating action, denoted by the * dtag_hasarg field. */ dtrace_aggregate(agg, buf, offs, aggbuf, v, val); continue; } switch (act->dta_kind) { case DTRACEACT_STOP: if (dtrace_priv_proc_destructive(state)) dtrace_action_stop(); continue; case DTRACEACT_BREAKPOINT: if (dtrace_priv_kernel_destructive(state)) dtrace_action_breakpoint(ecb); continue; case DTRACEACT_PANIC: if (dtrace_priv_kernel_destructive(state)) dtrace_action_panic(ecb); continue; case DTRACEACT_STACK: if (!dtrace_priv_kernel(state)) continue; dtrace_getpcstack((pc_t *)(tomax + valoffs), size / sizeof (pc_t), probe->dtpr_aframes, DTRACE_ANCHORED(probe) ? NULL : (uint32_t *)arg0); continue; case DTRACEACT_JSTACK: case DTRACEACT_USTACK: if (!dtrace_priv_proc(state)) continue; /* * See comment in DIF_VAR_PID. */ if (DTRACE_ANCHORED(mstate.dtms_probe) && CPU_ON_INTR(CPU)) { int depth = DTRACE_USTACK_NFRAMES( rec->dtrd_arg) + 1; dtrace_bzero((void *)(tomax + valoffs), DTRACE_USTACK_STRSIZE(rec->dtrd_arg) + depth * sizeof (uint64_t)); continue; } if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0 && curproc->p_dtrace_helpers != NULL) { /* * This is the slow path -- we have * allocated string space, and we're * getting the stack of a process that * has helpers. Call into a separate * routine to perform this processing. */ dtrace_action_ustack(&mstate, state, (uint64_t *)(tomax + valoffs), rec->dtrd_arg); continue; } DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); dtrace_getupcstack((uint64_t *) (tomax + valoffs), DTRACE_USTACK_NFRAMES(rec->dtrd_arg) + 1); DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); continue; default: break; } dp = act->dta_difo; ASSERT(dp != NULL); val = dtrace_dif_emulate(dp, &mstate, vstate, state); if (*flags & CPU_DTRACE_ERROR) continue; switch (act->dta_kind) { case DTRACEACT_SPECULATE: { dtrace_rechdr_t *dtrh; ASSERT(buf == &state->dts_buffer[cpuid]); buf = dtrace_speculation_buffer(state, cpuid, val); if (buf == NULL) { *flags |= CPU_DTRACE_DROP; continue; } offs = dtrace_buffer_reserve(buf, ecb->dte_needed, ecb->dte_alignment, state, NULL); if (offs < 0) { *flags |= CPU_DTRACE_DROP; continue; } tomax = buf->dtb_tomax; ASSERT(tomax != NULL); if (ecb->dte_size == 0) continue; ASSERT3U(ecb->dte_size, >=, sizeof (dtrace_rechdr_t)); dtrh = ((void *)(tomax + offs)); dtrh->dtrh_epid = ecb->dte_epid; /* * When the speculation is committed, all of * the records in the speculative buffer will * have their timestamps set to the commit * time. Until then, it is set to a sentinel * value, for debugability. */ DTRACE_RECORD_STORE_TIMESTAMP(dtrh, UINT64_MAX); continue; } case DTRACEACT_PRINTM: { /* The DIF returns a 'memref'. */ uintptr_t *memref = (uintptr_t *)(uintptr_t) val; /* Get the size from the memref. */ size = memref[1]; /* * Check if the size exceeds the allocated * buffer size. */ if (size + sizeof(uintptr_t) > dp->dtdo_rtype.dtdt_size) { /* Flag a drop! */ *flags |= CPU_DTRACE_DROP; continue; } /* Store the size in the buffer first. */ DTRACE_STORE(uintptr_t, tomax, valoffs, size); /* * Offset the buffer address to the start * of the data. */ valoffs += sizeof(uintptr_t); /* * Reset to the memory address rather than * the memref array, then let the BYREF * code below do the work to store the * memory data in the buffer. */ val = memref[0]; break; } case DTRACEACT_PRINTT: { /* The DIF returns a 'typeref'. */ uintptr_t *typeref = (uintptr_t *)(uintptr_t) val; char c = '\0' + 1; size_t s; /* * Get the type string length and round it * up so that the data that follows is * aligned for easy access. */ size_t typs = strlen((char *) typeref[2]) + 1; typs = roundup(typs, sizeof(uintptr_t)); /* *Get the size from the typeref using the * number of elements and the type size. */ size = typeref[1] * typeref[3]; /* * Check if the size exceeds the allocated * buffer size. */ if (size + typs + 2 * sizeof(uintptr_t) > dp->dtdo_rtype.dtdt_size) { /* Flag a drop! */ *flags |= CPU_DTRACE_DROP; } /* Store the size in the buffer first. */ DTRACE_STORE(uintptr_t, tomax, valoffs, size); valoffs += sizeof(uintptr_t); /* Store the type size in the buffer. */ DTRACE_STORE(uintptr_t, tomax, valoffs, typeref[3]); valoffs += sizeof(uintptr_t); val = typeref[2]; for (s = 0; s < typs; s++) { if (c != '\0') c = dtrace_load8(val++); DTRACE_STORE(uint8_t, tomax, valoffs++, c); } /* * Reset to the memory address rather than * the typeref array, then let the BYREF * code below do the work to store the * memory data in the buffer. */ val = typeref[0]; break; } case DTRACEACT_CHILL: if (dtrace_priv_kernel_destructive(state)) dtrace_action_chill(&mstate, val); continue; case DTRACEACT_RAISE: if (dtrace_priv_proc_destructive(state)) dtrace_action_raise(val); continue; case DTRACEACT_COMMIT: ASSERT(!committed); /* * We need to commit our buffer state. */ if (ecb->dte_size) buf->dtb_offset = offs + ecb->dte_size; buf = &state->dts_buffer[cpuid]; dtrace_speculation_commit(state, cpuid, val); committed = 1; continue; case DTRACEACT_DISCARD: dtrace_speculation_discard(state, cpuid, val); continue; case DTRACEACT_DIFEXPR: case DTRACEACT_LIBACT: case DTRACEACT_PRINTF: case DTRACEACT_PRINTA: case DTRACEACT_SYSTEM: case DTRACEACT_FREOPEN: case DTRACEACT_TRACEMEM: break; case DTRACEACT_TRACEMEM_DYNSIZE: tracememsize = val; break; case DTRACEACT_SYM: case DTRACEACT_MOD: if (!dtrace_priv_kernel(state)) continue; break; case DTRACEACT_USYM: case DTRACEACT_UMOD: case DTRACEACT_UADDR: { #ifdef illumos struct pid *pid = curthread->t_procp->p_pidp; #endif if (!dtrace_priv_proc(state)) continue; DTRACE_STORE(uint64_t, tomax, #ifdef illumos valoffs, (uint64_t)pid->pid_id); #else valoffs, (uint64_t) curproc->p_pid); #endif DTRACE_STORE(uint64_t, tomax, valoffs + sizeof (uint64_t), val); continue; } case DTRACEACT_EXIT: { /* * For the exit action, we are going to attempt * to atomically set our activity to be * draining. If this fails (either because * another CPU has beat us to the exit action, * or because our current activity is something * other than ACTIVE or WARMUP), we will * continue. This assures that the exit action * can be successfully recorded at most once * when we're in the ACTIVE state. If we're * encountering the exit() action while in * COOLDOWN, however, we want to honor the new * status code. (We know that we're the only * thread in COOLDOWN, so there is no race.) */ void *activity = &state->dts_activity; dtrace_activity_t current = state->dts_activity; if (current == DTRACE_ACTIVITY_COOLDOWN) break; if (current != DTRACE_ACTIVITY_WARMUP) current = DTRACE_ACTIVITY_ACTIVE; if (dtrace_cas32(activity, current, DTRACE_ACTIVITY_DRAINING) != current) { *flags |= CPU_DTRACE_DROP; continue; } break; } default: ASSERT(0); } if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF || dp->dtdo_rtype.dtdt_flags & DIF_TF_BYUREF) { uintptr_t end = valoffs + size; if (tracememsize != 0 && valoffs + tracememsize < end) { end = valoffs + tracememsize; tracememsize = 0; } if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF && !dtrace_vcanload((void *)(uintptr_t)val, &dp->dtdo_rtype, &mstate, vstate)) continue; dtrace_store_by_ref(dp, tomax, size, &valoffs, &val, end, act->dta_intuple, dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF ? DIF_TF_BYREF: DIF_TF_BYUREF); continue; } switch (size) { case 0: break; case sizeof (uint8_t): DTRACE_STORE(uint8_t, tomax, valoffs, val); break; case sizeof (uint16_t): DTRACE_STORE(uint16_t, tomax, valoffs, val); break; case sizeof (uint32_t): DTRACE_STORE(uint32_t, tomax, valoffs, val); break; case sizeof (uint64_t): DTRACE_STORE(uint64_t, tomax, valoffs, val); break; default: /* * Any other size should have been returned by * reference, not by value. */ ASSERT(0); break; } } if (*flags & CPU_DTRACE_DROP) continue; if (*flags & CPU_DTRACE_FAULT) { int ndx; dtrace_action_t *err; buf->dtb_errors++; if (probe->dtpr_id == dtrace_probeid_error) { /* * There's nothing we can do -- we had an * error on the error probe. We bump an * error counter to at least indicate that * this condition happened. */ dtrace_error(&state->dts_dblerrors); continue; } if (vtime) { /* * Before recursing on dtrace_probe(), we * need to explicitly clear out our start * time to prevent it from being accumulated * into t_dtrace_vtime. */ curthread->t_dtrace_start = 0; } /* * Iterate over the actions to figure out which action * we were processing when we experienced the error. * Note that act points _past_ the faulting action; if * act is ecb->dte_action, the fault was in the * predicate, if it's ecb->dte_action->dta_next it's * in action #1, and so on. */ for (err = ecb->dte_action, ndx = 0; err != act; err = err->dta_next, ndx++) continue; dtrace_probe_error(state, ecb->dte_epid, ndx, (mstate.dtms_present & DTRACE_MSTATE_FLTOFFS) ? mstate.dtms_fltoffs : -1, DTRACE_FLAGS2FLT(*flags), cpu_core[cpuid].cpuc_dtrace_illval); continue; } if (!committed) buf->dtb_offset = offs + ecb->dte_size; } if (vtime) curthread->t_dtrace_start = dtrace_gethrtime(); dtrace_interrupt_enable(cookie); } /* * DTrace Probe Hashing Functions * * The functions in this section (and indeed, the functions in remaining * sections) are not _called_ from probe context. (Any exceptions to this are * marked with a "Note:".) Rather, they are called from elsewhere in the * DTrace framework to look-up probes in, add probes to and remove probes from * the DTrace probe hashes. (Each probe is hashed by each element of the * probe tuple -- allowing for fast lookups, regardless of what was * specified.) */ static uint_t dtrace_hash_str(const char *p) { unsigned int g; uint_t hval = 0; while (*p) { hval = (hval << 4) + *p++; if ((g = (hval & 0xf0000000)) != 0) hval ^= g >> 24; hval &= ~g; } return (hval); } static dtrace_hash_t * dtrace_hash_create(uintptr_t stroffs, uintptr_t nextoffs, uintptr_t prevoffs) { dtrace_hash_t *hash = kmem_zalloc(sizeof (dtrace_hash_t), KM_SLEEP); hash->dth_stroffs = stroffs; hash->dth_nextoffs = nextoffs; hash->dth_prevoffs = prevoffs; hash->dth_size = 1; hash->dth_mask = hash->dth_size - 1; hash->dth_tab = kmem_zalloc(hash->dth_size * sizeof (dtrace_hashbucket_t *), KM_SLEEP); return (hash); } static void dtrace_hash_destroy(dtrace_hash_t *hash) { #ifdef DEBUG int i; for (i = 0; i < hash->dth_size; i++) ASSERT(hash->dth_tab[i] == NULL); #endif kmem_free(hash->dth_tab, hash->dth_size * sizeof (dtrace_hashbucket_t *)); kmem_free(hash, sizeof (dtrace_hash_t)); } static void dtrace_hash_resize(dtrace_hash_t *hash) { int size = hash->dth_size, i, ndx; int new_size = hash->dth_size << 1; int new_mask = new_size - 1; dtrace_hashbucket_t **new_tab, *bucket, *next; ASSERT((new_size & new_mask) == 0); new_tab = kmem_zalloc(new_size * sizeof (void *), KM_SLEEP); for (i = 0; i < size; i++) { for (bucket = hash->dth_tab[i]; bucket != NULL; bucket = next) { dtrace_probe_t *probe = bucket->dthb_chain; ASSERT(probe != NULL); ndx = DTRACE_HASHSTR(hash, probe) & new_mask; next = bucket->dthb_next; bucket->dthb_next = new_tab[ndx]; new_tab[ndx] = bucket; } } kmem_free(hash->dth_tab, hash->dth_size * sizeof (void *)); hash->dth_tab = new_tab; hash->dth_size = new_size; hash->dth_mask = new_mask; } static void dtrace_hash_add(dtrace_hash_t *hash, dtrace_probe_t *new) { int hashval = DTRACE_HASHSTR(hash, new); int ndx = hashval & hash->dth_mask; dtrace_hashbucket_t *bucket = hash->dth_tab[ndx]; dtrace_probe_t **nextp, **prevp; for (; bucket != NULL; bucket = bucket->dthb_next) { if (DTRACE_HASHEQ(hash, bucket->dthb_chain, new)) goto add; } if ((hash->dth_nbuckets >> 1) > hash->dth_size) { dtrace_hash_resize(hash); dtrace_hash_add(hash, new); return; } bucket = kmem_zalloc(sizeof (dtrace_hashbucket_t), KM_SLEEP); bucket->dthb_next = hash->dth_tab[ndx]; hash->dth_tab[ndx] = bucket; hash->dth_nbuckets++; add: nextp = DTRACE_HASHNEXT(hash, new); ASSERT(*nextp == NULL && *(DTRACE_HASHPREV(hash, new)) == NULL); *nextp = bucket->dthb_chain; if (bucket->dthb_chain != NULL) { prevp = DTRACE_HASHPREV(hash, bucket->dthb_chain); ASSERT(*prevp == NULL); *prevp = new; } bucket->dthb_chain = new; bucket->dthb_len++; } static dtrace_probe_t * dtrace_hash_lookup(dtrace_hash_t *hash, dtrace_probe_t *template) { int hashval = DTRACE_HASHSTR(hash, template); int ndx = hashval & hash->dth_mask; dtrace_hashbucket_t *bucket = hash->dth_tab[ndx]; for (; bucket != NULL; bucket = bucket->dthb_next) { if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template)) return (bucket->dthb_chain); } return (NULL); } static int dtrace_hash_collisions(dtrace_hash_t *hash, dtrace_probe_t *template) { int hashval = DTRACE_HASHSTR(hash, template); int ndx = hashval & hash->dth_mask; dtrace_hashbucket_t *bucket = hash->dth_tab[ndx]; for (; bucket != NULL; bucket = bucket->dthb_next) { if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template)) return (bucket->dthb_len); } return (0); } static void dtrace_hash_remove(dtrace_hash_t *hash, dtrace_probe_t *probe) { int ndx = DTRACE_HASHSTR(hash, probe) & hash->dth_mask; dtrace_hashbucket_t *bucket = hash->dth_tab[ndx]; dtrace_probe_t **prevp = DTRACE_HASHPREV(hash, probe); dtrace_probe_t **nextp = DTRACE_HASHNEXT(hash, probe); /* * Find the bucket that we're removing this probe from. */ for (; bucket != NULL; bucket = bucket->dthb_next) { if (DTRACE_HASHEQ(hash, bucket->dthb_chain, probe)) break; } ASSERT(bucket != NULL); if (*prevp == NULL) { if (*nextp == NULL) { /* * The removed probe was the only probe on this * bucket; we need to remove the bucket. */ dtrace_hashbucket_t *b = hash->dth_tab[ndx]; ASSERT(bucket->dthb_chain == probe); ASSERT(b != NULL); if (b == bucket) { hash->dth_tab[ndx] = bucket->dthb_next; } else { while (b->dthb_next != bucket) b = b->dthb_next; b->dthb_next = bucket->dthb_next; } ASSERT(hash->dth_nbuckets > 0); hash->dth_nbuckets--; kmem_free(bucket, sizeof (dtrace_hashbucket_t)); return; } bucket->dthb_chain = *nextp; } else { *(DTRACE_HASHNEXT(hash, *prevp)) = *nextp; } if (*nextp != NULL) *(DTRACE_HASHPREV(hash, *nextp)) = *prevp; } /* * DTrace Utility Functions * * These are random utility functions that are _not_ called from probe context. */ static int dtrace_badattr(const dtrace_attribute_t *a) { return (a->dtat_name > DTRACE_STABILITY_MAX || a->dtat_data > DTRACE_STABILITY_MAX || a->dtat_class > DTRACE_CLASS_MAX); } /* * Return a duplicate copy of a string. If the specified string is NULL, * this function returns a zero-length string. */ static char * dtrace_strdup(const char *str) { char *new = kmem_zalloc((str != NULL ? strlen(str) : 0) + 1, KM_SLEEP); if (str != NULL) (void) strcpy(new, str); return (new); } #define DTRACE_ISALPHA(c) \ (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z')) static int dtrace_badname(const char *s) { char c; if (s == NULL || (c = *s++) == '\0') return (0); if (!DTRACE_ISALPHA(c) && c != '-' && c != '_' && c != '.') return (1); while ((c = *s++) != '\0') { if (!DTRACE_ISALPHA(c) && (c < '0' || c > '9') && c != '-' && c != '_' && c != '.' && c != '`') return (1); } return (0); } static void dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp, zoneid_t *zoneidp) { uint32_t priv; #ifdef illumos if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) { /* * For DTRACE_PRIV_ALL, the uid and zoneid don't matter. */ priv = DTRACE_PRIV_ALL; } else { *uidp = crgetuid(cr); *zoneidp = crgetzoneid(cr); priv = 0; if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) priv |= DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER; else if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) priv |= DTRACE_PRIV_USER; if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) priv |= DTRACE_PRIV_PROC; if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) priv |= DTRACE_PRIV_OWNER; if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) priv |= DTRACE_PRIV_ZONEOWNER; } #else priv = DTRACE_PRIV_ALL; #endif *privp = priv; } #ifdef DTRACE_ERRDEBUG static void dtrace_errdebug(const char *str) { int hval = dtrace_hash_str(str) % DTRACE_ERRHASHSZ; int occupied = 0; mutex_enter(&dtrace_errlock); dtrace_errlast = str; dtrace_errthread = curthread; while (occupied++ < DTRACE_ERRHASHSZ) { if (dtrace_errhash[hval].dter_msg == str) { dtrace_errhash[hval].dter_count++; goto out; } if (dtrace_errhash[hval].dter_msg != NULL) { hval = (hval + 1) % DTRACE_ERRHASHSZ; continue; } dtrace_errhash[hval].dter_msg = str; dtrace_errhash[hval].dter_count = 1; goto out; } panic("dtrace: undersized error hash"); out: mutex_exit(&dtrace_errlock); } #endif /* * DTrace Matching Functions * * These functions are used to match groups of probes, given some elements of * a probe tuple, or some globbed expressions for elements of a probe tuple. */ static int dtrace_match_priv(const dtrace_probe_t *prp, uint32_t priv, uid_t uid, zoneid_t zoneid) { if (priv != DTRACE_PRIV_ALL) { uint32_t ppriv = prp->dtpr_provider->dtpv_priv.dtpp_flags; uint32_t match = priv & ppriv; /* * No PRIV_DTRACE_* privileges... */ if ((priv & (DTRACE_PRIV_PROC | DTRACE_PRIV_USER | DTRACE_PRIV_KERNEL)) == 0) return (0); /* * No matching bits, but there were bits to match... */ if (match == 0 && ppriv != 0) return (0); /* * Need to have permissions to the process, but don't... */ if (((ppriv & ~match) & DTRACE_PRIV_OWNER) != 0 && uid != prp->dtpr_provider->dtpv_priv.dtpp_uid) { return (0); } /* * Need to be in the same zone unless we possess the * privilege to examine all zones. */ if (((ppriv & ~match) & DTRACE_PRIV_ZONEOWNER) != 0 && zoneid != prp->dtpr_provider->dtpv_priv.dtpp_zoneid) { return (0); } } return (1); } /* * dtrace_match_probe compares a dtrace_probe_t to a pre-compiled key, which * consists of input pattern strings and an ops-vector to evaluate them. * This function returns >0 for match, 0 for no match, and <0 for error. */ static int dtrace_match_probe(const dtrace_probe_t *prp, const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid, zoneid_t zoneid) { dtrace_provider_t *pvp = prp->dtpr_provider; int rv; if (pvp->dtpv_defunct) return (0); if ((rv = pkp->dtpk_pmatch(pvp->dtpv_name, pkp->dtpk_prov, 0)) <= 0) return (rv); if ((rv = pkp->dtpk_mmatch(prp->dtpr_mod, pkp->dtpk_mod, 0)) <= 0) return (rv); if ((rv = pkp->dtpk_fmatch(prp->dtpr_func, pkp->dtpk_func, 0)) <= 0) return (rv); if ((rv = pkp->dtpk_nmatch(prp->dtpr_name, pkp->dtpk_name, 0)) <= 0) return (rv); if (dtrace_match_priv(prp, priv, uid, zoneid) == 0) return (0); return (rv); } /* * dtrace_match_glob() is a safe kernel implementation of the gmatch(3GEN) * interface for matching a glob pattern 'p' to an input string 's'. Unlike * libc's version, the kernel version only applies to 8-bit ASCII strings. * In addition, all of the recursion cases except for '*' matching have been * unwound. For '*', we still implement recursive evaluation, but a depth * counter is maintained and matching is aborted if we recurse too deep. * The function returns 0 if no match, >0 if match, and <0 if recursion error. */ static int dtrace_match_glob(const char *s, const char *p, int depth) { const char *olds; char s1, c; int gs; if (depth > DTRACE_PROBEKEY_MAXDEPTH) return (-1); if (s == NULL) s = ""; /* treat NULL as empty string */ top: olds = s; s1 = *s++; if (p == NULL) return (0); if ((c = *p++) == '\0') return (s1 == '\0'); switch (c) { case '[': { int ok = 0, notflag = 0; char lc = '\0'; if (s1 == '\0') return (0); if (*p == '!') { notflag = 1; p++; } if ((c = *p++) == '\0') return (0); do { if (c == '-' && lc != '\0' && *p != ']') { if ((c = *p++) == '\0') return (0); if (c == '\\' && (c = *p++) == '\0') return (0); if (notflag) { if (s1 < lc || s1 > c) ok++; else return (0); } else if (lc <= s1 && s1 <= c) ok++; } else if (c == '\\' && (c = *p++) == '\0') return (0); lc = c; /* save left-hand 'c' for next iteration */ if (notflag) { if (s1 != c) ok++; else return (0); } else if (s1 == c) ok++; if ((c = *p++) == '\0') return (0); } while (c != ']'); if (ok) goto top; return (0); } case '\\': if ((c = *p++) == '\0') return (0); /*FALLTHRU*/ default: if (c != s1) return (0); /*FALLTHRU*/ case '?': if (s1 != '\0') goto top; return (0); case '*': while (*p == '*') p++; /* consecutive *'s are identical to a single one */ if (*p == '\0') return (1); for (s = olds; *s != '\0'; s++) { if ((gs = dtrace_match_glob(s, p, depth + 1)) != 0) return (gs); } return (0); } } /*ARGSUSED*/ static int dtrace_match_string(const char *s, const char *p, int depth) { return (s != NULL && strcmp(s, p) == 0); } /*ARGSUSED*/ static int dtrace_match_nul(const char *s, const char *p, int depth) { return (1); /* always match the empty pattern */ } /*ARGSUSED*/ static int dtrace_match_nonzero(const char *s, const char *p, int depth) { return (s != NULL && s[0] != '\0'); } static int dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid, zoneid_t zoneid, int (*matched)(dtrace_probe_t *, void *), void *arg) { dtrace_probe_t template, *probe; dtrace_hash_t *hash = NULL; int len, best = INT_MAX, nmatched = 0; dtrace_id_t i; ASSERT(MUTEX_HELD(&dtrace_lock)); /* * If the probe ID is specified in the key, just lookup by ID and * invoke the match callback once if a matching probe is found. */ if (pkp->dtpk_id != DTRACE_IDNONE) { if ((probe = dtrace_probe_lookup_id(pkp->dtpk_id)) != NULL && dtrace_match_probe(probe, pkp, priv, uid, zoneid) > 0) { (void) (*matched)(probe, arg); nmatched++; } return (nmatched); } template.dtpr_mod = (char *)pkp->dtpk_mod; template.dtpr_func = (char *)pkp->dtpk_func; template.dtpr_name = (char *)pkp->dtpk_name; /* * We want to find the most distinct of the module name, function * name, and name. So for each one that is not a glob pattern or * empty string, we perform a lookup in the corresponding hash and * use the hash table with the fewest collisions to do our search. */ if (pkp->dtpk_mmatch == &dtrace_match_string && (len = dtrace_hash_collisions(dtrace_bymod, &template)) < best) { best = len; hash = dtrace_bymod; } if (pkp->dtpk_fmatch == &dtrace_match_string && (len = dtrace_hash_collisions(dtrace_byfunc, &template)) < best) { best = len; hash = dtrace_byfunc; } if (pkp->dtpk_nmatch == &dtrace_match_string && (len = dtrace_hash_collisions(dtrace_byname, &template)) < best) { best = len; hash = dtrace_byname; } /* * If we did not select a hash table, iterate over every probe and * invoke our callback for each one that matches our input probe key. */ if (hash == NULL) { for (i = 0; i < dtrace_nprobes; i++) { if ((probe = dtrace_probes[i]) == NULL || dtrace_match_probe(probe, pkp, priv, uid, zoneid) <= 0) continue; nmatched++; if ((*matched)(probe, arg) != DTRACE_MATCH_NEXT) break; } return (nmatched); } /* * If we selected a hash table, iterate over each probe of the same key * name and invoke the callback for every probe that matches the other * attributes of our input probe key. */ for (probe = dtrace_hash_lookup(hash, &template); probe != NULL; probe = *(DTRACE_HASHNEXT(hash, probe))) { if (dtrace_match_probe(probe, pkp, priv, uid, zoneid) <= 0) continue; nmatched++; if ((*matched)(probe, arg) != DTRACE_MATCH_NEXT) break; } return (nmatched); } /* * Return the function pointer dtrace_probecmp() should use to compare the * specified pattern with a string. For NULL or empty patterns, we select * dtrace_match_nul(). For glob pattern strings, we use dtrace_match_glob(). * For non-empty non-glob strings, we use dtrace_match_string(). */ static dtrace_probekey_f * dtrace_probekey_func(const char *p) { char c; if (p == NULL || *p == '\0') return (&dtrace_match_nul); while ((c = *p++) != '\0') { if (c == '[' || c == '?' || c == '*' || c == '\\') return (&dtrace_match_glob); } return (&dtrace_match_string); } /* * Build a probe comparison key for use with dtrace_match_probe() from the * given probe description. By convention, a null key only matches anchored * probes: if each field is the empty string, reset dtpk_fmatch to * dtrace_match_nonzero(). */ static void dtrace_probekey(dtrace_probedesc_t *pdp, dtrace_probekey_t *pkp) { pkp->dtpk_prov = pdp->dtpd_provider; pkp->dtpk_pmatch = dtrace_probekey_func(pdp->dtpd_provider); pkp->dtpk_mod = pdp->dtpd_mod; pkp->dtpk_mmatch = dtrace_probekey_func(pdp->dtpd_mod); pkp->dtpk_func = pdp->dtpd_func; pkp->dtpk_fmatch = dtrace_probekey_func(pdp->dtpd_func); pkp->dtpk_name = pdp->dtpd_name; pkp->dtpk_nmatch = dtrace_probekey_func(pdp->dtpd_name); pkp->dtpk_id = pdp->dtpd_id; if (pkp->dtpk_id == DTRACE_IDNONE && pkp->dtpk_pmatch == &dtrace_match_nul && pkp->dtpk_mmatch == &dtrace_match_nul && pkp->dtpk_fmatch == &dtrace_match_nul && pkp->dtpk_nmatch == &dtrace_match_nul) pkp->dtpk_fmatch = &dtrace_match_nonzero; } /* * DTrace Provider-to-Framework API Functions * * These functions implement much of the Provider-to-Framework API, as * described in . The parts of the API not in this section are * the functions in the API for probe management (found below), and * dtrace_probe() itself (found above). */ /* * Register the calling provider with the DTrace framework. This should * generally be called by DTrace providers in their attach(9E) entry point. */ int dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv, cred_t *cr, const dtrace_pops_t *pops, void *arg, dtrace_provider_id_t *idp) { dtrace_provider_t *provider; if (name == NULL || pap == NULL || pops == NULL || idp == NULL) { cmn_err(CE_WARN, "failed to register provider '%s': invalid " "arguments", name ? name : ""); return (EINVAL); } if (name[0] == '\0' || dtrace_badname(name)) { cmn_err(CE_WARN, "failed to register provider '%s': invalid " "provider name", name); return (EINVAL); } if ((pops->dtps_provide == NULL && pops->dtps_provide_module == NULL) || pops->dtps_enable == NULL || pops->dtps_disable == NULL || pops->dtps_destroy == NULL || ((pops->dtps_resume == NULL) != (pops->dtps_suspend == NULL))) { cmn_err(CE_WARN, "failed to register provider '%s': invalid " "provider ops", name); return (EINVAL); } if (dtrace_badattr(&pap->dtpa_provider) || dtrace_badattr(&pap->dtpa_mod) || dtrace_badattr(&pap->dtpa_func) || dtrace_badattr(&pap->dtpa_name) || dtrace_badattr(&pap->dtpa_args)) { cmn_err(CE_WARN, "failed to register provider '%s': invalid " "provider attributes", name); return (EINVAL); } if (priv & ~DTRACE_PRIV_ALL) { cmn_err(CE_WARN, "failed to register provider '%s': invalid " "privilege attributes", name); return (EINVAL); } if ((priv & DTRACE_PRIV_KERNEL) && (priv & (DTRACE_PRIV_USER | DTRACE_PRIV_OWNER)) && pops->dtps_usermode == NULL) { cmn_err(CE_WARN, "failed to register provider '%s': need " "dtps_usermode() op for given privilege attributes", name); return (EINVAL); } provider = kmem_zalloc(sizeof (dtrace_provider_t), KM_SLEEP); provider->dtpv_name = kmem_alloc(strlen(name) + 1, KM_SLEEP); (void) strcpy(provider->dtpv_name, name); provider->dtpv_attr = *pap; provider->dtpv_priv.dtpp_flags = priv; if (cr != NULL) { provider->dtpv_priv.dtpp_uid = crgetuid(cr); provider->dtpv_priv.dtpp_zoneid = crgetzoneid(cr); } provider->dtpv_pops = *pops; if (pops->dtps_provide == NULL) { ASSERT(pops->dtps_provide_module != NULL); provider->dtpv_pops.dtps_provide = (void (*)(void *, dtrace_probedesc_t *))dtrace_nullop; } if (pops->dtps_provide_module == NULL) { ASSERT(pops->dtps_provide != NULL); provider->dtpv_pops.dtps_provide_module = (void (*)(void *, modctl_t *))dtrace_nullop; } if (pops->dtps_suspend == NULL) { ASSERT(pops->dtps_resume == NULL); provider->dtpv_pops.dtps_suspend = (void (*)(void *, dtrace_id_t, void *))dtrace_nullop; provider->dtpv_pops.dtps_resume = (void (*)(void *, dtrace_id_t, void *))dtrace_nullop; } provider->dtpv_arg = arg; *idp = (dtrace_provider_id_t)provider; if (pops == &dtrace_provider_ops) { ASSERT(MUTEX_HELD(&dtrace_provider_lock)); ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(dtrace_anon.dta_enabling == NULL); /* * We make sure that the DTrace provider is at the head of * the provider chain. */ provider->dtpv_next = dtrace_provider; dtrace_provider = provider; return (0); } mutex_enter(&dtrace_provider_lock); mutex_enter(&dtrace_lock); /* * If there is at least one provider registered, we'll add this * provider after the first provider. */ if (dtrace_provider != NULL) { provider->dtpv_next = dtrace_provider->dtpv_next; dtrace_provider->dtpv_next = provider; } else { dtrace_provider = provider; } if (dtrace_retained != NULL) { dtrace_enabling_provide(provider); /* * Now we need to call dtrace_enabling_matchall() -- which * will acquire cpu_lock and dtrace_lock. We therefore need * to drop all of our locks before calling into it... */ mutex_exit(&dtrace_lock); mutex_exit(&dtrace_provider_lock); dtrace_enabling_matchall(); return (0); } mutex_exit(&dtrace_lock); mutex_exit(&dtrace_provider_lock); return (0); } /* * Unregister the specified provider from the DTrace framework. This should * generally be called by DTrace providers in their detach(9E) entry point. */ int dtrace_unregister(dtrace_provider_id_t id) { dtrace_provider_t *old = (dtrace_provider_t *)id; dtrace_provider_t *prev = NULL; int i, self = 0, noreap = 0; dtrace_probe_t *probe, *first = NULL; if (old->dtpv_pops.dtps_enable == (void (*)(void *, dtrace_id_t, void *))dtrace_nullop) { /* * If DTrace itself is the provider, we're called with locks * already held. */ ASSERT(old == dtrace_provider); #ifdef illumos ASSERT(dtrace_devi != NULL); #endif ASSERT(MUTEX_HELD(&dtrace_provider_lock)); ASSERT(MUTEX_HELD(&dtrace_lock)); self = 1; if (dtrace_provider->dtpv_next != NULL) { /* * There's another provider here; return failure. */ return (EBUSY); } } else { mutex_enter(&dtrace_provider_lock); #ifdef illumos mutex_enter(&mod_lock); #endif mutex_enter(&dtrace_lock); } /* * If anyone has /dev/dtrace open, or if there are anonymous enabled * probes, we refuse to let providers slither away, unless this * provider has already been explicitly invalidated. */ if (!old->dtpv_defunct && (dtrace_opens || (dtrace_anon.dta_state != NULL && dtrace_anon.dta_state->dts_necbs > 0))) { if (!self) { mutex_exit(&dtrace_lock); #ifdef illumos mutex_exit(&mod_lock); #endif mutex_exit(&dtrace_provider_lock); } return (EBUSY); } /* * Attempt to destroy the probes associated with this provider. */ for (i = 0; i < dtrace_nprobes; i++) { if ((probe = dtrace_probes[i]) == NULL) continue; if (probe->dtpr_provider != old) continue; if (probe->dtpr_ecb == NULL) continue; /* * If we are trying to unregister a defunct provider, and the * provider was made defunct within the interval dictated by * dtrace_unregister_defunct_reap, we'll (asynchronously) * attempt to reap our enablings. To denote that the provider * should reattempt to unregister itself at some point in the * future, we will return a differentiable error code (EAGAIN * instead of EBUSY) in this case. */ if (dtrace_gethrtime() - old->dtpv_defunct > dtrace_unregister_defunct_reap) noreap = 1; if (!self) { mutex_exit(&dtrace_lock); #ifdef illumos mutex_exit(&mod_lock); #endif mutex_exit(&dtrace_provider_lock); } if (noreap) return (EBUSY); (void) taskq_dispatch(dtrace_taskq, (task_func_t *)dtrace_enabling_reap, NULL, TQ_SLEEP); return (EAGAIN); } /* * All of the probes for this provider are disabled; we can safely * remove all of them from their hash chains and from the probe array. */ for (i = 0; i < dtrace_nprobes; i++) { if ((probe = dtrace_probes[i]) == NULL) continue; if (probe->dtpr_provider != old) continue; dtrace_probes[i] = NULL; dtrace_hash_remove(dtrace_bymod, probe); dtrace_hash_remove(dtrace_byfunc, probe); dtrace_hash_remove(dtrace_byname, probe); if (first == NULL) { first = probe; probe->dtpr_nextmod = NULL; } else { probe->dtpr_nextmod = first; first = probe; } } /* * The provider's probes have been removed from the hash chains and * from the probe array. Now issue a dtrace_sync() to be sure that * everyone has cleared out from any probe array processing. */ dtrace_sync(); for (probe = first; probe != NULL; probe = first) { first = probe->dtpr_nextmod; old->dtpv_pops.dtps_destroy(old->dtpv_arg, probe->dtpr_id, probe->dtpr_arg); kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1); kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1); kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1); #ifdef illumos vmem_free(dtrace_arena, (void *)(uintptr_t)(probe->dtpr_id), 1); #else free_unr(dtrace_arena, probe->dtpr_id); #endif kmem_free(probe, sizeof (dtrace_probe_t)); } if ((prev = dtrace_provider) == old) { #ifdef illumos ASSERT(self || dtrace_devi == NULL); ASSERT(old->dtpv_next == NULL || dtrace_devi == NULL); #endif dtrace_provider = old->dtpv_next; } else { while (prev != NULL && prev->dtpv_next != old) prev = prev->dtpv_next; if (prev == NULL) { panic("attempt to unregister non-existent " "dtrace provider %p\n", (void *)id); } prev->dtpv_next = old->dtpv_next; } if (!self) { mutex_exit(&dtrace_lock); #ifdef illumos mutex_exit(&mod_lock); #endif mutex_exit(&dtrace_provider_lock); } kmem_free(old->dtpv_name, strlen(old->dtpv_name) + 1); kmem_free(old, sizeof (dtrace_provider_t)); return (0); } /* * Invalidate the specified provider. All subsequent probe lookups for the * specified provider will fail, but its probes will not be removed. */ void dtrace_invalidate(dtrace_provider_id_t id) { dtrace_provider_t *pvp = (dtrace_provider_t *)id; ASSERT(pvp->dtpv_pops.dtps_enable != (void (*)(void *, dtrace_id_t, void *))dtrace_nullop); mutex_enter(&dtrace_provider_lock); mutex_enter(&dtrace_lock); pvp->dtpv_defunct = dtrace_gethrtime(); mutex_exit(&dtrace_lock); mutex_exit(&dtrace_provider_lock); } /* * Indicate whether or not DTrace has attached. */ int dtrace_attached(void) { /* * dtrace_provider will be non-NULL iff the DTrace driver has * attached. (It's non-NULL because DTrace is always itself a * provider.) */ return (dtrace_provider != NULL); } /* * Remove all the unenabled probes for the given provider. This function is * not unlike dtrace_unregister(), except that it doesn't remove the provider * -- just as many of its associated probes as it can. */ int dtrace_condense(dtrace_provider_id_t id) { dtrace_provider_t *prov = (dtrace_provider_t *)id; int i; dtrace_probe_t *probe; /* * Make sure this isn't the dtrace provider itself. */ ASSERT(prov->dtpv_pops.dtps_enable != (void (*)(void *, dtrace_id_t, void *))dtrace_nullop); mutex_enter(&dtrace_provider_lock); mutex_enter(&dtrace_lock); /* * Attempt to destroy the probes associated with this provider. */ for (i = 0; i < dtrace_nprobes; i++) { if ((probe = dtrace_probes[i]) == NULL) continue; if (probe->dtpr_provider != prov) continue; if (probe->dtpr_ecb != NULL) continue; dtrace_probes[i] = NULL; dtrace_hash_remove(dtrace_bymod, probe); dtrace_hash_remove(dtrace_byfunc, probe); dtrace_hash_remove(dtrace_byname, probe); prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, i + 1, probe->dtpr_arg); kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1); kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1); kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1); kmem_free(probe, sizeof (dtrace_probe_t)); #ifdef illumos vmem_free(dtrace_arena, (void *)((uintptr_t)i + 1), 1); #else free_unr(dtrace_arena, i + 1); #endif } mutex_exit(&dtrace_lock); mutex_exit(&dtrace_provider_lock); return (0); } /* * DTrace Probe Management Functions * * The functions in this section perform the DTrace probe management, * including functions to create probes, look-up probes, and call into the * providers to request that probes be provided. Some of these functions are * in the Provider-to-Framework API; these functions can be identified by the * fact that they are not declared "static". */ /* * Create a probe with the specified module name, function name, and name. */ dtrace_id_t dtrace_probe_create(dtrace_provider_id_t prov, const char *mod, const char *func, const char *name, int aframes, void *arg) { dtrace_probe_t *probe, **probes; dtrace_provider_t *provider = (dtrace_provider_t *)prov; dtrace_id_t id; if (provider == dtrace_provider) { ASSERT(MUTEX_HELD(&dtrace_lock)); } else { mutex_enter(&dtrace_lock); } #ifdef illumos id = (dtrace_id_t)(uintptr_t)vmem_alloc(dtrace_arena, 1, VM_BESTFIT | VM_SLEEP); #else id = alloc_unr(dtrace_arena); #endif probe = kmem_zalloc(sizeof (dtrace_probe_t), KM_SLEEP); probe->dtpr_id = id; probe->dtpr_gen = dtrace_probegen++; probe->dtpr_mod = dtrace_strdup(mod); probe->dtpr_func = dtrace_strdup(func); probe->dtpr_name = dtrace_strdup(name); probe->dtpr_arg = arg; probe->dtpr_aframes = aframes; probe->dtpr_provider = provider; dtrace_hash_add(dtrace_bymod, probe); dtrace_hash_add(dtrace_byfunc, probe); dtrace_hash_add(dtrace_byname, probe); if (id - 1 >= dtrace_nprobes) { size_t osize = dtrace_nprobes * sizeof (dtrace_probe_t *); size_t nsize = osize << 1; if (nsize == 0) { ASSERT(osize == 0); ASSERT(dtrace_probes == NULL); nsize = sizeof (dtrace_probe_t *); } probes = kmem_zalloc(nsize, KM_SLEEP); if (dtrace_probes == NULL) { ASSERT(osize == 0); dtrace_probes = probes; dtrace_nprobes = 1; } else { dtrace_probe_t **oprobes = dtrace_probes; bcopy(oprobes, probes, osize); dtrace_membar_producer(); dtrace_probes = probes; dtrace_sync(); /* * All CPUs are now seeing the new probes array; we can * safely free the old array. */ kmem_free(oprobes, osize); dtrace_nprobes <<= 1; } ASSERT(id - 1 < dtrace_nprobes); } ASSERT(dtrace_probes[id - 1] == NULL); dtrace_probes[id - 1] = probe; if (provider != dtrace_provider) mutex_exit(&dtrace_lock); return (id); } static dtrace_probe_t * dtrace_probe_lookup_id(dtrace_id_t id) { ASSERT(MUTEX_HELD(&dtrace_lock)); if (id == 0 || id > dtrace_nprobes) return (NULL); return (dtrace_probes[id - 1]); } static int dtrace_probe_lookup_match(dtrace_probe_t *probe, void *arg) { *((dtrace_id_t *)arg) = probe->dtpr_id; return (DTRACE_MATCH_DONE); } /* * Look up a probe based on provider and one or more of module name, function * name and probe name. */ dtrace_id_t dtrace_probe_lookup(dtrace_provider_id_t prid, char *mod, char *func, char *name) { dtrace_probekey_t pkey; dtrace_id_t id; int match; pkey.dtpk_prov = ((dtrace_provider_t *)prid)->dtpv_name; pkey.dtpk_pmatch = &dtrace_match_string; pkey.dtpk_mod = mod; pkey.dtpk_mmatch = mod ? &dtrace_match_string : &dtrace_match_nul; pkey.dtpk_func = func; pkey.dtpk_fmatch = func ? &dtrace_match_string : &dtrace_match_nul; pkey.dtpk_name = name; pkey.dtpk_nmatch = name ? &dtrace_match_string : &dtrace_match_nul; pkey.dtpk_id = DTRACE_IDNONE; mutex_enter(&dtrace_lock); match = dtrace_match(&pkey, DTRACE_PRIV_ALL, 0, 0, dtrace_probe_lookup_match, &id); mutex_exit(&dtrace_lock); ASSERT(match == 1 || match == 0); return (match ? id : 0); } /* * Returns the probe argument associated with the specified probe. */ void * dtrace_probe_arg(dtrace_provider_id_t id, dtrace_id_t pid) { dtrace_probe_t *probe; void *rval = NULL; mutex_enter(&dtrace_lock); if ((probe = dtrace_probe_lookup_id(pid)) != NULL && probe->dtpr_provider == (dtrace_provider_t *)id) rval = probe->dtpr_arg; mutex_exit(&dtrace_lock); return (rval); } /* * Copy a probe into a probe description. */ static void dtrace_probe_description(const dtrace_probe_t *prp, dtrace_probedesc_t *pdp) { bzero(pdp, sizeof (dtrace_probedesc_t)); pdp->dtpd_id = prp->dtpr_id; (void) strncpy(pdp->dtpd_provider, prp->dtpr_provider->dtpv_name, DTRACE_PROVNAMELEN - 1); (void) strncpy(pdp->dtpd_mod, prp->dtpr_mod, DTRACE_MODNAMELEN - 1); (void) strncpy(pdp->dtpd_func, prp->dtpr_func, DTRACE_FUNCNAMELEN - 1); (void) strncpy(pdp->dtpd_name, prp->dtpr_name, DTRACE_NAMELEN - 1); } /* * Called to indicate that a probe -- or probes -- should be provided by a * specfied provider. If the specified description is NULL, the provider will * be told to provide all of its probes. (This is done whenever a new * consumer comes along, or whenever a retained enabling is to be matched.) If * the specified description is non-NULL, the provider is given the * opportunity to dynamically provide the specified probe, allowing providers * to support the creation of probes on-the-fly. (So-called _autocreated_ * probes.) If the provider is NULL, the operations will be applied to all * providers; if the provider is non-NULL the operations will only be applied * to the specified provider. The dtrace_provider_lock must be held, and the * dtrace_lock must _not_ be held -- the provider's dtps_provide() operation * will need to grab the dtrace_lock when it reenters the framework through * dtrace_probe_lookup(), dtrace_probe_create(), etc. */ static void dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv) { #ifdef illumos modctl_t *ctl; #endif int all = 0; ASSERT(MUTEX_HELD(&dtrace_provider_lock)); if (prv == NULL) { all = 1; prv = dtrace_provider; } do { /* * First, call the blanket provide operation. */ prv->dtpv_pops.dtps_provide(prv->dtpv_arg, desc); #ifdef illumos /* * Now call the per-module provide operation. We will grab * mod_lock to prevent the list from being modified. Note * that this also prevents the mod_busy bits from changing. * (mod_busy can only be changed with mod_lock held.) */ mutex_enter(&mod_lock); ctl = &modules; do { if (ctl->mod_busy || ctl->mod_mp == NULL) continue; prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl); } while ((ctl = ctl->mod_next) != &modules); mutex_exit(&mod_lock); #endif } while (all && (prv = prv->dtpv_next) != NULL); } #ifdef illumos /* * Iterate over each probe, and call the Framework-to-Provider API function * denoted by offs. */ static void dtrace_probe_foreach(uintptr_t offs) { dtrace_provider_t *prov; void (*func)(void *, dtrace_id_t, void *); dtrace_probe_t *probe; dtrace_icookie_t cookie; int i; /* * We disable interrupts to walk through the probe array. This is * safe -- the dtrace_sync() in dtrace_unregister() assures that we * won't see stale data. */ cookie = dtrace_interrupt_disable(); for (i = 0; i < dtrace_nprobes; i++) { if ((probe = dtrace_probes[i]) == NULL) continue; if (probe->dtpr_ecb == NULL) { /* * This probe isn't enabled -- don't call the function. */ continue; } prov = probe->dtpr_provider; func = *((void(**)(void *, dtrace_id_t, void *)) ((uintptr_t)&prov->dtpv_pops + offs)); func(prov->dtpv_arg, i + 1, probe->dtpr_arg); } dtrace_interrupt_enable(cookie); } #endif static int dtrace_probe_enable(dtrace_probedesc_t *desc, dtrace_enabling_t *enab) { dtrace_probekey_t pkey; uint32_t priv; uid_t uid; zoneid_t zoneid; ASSERT(MUTEX_HELD(&dtrace_lock)); dtrace_ecb_create_cache = NULL; if (desc == NULL) { /* * If we're passed a NULL description, we're being asked to * create an ECB with a NULL probe. */ (void) dtrace_ecb_create_enable(NULL, enab); return (0); } dtrace_probekey(desc, &pkey); dtrace_cred2priv(enab->dten_vstate->dtvs_state->dts_cred.dcr_cred, &priv, &uid, &zoneid); return (dtrace_match(&pkey, priv, uid, zoneid, dtrace_ecb_create_enable, enab)); } /* * DTrace Helper Provider Functions */ static void dtrace_dofattr2attr(dtrace_attribute_t *attr, const dof_attr_t dofattr) { attr->dtat_name = DOF_ATTR_NAME(dofattr); attr->dtat_data = DOF_ATTR_DATA(dofattr); attr->dtat_class = DOF_ATTR_CLASS(dofattr); } static void dtrace_dofprov2hprov(dtrace_helper_provdesc_t *hprov, const dof_provider_t *dofprov, char *strtab) { hprov->dthpv_provname = strtab + dofprov->dofpv_name; dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_provider, dofprov->dofpv_provattr); dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_mod, dofprov->dofpv_modattr); dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_func, dofprov->dofpv_funcattr); dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_name, dofprov->dofpv_nameattr); dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_args, dofprov->dofpv_argsattr); } static void dtrace_helper_provide_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid) { uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; dof_hdr_t *dof = (dof_hdr_t *)daddr; dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec; dof_provider_t *provider; dof_probe_t *probe; uint32_t *off, *enoff; uint8_t *arg; char *strtab; uint_t i, nprobes; dtrace_helper_provdesc_t dhpv; dtrace_helper_probedesc_t dhpb; dtrace_meta_t *meta = dtrace_meta_pid; dtrace_mops_t *mops = &meta->dtm_mops; void *parg; provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset); str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + provider->dofpv_strtab * dof->dofh_secsize); prb_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + provider->dofpv_probes * dof->dofh_secsize); arg_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + provider->dofpv_prargs * dof->dofh_secsize); off_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + provider->dofpv_proffs * dof->dofh_secsize); strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset); off = (uint32_t *)(uintptr_t)(daddr + off_sec->dofs_offset); arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset); enoff = NULL; /* * See dtrace_helper_provider_validate(). */ if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 && provider->dofpv_prenoffs != DOF_SECT_NONE) { enoff_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + provider->dofpv_prenoffs * dof->dofh_secsize); enoff = (uint32_t *)(uintptr_t)(daddr + enoff_sec->dofs_offset); } nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize; /* * Create the provider. */ dtrace_dofprov2hprov(&dhpv, provider, strtab); if ((parg = mops->dtms_provide_pid(meta->dtm_arg, &dhpv, pid)) == NULL) return; meta->dtm_count++; /* * Create the probes. */ for (i = 0; i < nprobes; i++) { probe = (dof_probe_t *)(uintptr_t)(daddr + prb_sec->dofs_offset + i * prb_sec->dofs_entsize); dhpb.dthpb_mod = dhp->dofhp_mod; dhpb.dthpb_func = strtab + probe->dofpr_func; dhpb.dthpb_name = strtab + probe->dofpr_name; dhpb.dthpb_base = probe->dofpr_addr; dhpb.dthpb_offs = off + probe->dofpr_offidx; dhpb.dthpb_noffs = probe->dofpr_noffs; if (enoff != NULL) { dhpb.dthpb_enoffs = enoff + probe->dofpr_enoffidx; dhpb.dthpb_nenoffs = probe->dofpr_nenoffs; } else { dhpb.dthpb_enoffs = NULL; dhpb.dthpb_nenoffs = 0; } dhpb.dthpb_args = arg + probe->dofpr_argidx; dhpb.dthpb_nargc = probe->dofpr_nargc; dhpb.dthpb_xargc = probe->dofpr_xargc; dhpb.dthpb_ntypes = strtab + probe->dofpr_nargv; dhpb.dthpb_xtypes = strtab + probe->dofpr_xargv; mops->dtms_create_probe(meta->dtm_arg, parg, &dhpb); } } static void dtrace_helper_provide(dof_helper_t *dhp, pid_t pid) { uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; dof_hdr_t *dof = (dof_hdr_t *)daddr; int i; ASSERT(MUTEX_HELD(&dtrace_meta_lock)); for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + i * dof->dofh_secsize); if (sec->dofs_type != DOF_SECT_PROVIDER) continue; dtrace_helper_provide_one(dhp, sec, pid); } /* * We may have just created probes, so we must now rematch against * any retained enablings. Note that this call will acquire both * cpu_lock and dtrace_lock; the fact that we are holding * dtrace_meta_lock now is what defines the ordering with respect to * these three locks. */ dtrace_enabling_matchall(); } static void dtrace_helper_provider_remove_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid) { uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; dof_hdr_t *dof = (dof_hdr_t *)daddr; dof_sec_t *str_sec; dof_provider_t *provider; char *strtab; dtrace_helper_provdesc_t dhpv; dtrace_meta_t *meta = dtrace_meta_pid; dtrace_mops_t *mops = &meta->dtm_mops; provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset); str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + provider->dofpv_strtab * dof->dofh_secsize); strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset); /* * Create the provider. */ dtrace_dofprov2hprov(&dhpv, provider, strtab); mops->dtms_remove_pid(meta->dtm_arg, &dhpv, pid); meta->dtm_count--; } static void dtrace_helper_provider_remove(dof_helper_t *dhp, pid_t pid) { uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; dof_hdr_t *dof = (dof_hdr_t *)daddr; int i; ASSERT(MUTEX_HELD(&dtrace_meta_lock)); for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + i * dof->dofh_secsize); if (sec->dofs_type != DOF_SECT_PROVIDER) continue; dtrace_helper_provider_remove_one(dhp, sec, pid); } } /* * DTrace Meta Provider-to-Framework API Functions * * These functions implement the Meta Provider-to-Framework API, as described * in . */ int dtrace_meta_register(const char *name, const dtrace_mops_t *mops, void *arg, dtrace_meta_provider_id_t *idp) { dtrace_meta_t *meta; dtrace_helpers_t *help, *next; int i; *idp = DTRACE_METAPROVNONE; /* * We strictly don't need the name, but we hold onto it for * debuggability. All hail error queues! */ if (name == NULL) { cmn_err(CE_WARN, "failed to register meta-provider: " "invalid name"); return (EINVAL); } if (mops == NULL || mops->dtms_create_probe == NULL || mops->dtms_provide_pid == NULL || mops->dtms_remove_pid == NULL) { cmn_err(CE_WARN, "failed to register meta-register %s: " "invalid ops", name); return (EINVAL); } meta = kmem_zalloc(sizeof (dtrace_meta_t), KM_SLEEP); meta->dtm_mops = *mops; meta->dtm_name = kmem_alloc(strlen(name) + 1, KM_SLEEP); (void) strcpy(meta->dtm_name, name); meta->dtm_arg = arg; mutex_enter(&dtrace_meta_lock); mutex_enter(&dtrace_lock); if (dtrace_meta_pid != NULL) { mutex_exit(&dtrace_lock); mutex_exit(&dtrace_meta_lock); cmn_err(CE_WARN, "failed to register meta-register %s: " "user-land meta-provider exists", name); kmem_free(meta->dtm_name, strlen(meta->dtm_name) + 1); kmem_free(meta, sizeof (dtrace_meta_t)); return (EINVAL); } dtrace_meta_pid = meta; *idp = (dtrace_meta_provider_id_t)meta; /* * If there are providers and probes ready to go, pass them * off to the new meta provider now. */ help = dtrace_deferred_pid; dtrace_deferred_pid = NULL; mutex_exit(&dtrace_lock); while (help != NULL) { for (i = 0; i < help->dthps_nprovs; i++) { dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov, help->dthps_pid); } next = help->dthps_next; help->dthps_next = NULL; help->dthps_prev = NULL; help->dthps_deferred = 0; help = next; } mutex_exit(&dtrace_meta_lock); return (0); } int dtrace_meta_unregister(dtrace_meta_provider_id_t id) { dtrace_meta_t **pp, *old = (dtrace_meta_t *)id; mutex_enter(&dtrace_meta_lock); mutex_enter(&dtrace_lock); if (old == dtrace_meta_pid) { pp = &dtrace_meta_pid; } else { panic("attempt to unregister non-existent " "dtrace meta-provider %p\n", (void *)old); } if (old->dtm_count != 0) { mutex_exit(&dtrace_lock); mutex_exit(&dtrace_meta_lock); return (EBUSY); } *pp = NULL; mutex_exit(&dtrace_lock); mutex_exit(&dtrace_meta_lock); kmem_free(old->dtm_name, strlen(old->dtm_name) + 1); kmem_free(old, sizeof (dtrace_meta_t)); return (0); } /* * DTrace DIF Object Functions */ static int dtrace_difo_err(uint_t pc, const char *format, ...) { if (dtrace_err_verbose) { va_list alist; (void) uprintf("dtrace DIF object error: [%u]: ", pc); va_start(alist, format); (void) vuprintf(format, alist); va_end(alist); } #ifdef DTRACE_ERRDEBUG dtrace_errdebug(format); #endif return (1); } /* * Validate a DTrace DIF object by checking the IR instructions. The following * rules are currently enforced by dtrace_difo_validate(): * * 1. Each instruction must have a valid opcode * 2. Each register, string, variable, or subroutine reference must be valid * 3. No instruction can modify register %r0 (must be zero) * 4. All instruction reserved bits must be set to zero * 5. The last instruction must be a "ret" instruction * 6. All branch targets must reference a valid instruction _after_ the branch */ static int dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs, cred_t *cr) { int err = 0, i; int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err; int kcheckload; uint_t pc; kcheckload = cr == NULL || (vstate->dtvs_state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) == 0; dp->dtdo_destructive = 0; for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) { dif_instr_t instr = dp->dtdo_buf[pc]; uint_t r1 = DIF_INSTR_R1(instr); uint_t r2 = DIF_INSTR_R2(instr); uint_t rd = DIF_INSTR_RD(instr); uint_t rs = DIF_INSTR_RS(instr); uint_t label = DIF_INSTR_LABEL(instr); uint_t v = DIF_INSTR_VAR(instr); uint_t subr = DIF_INSTR_SUBR(instr); uint_t type = DIF_INSTR_TYPE(instr); uint_t op = DIF_INSTR_OP(instr); switch (op) { case DIF_OP_OR: case DIF_OP_XOR: case DIF_OP_AND: case DIF_OP_SLL: case DIF_OP_SRL: case DIF_OP_SRA: case DIF_OP_SUB: case DIF_OP_ADD: case DIF_OP_MUL: case DIF_OP_SDIV: case DIF_OP_UDIV: case DIF_OP_SREM: case DIF_OP_UREM: case DIF_OP_COPYS: if (r1 >= nregs) err += efunc(pc, "invalid register %u\n", r1); if (r2 >= nregs) err += efunc(pc, "invalid register %u\n", r2); if (rd >= nregs) err += efunc(pc, "invalid register %u\n", rd); if (rd == 0) err += efunc(pc, "cannot write to %r0\n"); break; case DIF_OP_NOT: case DIF_OP_MOV: case DIF_OP_ALLOCS: if (r1 >= nregs) err += efunc(pc, "invalid register %u\n", r1); if (r2 != 0) err += efunc(pc, "non-zero reserved bits\n"); if (rd >= nregs) err += efunc(pc, "invalid register %u\n", rd); if (rd == 0) err += efunc(pc, "cannot write to %r0\n"); break; case DIF_OP_LDSB: case DIF_OP_LDSH: case DIF_OP_LDSW: case DIF_OP_LDUB: case DIF_OP_LDUH: case DIF_OP_LDUW: case DIF_OP_LDX: if (r1 >= nregs) err += efunc(pc, "invalid register %u\n", r1); if (r2 != 0) err += efunc(pc, "non-zero reserved bits\n"); if (rd >= nregs) err += efunc(pc, "invalid register %u\n", rd); if (rd == 0) err += efunc(pc, "cannot write to %r0\n"); if (kcheckload) dp->dtdo_buf[pc] = DIF_INSTR_LOAD(op + DIF_OP_RLDSB - DIF_OP_LDSB, r1, rd); break; case DIF_OP_RLDSB: case DIF_OP_RLDSH: case DIF_OP_RLDSW: case DIF_OP_RLDUB: case DIF_OP_RLDUH: case DIF_OP_RLDUW: case DIF_OP_RLDX: if (r1 >= nregs) err += efunc(pc, "invalid register %u\n", r1); if (r2 != 0) err += efunc(pc, "non-zero reserved bits\n"); if (rd >= nregs) err += efunc(pc, "invalid register %u\n", rd); if (rd == 0) err += efunc(pc, "cannot write to %r0\n"); break; case DIF_OP_ULDSB: case DIF_OP_ULDSH: case DIF_OP_ULDSW: case DIF_OP_ULDUB: case DIF_OP_ULDUH: case DIF_OP_ULDUW: case DIF_OP_ULDX: if (r1 >= nregs) err += efunc(pc, "invalid register %u\n", r1); if (r2 != 0) err += efunc(pc, "non-zero reserved bits\n"); if (rd >= nregs) err += efunc(pc, "invalid register %u\n", rd); if (rd == 0) err += efunc(pc, "cannot write to %r0\n"); break; case DIF_OP_STB: case DIF_OP_STH: case DIF_OP_STW: case DIF_OP_STX: if (r1 >= nregs) err += efunc(pc, "invalid register %u\n", r1); if (r2 != 0) err += efunc(pc, "non-zero reserved bits\n"); if (rd >= nregs) err += efunc(pc, "invalid register %u\n", rd); if (rd == 0) err += efunc(pc, "cannot write to 0 address\n"); break; case DIF_OP_CMP: case DIF_OP_SCMP: if (r1 >= nregs) err += efunc(pc, "invalid register %u\n", r1); if (r2 >= nregs) err += efunc(pc, "invalid register %u\n", r2); if (rd != 0) err += efunc(pc, "non-zero reserved bits\n"); break; case DIF_OP_TST: if (r1 >= nregs) err += efunc(pc, "invalid register %u\n", r1); if (r2 != 0 || rd != 0) err += efunc(pc, "non-zero reserved bits\n"); break; case DIF_OP_BA: case DIF_OP_BE: case DIF_OP_BNE: case DIF_OP_BG: case DIF_OP_BGU: case DIF_OP_BGE: case DIF_OP_BGEU: case DIF_OP_BL: case DIF_OP_BLU: case DIF_OP_BLE: case DIF_OP_BLEU: if (label >= dp->dtdo_len) { err += efunc(pc, "invalid branch target %u\n", label); } if (label <= pc) { err += efunc(pc, "backward branch to %u\n", label); } break; case DIF_OP_RET: if (r1 != 0 || r2 != 0) err += efunc(pc, "non-zero reserved bits\n"); if (rd >= nregs) err += efunc(pc, "invalid register %u\n", rd); break; case DIF_OP_NOP: case DIF_OP_POPTS: case DIF_OP_FLUSHTS: if (r1 != 0 || r2 != 0 || rd != 0) err += efunc(pc, "non-zero reserved bits\n"); break; case DIF_OP_SETX: if (DIF_INSTR_INTEGER(instr) >= dp->dtdo_intlen) { err += efunc(pc, "invalid integer ref %u\n", DIF_INSTR_INTEGER(instr)); } if (rd >= nregs) err += efunc(pc, "invalid register %u\n", rd); if (rd == 0) err += efunc(pc, "cannot write to %r0\n"); break; case DIF_OP_SETS: if (DIF_INSTR_STRING(instr) >= dp->dtdo_strlen) { err += efunc(pc, "invalid string ref %u\n", DIF_INSTR_STRING(instr)); } if (rd >= nregs) err += efunc(pc, "invalid register %u\n", rd); if (rd == 0) err += efunc(pc, "cannot write to %r0\n"); break; case DIF_OP_LDGA: case DIF_OP_LDTA: if (r1 > DIF_VAR_ARRAY_MAX) err += efunc(pc, "invalid array %u\n", r1); if (r2 >= nregs) err += efunc(pc, "invalid register %u\n", r2); if (rd >= nregs) err += efunc(pc, "invalid register %u\n", rd); if (rd == 0) err += efunc(pc, "cannot write to %r0\n"); break; case DIF_OP_LDGS: case DIF_OP_LDTS: case DIF_OP_LDLS: case DIF_OP_LDGAA: case DIF_OP_LDTAA: if (v < DIF_VAR_OTHER_MIN || v > DIF_VAR_OTHER_MAX) err += efunc(pc, "invalid variable %u\n", v); if (rd >= nregs) err += efunc(pc, "invalid register %u\n", rd); if (rd == 0) err += efunc(pc, "cannot write to %r0\n"); break; case DIF_OP_STGS: case DIF_OP_STTS: case DIF_OP_STLS: case DIF_OP_STGAA: case DIF_OP_STTAA: if (v < DIF_VAR_OTHER_UBASE || v > DIF_VAR_OTHER_MAX) err += efunc(pc, "invalid variable %u\n", v); if (rs >= nregs) err += efunc(pc, "invalid register %u\n", rd); break; case DIF_OP_CALL: if (subr > DIF_SUBR_MAX) err += efunc(pc, "invalid subr %u\n", subr); if (rd >= nregs) err += efunc(pc, "invalid register %u\n", rd); if (rd == 0) err += efunc(pc, "cannot write to %r0\n"); if (subr == DIF_SUBR_COPYOUT || subr == DIF_SUBR_COPYOUTSTR) { dp->dtdo_destructive = 1; } if (subr == DIF_SUBR_GETF) { /* * If we have a getf() we need to record that * in our state. Note that our state can be * NULL if this is a helper -- but in that * case, the call to getf() is itself illegal, * and will be caught (slightly later) when * the helper is validated. */ if (vstate->dtvs_state != NULL) vstate->dtvs_state->dts_getf++; } break; case DIF_OP_PUSHTR: if (type != DIF_TYPE_STRING && type != DIF_TYPE_CTF) err += efunc(pc, "invalid ref type %u\n", type); if (r2 >= nregs) err += efunc(pc, "invalid register %u\n", r2); if (rs >= nregs) err += efunc(pc, "invalid register %u\n", rs); break; case DIF_OP_PUSHTV: if (type != DIF_TYPE_CTF) err += efunc(pc, "invalid val type %u\n", type); if (r2 >= nregs) err += efunc(pc, "invalid register %u\n", r2); if (rs >= nregs) err += efunc(pc, "invalid register %u\n", rs); break; default: err += efunc(pc, "invalid opcode %u\n", DIF_INSTR_OP(instr)); } } if (dp->dtdo_len != 0 && DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) { err += efunc(dp->dtdo_len - 1, "expected 'ret' as last DIF instruction\n"); } if (!(dp->dtdo_rtype.dtdt_flags & (DIF_TF_BYREF | DIF_TF_BYUREF))) { /* * If we're not returning by reference, the size must be either * 0 or the size of one of the base types. */ switch (dp->dtdo_rtype.dtdt_size) { case 0: case sizeof (uint8_t): case sizeof (uint16_t): case sizeof (uint32_t): case sizeof (uint64_t): break; default: err += efunc(dp->dtdo_len - 1, "bad return size\n"); } } for (i = 0; i < dp->dtdo_varlen && err == 0; i++) { dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL; dtrace_diftype_t *vt, *et; uint_t id, ndx; if (v->dtdv_scope != DIFV_SCOPE_GLOBAL && v->dtdv_scope != DIFV_SCOPE_THREAD && v->dtdv_scope != DIFV_SCOPE_LOCAL) { err += efunc(i, "unrecognized variable scope %d\n", v->dtdv_scope); break; } if (v->dtdv_kind != DIFV_KIND_ARRAY && v->dtdv_kind != DIFV_KIND_SCALAR) { err += efunc(i, "unrecognized variable type %d\n", v->dtdv_kind); break; } if ((id = v->dtdv_id) > DIF_VARIABLE_MAX) { err += efunc(i, "%d exceeds variable id limit\n", id); break; } if (id < DIF_VAR_OTHER_UBASE) continue; /* * For user-defined variables, we need to check that this * definition is identical to any previous definition that we * encountered. */ ndx = id - DIF_VAR_OTHER_UBASE; switch (v->dtdv_scope) { case DIFV_SCOPE_GLOBAL: if (ndx < vstate->dtvs_nglobals) { dtrace_statvar_t *svar; if ((svar = vstate->dtvs_globals[ndx]) != NULL) existing = &svar->dtsv_var; } break; case DIFV_SCOPE_THREAD: if (ndx < vstate->dtvs_ntlocals) existing = &vstate->dtvs_tlocals[ndx]; break; case DIFV_SCOPE_LOCAL: if (ndx < vstate->dtvs_nlocals) { dtrace_statvar_t *svar; if ((svar = vstate->dtvs_locals[ndx]) != NULL) existing = &svar->dtsv_var; } break; } vt = &v->dtdv_type; if (vt->dtdt_flags & DIF_TF_BYREF) { if (vt->dtdt_size == 0) { err += efunc(i, "zero-sized variable\n"); break; } if (v->dtdv_scope == DIFV_SCOPE_GLOBAL && vt->dtdt_size > dtrace_global_maxsize) { err += efunc(i, "oversized by-ref global\n"); break; } } if (existing == NULL || existing->dtdv_id == 0) continue; ASSERT(existing->dtdv_id == v->dtdv_id); ASSERT(existing->dtdv_scope == v->dtdv_scope); if (existing->dtdv_kind != v->dtdv_kind) err += efunc(i, "%d changed variable kind\n", id); et = &existing->dtdv_type; if (vt->dtdt_flags != et->dtdt_flags) { err += efunc(i, "%d changed variable type flags\n", id); break; } if (vt->dtdt_size != 0 && vt->dtdt_size != et->dtdt_size) { err += efunc(i, "%d changed variable type size\n", id); break; } } return (err); } /* * Validate a DTrace DIF object that it is to be used as a helper. Helpers * are much more constrained than normal DIFOs. Specifically, they may * not: * * 1. Make calls to subroutines other than copyin(), copyinstr() or * miscellaneous string routines * 2. Access DTrace variables other than the args[] array, and the * curthread, pid, ppid, tid, execname, zonename, uid and gid variables. * 3. Have thread-local variables. * 4. Have dynamic variables. */ static int dtrace_difo_validate_helper(dtrace_difo_t *dp) { int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err; int err = 0; uint_t pc; for (pc = 0; pc < dp->dtdo_len; pc++) { dif_instr_t instr = dp->dtdo_buf[pc]; uint_t v = DIF_INSTR_VAR(instr); uint_t subr = DIF_INSTR_SUBR(instr); uint_t op = DIF_INSTR_OP(instr); switch (op) { case DIF_OP_OR: case DIF_OP_XOR: case DIF_OP_AND: case DIF_OP_SLL: case DIF_OP_SRL: case DIF_OP_SRA: case DIF_OP_SUB: case DIF_OP_ADD: case DIF_OP_MUL: case DIF_OP_SDIV: case DIF_OP_UDIV: case DIF_OP_SREM: case DIF_OP_UREM: case DIF_OP_COPYS: case DIF_OP_NOT: case DIF_OP_MOV: case DIF_OP_RLDSB: case DIF_OP_RLDSH: case DIF_OP_RLDSW: case DIF_OP_RLDUB: case DIF_OP_RLDUH: case DIF_OP_RLDUW: case DIF_OP_RLDX: case DIF_OP_ULDSB: case DIF_OP_ULDSH: case DIF_OP_ULDSW: case DIF_OP_ULDUB: case DIF_OP_ULDUH: case DIF_OP_ULDUW: case DIF_OP_ULDX: case DIF_OP_STB: case DIF_OP_STH: case DIF_OP_STW: case DIF_OP_STX: case DIF_OP_ALLOCS: case DIF_OP_CMP: case DIF_OP_SCMP: case DIF_OP_TST: case DIF_OP_BA: case DIF_OP_BE: case DIF_OP_BNE: case DIF_OP_BG: case DIF_OP_BGU: case DIF_OP_BGE: case DIF_OP_BGEU: case DIF_OP_BL: case DIF_OP_BLU: case DIF_OP_BLE: case DIF_OP_BLEU: case DIF_OP_RET: case DIF_OP_NOP: case DIF_OP_POPTS: case DIF_OP_FLUSHTS: case DIF_OP_SETX: case DIF_OP_SETS: case DIF_OP_LDGA: case DIF_OP_LDLS: case DIF_OP_STGS: case DIF_OP_STLS: case DIF_OP_PUSHTR: case DIF_OP_PUSHTV: break; case DIF_OP_LDGS: if (v >= DIF_VAR_OTHER_UBASE) break; if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) break; if (v == DIF_VAR_CURTHREAD || v == DIF_VAR_PID || v == DIF_VAR_PPID || v == DIF_VAR_TID || v == DIF_VAR_EXECARGS || v == DIF_VAR_EXECNAME || v == DIF_VAR_ZONENAME || v == DIF_VAR_UID || v == DIF_VAR_GID) break; err += efunc(pc, "illegal variable %u\n", v); break; case DIF_OP_LDTA: case DIF_OP_LDTS: case DIF_OP_LDGAA: case DIF_OP_LDTAA: err += efunc(pc, "illegal dynamic variable load\n"); break; case DIF_OP_STTS: case DIF_OP_STGAA: case DIF_OP_STTAA: err += efunc(pc, "illegal dynamic variable store\n"); break; case DIF_OP_CALL: if (subr == DIF_SUBR_ALLOCA || subr == DIF_SUBR_BCOPY || subr == DIF_SUBR_COPYIN || subr == DIF_SUBR_COPYINTO || subr == DIF_SUBR_COPYINSTR || subr == DIF_SUBR_INDEX || subr == DIF_SUBR_INET_NTOA || subr == DIF_SUBR_INET_NTOA6 || subr == DIF_SUBR_INET_NTOP || subr == DIF_SUBR_JSON || subr == DIF_SUBR_LLTOSTR || subr == DIF_SUBR_STRTOLL || subr == DIF_SUBR_RINDEX || subr == DIF_SUBR_STRCHR || subr == DIF_SUBR_STRJOIN || subr == DIF_SUBR_STRRCHR || subr == DIF_SUBR_STRSTR || subr == DIF_SUBR_HTONS || subr == DIF_SUBR_HTONL || subr == DIF_SUBR_HTONLL || subr == DIF_SUBR_NTOHS || subr == DIF_SUBR_NTOHL || subr == DIF_SUBR_NTOHLL || subr == DIF_SUBR_MEMREF || #ifndef illumos subr == DIF_SUBR_MEMSTR || #endif subr == DIF_SUBR_TYPEREF) break; err += efunc(pc, "invalid subr %u\n", subr); break; default: err += efunc(pc, "invalid opcode %u\n", DIF_INSTR_OP(instr)); } } return (err); } /* * Returns 1 if the expression in the DIF object can be cached on a per-thread * basis; 0 if not. */ static int dtrace_difo_cacheable(dtrace_difo_t *dp) { int i; if (dp == NULL) return (0); for (i = 0; i < dp->dtdo_varlen; i++) { dtrace_difv_t *v = &dp->dtdo_vartab[i]; if (v->dtdv_scope != DIFV_SCOPE_GLOBAL) continue; switch (v->dtdv_id) { case DIF_VAR_CURTHREAD: case DIF_VAR_PID: case DIF_VAR_TID: case DIF_VAR_EXECARGS: case DIF_VAR_EXECNAME: case DIF_VAR_ZONENAME: break; default: return (0); } } /* * This DIF object may be cacheable. Now we need to look for any * array loading instructions, any memory loading instructions, or * any stores to thread-local variables. */ for (i = 0; i < dp->dtdo_len; i++) { uint_t op = DIF_INSTR_OP(dp->dtdo_buf[i]); if ((op >= DIF_OP_LDSB && op <= DIF_OP_LDX) || (op >= DIF_OP_ULDSB && op <= DIF_OP_ULDX) || (op >= DIF_OP_RLDSB && op <= DIF_OP_RLDX) || op == DIF_OP_LDGA || op == DIF_OP_STTS) return (0); } return (1); } static void dtrace_difo_hold(dtrace_difo_t *dp) { int i; ASSERT(MUTEX_HELD(&dtrace_lock)); dp->dtdo_refcnt++; ASSERT(dp->dtdo_refcnt != 0); /* * We need to check this DIF object for references to the variable * DIF_VAR_VTIMESTAMP. */ for (i = 0; i < dp->dtdo_varlen; i++) { dtrace_difv_t *v = &dp->dtdo_vartab[i]; if (v->dtdv_id != DIF_VAR_VTIMESTAMP) continue; if (dtrace_vtime_references++ == 0) dtrace_vtime_enable(); } } /* * This routine calculates the dynamic variable chunksize for a given DIF * object. The calculation is not fool-proof, and can probably be tricked by * malicious DIF -- but it works for all compiler-generated DIF. Because this * calculation is likely imperfect, dtrace_dynvar() is able to gracefully fail * if a dynamic variable size exceeds the chunksize. */ static void dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate) { uint64_t sval = 0; dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */ const dif_instr_t *text = dp->dtdo_buf; uint_t pc, srd = 0; uint_t ttop = 0; size_t size, ksize; uint_t id, i; for (pc = 0; pc < dp->dtdo_len; pc++) { dif_instr_t instr = text[pc]; uint_t op = DIF_INSTR_OP(instr); uint_t rd = DIF_INSTR_RD(instr); uint_t r1 = DIF_INSTR_R1(instr); uint_t nkeys = 0; uchar_t scope = 0; dtrace_key_t *key = tupregs; switch (op) { case DIF_OP_SETX: sval = dp->dtdo_inttab[DIF_INSTR_INTEGER(instr)]; srd = rd; continue; case DIF_OP_STTS: key = &tupregs[DIF_DTR_NREGS]; key[0].dttk_size = 0; key[1].dttk_size = 0; nkeys = 2; scope = DIFV_SCOPE_THREAD; break; case DIF_OP_STGAA: case DIF_OP_STTAA: nkeys = ttop; if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) key[nkeys++].dttk_size = 0; key[nkeys++].dttk_size = 0; if (op == DIF_OP_STTAA) { scope = DIFV_SCOPE_THREAD; } else { scope = DIFV_SCOPE_GLOBAL; } break; case DIF_OP_PUSHTR: if (ttop == DIF_DTR_NREGS) return; if ((srd == 0 || sval == 0) && r1 == DIF_TYPE_STRING) { /* * If the register for the size of the "pushtr" * is %r0 (or the value is 0) and the type is * a string, we'll use the system-wide default * string size. */ tupregs[ttop++].dttk_size = dtrace_strsize_default; } else { if (srd == 0) return; tupregs[ttop++].dttk_size = sval; } break; case DIF_OP_PUSHTV: if (ttop == DIF_DTR_NREGS) return; tupregs[ttop++].dttk_size = 0; break; case DIF_OP_FLUSHTS: ttop = 0; break; case DIF_OP_POPTS: if (ttop != 0) ttop--; break; } sval = 0; srd = 0; if (nkeys == 0) continue; /* * We have a dynamic variable allocation; calculate its size. */ for (ksize = 0, i = 0; i < nkeys; i++) ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t)); size = sizeof (dtrace_dynvar_t); size += sizeof (dtrace_key_t) * (nkeys - 1); size += ksize; /* * Now we need to determine the size of the stored data. */ id = DIF_INSTR_VAR(instr); for (i = 0; i < dp->dtdo_varlen; i++) { dtrace_difv_t *v = &dp->dtdo_vartab[i]; if (v->dtdv_id == id && v->dtdv_scope == scope) { size += v->dtdv_type.dtdt_size; break; } } if (i == dp->dtdo_varlen) return; /* * We have the size. If this is larger than the chunk size * for our dynamic variable state, reset the chunk size. */ size = P2ROUNDUP(size, sizeof (uint64_t)); if (size > vstate->dtvs_dynvars.dtds_chunksize) vstate->dtvs_dynvars.dtds_chunksize = size; } } static void dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate) { int i, oldsvars, osz, nsz, otlocals, ntlocals; uint_t id; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0); for (i = 0; i < dp->dtdo_varlen; i++) { dtrace_difv_t *v = &dp->dtdo_vartab[i]; dtrace_statvar_t *svar, ***svarp = NULL; size_t dsize = 0; uint8_t scope = v->dtdv_scope; int *np = NULL; if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE) continue; id -= DIF_VAR_OTHER_UBASE; switch (scope) { case DIFV_SCOPE_THREAD: while (id >= (otlocals = vstate->dtvs_ntlocals)) { dtrace_difv_t *tlocals; if ((ntlocals = (otlocals << 1)) == 0) ntlocals = 1; osz = otlocals * sizeof (dtrace_difv_t); nsz = ntlocals * sizeof (dtrace_difv_t); tlocals = kmem_zalloc(nsz, KM_SLEEP); if (osz != 0) { bcopy(vstate->dtvs_tlocals, tlocals, osz); kmem_free(vstate->dtvs_tlocals, osz); } vstate->dtvs_tlocals = tlocals; vstate->dtvs_ntlocals = ntlocals; } vstate->dtvs_tlocals[id] = *v; continue; case DIFV_SCOPE_LOCAL: np = &vstate->dtvs_nlocals; svarp = &vstate->dtvs_locals; if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) dsize = NCPU * (v->dtdv_type.dtdt_size + sizeof (uint64_t)); else dsize = NCPU * sizeof (uint64_t); break; case DIFV_SCOPE_GLOBAL: np = &vstate->dtvs_nglobals; svarp = &vstate->dtvs_globals; if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) dsize = v->dtdv_type.dtdt_size + sizeof (uint64_t); break; default: ASSERT(0); } while (id >= (oldsvars = *np)) { dtrace_statvar_t **statics; int newsvars, oldsize, newsize; if ((newsvars = (oldsvars << 1)) == 0) newsvars = 1; oldsize = oldsvars * sizeof (dtrace_statvar_t *); newsize = newsvars * sizeof (dtrace_statvar_t *); statics = kmem_zalloc(newsize, KM_SLEEP); if (oldsize != 0) { bcopy(*svarp, statics, oldsize); kmem_free(*svarp, oldsize); } *svarp = statics; *np = newsvars; } if ((svar = (*svarp)[id]) == NULL) { svar = kmem_zalloc(sizeof (dtrace_statvar_t), KM_SLEEP); svar->dtsv_var = *v; if ((svar->dtsv_size = dsize) != 0) { svar->dtsv_data = (uint64_t)(uintptr_t) kmem_zalloc(dsize, KM_SLEEP); } (*svarp)[id] = svar; } svar->dtsv_refcnt++; } dtrace_difo_chunksize(dp, vstate); dtrace_difo_hold(dp); } static dtrace_difo_t * dtrace_difo_duplicate(dtrace_difo_t *dp, dtrace_vstate_t *vstate) { dtrace_difo_t *new; size_t sz; ASSERT(dp->dtdo_buf != NULL); ASSERT(dp->dtdo_refcnt != 0); new = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP); ASSERT(dp->dtdo_buf != NULL); sz = dp->dtdo_len * sizeof (dif_instr_t); new->dtdo_buf = kmem_alloc(sz, KM_SLEEP); bcopy(dp->dtdo_buf, new->dtdo_buf, sz); new->dtdo_len = dp->dtdo_len; if (dp->dtdo_strtab != NULL) { ASSERT(dp->dtdo_strlen != 0); new->dtdo_strtab = kmem_alloc(dp->dtdo_strlen, KM_SLEEP); bcopy(dp->dtdo_strtab, new->dtdo_strtab, dp->dtdo_strlen); new->dtdo_strlen = dp->dtdo_strlen; } if (dp->dtdo_inttab != NULL) { ASSERT(dp->dtdo_intlen != 0); sz = dp->dtdo_intlen * sizeof (uint64_t); new->dtdo_inttab = kmem_alloc(sz, KM_SLEEP); bcopy(dp->dtdo_inttab, new->dtdo_inttab, sz); new->dtdo_intlen = dp->dtdo_intlen; } if (dp->dtdo_vartab != NULL) { ASSERT(dp->dtdo_varlen != 0); sz = dp->dtdo_varlen * sizeof (dtrace_difv_t); new->dtdo_vartab = kmem_alloc(sz, KM_SLEEP); bcopy(dp->dtdo_vartab, new->dtdo_vartab, sz); new->dtdo_varlen = dp->dtdo_varlen; } dtrace_difo_init(new, vstate); return (new); } static void dtrace_difo_destroy(dtrace_difo_t *dp, dtrace_vstate_t *vstate) { int i; ASSERT(dp->dtdo_refcnt == 0); for (i = 0; i < dp->dtdo_varlen; i++) { dtrace_difv_t *v = &dp->dtdo_vartab[i]; dtrace_statvar_t *svar, **svarp = NULL; uint_t id; uint8_t scope = v->dtdv_scope; int *np = NULL; switch (scope) { case DIFV_SCOPE_THREAD: continue; case DIFV_SCOPE_LOCAL: np = &vstate->dtvs_nlocals; svarp = vstate->dtvs_locals; break; case DIFV_SCOPE_GLOBAL: np = &vstate->dtvs_nglobals; svarp = vstate->dtvs_globals; break; default: ASSERT(0); } if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE) continue; id -= DIF_VAR_OTHER_UBASE; ASSERT(id < *np); svar = svarp[id]; ASSERT(svar != NULL); ASSERT(svar->dtsv_refcnt > 0); if (--svar->dtsv_refcnt > 0) continue; if (svar->dtsv_size != 0) { ASSERT(svar->dtsv_data != 0); kmem_free((void *)(uintptr_t)svar->dtsv_data, svar->dtsv_size); } kmem_free(svar, sizeof (dtrace_statvar_t)); svarp[id] = NULL; } if (dp->dtdo_buf != NULL) kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t)); if (dp->dtdo_inttab != NULL) kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t)); if (dp->dtdo_strtab != NULL) kmem_free(dp->dtdo_strtab, dp->dtdo_strlen); if (dp->dtdo_vartab != NULL) kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t)); kmem_free(dp, sizeof (dtrace_difo_t)); } static void dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate) { int i; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(dp->dtdo_refcnt != 0); for (i = 0; i < dp->dtdo_varlen; i++) { dtrace_difv_t *v = &dp->dtdo_vartab[i]; if (v->dtdv_id != DIF_VAR_VTIMESTAMP) continue; ASSERT(dtrace_vtime_references > 0); if (--dtrace_vtime_references == 0) dtrace_vtime_disable(); } if (--dp->dtdo_refcnt == 0) dtrace_difo_destroy(dp, vstate); } /* * DTrace Format Functions */ static uint16_t dtrace_format_add(dtrace_state_t *state, char *str) { char *fmt, **new; uint16_t ndx, len = strlen(str) + 1; fmt = kmem_zalloc(len, KM_SLEEP); bcopy(str, fmt, len); for (ndx = 0; ndx < state->dts_nformats; ndx++) { if (state->dts_formats[ndx] == NULL) { state->dts_formats[ndx] = fmt; return (ndx + 1); } } if (state->dts_nformats == USHRT_MAX) { /* * This is only likely if a denial-of-service attack is being * attempted. As such, it's okay to fail silently here. */ kmem_free(fmt, len); return (0); } /* * For simplicity, we always resize the formats array to be exactly the * number of formats. */ ndx = state->dts_nformats++; new = kmem_alloc((ndx + 1) * sizeof (char *), KM_SLEEP); if (state->dts_formats != NULL) { ASSERT(ndx != 0); bcopy(state->dts_formats, new, ndx * sizeof (char *)); kmem_free(state->dts_formats, ndx * sizeof (char *)); } state->dts_formats = new; state->dts_formats[ndx] = fmt; return (ndx + 1); } static void dtrace_format_remove(dtrace_state_t *state, uint16_t format) { char *fmt; ASSERT(state->dts_formats != NULL); ASSERT(format <= state->dts_nformats); ASSERT(state->dts_formats[format - 1] != NULL); fmt = state->dts_formats[format - 1]; kmem_free(fmt, strlen(fmt) + 1); state->dts_formats[format - 1] = NULL; } static void dtrace_format_destroy(dtrace_state_t *state) { int i; if (state->dts_nformats == 0) { ASSERT(state->dts_formats == NULL); return; } ASSERT(state->dts_formats != NULL); for (i = 0; i < state->dts_nformats; i++) { char *fmt = state->dts_formats[i]; if (fmt == NULL) continue; kmem_free(fmt, strlen(fmt) + 1); } kmem_free(state->dts_formats, state->dts_nformats * sizeof (char *)); state->dts_nformats = 0; state->dts_formats = NULL; } /* * DTrace Predicate Functions */ static dtrace_predicate_t * dtrace_predicate_create(dtrace_difo_t *dp) { dtrace_predicate_t *pred; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(dp->dtdo_refcnt != 0); pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP); pred->dtp_difo = dp; pred->dtp_refcnt = 1; if (!dtrace_difo_cacheable(dp)) return (pred); if (dtrace_predcache_id == DTRACE_CACHEIDNONE) { /* * This is only theoretically possible -- we have had 2^32 * cacheable predicates on this machine. We cannot allow any * more predicates to become cacheable: as unlikely as it is, * there may be a thread caching a (now stale) predicate cache * ID. (N.B.: the temptation is being successfully resisted to * have this cmn_err() "Holy shit -- we executed this code!") */ return (pred); } pred->dtp_cacheid = dtrace_predcache_id++; return (pred); } static void dtrace_predicate_hold(dtrace_predicate_t *pred) { ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0); ASSERT(pred->dtp_refcnt > 0); pred->dtp_refcnt++; } static void dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate) { dtrace_difo_t *dp = pred->dtp_difo; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(dp != NULL && dp->dtdo_refcnt != 0); ASSERT(pred->dtp_refcnt > 0); if (--pred->dtp_refcnt == 0) { dtrace_difo_release(pred->dtp_difo, vstate); kmem_free(pred, sizeof (dtrace_predicate_t)); } } /* * DTrace Action Description Functions */ static dtrace_actdesc_t * dtrace_actdesc_create(dtrace_actkind_t kind, uint32_t ntuple, uint64_t uarg, uint64_t arg) { dtrace_actdesc_t *act; #ifdef illumos ASSERT(!DTRACEACT_ISPRINTFLIKE(kind) || (arg != NULL && arg >= KERNELBASE) || (arg == NULL && kind == DTRACEACT_PRINTA)); #endif act = kmem_zalloc(sizeof (dtrace_actdesc_t), KM_SLEEP); act->dtad_kind = kind; act->dtad_ntuple = ntuple; act->dtad_uarg = uarg; act->dtad_arg = arg; act->dtad_refcnt = 1; return (act); } static void dtrace_actdesc_hold(dtrace_actdesc_t *act) { ASSERT(act->dtad_refcnt >= 1); act->dtad_refcnt++; } static void dtrace_actdesc_release(dtrace_actdesc_t *act, dtrace_vstate_t *vstate) { dtrace_actkind_t kind = act->dtad_kind; dtrace_difo_t *dp; ASSERT(act->dtad_refcnt >= 1); if (--act->dtad_refcnt != 0) return; if ((dp = act->dtad_difo) != NULL) dtrace_difo_release(dp, vstate); if (DTRACEACT_ISPRINTFLIKE(kind)) { char *str = (char *)(uintptr_t)act->dtad_arg; #ifdef illumos ASSERT((str != NULL && (uintptr_t)str >= KERNELBASE) || (str == NULL && act->dtad_kind == DTRACEACT_PRINTA)); #endif if (str != NULL) kmem_free(str, strlen(str) + 1); } kmem_free(act, sizeof (dtrace_actdesc_t)); } /* * DTrace ECB Functions */ static dtrace_ecb_t * dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe) { dtrace_ecb_t *ecb; dtrace_epid_t epid; ASSERT(MUTEX_HELD(&dtrace_lock)); ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP); ecb->dte_predicate = NULL; ecb->dte_probe = probe; /* * The default size is the size of the default action: recording * the header. */ ecb->dte_size = ecb->dte_needed = sizeof (dtrace_rechdr_t); ecb->dte_alignment = sizeof (dtrace_epid_t); epid = state->dts_epid++; if (epid - 1 >= state->dts_necbs) { dtrace_ecb_t **oecbs = state->dts_ecbs, **ecbs; int necbs = state->dts_necbs << 1; ASSERT(epid == state->dts_necbs + 1); if (necbs == 0) { ASSERT(oecbs == NULL); necbs = 1; } ecbs = kmem_zalloc(necbs * sizeof (*ecbs), KM_SLEEP); if (oecbs != NULL) bcopy(oecbs, ecbs, state->dts_necbs * sizeof (*ecbs)); dtrace_membar_producer(); state->dts_ecbs = ecbs; if (oecbs != NULL) { /* * If this state is active, we must dtrace_sync() * before we can free the old dts_ecbs array: we're * coming in hot, and there may be active ring * buffer processing (which indexes into the dts_ecbs * array) on another CPU. */ if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) dtrace_sync(); kmem_free(oecbs, state->dts_necbs * sizeof (*ecbs)); } dtrace_membar_producer(); state->dts_necbs = necbs; } ecb->dte_state = state; ASSERT(state->dts_ecbs[epid - 1] == NULL); dtrace_membar_producer(); state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb; return (ecb); } static void dtrace_ecb_enable(dtrace_ecb_t *ecb) { dtrace_probe_t *probe = ecb->dte_probe; ASSERT(MUTEX_HELD(&cpu_lock)); ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(ecb->dte_next == NULL); if (probe == NULL) { /* * This is the NULL probe -- there's nothing to do. */ return; } if (probe->dtpr_ecb == NULL) { dtrace_provider_t *prov = probe->dtpr_provider; /* * We're the first ECB on this probe. */ probe->dtpr_ecb = probe->dtpr_ecb_last = ecb; if (ecb->dte_predicate != NULL) probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid; prov->dtpv_pops.dtps_enable(prov->dtpv_arg, probe->dtpr_id, probe->dtpr_arg); } else { /* * This probe is already active. Swing the last pointer to * point to the new ECB, and issue a dtrace_sync() to assure * that all CPUs have seen the change. */ ASSERT(probe->dtpr_ecb_last != NULL); probe->dtpr_ecb_last->dte_next = ecb; probe->dtpr_ecb_last = ecb; probe->dtpr_predcache = 0; dtrace_sync(); } } static void dtrace_ecb_resize(dtrace_ecb_t *ecb) { dtrace_action_t *act; uint32_t curneeded = UINT32_MAX; uint32_t aggbase = UINT32_MAX; /* * If we record anything, we always record the dtrace_rechdr_t. (And * we always record it first.) */ ecb->dte_size = sizeof (dtrace_rechdr_t); ecb->dte_alignment = sizeof (dtrace_epid_t); for (act = ecb->dte_action; act != NULL; act = act->dta_next) { dtrace_recdesc_t *rec = &act->dta_rec; ASSERT(rec->dtrd_size > 0 || rec->dtrd_alignment == 1); ecb->dte_alignment = MAX(ecb->dte_alignment, rec->dtrd_alignment); if (DTRACEACT_ISAGG(act->dta_kind)) { dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act; ASSERT(rec->dtrd_size != 0); ASSERT(agg->dtag_first != NULL); ASSERT(act->dta_prev->dta_intuple); ASSERT(aggbase != UINT32_MAX); ASSERT(curneeded != UINT32_MAX); agg->dtag_base = aggbase; curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment); rec->dtrd_offset = curneeded; curneeded += rec->dtrd_size; ecb->dte_needed = MAX(ecb->dte_needed, curneeded); aggbase = UINT32_MAX; curneeded = UINT32_MAX; } else if (act->dta_intuple) { if (curneeded == UINT32_MAX) { /* * This is the first record in a tuple. Align * curneeded to be at offset 4 in an 8-byte * aligned block. */ ASSERT(act->dta_prev == NULL || !act->dta_prev->dta_intuple); ASSERT3U(aggbase, ==, UINT32_MAX); curneeded = P2PHASEUP(ecb->dte_size, sizeof (uint64_t), sizeof (dtrace_aggid_t)); aggbase = curneeded - sizeof (dtrace_aggid_t); ASSERT(IS_P2ALIGNED(aggbase, sizeof (uint64_t))); } curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment); rec->dtrd_offset = curneeded; curneeded += rec->dtrd_size; } else { /* tuples must be followed by an aggregation */ ASSERT(act->dta_prev == NULL || !act->dta_prev->dta_intuple); ecb->dte_size = P2ROUNDUP(ecb->dte_size, rec->dtrd_alignment); rec->dtrd_offset = ecb->dte_size; ecb->dte_size += rec->dtrd_size; ecb->dte_needed = MAX(ecb->dte_needed, ecb->dte_size); } } if ((act = ecb->dte_action) != NULL && !(act->dta_kind == DTRACEACT_SPECULATE && act->dta_next == NULL) && ecb->dte_size == sizeof (dtrace_rechdr_t)) { /* * If the size is still sizeof (dtrace_rechdr_t), then all * actions store no data; set the size to 0. */ ecb->dte_size = 0; } ecb->dte_size = P2ROUNDUP(ecb->dte_size, sizeof (dtrace_epid_t)); ecb->dte_needed = P2ROUNDUP(ecb->dte_needed, (sizeof (dtrace_epid_t))); ecb->dte_state->dts_needed = MAX(ecb->dte_state->dts_needed, ecb->dte_needed); } static dtrace_action_t * dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc) { dtrace_aggregation_t *agg; size_t size = sizeof (uint64_t); int ntuple = desc->dtad_ntuple; dtrace_action_t *act; dtrace_recdesc_t *frec; dtrace_aggid_t aggid; dtrace_state_t *state = ecb->dte_state; agg = kmem_zalloc(sizeof (dtrace_aggregation_t), KM_SLEEP); agg->dtag_ecb = ecb; ASSERT(DTRACEACT_ISAGG(desc->dtad_kind)); switch (desc->dtad_kind) { case DTRACEAGG_MIN: agg->dtag_initial = INT64_MAX; agg->dtag_aggregate = dtrace_aggregate_min; break; case DTRACEAGG_MAX: agg->dtag_initial = INT64_MIN; agg->dtag_aggregate = dtrace_aggregate_max; break; case DTRACEAGG_COUNT: agg->dtag_aggregate = dtrace_aggregate_count; break; case DTRACEAGG_QUANTIZE: agg->dtag_aggregate = dtrace_aggregate_quantize; size = (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1) * sizeof (uint64_t); break; case DTRACEAGG_LQUANTIZE: { uint16_t step = DTRACE_LQUANTIZE_STEP(desc->dtad_arg); uint16_t levels = DTRACE_LQUANTIZE_LEVELS(desc->dtad_arg); agg->dtag_initial = desc->dtad_arg; agg->dtag_aggregate = dtrace_aggregate_lquantize; if (step == 0 || levels == 0) goto err; size = levels * sizeof (uint64_t) + 3 * sizeof (uint64_t); break; } case DTRACEAGG_LLQUANTIZE: { uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(desc->dtad_arg); uint16_t low = DTRACE_LLQUANTIZE_LOW(desc->dtad_arg); uint16_t high = DTRACE_LLQUANTIZE_HIGH(desc->dtad_arg); uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(desc->dtad_arg); int64_t v; agg->dtag_initial = desc->dtad_arg; agg->dtag_aggregate = dtrace_aggregate_llquantize; if (factor < 2 || low >= high || nsteps < factor) goto err; /* * Now check that the number of steps evenly divides a power * of the factor. (This assures both integer bucket size and * linearity within each magnitude.) */ for (v = factor; v < nsteps; v *= factor) continue; if ((v % nsteps) || (nsteps % factor)) goto err; size = (dtrace_aggregate_llquantize_bucket(factor, low, high, nsteps, INT64_MAX) + 2) * sizeof (uint64_t); break; } case DTRACEAGG_AVG: agg->dtag_aggregate = dtrace_aggregate_avg; size = sizeof (uint64_t) * 2; break; case DTRACEAGG_STDDEV: agg->dtag_aggregate = dtrace_aggregate_stddev; size = sizeof (uint64_t) * 4; break; case DTRACEAGG_SUM: agg->dtag_aggregate = dtrace_aggregate_sum; break; default: goto err; } agg->dtag_action.dta_rec.dtrd_size = size; if (ntuple == 0) goto err; /* * We must make sure that we have enough actions for the n-tuple. */ for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) { if (DTRACEACT_ISAGG(act->dta_kind)) break; if (--ntuple == 0) { /* * This is the action with which our n-tuple begins. */ agg->dtag_first = act; goto success; } } /* * This n-tuple is short by ntuple elements. Return failure. */ ASSERT(ntuple != 0); err: kmem_free(agg, sizeof (dtrace_aggregation_t)); return (NULL); success: /* * If the last action in the tuple has a size of zero, it's actually * an expression argument for the aggregating action. */ ASSERT(ecb->dte_action_last != NULL); act = ecb->dte_action_last; if (act->dta_kind == DTRACEACT_DIFEXPR) { ASSERT(act->dta_difo != NULL); if (act->dta_difo->dtdo_rtype.dtdt_size == 0) agg->dtag_hasarg = 1; } /* * We need to allocate an id for this aggregation. */ #ifdef illumos aggid = (dtrace_aggid_t)(uintptr_t)vmem_alloc(state->dts_aggid_arena, 1, VM_BESTFIT | VM_SLEEP); #else aggid = alloc_unr(state->dts_aggid_arena); #endif if (aggid - 1 >= state->dts_naggregations) { dtrace_aggregation_t **oaggs = state->dts_aggregations; dtrace_aggregation_t **aggs; int naggs = state->dts_naggregations << 1; int onaggs = state->dts_naggregations; ASSERT(aggid == state->dts_naggregations + 1); if (naggs == 0) { ASSERT(oaggs == NULL); naggs = 1; } aggs = kmem_zalloc(naggs * sizeof (*aggs), KM_SLEEP); if (oaggs != NULL) { bcopy(oaggs, aggs, onaggs * sizeof (*aggs)); kmem_free(oaggs, onaggs * sizeof (*aggs)); } state->dts_aggregations = aggs; state->dts_naggregations = naggs; } ASSERT(state->dts_aggregations[aggid - 1] == NULL); state->dts_aggregations[(agg->dtag_id = aggid) - 1] = agg; frec = &agg->dtag_first->dta_rec; if (frec->dtrd_alignment < sizeof (dtrace_aggid_t)) frec->dtrd_alignment = sizeof (dtrace_aggid_t); for (act = agg->dtag_first; act != NULL; act = act->dta_next) { ASSERT(!act->dta_intuple); act->dta_intuple = 1; } return (&agg->dtag_action); } static void dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act) { dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act; dtrace_state_t *state = ecb->dte_state; dtrace_aggid_t aggid = agg->dtag_id; ASSERT(DTRACEACT_ISAGG(act->dta_kind)); #ifdef illumos vmem_free(state->dts_aggid_arena, (void *)(uintptr_t)aggid, 1); #else free_unr(state->dts_aggid_arena, aggid); #endif ASSERT(state->dts_aggregations[aggid - 1] == agg); state->dts_aggregations[aggid - 1] = NULL; kmem_free(agg, sizeof (dtrace_aggregation_t)); } static int dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc) { dtrace_action_t *action, *last; dtrace_difo_t *dp = desc->dtad_difo; uint32_t size = 0, align = sizeof (uint8_t), mask; uint16_t format = 0; dtrace_recdesc_t *rec; dtrace_state_t *state = ecb->dte_state; dtrace_optval_t *opt = state->dts_options, nframes = 0, strsize; uint64_t arg = desc->dtad_arg; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1); if (DTRACEACT_ISAGG(desc->dtad_kind)) { /* * If this is an aggregating action, there must be neither * a speculate nor a commit on the action chain. */ dtrace_action_t *act; for (act = ecb->dte_action; act != NULL; act = act->dta_next) { if (act->dta_kind == DTRACEACT_COMMIT) return (EINVAL); if (act->dta_kind == DTRACEACT_SPECULATE) return (EINVAL); } action = dtrace_ecb_aggregation_create(ecb, desc); if (action == NULL) return (EINVAL); } else { if (DTRACEACT_ISDESTRUCTIVE(desc->dtad_kind) || (desc->dtad_kind == DTRACEACT_DIFEXPR && dp != NULL && dp->dtdo_destructive)) { state->dts_destructive = 1; } switch (desc->dtad_kind) { case DTRACEACT_PRINTF: case DTRACEACT_PRINTA: case DTRACEACT_SYSTEM: case DTRACEACT_FREOPEN: case DTRACEACT_DIFEXPR: /* * We know that our arg is a string -- turn it into a * format. */ if (arg == 0) { ASSERT(desc->dtad_kind == DTRACEACT_PRINTA || desc->dtad_kind == DTRACEACT_DIFEXPR); format = 0; } else { ASSERT(arg != 0); #ifdef illumos ASSERT(arg > KERNELBASE); #endif format = dtrace_format_add(state, (char *)(uintptr_t)arg); } /*FALLTHROUGH*/ case DTRACEACT_LIBACT: case DTRACEACT_TRACEMEM: case DTRACEACT_TRACEMEM_DYNSIZE: if (dp == NULL) return (EINVAL); if ((size = dp->dtdo_rtype.dtdt_size) != 0) break; if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) { if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) return (EINVAL); size = opt[DTRACEOPT_STRSIZE]; } break; case DTRACEACT_STACK: if ((nframes = arg) == 0) { nframes = opt[DTRACEOPT_STACKFRAMES]; ASSERT(nframes > 0); arg = nframes; } size = nframes * sizeof (pc_t); break; case DTRACEACT_JSTACK: if ((strsize = DTRACE_USTACK_STRSIZE(arg)) == 0) strsize = opt[DTRACEOPT_JSTACKSTRSIZE]; if ((nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) nframes = opt[DTRACEOPT_JSTACKFRAMES]; arg = DTRACE_USTACK_ARG(nframes, strsize); /*FALLTHROUGH*/ case DTRACEACT_USTACK: if (desc->dtad_kind != DTRACEACT_JSTACK && (nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) { strsize = DTRACE_USTACK_STRSIZE(arg); nframes = opt[DTRACEOPT_USTACKFRAMES]; ASSERT(nframes > 0); arg = DTRACE_USTACK_ARG(nframes, strsize); } /* * Save a slot for the pid. */ size = (nframes + 1) * sizeof (uint64_t); size += DTRACE_USTACK_STRSIZE(arg); size = P2ROUNDUP(size, (uint32_t)(sizeof (uintptr_t))); break; case DTRACEACT_SYM: case DTRACEACT_MOD: if (dp == NULL || ((size = dp->dtdo_rtype.dtdt_size) != sizeof (uint64_t)) || (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) return (EINVAL); break; case DTRACEACT_USYM: case DTRACEACT_UMOD: case DTRACEACT_UADDR: if (dp == NULL || (dp->dtdo_rtype.dtdt_size != sizeof (uint64_t)) || (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) return (EINVAL); /* * We have a slot for the pid, plus a slot for the * argument. To keep things simple (aligned with * bitness-neutral sizing), we store each as a 64-bit * quantity. */ size = 2 * sizeof (uint64_t); break; case DTRACEACT_STOP: case DTRACEACT_BREAKPOINT: case DTRACEACT_PANIC: break; case DTRACEACT_CHILL: case DTRACEACT_DISCARD: case DTRACEACT_RAISE: if (dp == NULL) return (EINVAL); break; case DTRACEACT_EXIT: if (dp == NULL || (size = dp->dtdo_rtype.dtdt_size) != sizeof (int) || (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) return (EINVAL); break; case DTRACEACT_SPECULATE: if (ecb->dte_size > sizeof (dtrace_rechdr_t)) return (EINVAL); if (dp == NULL) return (EINVAL); state->dts_speculates = 1; break; case DTRACEACT_PRINTM: size = dp->dtdo_rtype.dtdt_size; break; case DTRACEACT_PRINTT: size = dp->dtdo_rtype.dtdt_size; break; case DTRACEACT_COMMIT: { dtrace_action_t *act = ecb->dte_action; for (; act != NULL; act = act->dta_next) { if (act->dta_kind == DTRACEACT_COMMIT) return (EINVAL); } if (dp == NULL) return (EINVAL); break; } default: return (EINVAL); } if (size != 0 || desc->dtad_kind == DTRACEACT_SPECULATE) { /* * If this is a data-storing action or a speculate, * we must be sure that there isn't a commit on the * action chain. */ dtrace_action_t *act = ecb->dte_action; for (; act != NULL; act = act->dta_next) { if (act->dta_kind == DTRACEACT_COMMIT) return (EINVAL); } } action = kmem_zalloc(sizeof (dtrace_action_t), KM_SLEEP); action->dta_rec.dtrd_size = size; } action->dta_refcnt = 1; rec = &action->dta_rec; size = rec->dtrd_size; for (mask = sizeof (uint64_t) - 1; size != 0 && mask > 0; mask >>= 1) { if (!(size & mask)) { align = mask + 1; break; } } action->dta_kind = desc->dtad_kind; if ((action->dta_difo = dp) != NULL) dtrace_difo_hold(dp); rec->dtrd_action = action->dta_kind; rec->dtrd_arg = arg; rec->dtrd_uarg = desc->dtad_uarg; rec->dtrd_alignment = (uint16_t)align; rec->dtrd_format = format; if ((last = ecb->dte_action_last) != NULL) { ASSERT(ecb->dte_action != NULL); action->dta_prev = last; last->dta_next = action; } else { ASSERT(ecb->dte_action == NULL); ecb->dte_action = action; } ecb->dte_action_last = action; return (0); } static void dtrace_ecb_action_remove(dtrace_ecb_t *ecb) { dtrace_action_t *act = ecb->dte_action, *next; dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate; dtrace_difo_t *dp; uint16_t format; if (act != NULL && act->dta_refcnt > 1) { ASSERT(act->dta_next == NULL || act->dta_next->dta_refcnt == 1); act->dta_refcnt--; } else { for (; act != NULL; act = next) { next = act->dta_next; ASSERT(next != NULL || act == ecb->dte_action_last); ASSERT(act->dta_refcnt == 1); if ((format = act->dta_rec.dtrd_format) != 0) dtrace_format_remove(ecb->dte_state, format); if ((dp = act->dta_difo) != NULL) dtrace_difo_release(dp, vstate); if (DTRACEACT_ISAGG(act->dta_kind)) { dtrace_ecb_aggregation_destroy(ecb, act); } else { kmem_free(act, sizeof (dtrace_action_t)); } } } ecb->dte_action = NULL; ecb->dte_action_last = NULL; ecb->dte_size = 0; } static void dtrace_ecb_disable(dtrace_ecb_t *ecb) { /* * We disable the ECB by removing it from its probe. */ dtrace_ecb_t *pecb, *prev = NULL; dtrace_probe_t *probe = ecb->dte_probe; ASSERT(MUTEX_HELD(&dtrace_lock)); if (probe == NULL) { /* * This is the NULL probe; there is nothing to disable. */ return; } for (pecb = probe->dtpr_ecb; pecb != NULL; pecb = pecb->dte_next) { if (pecb == ecb) break; prev = pecb; } ASSERT(pecb != NULL); if (prev == NULL) { probe->dtpr_ecb = ecb->dte_next; } else { prev->dte_next = ecb->dte_next; } if (ecb == probe->dtpr_ecb_last) { ASSERT(ecb->dte_next == NULL); probe->dtpr_ecb_last = prev; } /* * The ECB has been disconnected from the probe; now sync to assure * that all CPUs have seen the change before returning. */ dtrace_sync(); if (probe->dtpr_ecb == NULL) { /* * That was the last ECB on the probe; clear the predicate * cache ID for the probe, disable it and sync one more time * to assure that we'll never hit it again. */ dtrace_provider_t *prov = probe->dtpr_provider; ASSERT(ecb->dte_next == NULL); ASSERT(probe->dtpr_ecb_last == NULL); probe->dtpr_predcache = DTRACE_CACHEIDNONE; prov->dtpv_pops.dtps_disable(prov->dtpv_arg, probe->dtpr_id, probe->dtpr_arg); dtrace_sync(); } else { /* * There is at least one ECB remaining on the probe. If there * is _exactly_ one, set the probe's predicate cache ID to be * the predicate cache ID of the remaining ECB. */ ASSERT(probe->dtpr_ecb_last != NULL); ASSERT(probe->dtpr_predcache == DTRACE_CACHEIDNONE); if (probe->dtpr_ecb == probe->dtpr_ecb_last) { dtrace_predicate_t *p = probe->dtpr_ecb->dte_predicate; ASSERT(probe->dtpr_ecb->dte_next == NULL); if (p != NULL) probe->dtpr_predcache = p->dtp_cacheid; } ecb->dte_next = NULL; } } static void dtrace_ecb_destroy(dtrace_ecb_t *ecb) { dtrace_state_t *state = ecb->dte_state; dtrace_vstate_t *vstate = &state->dts_vstate; dtrace_predicate_t *pred; dtrace_epid_t epid = ecb->dte_epid; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(ecb->dte_next == NULL); ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb); if ((pred = ecb->dte_predicate) != NULL) dtrace_predicate_release(pred, vstate); dtrace_ecb_action_remove(ecb); ASSERT(state->dts_ecbs[epid - 1] == ecb); state->dts_ecbs[epid - 1] = NULL; kmem_free(ecb, sizeof (dtrace_ecb_t)); } static dtrace_ecb_t * dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe, dtrace_enabling_t *enab) { dtrace_ecb_t *ecb; dtrace_predicate_t *pred; dtrace_actdesc_t *act; dtrace_provider_t *prov; dtrace_ecbdesc_t *desc = enab->dten_current; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(state != NULL); ecb = dtrace_ecb_add(state, probe); ecb->dte_uarg = desc->dted_uarg; if ((pred = desc->dted_pred.dtpdd_predicate) != NULL) { dtrace_predicate_hold(pred); ecb->dte_predicate = pred; } if (probe != NULL) { /* * If the provider shows more leg than the consumer is old * enough to see, we need to enable the appropriate implicit * predicate bits to prevent the ecb from activating at * revealing times. * * Providers specifying DTRACE_PRIV_USER at register time * are stating that they need the /proc-style privilege * model to be enforced, and this is what DTRACE_COND_OWNER * and DTRACE_COND_ZONEOWNER will then do at probe time. */ prov = probe->dtpr_provider; if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLPROC) && (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER)) ecb->dte_cond |= DTRACE_COND_OWNER; if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLZONE) && (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER)) ecb->dte_cond |= DTRACE_COND_ZONEOWNER; /* * If the provider shows us kernel innards and the user * is lacking sufficient privilege, enable the * DTRACE_COND_USERMODE implicit predicate. */ if (!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) && (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_KERNEL)) ecb->dte_cond |= DTRACE_COND_USERMODE; } if (dtrace_ecb_create_cache != NULL) { /* * If we have a cached ecb, we'll use its action list instead * of creating our own (saving both time and space). */ dtrace_ecb_t *cached = dtrace_ecb_create_cache; dtrace_action_t *act = cached->dte_action; if (act != NULL) { ASSERT(act->dta_refcnt > 0); act->dta_refcnt++; ecb->dte_action = act; ecb->dte_action_last = cached->dte_action_last; ecb->dte_needed = cached->dte_needed; ecb->dte_size = cached->dte_size; ecb->dte_alignment = cached->dte_alignment; } return (ecb); } for (act = desc->dted_action; act != NULL; act = act->dtad_next) { if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) { dtrace_ecb_destroy(ecb); return (NULL); } } dtrace_ecb_resize(ecb); return (dtrace_ecb_create_cache = ecb); } static int dtrace_ecb_create_enable(dtrace_probe_t *probe, void *arg) { dtrace_ecb_t *ecb; dtrace_enabling_t *enab = arg; dtrace_state_t *state = enab->dten_vstate->dtvs_state; ASSERT(state != NULL); if (probe != NULL && probe->dtpr_gen < enab->dten_probegen) { /* * This probe was created in a generation for which this * enabling has previously created ECBs; we don't want to * enable it again, so just kick out. */ return (DTRACE_MATCH_NEXT); } if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL) return (DTRACE_MATCH_DONE); dtrace_ecb_enable(ecb); return (DTRACE_MATCH_NEXT); } static dtrace_ecb_t * dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id) { dtrace_ecb_t *ecb; ASSERT(MUTEX_HELD(&dtrace_lock)); if (id == 0 || id > state->dts_necbs) return (NULL); ASSERT(state->dts_necbs > 0 && state->dts_ecbs != NULL); ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id); return (state->dts_ecbs[id - 1]); } static dtrace_aggregation_t * dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id) { dtrace_aggregation_t *agg; ASSERT(MUTEX_HELD(&dtrace_lock)); if (id == 0 || id > state->dts_naggregations) return (NULL); ASSERT(state->dts_naggregations > 0 && state->dts_aggregations != NULL); ASSERT((agg = state->dts_aggregations[id - 1]) == NULL || agg->dtag_id == id); return (state->dts_aggregations[id - 1]); } /* * DTrace Buffer Functions * * The following functions manipulate DTrace buffers. Most of these functions * are called in the context of establishing or processing consumer state; * exceptions are explicitly noted. */ /* * Note: called from cross call context. This function switches the two * buffers on a given CPU. The atomicity of this operation is assured by * disabling interrupts while the actual switch takes place; the disabling of * interrupts serializes the execution with any execution of dtrace_probe() on * the same CPU. */ static void dtrace_buffer_switch(dtrace_buffer_t *buf) { caddr_t tomax = buf->dtb_tomax; caddr_t xamot = buf->dtb_xamot; dtrace_icookie_t cookie; hrtime_t now; ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH)); ASSERT(!(buf->dtb_flags & DTRACEBUF_RING)); cookie = dtrace_interrupt_disable(); now = dtrace_gethrtime(); buf->dtb_tomax = xamot; buf->dtb_xamot = tomax; buf->dtb_xamot_drops = buf->dtb_drops; buf->dtb_xamot_offset = buf->dtb_offset; buf->dtb_xamot_errors = buf->dtb_errors; buf->dtb_xamot_flags = buf->dtb_flags; buf->dtb_offset = 0; buf->dtb_drops = 0; buf->dtb_errors = 0; buf->dtb_flags &= ~(DTRACEBUF_ERROR | DTRACEBUF_DROPPED); buf->dtb_interval = now - buf->dtb_switched; buf->dtb_switched = now; dtrace_interrupt_enable(cookie); } /* * Note: called from cross call context. This function activates a buffer * on a CPU. As with dtrace_buffer_switch(), the atomicity of the operation * is guaranteed by the disabling of interrupts. */ static void dtrace_buffer_activate(dtrace_state_t *state) { dtrace_buffer_t *buf; dtrace_icookie_t cookie = dtrace_interrupt_disable(); buf = &state->dts_buffer[curcpu]; if (buf->dtb_tomax != NULL) { /* * We might like to assert that the buffer is marked inactive, * but this isn't necessarily true: the buffer for the CPU * that processes the BEGIN probe has its buffer activated * manually. In this case, we take the (harmless) action * re-clearing the bit INACTIVE bit. */ buf->dtb_flags &= ~DTRACEBUF_INACTIVE; } dtrace_interrupt_enable(cookie); } static int dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t size, int flags, processorid_t cpu, int *factor) { #ifdef illumos cpu_t *cp; #endif dtrace_buffer_t *buf; int allocated = 0, desired = 0; #ifdef illumos ASSERT(MUTEX_HELD(&cpu_lock)); ASSERT(MUTEX_HELD(&dtrace_lock)); *factor = 1; if (size > dtrace_nonroot_maxsize && !PRIV_POLICY_CHOICE(CRED(), PRIV_ALL, B_FALSE)) return (EFBIG); cp = cpu_list; do { if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id) continue; buf = &bufs[cp->cpu_id]; /* * If there is already a buffer allocated for this CPU, it * is only possible that this is a DR event. In this case, */ if (buf->dtb_tomax != NULL) { ASSERT(buf->dtb_size == size); continue; } ASSERT(buf->dtb_xamot == NULL); if ((buf->dtb_tomax = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL) goto err; buf->dtb_size = size; buf->dtb_flags = flags; buf->dtb_offset = 0; buf->dtb_drops = 0; if (flags & DTRACEBUF_NOSWITCH) continue; if ((buf->dtb_xamot = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL) goto err; } while ((cp = cp->cpu_next) != cpu_list); return (0); err: cp = cpu_list; do { if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id) continue; buf = &bufs[cp->cpu_id]; desired += 2; if (buf->dtb_xamot != NULL) { ASSERT(buf->dtb_tomax != NULL); ASSERT(buf->dtb_size == size); kmem_free(buf->dtb_xamot, size); allocated++; } if (buf->dtb_tomax != NULL) { ASSERT(buf->dtb_size == size); kmem_free(buf->dtb_tomax, size); allocated++; } buf->dtb_tomax = NULL; buf->dtb_xamot = NULL; buf->dtb_size = 0; } while ((cp = cp->cpu_next) != cpu_list); #else int i; *factor = 1; -#if defined(__amd64__) || defined(__mips__) || defined(__powerpc__) +#if defined(__amd64__) || defined(__arm__) || defined(__mips__) || defined(__powerpc__) /* * FreeBSD isn't good at limiting the amount of memory we * ask to malloc, so let's place a limit here before trying * to do something that might well end in tears at bedtime. */ if (size > physmem * PAGE_SIZE / (128 * (mp_maxid + 1))) return (ENOMEM); #endif ASSERT(MUTEX_HELD(&dtrace_lock)); CPU_FOREACH(i) { if (cpu != DTRACE_CPUALL && cpu != i) continue; buf = &bufs[i]; /* * If there is already a buffer allocated for this CPU, it * is only possible that this is a DR event. In this case, * the buffer size must match our specified size. */ if (buf->dtb_tomax != NULL) { ASSERT(buf->dtb_size == size); continue; } ASSERT(buf->dtb_xamot == NULL); if ((buf->dtb_tomax = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL) goto err; buf->dtb_size = size; buf->dtb_flags = flags; buf->dtb_offset = 0; buf->dtb_drops = 0; if (flags & DTRACEBUF_NOSWITCH) continue; if ((buf->dtb_xamot = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL) goto err; } return (0); err: /* * Error allocating memory, so free the buffers that were * allocated before the failed allocation. */ CPU_FOREACH(i) { if (cpu != DTRACE_CPUALL && cpu != i) continue; buf = &bufs[i]; desired += 2; if (buf->dtb_xamot != NULL) { ASSERT(buf->dtb_tomax != NULL); ASSERT(buf->dtb_size == size); kmem_free(buf->dtb_xamot, size); allocated++; } if (buf->dtb_tomax != NULL) { ASSERT(buf->dtb_size == size); kmem_free(buf->dtb_tomax, size); allocated++; } buf->dtb_tomax = NULL; buf->dtb_xamot = NULL; buf->dtb_size = 0; } #endif *factor = desired / (allocated > 0 ? allocated : 1); return (ENOMEM); } /* * Note: called from probe context. This function just increments the drop * count on a buffer. It has been made a function to allow for the * possibility of understanding the source of mysterious drop counts. (A * problem for which one may be particularly disappointed that DTrace cannot * be used to understand DTrace.) */ static void dtrace_buffer_drop(dtrace_buffer_t *buf) { buf->dtb_drops++; } /* * Note: called from probe context. This function is called to reserve space * in a buffer. If mstate is non-NULL, sets the scratch base and size in the * mstate. Returns the new offset in the buffer, or a negative value if an * error has occurred. */ static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *buf, size_t needed, size_t align, dtrace_state_t *state, dtrace_mstate_t *mstate) { intptr_t offs = buf->dtb_offset, soffs; intptr_t woffs; caddr_t tomax; size_t total; if (buf->dtb_flags & DTRACEBUF_INACTIVE) return (-1); if ((tomax = buf->dtb_tomax) == NULL) { dtrace_buffer_drop(buf); return (-1); } if (!(buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL))) { while (offs & (align - 1)) { /* * Assert that our alignment is off by a number which * is itself sizeof (uint32_t) aligned. */ ASSERT(!((align - (offs & (align - 1))) & (sizeof (uint32_t) - 1))); DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE); offs += sizeof (uint32_t); } if ((soffs = offs + needed) > buf->dtb_size) { dtrace_buffer_drop(buf); return (-1); } if (mstate == NULL) return (offs); mstate->dtms_scratch_base = (uintptr_t)tomax + soffs; mstate->dtms_scratch_size = buf->dtb_size - soffs; mstate->dtms_scratch_ptr = mstate->dtms_scratch_base; return (offs); } if (buf->dtb_flags & DTRACEBUF_FILL) { if (state->dts_activity != DTRACE_ACTIVITY_COOLDOWN && (buf->dtb_flags & DTRACEBUF_FULL)) return (-1); goto out; } total = needed + (offs & (align - 1)); /* * For a ring buffer, life is quite a bit more complicated. Before * we can store any padding, we need to adjust our wrapping offset. * (If we've never before wrapped or we're not about to, no adjustment * is required.) */ if ((buf->dtb_flags & DTRACEBUF_WRAPPED) || offs + total > buf->dtb_size) { woffs = buf->dtb_xamot_offset; if (offs + total > buf->dtb_size) { /* * We can't fit in the end of the buffer. First, a * sanity check that we can fit in the buffer at all. */ if (total > buf->dtb_size) { dtrace_buffer_drop(buf); return (-1); } /* * We're going to be storing at the top of the buffer, * so now we need to deal with the wrapped offset. We * only reset our wrapped offset to 0 if it is * currently greater than the current offset. If it * is less than the current offset, it is because a * previous allocation induced a wrap -- but the * allocation didn't subsequently take the space due * to an error or false predicate evaluation. In this * case, we'll just leave the wrapped offset alone: if * the wrapped offset hasn't been advanced far enough * for this allocation, it will be adjusted in the * lower loop. */ if (buf->dtb_flags & DTRACEBUF_WRAPPED) { if (woffs >= offs) woffs = 0; } else { woffs = 0; } /* * Now we know that we're going to be storing to the * top of the buffer and that there is room for us * there. We need to clear the buffer from the current * offset to the end (there may be old gunk there). */ while (offs < buf->dtb_size) tomax[offs++] = 0; /* * We need to set our offset to zero. And because we * are wrapping, we need to set the bit indicating as * much. We can also adjust our needed space back * down to the space required by the ECB -- we know * that the top of the buffer is aligned. */ offs = 0; total = needed; buf->dtb_flags |= DTRACEBUF_WRAPPED; } else { /* * There is room for us in the buffer, so we simply * need to check the wrapped offset. */ if (woffs < offs) { /* * The wrapped offset is less than the offset. * This can happen if we allocated buffer space * that induced a wrap, but then we didn't * subsequently take the space due to an error * or false predicate evaluation. This is * okay; we know that _this_ allocation isn't * going to induce a wrap. We still can't * reset the wrapped offset to be zero, * however: the space may have been trashed in * the previous failed probe attempt. But at * least the wrapped offset doesn't need to * be adjusted at all... */ goto out; } } while (offs + total > woffs) { dtrace_epid_t epid = *(uint32_t *)(tomax + woffs); size_t size; if (epid == DTRACE_EPIDNONE) { size = sizeof (uint32_t); } else { ASSERT3U(epid, <=, state->dts_necbs); ASSERT(state->dts_ecbs[epid - 1] != NULL); size = state->dts_ecbs[epid - 1]->dte_size; } ASSERT(woffs + size <= buf->dtb_size); ASSERT(size != 0); if (woffs + size == buf->dtb_size) { /* * We've reached the end of the buffer; we want * to set the wrapped offset to 0 and break * out. However, if the offs is 0, then we're * in a strange edge-condition: the amount of * space that we want to reserve plus the size * of the record that we're overwriting is * greater than the size of the buffer. This * is problematic because if we reserve the * space but subsequently don't consume it (due * to a failed predicate or error) the wrapped * offset will be 0 -- yet the EPID at offset 0 * will not be committed. This situation is * relatively easy to deal with: if we're in * this case, the buffer is indistinguishable * from one that hasn't wrapped; we need only * finish the job by clearing the wrapped bit, * explicitly setting the offset to be 0, and * zero'ing out the old data in the buffer. */ if (offs == 0) { buf->dtb_flags &= ~DTRACEBUF_WRAPPED; buf->dtb_offset = 0; woffs = total; while (woffs < buf->dtb_size) tomax[woffs++] = 0; } woffs = 0; break; } woffs += size; } /* * We have a wrapped offset. It may be that the wrapped offset * has become zero -- that's okay. */ buf->dtb_xamot_offset = woffs; } out: /* * Now we can plow the buffer with any necessary padding. */ while (offs & (align - 1)) { /* * Assert that our alignment is off by a number which * is itself sizeof (uint32_t) aligned. */ ASSERT(!((align - (offs & (align - 1))) & (sizeof (uint32_t) - 1))); DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE); offs += sizeof (uint32_t); } if (buf->dtb_flags & DTRACEBUF_FILL) { if (offs + needed > buf->dtb_size - state->dts_reserve) { buf->dtb_flags |= DTRACEBUF_FULL; return (-1); } } if (mstate == NULL) return (offs); /* * For ring buffers and fill buffers, the scratch space is always * the inactive buffer. */ mstate->dtms_scratch_base = (uintptr_t)buf->dtb_xamot; mstate->dtms_scratch_size = buf->dtb_size; mstate->dtms_scratch_ptr = mstate->dtms_scratch_base; return (offs); } static void dtrace_buffer_polish(dtrace_buffer_t *buf) { ASSERT(buf->dtb_flags & DTRACEBUF_RING); ASSERT(MUTEX_HELD(&dtrace_lock)); if (!(buf->dtb_flags & DTRACEBUF_WRAPPED)) return; /* * We need to polish the ring buffer. There are three cases: * * - The first (and presumably most common) is that there is no gap * between the buffer offset and the wrapped offset. In this case, * there is nothing in the buffer that isn't valid data; we can * mark the buffer as polished and return. * * - The second (less common than the first but still more common * than the third) is that there is a gap between the buffer offset * and the wrapped offset, and the wrapped offset is larger than the * buffer offset. This can happen because of an alignment issue, or * can happen because of a call to dtrace_buffer_reserve() that * didn't subsequently consume the buffer space. In this case, * we need to zero the data from the buffer offset to the wrapped * offset. * * - The third (and least common) is that there is a gap between the * buffer offset and the wrapped offset, but the wrapped offset is * _less_ than the buffer offset. This can only happen because a * call to dtrace_buffer_reserve() induced a wrap, but the space * was not subsequently consumed. In this case, we need to zero the * space from the offset to the end of the buffer _and_ from the * top of the buffer to the wrapped offset. */ if (buf->dtb_offset < buf->dtb_xamot_offset) { bzero(buf->dtb_tomax + buf->dtb_offset, buf->dtb_xamot_offset - buf->dtb_offset); } if (buf->dtb_offset > buf->dtb_xamot_offset) { bzero(buf->dtb_tomax + buf->dtb_offset, buf->dtb_size - buf->dtb_offset); bzero(buf->dtb_tomax, buf->dtb_xamot_offset); } } /* * This routine determines if data generated at the specified time has likely * been entirely consumed at user-level. This routine is called to determine * if an ECB on a defunct probe (but for an active enabling) can be safely * disabled and destroyed. */ static int dtrace_buffer_consumed(dtrace_buffer_t *bufs, hrtime_t when) { int i; for (i = 0; i < NCPU; i++) { dtrace_buffer_t *buf = &bufs[i]; if (buf->dtb_size == 0) continue; if (buf->dtb_flags & DTRACEBUF_RING) return (0); if (!buf->dtb_switched && buf->dtb_offset != 0) return (0); if (buf->dtb_switched - buf->dtb_interval < when) return (0); } return (1); } static void dtrace_buffer_free(dtrace_buffer_t *bufs) { int i; for (i = 0; i < NCPU; i++) { dtrace_buffer_t *buf = &bufs[i]; if (buf->dtb_tomax == NULL) { ASSERT(buf->dtb_xamot == NULL); ASSERT(buf->dtb_size == 0); continue; } if (buf->dtb_xamot != NULL) { ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH)); kmem_free(buf->dtb_xamot, buf->dtb_size); } kmem_free(buf->dtb_tomax, buf->dtb_size); buf->dtb_size = 0; buf->dtb_tomax = NULL; buf->dtb_xamot = NULL; } } /* * DTrace Enabling Functions */ static dtrace_enabling_t * dtrace_enabling_create(dtrace_vstate_t *vstate) { dtrace_enabling_t *enab; enab = kmem_zalloc(sizeof (dtrace_enabling_t), KM_SLEEP); enab->dten_vstate = vstate; return (enab); } static void dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb) { dtrace_ecbdesc_t **ndesc; size_t osize, nsize; /* * We can't add to enablings after we've enabled them, or after we've * retained them. */ ASSERT(enab->dten_probegen == 0); ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL); if (enab->dten_ndesc < enab->dten_maxdesc) { enab->dten_desc[enab->dten_ndesc++] = ecb; return; } osize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *); if (enab->dten_maxdesc == 0) { enab->dten_maxdesc = 1; } else { enab->dten_maxdesc <<= 1; } ASSERT(enab->dten_ndesc < enab->dten_maxdesc); nsize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *); ndesc = kmem_zalloc(nsize, KM_SLEEP); bcopy(enab->dten_desc, ndesc, osize); if (enab->dten_desc != NULL) kmem_free(enab->dten_desc, osize); enab->dten_desc = ndesc; enab->dten_desc[enab->dten_ndesc++] = ecb; } static void dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb, dtrace_probedesc_t *pd) { dtrace_ecbdesc_t *new; dtrace_predicate_t *pred; dtrace_actdesc_t *act; /* * We're going to create a new ECB description that matches the * specified ECB in every way, but has the specified probe description. */ new = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP); if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL) dtrace_predicate_hold(pred); for (act = ecb->dted_action; act != NULL; act = act->dtad_next) dtrace_actdesc_hold(act); new->dted_action = ecb->dted_action; new->dted_pred = ecb->dted_pred; new->dted_probe = *pd; new->dted_uarg = ecb->dted_uarg; dtrace_enabling_add(enab, new); } static void dtrace_enabling_dump(dtrace_enabling_t *enab) { int i; for (i = 0; i < enab->dten_ndesc; i++) { dtrace_probedesc_t *desc = &enab->dten_desc[i]->dted_probe; cmn_err(CE_NOTE, "enabling probe %d (%s:%s:%s:%s)", i, desc->dtpd_provider, desc->dtpd_mod, desc->dtpd_func, desc->dtpd_name); } } static void dtrace_enabling_destroy(dtrace_enabling_t *enab) { int i; dtrace_ecbdesc_t *ep; dtrace_vstate_t *vstate = enab->dten_vstate; ASSERT(MUTEX_HELD(&dtrace_lock)); for (i = 0; i < enab->dten_ndesc; i++) { dtrace_actdesc_t *act, *next; dtrace_predicate_t *pred; ep = enab->dten_desc[i]; if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) dtrace_predicate_release(pred, vstate); for (act = ep->dted_action; act != NULL; act = next) { next = act->dtad_next; dtrace_actdesc_release(act, vstate); } kmem_free(ep, sizeof (dtrace_ecbdesc_t)); } if (enab->dten_desc != NULL) kmem_free(enab->dten_desc, enab->dten_maxdesc * sizeof (dtrace_enabling_t *)); /* * If this was a retained enabling, decrement the dts_nretained count * and take it off of the dtrace_retained list. */ if (enab->dten_prev != NULL || enab->dten_next != NULL || dtrace_retained == enab) { ASSERT(enab->dten_vstate->dtvs_state != NULL); ASSERT(enab->dten_vstate->dtvs_state->dts_nretained > 0); enab->dten_vstate->dtvs_state->dts_nretained--; dtrace_retained_gen++; } if (enab->dten_prev == NULL) { if (dtrace_retained == enab) { dtrace_retained = enab->dten_next; if (dtrace_retained != NULL) dtrace_retained->dten_prev = NULL; } } else { ASSERT(enab != dtrace_retained); ASSERT(dtrace_retained != NULL); enab->dten_prev->dten_next = enab->dten_next; } if (enab->dten_next != NULL) { ASSERT(dtrace_retained != NULL); enab->dten_next->dten_prev = enab->dten_prev; } kmem_free(enab, sizeof (dtrace_enabling_t)); } static int dtrace_enabling_retain(dtrace_enabling_t *enab) { dtrace_state_t *state; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL); ASSERT(enab->dten_vstate != NULL); state = enab->dten_vstate->dtvs_state; ASSERT(state != NULL); /* * We only allow each state to retain dtrace_retain_max enablings. */ if (state->dts_nretained >= dtrace_retain_max) return (ENOSPC); state->dts_nretained++; dtrace_retained_gen++; if (dtrace_retained == NULL) { dtrace_retained = enab; return (0); } enab->dten_next = dtrace_retained; dtrace_retained->dten_prev = enab; dtrace_retained = enab; return (0); } static int dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match, dtrace_probedesc_t *create) { dtrace_enabling_t *new, *enab; int found = 0, err = ENOENT; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN); ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN); ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN); ASSERT(strlen(match->dtpd_name) < DTRACE_NAMELEN); new = dtrace_enabling_create(&state->dts_vstate); /* * Iterate over all retained enablings, looking for enablings that * match the specified state. */ for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) { int i; /* * dtvs_state can only be NULL for helper enablings -- and * helper enablings can't be retained. */ ASSERT(enab->dten_vstate->dtvs_state != NULL); if (enab->dten_vstate->dtvs_state != state) continue; /* * Now iterate over each probe description; we're looking for * an exact match to the specified probe description. */ for (i = 0; i < enab->dten_ndesc; i++) { dtrace_ecbdesc_t *ep = enab->dten_desc[i]; dtrace_probedesc_t *pd = &ep->dted_probe; if (strcmp(pd->dtpd_provider, match->dtpd_provider)) continue; if (strcmp(pd->dtpd_mod, match->dtpd_mod)) continue; if (strcmp(pd->dtpd_func, match->dtpd_func)) continue; if (strcmp(pd->dtpd_name, match->dtpd_name)) continue; /* * We have a winning probe! Add it to our growing * enabling. */ found = 1; dtrace_enabling_addlike(new, ep, create); } } if (!found || (err = dtrace_enabling_retain(new)) != 0) { dtrace_enabling_destroy(new); return (err); } return (0); } static void dtrace_enabling_retract(dtrace_state_t *state) { dtrace_enabling_t *enab, *next; ASSERT(MUTEX_HELD(&dtrace_lock)); /* * Iterate over all retained enablings, destroy the enablings retained * for the specified state. */ for (enab = dtrace_retained; enab != NULL; enab = next) { next = enab->dten_next; /* * dtvs_state can only be NULL for helper enablings -- and * helper enablings can't be retained. */ ASSERT(enab->dten_vstate->dtvs_state != NULL); if (enab->dten_vstate->dtvs_state == state) { ASSERT(state->dts_nretained > 0); dtrace_enabling_destroy(enab); } } ASSERT(state->dts_nretained == 0); } static int dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched) { int i = 0; int matched = 0; ASSERT(MUTEX_HELD(&cpu_lock)); ASSERT(MUTEX_HELD(&dtrace_lock)); for (i = 0; i < enab->dten_ndesc; i++) { dtrace_ecbdesc_t *ep = enab->dten_desc[i]; enab->dten_current = ep; enab->dten_error = 0; matched += dtrace_probe_enable(&ep->dted_probe, enab); if (enab->dten_error != 0) { /* * If we get an error half-way through enabling the * probes, we kick out -- perhaps with some number of * them enabled. Leaving enabled probes enabled may * be slightly confusing for user-level, but we expect * that no one will attempt to actually drive on in * the face of such errors. If this is an anonymous * enabling (indicated with a NULL nmatched pointer), * we cmn_err() a message. We aren't expecting to * get such an error -- such as it can exist at all, * it would be a result of corrupted DOF in the driver * properties. */ if (nmatched == NULL) { cmn_err(CE_WARN, "dtrace_enabling_match() " "error on %p: %d", (void *)ep, enab->dten_error); } return (enab->dten_error); } } enab->dten_probegen = dtrace_probegen; if (nmatched != NULL) *nmatched = matched; return (0); } static void dtrace_enabling_matchall(void) { dtrace_enabling_t *enab; mutex_enter(&cpu_lock); mutex_enter(&dtrace_lock); /* * Iterate over all retained enablings to see if any probes match * against them. We only perform this operation on enablings for which * we have sufficient permissions by virtue of being in the global zone * or in the same zone as the DTrace client. Because we can be called * after dtrace_detach() has been called, we cannot assert that there * are retained enablings. We can safely load from dtrace_retained, * however: the taskq_destroy() at the end of dtrace_detach() will * block pending our completion. */ for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) { #ifdef illumos cred_t *cr = enab->dten_vstate->dtvs_state->dts_cred.dcr_cred; if (INGLOBALZONE(curproc) || cr != NULL && getzoneid() == crgetzoneid(cr)) #endif (void) dtrace_enabling_match(enab, NULL); } mutex_exit(&dtrace_lock); mutex_exit(&cpu_lock); } /* * If an enabling is to be enabled without having matched probes (that is, if * dtrace_state_go() is to be called on the underlying dtrace_state_t), the * enabling must be _primed_ by creating an ECB for every ECB description. * This must be done to assure that we know the number of speculations, the * number of aggregations, the minimum buffer size needed, etc. before we * transition out of DTRACE_ACTIVITY_INACTIVE. To do this without actually * enabling any probes, we create ECBs for every ECB decription, but with a * NULL probe -- which is exactly what this function does. */ static void dtrace_enabling_prime(dtrace_state_t *state) { dtrace_enabling_t *enab; int i; for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) { ASSERT(enab->dten_vstate->dtvs_state != NULL); if (enab->dten_vstate->dtvs_state != state) continue; /* * We don't want to prime an enabling more than once, lest * we allow a malicious user to induce resource exhaustion. * (The ECBs that result from priming an enabling aren't * leaked -- but they also aren't deallocated until the * consumer state is destroyed.) */ if (enab->dten_primed) continue; for (i = 0; i < enab->dten_ndesc; i++) { enab->dten_current = enab->dten_desc[i]; (void) dtrace_probe_enable(NULL, enab); } enab->dten_primed = 1; } } /* * Called to indicate that probes should be provided due to retained * enablings. This is implemented in terms of dtrace_probe_provide(), but it * must take an initial lap through the enabling calling the dtps_provide() * entry point explicitly to allow for autocreated probes. */ static void dtrace_enabling_provide(dtrace_provider_t *prv) { int i, all = 0; dtrace_probedesc_t desc; dtrace_genid_t gen; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(MUTEX_HELD(&dtrace_provider_lock)); if (prv == NULL) { all = 1; prv = dtrace_provider; } do { dtrace_enabling_t *enab; void *parg = prv->dtpv_arg; retry: gen = dtrace_retained_gen; for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) { for (i = 0; i < enab->dten_ndesc; i++) { desc = enab->dten_desc[i]->dted_probe; mutex_exit(&dtrace_lock); prv->dtpv_pops.dtps_provide(parg, &desc); mutex_enter(&dtrace_lock); /* * Process the retained enablings again if * they have changed while we weren't holding * dtrace_lock. */ if (gen != dtrace_retained_gen) goto retry; } } } while (all && (prv = prv->dtpv_next) != NULL); mutex_exit(&dtrace_lock); dtrace_probe_provide(NULL, all ? NULL : prv); mutex_enter(&dtrace_lock); } /* * Called to reap ECBs that are attached to probes from defunct providers. */ static void dtrace_enabling_reap(void) { dtrace_provider_t *prov; dtrace_probe_t *probe; dtrace_ecb_t *ecb; hrtime_t when; int i; mutex_enter(&cpu_lock); mutex_enter(&dtrace_lock); for (i = 0; i < dtrace_nprobes; i++) { if ((probe = dtrace_probes[i]) == NULL) continue; if (probe->dtpr_ecb == NULL) continue; prov = probe->dtpr_provider; if ((when = prov->dtpv_defunct) == 0) continue; /* * We have ECBs on a defunct provider: we want to reap these * ECBs to allow the provider to unregister. The destruction * of these ECBs must be done carefully: if we destroy the ECB * and the consumer later wishes to consume an EPID that * corresponds to the destroyed ECB (and if the EPID metadata * has not been previously consumed), the consumer will abort * processing on the unknown EPID. To reduce (but not, sadly, * eliminate) the possibility of this, we will only destroy an * ECB for a defunct provider if, for the state that * corresponds to the ECB: * * (a) There is no speculative tracing (which can effectively * cache an EPID for an arbitrary amount of time). * * (b) The principal buffers have been switched twice since the * provider became defunct. * * (c) The aggregation buffers are of zero size or have been * switched twice since the provider became defunct. * * We use dts_speculates to determine (a) and call a function * (dtrace_buffer_consumed()) to determine (b) and (c). Note * that as soon as we've been unable to destroy one of the ECBs * associated with the probe, we quit trying -- reaping is only * fruitful in as much as we can destroy all ECBs associated * with the defunct provider's probes. */ while ((ecb = probe->dtpr_ecb) != NULL) { dtrace_state_t *state = ecb->dte_state; dtrace_buffer_t *buf = state->dts_buffer; dtrace_buffer_t *aggbuf = state->dts_aggbuffer; if (state->dts_speculates) break; if (!dtrace_buffer_consumed(buf, when)) break; if (!dtrace_buffer_consumed(aggbuf, when)) break; dtrace_ecb_disable(ecb); ASSERT(probe->dtpr_ecb != ecb); dtrace_ecb_destroy(ecb); } } mutex_exit(&dtrace_lock); mutex_exit(&cpu_lock); } /* * DTrace DOF Functions */ /*ARGSUSED*/ static void dtrace_dof_error(dof_hdr_t *dof, const char *str) { if (dtrace_err_verbose) cmn_err(CE_WARN, "failed to process DOF: %s", str); #ifdef DTRACE_ERRDEBUG dtrace_errdebug(str); #endif } /* * Create DOF out of a currently enabled state. Right now, we only create * DOF containing the run-time options -- but this could be expanded to create * complete DOF representing the enabled state. */ static dof_hdr_t * dtrace_dof_create(dtrace_state_t *state) { dof_hdr_t *dof; dof_sec_t *sec; dof_optdesc_t *opt; int i, len = sizeof (dof_hdr_t) + roundup(sizeof (dof_sec_t), sizeof (uint64_t)) + sizeof (dof_optdesc_t) * DTRACEOPT_MAX; ASSERT(MUTEX_HELD(&dtrace_lock)); dof = kmem_zalloc(len, KM_SLEEP); dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0; dof->dofh_ident[DOF_ID_MAG1] = DOF_MAG_MAG1; dof->dofh_ident[DOF_ID_MAG2] = DOF_MAG_MAG2; dof->dofh_ident[DOF_ID_MAG3] = DOF_MAG_MAG3; dof->dofh_ident[DOF_ID_MODEL] = DOF_MODEL_NATIVE; dof->dofh_ident[DOF_ID_ENCODING] = DOF_ENCODE_NATIVE; dof->dofh_ident[DOF_ID_VERSION] = DOF_VERSION; dof->dofh_ident[DOF_ID_DIFVERS] = DIF_VERSION; dof->dofh_ident[DOF_ID_DIFIREG] = DIF_DIR_NREGS; dof->dofh_ident[DOF_ID_DIFTREG] = DIF_DTR_NREGS; dof->dofh_flags = 0; dof->dofh_hdrsize = sizeof (dof_hdr_t); dof->dofh_secsize = sizeof (dof_sec_t); dof->dofh_secnum = 1; /* only DOF_SECT_OPTDESC */ dof->dofh_secoff = sizeof (dof_hdr_t); dof->dofh_loadsz = len; dof->dofh_filesz = len; dof->dofh_pad = 0; /* * Fill in the option section header... */ sec = (dof_sec_t *)((uintptr_t)dof + sizeof (dof_hdr_t)); sec->dofs_type = DOF_SECT_OPTDESC; sec->dofs_align = sizeof (uint64_t); sec->dofs_flags = DOF_SECF_LOAD; sec->dofs_entsize = sizeof (dof_optdesc_t); opt = (dof_optdesc_t *)((uintptr_t)sec + roundup(sizeof (dof_sec_t), sizeof (uint64_t))); sec->dofs_offset = (uintptr_t)opt - (uintptr_t)dof; sec->dofs_size = sizeof (dof_optdesc_t) * DTRACEOPT_MAX; for (i = 0; i < DTRACEOPT_MAX; i++) { opt[i].dofo_option = i; opt[i].dofo_strtab = DOF_SECIDX_NONE; opt[i].dofo_value = state->dts_options[i]; } return (dof); } static dof_hdr_t * dtrace_dof_copyin(uintptr_t uarg, int *errp) { dof_hdr_t hdr, *dof; ASSERT(!MUTEX_HELD(&dtrace_lock)); /* * First, we're going to copyin() the sizeof (dof_hdr_t). */ if (copyin((void *)uarg, &hdr, sizeof (hdr)) != 0) { dtrace_dof_error(NULL, "failed to copyin DOF header"); *errp = EFAULT; return (NULL); } /* * Now we'll allocate the entire DOF and copy it in -- provided * that the length isn't outrageous. */ if (hdr.dofh_loadsz >= dtrace_dof_maxsize) { dtrace_dof_error(&hdr, "load size exceeds maximum"); *errp = E2BIG; return (NULL); } if (hdr.dofh_loadsz < sizeof (hdr)) { dtrace_dof_error(&hdr, "invalid load size"); *errp = EINVAL; return (NULL); } dof = kmem_alloc(hdr.dofh_loadsz, KM_SLEEP); if (copyin((void *)uarg, dof, hdr.dofh_loadsz) != 0 || dof->dofh_loadsz != hdr.dofh_loadsz) { kmem_free(dof, hdr.dofh_loadsz); *errp = EFAULT; return (NULL); } return (dof); } #ifndef illumos static __inline uchar_t dtrace_dof_char(char c) { switch (c) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': return (c - '0'); case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': return (c - 'A' + 10); case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': return (c - 'a' + 10); } /* Should not reach here. */ return (0); } #endif static dof_hdr_t * dtrace_dof_property(const char *name) { uchar_t *buf; uint64_t loadsz; unsigned int len, i; dof_hdr_t *dof; #ifdef illumos /* * Unfortunately, array of values in .conf files are always (and * only) interpreted to be integer arrays. We must read our DOF * as an integer array, and then squeeze it into a byte array. */ if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dtrace_devi, 0, (char *)name, (int **)&buf, &len) != DDI_PROP_SUCCESS) return (NULL); for (i = 0; i < len; i++) buf[i] = (uchar_t)(((int *)buf)[i]); if (len < sizeof (dof_hdr_t)) { ddi_prop_free(buf); dtrace_dof_error(NULL, "truncated header"); return (NULL); } if (len < (loadsz = ((dof_hdr_t *)buf)->dofh_loadsz)) { ddi_prop_free(buf); dtrace_dof_error(NULL, "truncated DOF"); return (NULL); } if (loadsz >= dtrace_dof_maxsize) { ddi_prop_free(buf); dtrace_dof_error(NULL, "oversized DOF"); return (NULL); } dof = kmem_alloc(loadsz, KM_SLEEP); bcopy(buf, dof, loadsz); ddi_prop_free(buf); #else char *p; char *p_env; if ((p_env = kern_getenv(name)) == NULL) return (NULL); len = strlen(p_env) / 2; buf = kmem_alloc(len, KM_SLEEP); dof = (dof_hdr_t *) buf; p = p_env; for (i = 0; i < len; i++) { buf[i] = (dtrace_dof_char(p[0]) << 4) | dtrace_dof_char(p[1]); p += 2; } freeenv(p_env); if (len < sizeof (dof_hdr_t)) { kmem_free(buf, 0); dtrace_dof_error(NULL, "truncated header"); return (NULL); } if (len < (loadsz = dof->dofh_loadsz)) { kmem_free(buf, 0); dtrace_dof_error(NULL, "truncated DOF"); return (NULL); } if (loadsz >= dtrace_dof_maxsize) { kmem_free(buf, 0); dtrace_dof_error(NULL, "oversized DOF"); return (NULL); } #endif return (dof); } static void dtrace_dof_destroy(dof_hdr_t *dof) { kmem_free(dof, dof->dofh_loadsz); } /* * Return the dof_sec_t pointer corresponding to a given section index. If the * index is not valid, dtrace_dof_error() is called and NULL is returned. If * a type other than DOF_SECT_NONE is specified, the header is checked against * this type and NULL is returned if the types do not match. */ static dof_sec_t * dtrace_dof_sect(dof_hdr_t *dof, uint32_t type, dof_secidx_t i) { dof_sec_t *sec = (dof_sec_t *)(uintptr_t) ((uintptr_t)dof + dof->dofh_secoff + i * dof->dofh_secsize); if (i >= dof->dofh_secnum) { dtrace_dof_error(dof, "referenced section index is invalid"); return (NULL); } if (!(sec->dofs_flags & DOF_SECF_LOAD)) { dtrace_dof_error(dof, "referenced section is not loadable"); return (NULL); } if (type != DOF_SECT_NONE && type != sec->dofs_type) { dtrace_dof_error(dof, "referenced section is the wrong type"); return (NULL); } return (sec); } static dtrace_probedesc_t * dtrace_dof_probedesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_probedesc_t *desc) { dof_probedesc_t *probe; dof_sec_t *strtab; uintptr_t daddr = (uintptr_t)dof; uintptr_t str; size_t size; if (sec->dofs_type != DOF_SECT_PROBEDESC) { dtrace_dof_error(dof, "invalid probe section"); return (NULL); } if (sec->dofs_align != sizeof (dof_secidx_t)) { dtrace_dof_error(dof, "bad alignment in probe description"); return (NULL); } if (sec->dofs_offset + sizeof (dof_probedesc_t) > dof->dofh_loadsz) { dtrace_dof_error(dof, "truncated probe description"); return (NULL); } probe = (dof_probedesc_t *)(uintptr_t)(daddr + sec->dofs_offset); strtab = dtrace_dof_sect(dof, DOF_SECT_STRTAB, probe->dofp_strtab); if (strtab == NULL) return (NULL); str = daddr + strtab->dofs_offset; size = strtab->dofs_size; if (probe->dofp_provider >= strtab->dofs_size) { dtrace_dof_error(dof, "corrupt probe provider"); return (NULL); } (void) strncpy(desc->dtpd_provider, (char *)(str + probe->dofp_provider), MIN(DTRACE_PROVNAMELEN - 1, size - probe->dofp_provider)); if (probe->dofp_mod >= strtab->dofs_size) { dtrace_dof_error(dof, "corrupt probe module"); return (NULL); } (void) strncpy(desc->dtpd_mod, (char *)(str + probe->dofp_mod), MIN(DTRACE_MODNAMELEN - 1, size - probe->dofp_mod)); if (probe->dofp_func >= strtab->dofs_size) { dtrace_dof_error(dof, "corrupt probe function"); return (NULL); } (void) strncpy(desc->dtpd_func, (char *)(str + probe->dofp_func), MIN(DTRACE_FUNCNAMELEN - 1, size - probe->dofp_func)); if (probe->dofp_name >= strtab->dofs_size) { dtrace_dof_error(dof, "corrupt probe name"); return (NULL); } (void) strncpy(desc->dtpd_name, (char *)(str + probe->dofp_name), MIN(DTRACE_NAMELEN - 1, size - probe->dofp_name)); return (desc); } static dtrace_difo_t * dtrace_dof_difo(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate, cred_t *cr) { dtrace_difo_t *dp; size_t ttl = 0; dof_difohdr_t *dofd; uintptr_t daddr = (uintptr_t)dof; size_t max = dtrace_difo_maxsize; int i, l, n; static const struct { int section; int bufoffs; int lenoffs; int entsize; int align; const char *msg; } difo[] = { { DOF_SECT_DIF, offsetof(dtrace_difo_t, dtdo_buf), offsetof(dtrace_difo_t, dtdo_len), sizeof (dif_instr_t), sizeof (dif_instr_t), "multiple DIF sections" }, { DOF_SECT_INTTAB, offsetof(dtrace_difo_t, dtdo_inttab), offsetof(dtrace_difo_t, dtdo_intlen), sizeof (uint64_t), sizeof (uint64_t), "multiple integer tables" }, { DOF_SECT_STRTAB, offsetof(dtrace_difo_t, dtdo_strtab), offsetof(dtrace_difo_t, dtdo_strlen), 0, sizeof (char), "multiple string tables" }, { DOF_SECT_VARTAB, offsetof(dtrace_difo_t, dtdo_vartab), offsetof(dtrace_difo_t, dtdo_varlen), sizeof (dtrace_difv_t), sizeof (uint_t), "multiple variable tables" }, { DOF_SECT_NONE, 0, 0, 0, 0, NULL } }; if (sec->dofs_type != DOF_SECT_DIFOHDR) { dtrace_dof_error(dof, "invalid DIFO header section"); return (NULL); } if (sec->dofs_align != sizeof (dof_secidx_t)) { dtrace_dof_error(dof, "bad alignment in DIFO header"); return (NULL); } if (sec->dofs_size < sizeof (dof_difohdr_t) || sec->dofs_size % sizeof (dof_secidx_t)) { dtrace_dof_error(dof, "bad size in DIFO header"); return (NULL); } dofd = (dof_difohdr_t *)(uintptr_t)(daddr + sec->dofs_offset); n = (sec->dofs_size - sizeof (*dofd)) / sizeof (dof_secidx_t) + 1; dp = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP); dp->dtdo_rtype = dofd->dofd_rtype; for (l = 0; l < n; l++) { dof_sec_t *subsec; void **bufp; uint32_t *lenp; if ((subsec = dtrace_dof_sect(dof, DOF_SECT_NONE, dofd->dofd_links[l])) == NULL) goto err; /* invalid section link */ if (ttl + subsec->dofs_size > max) { dtrace_dof_error(dof, "exceeds maximum size"); goto err; } ttl += subsec->dofs_size; for (i = 0; difo[i].section != DOF_SECT_NONE; i++) { if (subsec->dofs_type != difo[i].section) continue; if (!(subsec->dofs_flags & DOF_SECF_LOAD)) { dtrace_dof_error(dof, "section not loaded"); goto err; } if (subsec->dofs_align != difo[i].align) { dtrace_dof_error(dof, "bad alignment"); goto err; } bufp = (void **)((uintptr_t)dp + difo[i].bufoffs); lenp = (uint32_t *)((uintptr_t)dp + difo[i].lenoffs); if (*bufp != NULL) { dtrace_dof_error(dof, difo[i].msg); goto err; } if (difo[i].entsize != subsec->dofs_entsize) { dtrace_dof_error(dof, "entry size mismatch"); goto err; } if (subsec->dofs_entsize != 0 && (subsec->dofs_size % subsec->dofs_entsize) != 0) { dtrace_dof_error(dof, "corrupt entry size"); goto err; } *lenp = subsec->dofs_size; *bufp = kmem_alloc(subsec->dofs_size, KM_SLEEP); bcopy((char *)(uintptr_t)(daddr + subsec->dofs_offset), *bufp, subsec->dofs_size); if (subsec->dofs_entsize != 0) *lenp /= subsec->dofs_entsize; break; } /* * If we encounter a loadable DIFO sub-section that is not * known to us, assume this is a broken program and fail. */ if (difo[i].section == DOF_SECT_NONE && (subsec->dofs_flags & DOF_SECF_LOAD)) { dtrace_dof_error(dof, "unrecognized DIFO subsection"); goto err; } } if (dp->dtdo_buf == NULL) { /* * We can't have a DIF object without DIF text. */ dtrace_dof_error(dof, "missing DIF text"); goto err; } /* * Before we validate the DIF object, run through the variable table * looking for the strings -- if any of their size are under, we'll set * their size to be the system-wide default string size. Note that * this should _not_ happen if the "strsize" option has been set -- * in this case, the compiler should have set the size to reflect the * setting of the option. */ for (i = 0; i < dp->dtdo_varlen; i++) { dtrace_difv_t *v = &dp->dtdo_vartab[i]; dtrace_diftype_t *t = &v->dtdv_type; if (v->dtdv_id < DIF_VAR_OTHER_UBASE) continue; if (t->dtdt_kind == DIF_TYPE_STRING && t->dtdt_size == 0) t->dtdt_size = dtrace_strsize_default; } if (dtrace_difo_validate(dp, vstate, DIF_DIR_NREGS, cr) != 0) goto err; dtrace_difo_init(dp, vstate); return (dp); err: kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t)); kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t)); kmem_free(dp->dtdo_strtab, dp->dtdo_strlen); kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t)); kmem_free(dp, sizeof (dtrace_difo_t)); return (NULL); } static dtrace_predicate_t * dtrace_dof_predicate(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate, cred_t *cr) { dtrace_difo_t *dp; if ((dp = dtrace_dof_difo(dof, sec, vstate, cr)) == NULL) return (NULL); return (dtrace_predicate_create(dp)); } static dtrace_actdesc_t * dtrace_dof_actdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate, cred_t *cr) { dtrace_actdesc_t *act, *first = NULL, *last = NULL, *next; dof_actdesc_t *desc; dof_sec_t *difosec; size_t offs; uintptr_t daddr = (uintptr_t)dof; uint64_t arg; dtrace_actkind_t kind; if (sec->dofs_type != DOF_SECT_ACTDESC) { dtrace_dof_error(dof, "invalid action section"); return (NULL); } if (sec->dofs_offset + sizeof (dof_actdesc_t) > dof->dofh_loadsz) { dtrace_dof_error(dof, "truncated action description"); return (NULL); } if (sec->dofs_align != sizeof (uint64_t)) { dtrace_dof_error(dof, "bad alignment in action description"); return (NULL); } if (sec->dofs_size < sec->dofs_entsize) { dtrace_dof_error(dof, "section entry size exceeds total size"); return (NULL); } if (sec->dofs_entsize != sizeof (dof_actdesc_t)) { dtrace_dof_error(dof, "bad entry size in action description"); return (NULL); } if (sec->dofs_size / sec->dofs_entsize > dtrace_actions_max) { dtrace_dof_error(dof, "actions exceed dtrace_actions_max"); return (NULL); } for (offs = 0; offs < sec->dofs_size; offs += sec->dofs_entsize) { desc = (dof_actdesc_t *)(daddr + (uintptr_t)sec->dofs_offset + offs); kind = (dtrace_actkind_t)desc->dofa_kind; if ((DTRACEACT_ISPRINTFLIKE(kind) && (kind != DTRACEACT_PRINTA || desc->dofa_strtab != DOF_SECIDX_NONE)) || (kind == DTRACEACT_DIFEXPR && desc->dofa_strtab != DOF_SECIDX_NONE)) { dof_sec_t *strtab; char *str, *fmt; uint64_t i; /* * The argument to these actions is an index into the * DOF string table. For printf()-like actions, this * is the format string. For print(), this is the * CTF type of the expression result. */ if ((strtab = dtrace_dof_sect(dof, DOF_SECT_STRTAB, desc->dofa_strtab)) == NULL) goto err; str = (char *)((uintptr_t)dof + (uintptr_t)strtab->dofs_offset); for (i = desc->dofa_arg; i < strtab->dofs_size; i++) { if (str[i] == '\0') break; } if (i >= strtab->dofs_size) { dtrace_dof_error(dof, "bogus format string"); goto err; } if (i == desc->dofa_arg) { dtrace_dof_error(dof, "empty format string"); goto err; } i -= desc->dofa_arg; fmt = kmem_alloc(i + 1, KM_SLEEP); bcopy(&str[desc->dofa_arg], fmt, i + 1); arg = (uint64_t)(uintptr_t)fmt; } else { if (kind == DTRACEACT_PRINTA) { ASSERT(desc->dofa_strtab == DOF_SECIDX_NONE); arg = 0; } else { arg = desc->dofa_arg; } } act = dtrace_actdesc_create(kind, desc->dofa_ntuple, desc->dofa_uarg, arg); if (last != NULL) { last->dtad_next = act; } else { first = act; } last = act; if (desc->dofa_difo == DOF_SECIDX_NONE) continue; if ((difosec = dtrace_dof_sect(dof, DOF_SECT_DIFOHDR, desc->dofa_difo)) == NULL) goto err; act->dtad_difo = dtrace_dof_difo(dof, difosec, vstate, cr); if (act->dtad_difo == NULL) goto err; } ASSERT(first != NULL); return (first); err: for (act = first; act != NULL; act = next) { next = act->dtad_next; dtrace_actdesc_release(act, vstate); } return (NULL); } static dtrace_ecbdesc_t * dtrace_dof_ecbdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate, cred_t *cr) { dtrace_ecbdesc_t *ep; dof_ecbdesc_t *ecb; dtrace_probedesc_t *desc; dtrace_predicate_t *pred = NULL; if (sec->dofs_size < sizeof (dof_ecbdesc_t)) { dtrace_dof_error(dof, "truncated ECB description"); return (NULL); } if (sec->dofs_align != sizeof (uint64_t)) { dtrace_dof_error(dof, "bad alignment in ECB description"); return (NULL); } ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset); sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes); if (sec == NULL) return (NULL); ep = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP); ep->dted_uarg = ecb->dofe_uarg; desc = &ep->dted_probe; if (dtrace_dof_probedesc(dof, sec, desc) == NULL) goto err; if (ecb->dofe_pred != DOF_SECIDX_NONE) { if ((sec = dtrace_dof_sect(dof, DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL) goto err; if ((pred = dtrace_dof_predicate(dof, sec, vstate, cr)) == NULL) goto err; ep->dted_pred.dtpdd_predicate = pred; } if (ecb->dofe_actions != DOF_SECIDX_NONE) { if ((sec = dtrace_dof_sect(dof, DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL) goto err; ep->dted_action = dtrace_dof_actdesc(dof, sec, vstate, cr); if (ep->dted_action == NULL) goto err; } return (ep); err: if (pred != NULL) dtrace_predicate_release(pred, vstate); kmem_free(ep, sizeof (dtrace_ecbdesc_t)); return (NULL); } /* * Apply the relocations from the specified 'sec' (a DOF_SECT_URELHDR) to the * specified DOF. At present, this amounts to simply adding 'ubase' to the * site of any user SETX relocations to account for load object base address. * In the future, if we need other relocations, this function can be extended. */ static int dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase) { uintptr_t daddr = (uintptr_t)dof; dof_relohdr_t *dofr = (dof_relohdr_t *)(uintptr_t)(daddr + sec->dofs_offset); dof_sec_t *ss, *rs, *ts; dof_relodesc_t *r; uint_t i, n; if (sec->dofs_size < sizeof (dof_relohdr_t) || sec->dofs_align != sizeof (dof_secidx_t)) { dtrace_dof_error(dof, "invalid relocation header"); return (-1); } ss = dtrace_dof_sect(dof, DOF_SECT_STRTAB, dofr->dofr_strtab); rs = dtrace_dof_sect(dof, DOF_SECT_RELTAB, dofr->dofr_relsec); ts = dtrace_dof_sect(dof, DOF_SECT_NONE, dofr->dofr_tgtsec); if (ss == NULL || rs == NULL || ts == NULL) return (-1); /* dtrace_dof_error() has been called already */ if (rs->dofs_entsize < sizeof (dof_relodesc_t) || rs->dofs_align != sizeof (uint64_t)) { dtrace_dof_error(dof, "invalid relocation section"); return (-1); } r = (dof_relodesc_t *)(uintptr_t)(daddr + rs->dofs_offset); n = rs->dofs_size / rs->dofs_entsize; for (i = 0; i < n; i++) { uintptr_t taddr = daddr + ts->dofs_offset + r->dofr_offset; switch (r->dofr_type) { case DOF_RELO_NONE: break; case DOF_RELO_SETX: if (r->dofr_offset >= ts->dofs_size || r->dofr_offset + sizeof (uint64_t) > ts->dofs_size) { dtrace_dof_error(dof, "bad relocation offset"); return (-1); } if (!IS_P2ALIGNED(taddr, sizeof (uint64_t))) { dtrace_dof_error(dof, "misaligned setx relo"); return (-1); } *(uint64_t *)taddr += ubase; break; default: dtrace_dof_error(dof, "invalid relocation type"); return (-1); } r = (dof_relodesc_t *)((uintptr_t)r + rs->dofs_entsize); } return (0); } /* * The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated * header: it should be at the front of a memory region that is at least * sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in * size. It need not be validated in any other way. */ static int dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr, dtrace_enabling_t **enabp, uint64_t ubase, int noprobes) { uint64_t len = dof->dofh_loadsz, seclen; uintptr_t daddr = (uintptr_t)dof; dtrace_ecbdesc_t *ep; dtrace_enabling_t *enab; uint_t i; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t)); /* * Check the DOF header identification bytes. In addition to checking * valid settings, we also verify that unused bits/bytes are zeroed so * we can use them later without fear of regressing existing binaries. */ if (bcmp(&dof->dofh_ident[DOF_ID_MAG0], DOF_MAG_STRING, DOF_MAG_STRLEN) != 0) { dtrace_dof_error(dof, "DOF magic string mismatch"); return (-1); } if (dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_ILP32 && dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_LP64) { dtrace_dof_error(dof, "DOF has invalid data model"); return (-1); } if (dof->dofh_ident[DOF_ID_ENCODING] != DOF_ENCODE_NATIVE) { dtrace_dof_error(dof, "DOF encoding mismatch"); return (-1); } if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 && dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_2) { dtrace_dof_error(dof, "DOF version mismatch"); return (-1); } if (dof->dofh_ident[DOF_ID_DIFVERS] != DIF_VERSION_2) { dtrace_dof_error(dof, "DOF uses unsupported instruction set"); return (-1); } if (dof->dofh_ident[DOF_ID_DIFIREG] > DIF_DIR_NREGS) { dtrace_dof_error(dof, "DOF uses too many integer registers"); return (-1); } if (dof->dofh_ident[DOF_ID_DIFTREG] > DIF_DTR_NREGS) { dtrace_dof_error(dof, "DOF uses too many tuple registers"); return (-1); } for (i = DOF_ID_PAD; i < DOF_ID_SIZE; i++) { if (dof->dofh_ident[i] != 0) { dtrace_dof_error(dof, "DOF has invalid ident byte set"); return (-1); } } if (dof->dofh_flags & ~DOF_FL_VALID) { dtrace_dof_error(dof, "DOF has invalid flag bits set"); return (-1); } if (dof->dofh_secsize == 0) { dtrace_dof_error(dof, "zero section header size"); return (-1); } /* * Check that the section headers don't exceed the amount of DOF * data. Note that we cast the section size and number of sections * to uint64_t's to prevent possible overflow in the multiplication. */ seclen = (uint64_t)dof->dofh_secnum * (uint64_t)dof->dofh_secsize; if (dof->dofh_secoff > len || seclen > len || dof->dofh_secoff + seclen > len) { dtrace_dof_error(dof, "truncated section headers"); return (-1); } if (!IS_P2ALIGNED(dof->dofh_secoff, sizeof (uint64_t))) { dtrace_dof_error(dof, "misaligned section headers"); return (-1); } if (!IS_P2ALIGNED(dof->dofh_secsize, sizeof (uint64_t))) { dtrace_dof_error(dof, "misaligned section size"); return (-1); } /* * Take an initial pass through the section headers to be sure that * the headers don't have stray offsets. If the 'noprobes' flag is * set, do not permit sections relating to providers, probes, or args. */ for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)(daddr + (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize); if (noprobes) { switch (sec->dofs_type) { case DOF_SECT_PROVIDER: case DOF_SECT_PROBES: case DOF_SECT_PRARGS: case DOF_SECT_PROFFS: dtrace_dof_error(dof, "illegal sections " "for enabling"); return (-1); } } if (DOF_SEC_ISLOADABLE(sec->dofs_type) && !(sec->dofs_flags & DOF_SECF_LOAD)) { dtrace_dof_error(dof, "loadable section with load " "flag unset"); return (-1); } if (!(sec->dofs_flags & DOF_SECF_LOAD)) continue; /* just ignore non-loadable sections */ if (!ISP2(sec->dofs_align)) { dtrace_dof_error(dof, "bad section alignment"); return (-1); } if (sec->dofs_offset & (sec->dofs_align - 1)) { dtrace_dof_error(dof, "misaligned section"); return (-1); } if (sec->dofs_offset > len || sec->dofs_size > len || sec->dofs_offset + sec->dofs_size > len) { dtrace_dof_error(dof, "corrupt section header"); return (-1); } if (sec->dofs_type == DOF_SECT_STRTAB && *((char *)daddr + sec->dofs_offset + sec->dofs_size - 1) != '\0') { dtrace_dof_error(dof, "non-terminating string table"); return (-1); } } /* * Take a second pass through the sections and locate and perform any * relocations that are present. We do this after the first pass to * be sure that all sections have had their headers validated. */ for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)(daddr + (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize); if (!(sec->dofs_flags & DOF_SECF_LOAD)) continue; /* skip sections that are not loadable */ switch (sec->dofs_type) { case DOF_SECT_URELHDR: if (dtrace_dof_relocate(dof, sec, ubase) != 0) return (-1); break; } } if ((enab = *enabp) == NULL) enab = *enabp = dtrace_enabling_create(vstate); for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)(daddr + (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize); if (sec->dofs_type != DOF_SECT_ECBDESC) continue; if ((ep = dtrace_dof_ecbdesc(dof, sec, vstate, cr)) == NULL) { dtrace_enabling_destroy(enab); *enabp = NULL; return (-1); } dtrace_enabling_add(enab, ep); } return (0); } /* * Process DOF for any options. This routine assumes that the DOF has been * at least processed by dtrace_dof_slurp(). */ static int dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state) { int i, rval; uint32_t entsize; size_t offs; dof_optdesc_t *desc; for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)((uintptr_t)dof + (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize); if (sec->dofs_type != DOF_SECT_OPTDESC) continue; if (sec->dofs_align != sizeof (uint64_t)) { dtrace_dof_error(dof, "bad alignment in " "option description"); return (EINVAL); } if ((entsize = sec->dofs_entsize) == 0) { dtrace_dof_error(dof, "zeroed option entry size"); return (EINVAL); } if (entsize < sizeof (dof_optdesc_t)) { dtrace_dof_error(dof, "bad option entry size"); return (EINVAL); } for (offs = 0; offs < sec->dofs_size; offs += entsize) { desc = (dof_optdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset + offs); if (desc->dofo_strtab != DOF_SECIDX_NONE) { dtrace_dof_error(dof, "non-zero option string"); return (EINVAL); } if (desc->dofo_value == DTRACEOPT_UNSET) { dtrace_dof_error(dof, "unset option"); return (EINVAL); } if ((rval = dtrace_state_option(state, desc->dofo_option, desc->dofo_value)) != 0) { dtrace_dof_error(dof, "rejected option"); return (rval); } } } return (0); } /* * DTrace Consumer State Functions */ static int dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size) { size_t hashsize, maxper, min, chunksize = dstate->dtds_chunksize; void *base; uintptr_t limit; dtrace_dynvar_t *dvar, *next, *start; int i; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL); bzero(dstate, sizeof (dtrace_dstate_t)); if ((dstate->dtds_chunksize = chunksize) == 0) dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE; if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t))) size = min; if ((base = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL) return (ENOMEM); dstate->dtds_size = size; dstate->dtds_base = base; dstate->dtds_percpu = kmem_cache_alloc(dtrace_state_cache, KM_SLEEP); bzero(dstate->dtds_percpu, NCPU * sizeof (dtrace_dstate_percpu_t)); hashsize = size / (dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)); if (hashsize != 1 && (hashsize & 1)) hashsize--; dstate->dtds_hashsize = hashsize; dstate->dtds_hash = dstate->dtds_base; /* * Set all of our hash buckets to point to the single sink, and (if * it hasn't already been set), set the sink's hash value to be the * sink sentinel value. The sink is needed for dynamic variable * lookups to know that they have iterated over an entire, valid hash * chain. */ for (i = 0; i < hashsize; i++) dstate->dtds_hash[i].dtdh_chain = &dtrace_dynhash_sink; if (dtrace_dynhash_sink.dtdv_hashval != DTRACE_DYNHASH_SINK) dtrace_dynhash_sink.dtdv_hashval = DTRACE_DYNHASH_SINK; /* * Determine number of active CPUs. Divide free list evenly among * active CPUs. */ start = (dtrace_dynvar_t *) ((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t)); limit = (uintptr_t)base + size; maxper = (limit - (uintptr_t)start) / NCPU; maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize; #ifndef illumos CPU_FOREACH(i) { #else for (i = 0; i < NCPU; i++) { #endif dstate->dtds_percpu[i].dtdsc_free = dvar = start; /* * If we don't even have enough chunks to make it once through * NCPUs, we're just going to allocate everything to the first * CPU. And if we're on the last CPU, we're going to allocate * whatever is left over. In either case, we set the limit to * be the limit of the dynamic variable space. */ if (maxper == 0 || i == NCPU - 1) { limit = (uintptr_t)base + size; start = NULL; } else { limit = (uintptr_t)start + maxper; start = (dtrace_dynvar_t *)limit; } ASSERT(limit <= (uintptr_t)base + size); for (;;) { next = (dtrace_dynvar_t *)((uintptr_t)dvar + dstate->dtds_chunksize); if ((uintptr_t)next + dstate->dtds_chunksize >= limit) break; dvar->dtdv_next = next; dvar = next; } if (maxper == 0) break; } return (0); } static void dtrace_dstate_fini(dtrace_dstate_t *dstate) { ASSERT(MUTEX_HELD(&cpu_lock)); if (dstate->dtds_base == NULL) return; kmem_free(dstate->dtds_base, dstate->dtds_size); kmem_cache_free(dtrace_state_cache, dstate->dtds_percpu); } static void dtrace_vstate_fini(dtrace_vstate_t *vstate) { /* * Logical XOR, where are you? */ ASSERT((vstate->dtvs_nglobals == 0) ^ (vstate->dtvs_globals != NULL)); if (vstate->dtvs_nglobals > 0) { kmem_free(vstate->dtvs_globals, vstate->dtvs_nglobals * sizeof (dtrace_statvar_t *)); } if (vstate->dtvs_ntlocals > 0) { kmem_free(vstate->dtvs_tlocals, vstate->dtvs_ntlocals * sizeof (dtrace_difv_t)); } ASSERT((vstate->dtvs_nlocals == 0) ^ (vstate->dtvs_locals != NULL)); if (vstate->dtvs_nlocals > 0) { kmem_free(vstate->dtvs_locals, vstate->dtvs_nlocals * sizeof (dtrace_statvar_t *)); } } #ifdef illumos static void dtrace_state_clean(dtrace_state_t *state) { if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) return; dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars); dtrace_speculation_clean(state); } static void dtrace_state_deadman(dtrace_state_t *state) { hrtime_t now; dtrace_sync(); now = dtrace_gethrtime(); if (state != dtrace_anon.dta_state && now - state->dts_laststatus >= dtrace_deadman_user) return; /* * We must be sure that dts_alive never appears to be less than the * value upon entry to dtrace_state_deadman(), and because we lack a * dtrace_cas64(), we cannot store to it atomically. We thus instead * store INT64_MAX to it, followed by a memory barrier, followed by * the new value. This assures that dts_alive never appears to be * less than its true value, regardless of the order in which the * stores to the underlying storage are issued. */ state->dts_alive = INT64_MAX; dtrace_membar_producer(); state->dts_alive = now; } #else /* !illumos */ static void dtrace_state_clean(void *arg) { dtrace_state_t *state = arg; dtrace_optval_t *opt = state->dts_options; if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) return; dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars); dtrace_speculation_clean(state); callout_reset(&state->dts_cleaner, hz * opt[DTRACEOPT_CLEANRATE] / NANOSEC, dtrace_state_clean, state); } static void dtrace_state_deadman(void *arg) { dtrace_state_t *state = arg; hrtime_t now; dtrace_sync(); dtrace_debug_output(); now = dtrace_gethrtime(); if (state != dtrace_anon.dta_state && now - state->dts_laststatus >= dtrace_deadman_user) return; /* * We must be sure that dts_alive never appears to be less than the * value upon entry to dtrace_state_deadman(), and because we lack a * dtrace_cas64(), we cannot store to it atomically. We thus instead * store INT64_MAX to it, followed by a memory barrier, followed by * the new value. This assures that dts_alive never appears to be * less than its true value, regardless of the order in which the * stores to the underlying storage are issued. */ state->dts_alive = INT64_MAX; dtrace_membar_producer(); state->dts_alive = now; callout_reset(&state->dts_deadman, hz * dtrace_deadman_interval / NANOSEC, dtrace_state_deadman, state); } #endif /* illumos */ static dtrace_state_t * #ifdef illumos dtrace_state_create(dev_t *devp, cred_t *cr) #else dtrace_state_create(struct cdev *dev) #endif { #ifdef illumos minor_t minor; major_t major; #else cred_t *cr = NULL; int m = 0; #endif char c[30]; dtrace_state_t *state; dtrace_optval_t *opt; int bufsize = NCPU * sizeof (dtrace_buffer_t), i; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(MUTEX_HELD(&cpu_lock)); #ifdef illumos minor = (minor_t)(uintptr_t)vmem_alloc(dtrace_minor, 1, VM_BESTFIT | VM_SLEEP); if (ddi_soft_state_zalloc(dtrace_softstate, minor) != DDI_SUCCESS) { vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1); return (NULL); } state = ddi_get_soft_state(dtrace_softstate, minor); #else if (dev != NULL) { cr = dev->si_cred; m = dev2unit(dev); } /* Allocate memory for the state. */ state = kmem_zalloc(sizeof(dtrace_state_t), KM_SLEEP); #endif state->dts_epid = DTRACE_EPIDNONE + 1; (void) snprintf(c, sizeof (c), "dtrace_aggid_%d", m); #ifdef illumos state->dts_aggid_arena = vmem_create(c, (void *)1, UINT32_MAX, 1, NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); if (devp != NULL) { major = getemajor(*devp); } else { major = ddi_driver_major(dtrace_devi); } state->dts_dev = makedevice(major, minor); if (devp != NULL) *devp = state->dts_dev; #else state->dts_aggid_arena = new_unrhdr(1, INT_MAX, &dtrace_unr_mtx); state->dts_dev = dev; #endif /* * We allocate NCPU buffers. On the one hand, this can be quite * a bit of memory per instance (nearly 36K on a Starcat). On the * other hand, it saves an additional memory reference in the probe * path. */ state->dts_buffer = kmem_zalloc(bufsize, KM_SLEEP); state->dts_aggbuffer = kmem_zalloc(bufsize, KM_SLEEP); #ifdef illumos state->dts_cleaner = CYCLIC_NONE; state->dts_deadman = CYCLIC_NONE; #else callout_init(&state->dts_cleaner, CALLOUT_MPSAFE); callout_init(&state->dts_deadman, CALLOUT_MPSAFE); #endif state->dts_vstate.dtvs_state = state; for (i = 0; i < DTRACEOPT_MAX; i++) state->dts_options[i] = DTRACEOPT_UNSET; /* * Set the default options. */ opt = state->dts_options; opt[DTRACEOPT_BUFPOLICY] = DTRACEOPT_BUFPOLICY_SWITCH; opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_AUTO; opt[DTRACEOPT_NSPEC] = dtrace_nspec_default; opt[DTRACEOPT_SPECSIZE] = dtrace_specsize_default; opt[DTRACEOPT_CPU] = (dtrace_optval_t)DTRACE_CPUALL; opt[DTRACEOPT_STRSIZE] = dtrace_strsize_default; opt[DTRACEOPT_STACKFRAMES] = dtrace_stackframes_default; opt[DTRACEOPT_USTACKFRAMES] = dtrace_ustackframes_default; opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_default; opt[DTRACEOPT_AGGRATE] = dtrace_aggrate_default; opt[DTRACEOPT_SWITCHRATE] = dtrace_switchrate_default; opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_default; opt[DTRACEOPT_JSTACKFRAMES] = dtrace_jstackframes_default; opt[DTRACEOPT_JSTACKSTRSIZE] = dtrace_jstackstrsize_default; state->dts_activity = DTRACE_ACTIVITY_INACTIVE; /* * Depending on the user credentials, we set flag bits which alter probe * visibility or the amount of destructiveness allowed. In the case of * actual anonymous tracing, or the possession of all privileges, all of * the normal checks are bypassed. */ if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) { state->dts_cred.dcr_visible = DTRACE_CRV_ALL; state->dts_cred.dcr_action = DTRACE_CRA_ALL; } else { /* * Set up the credentials for this instantiation. We take a * hold on the credential to prevent it from disappearing on * us; this in turn prevents the zone_t referenced by this * credential from disappearing. This means that we can * examine the credential and the zone from probe context. */ crhold(cr); state->dts_cred.dcr_cred = cr; /* * CRA_PROC means "we have *some* privilege for dtrace" and * unlocks the use of variables like pid, zonename, etc. */ if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE) || PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) { state->dts_cred.dcr_action |= DTRACE_CRA_PROC; } /* * dtrace_user allows use of syscall and profile providers. * If the user also has proc_owner and/or proc_zone, we * extend the scope to include additional visibility and * destructive power. */ if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) { if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) { state->dts_cred.dcr_visible |= DTRACE_CRV_ALLPROC; state->dts_cred.dcr_action |= DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER; } if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) { state->dts_cred.dcr_visible |= DTRACE_CRV_ALLZONE; state->dts_cred.dcr_action |= DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE; } /* * If we have all privs in whatever zone this is, * we can do destructive things to processes which * have altered credentials. */ #ifdef illumos if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE), cr->cr_zone->zone_privset)) { state->dts_cred.dcr_action |= DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG; } #endif } /* * Holding the dtrace_kernel privilege also implies that * the user has the dtrace_user privilege from a visibility * perspective. But without further privileges, some * destructive actions are not available. */ if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) { /* * Make all probes in all zones visible. However, * this doesn't mean that all actions become available * to all zones. */ state->dts_cred.dcr_visible |= DTRACE_CRV_KERNEL | DTRACE_CRV_ALLPROC | DTRACE_CRV_ALLZONE; state->dts_cred.dcr_action |= DTRACE_CRA_KERNEL | DTRACE_CRA_PROC; /* * Holding proc_owner means that destructive actions * for *this* zone are allowed. */ if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) state->dts_cred.dcr_action |= DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER; /* * Holding proc_zone means that destructive actions * for this user/group ID in all zones is allowed. */ if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) state->dts_cred.dcr_action |= DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE; #ifdef illumos /* * If we have all privs in whatever zone this is, * we can do destructive things to processes which * have altered credentials. */ if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE), cr->cr_zone->zone_privset)) { state->dts_cred.dcr_action |= DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG; } #endif } /* * Holding the dtrace_proc privilege gives control over fasttrap * and pid providers. We need to grant wider destructive * privileges in the event that the user has proc_owner and/or * proc_zone. */ if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) { if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) state->dts_cred.dcr_action |= DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER; if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) state->dts_cred.dcr_action |= DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE; } } return (state); } static int dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which) { dtrace_optval_t *opt = state->dts_options, size; processorid_t cpu = 0;; int flags = 0, rval, factor, divisor = 1; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(MUTEX_HELD(&cpu_lock)); ASSERT(which < DTRACEOPT_MAX); ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE || (state == dtrace_anon.dta_state && state->dts_activity == DTRACE_ACTIVITY_ACTIVE)); if (opt[which] == DTRACEOPT_UNSET || opt[which] == 0) return (0); if (opt[DTRACEOPT_CPU] != DTRACEOPT_UNSET) cpu = opt[DTRACEOPT_CPU]; if (which == DTRACEOPT_SPECSIZE) flags |= DTRACEBUF_NOSWITCH; if (which == DTRACEOPT_BUFSIZE) { if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_RING) flags |= DTRACEBUF_RING; if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_FILL) flags |= DTRACEBUF_FILL; if (state != dtrace_anon.dta_state || state->dts_activity != DTRACE_ACTIVITY_ACTIVE) flags |= DTRACEBUF_INACTIVE; } for (size = opt[which]; size >= sizeof (uint64_t); size /= divisor) { /* * The size must be 8-byte aligned. If the size is not 8-byte * aligned, drop it down by the difference. */ if (size & (sizeof (uint64_t) - 1)) size -= size & (sizeof (uint64_t) - 1); if (size < state->dts_reserve) { /* * Buffers always must be large enough to accommodate * their prereserved space. We return E2BIG instead * of ENOMEM in this case to allow for user-level * software to differentiate the cases. */ return (E2BIG); } rval = dtrace_buffer_alloc(buf, size, flags, cpu, &factor); if (rval != ENOMEM) { opt[which] = size; return (rval); } if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL) return (rval); for (divisor = 2; divisor < factor; divisor <<= 1) continue; } return (ENOMEM); } static int dtrace_state_buffers(dtrace_state_t *state) { dtrace_speculation_t *spec = state->dts_speculations; int rval, i; if ((rval = dtrace_state_buffer(state, state->dts_buffer, DTRACEOPT_BUFSIZE)) != 0) return (rval); if ((rval = dtrace_state_buffer(state, state->dts_aggbuffer, DTRACEOPT_AGGSIZE)) != 0) return (rval); for (i = 0; i < state->dts_nspeculations; i++) { if ((rval = dtrace_state_buffer(state, spec[i].dtsp_buffer, DTRACEOPT_SPECSIZE)) != 0) return (rval); } return (0); } static void dtrace_state_prereserve(dtrace_state_t *state) { dtrace_ecb_t *ecb; dtrace_probe_t *probe; state->dts_reserve = 0; if (state->dts_options[DTRACEOPT_BUFPOLICY] != DTRACEOPT_BUFPOLICY_FILL) return; /* * If our buffer policy is a "fill" buffer policy, we need to set the * prereserved space to be the space required by the END probes. */ probe = dtrace_probes[dtrace_probeid_end - 1]; ASSERT(probe != NULL); for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) { if (ecb->dte_state != state) continue; state->dts_reserve += ecb->dte_needed + ecb->dte_alignment; } } static int dtrace_state_go(dtrace_state_t *state, processorid_t *cpu) { dtrace_optval_t *opt = state->dts_options, sz, nspec; dtrace_speculation_t *spec; dtrace_buffer_t *buf; #ifdef illumos cyc_handler_t hdlr; cyc_time_t when; #endif int rval = 0, i, bufsize = NCPU * sizeof (dtrace_buffer_t); dtrace_icookie_t cookie; mutex_enter(&cpu_lock); mutex_enter(&dtrace_lock); if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) { rval = EBUSY; goto out; } /* * Before we can perform any checks, we must prime all of the * retained enablings that correspond to this state. */ dtrace_enabling_prime(state); if (state->dts_destructive && !state->dts_cred.dcr_destructive) { rval = EACCES; goto out; } dtrace_state_prereserve(state); /* * Now we want to do is try to allocate our speculations. * We do not automatically resize the number of speculations; if * this fails, we will fail the operation. */ nspec = opt[DTRACEOPT_NSPEC]; ASSERT(nspec != DTRACEOPT_UNSET); if (nspec > INT_MAX) { rval = ENOMEM; goto out; } spec = kmem_zalloc(nspec * sizeof (dtrace_speculation_t), KM_NOSLEEP | KM_NORMALPRI); if (spec == NULL) { rval = ENOMEM; goto out; } state->dts_speculations = spec; state->dts_nspeculations = (int)nspec; for (i = 0; i < nspec; i++) { if ((buf = kmem_zalloc(bufsize, KM_NOSLEEP | KM_NORMALPRI)) == NULL) { rval = ENOMEM; goto err; } spec[i].dtsp_buffer = buf; } if (opt[DTRACEOPT_GRABANON] != DTRACEOPT_UNSET) { if (dtrace_anon.dta_state == NULL) { rval = ENOENT; goto out; } if (state->dts_necbs != 0) { rval = EALREADY; goto out; } state->dts_anon = dtrace_anon_grab(); ASSERT(state->dts_anon != NULL); state = state->dts_anon; /* * We want "grabanon" to be set in the grabbed state, so we'll * copy that option value from the grabbing state into the * grabbed state. */ state->dts_options[DTRACEOPT_GRABANON] = opt[DTRACEOPT_GRABANON]; *cpu = dtrace_anon.dta_beganon; /* * If the anonymous state is active (as it almost certainly * is if the anonymous enabling ultimately matched anything), * we don't allow any further option processing -- but we * don't return failure. */ if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) goto out; } if (opt[DTRACEOPT_AGGSIZE] != DTRACEOPT_UNSET && opt[DTRACEOPT_AGGSIZE] != 0) { if (state->dts_aggregations == NULL) { /* * We're not going to create an aggregation buffer * because we don't have any ECBs that contain * aggregations -- set this option to 0. */ opt[DTRACEOPT_AGGSIZE] = 0; } else { /* * If we have an aggregation buffer, we must also have * a buffer to use as scratch. */ if (opt[DTRACEOPT_BUFSIZE] == DTRACEOPT_UNSET || opt[DTRACEOPT_BUFSIZE] < state->dts_needed) { opt[DTRACEOPT_BUFSIZE] = state->dts_needed; } } } if (opt[DTRACEOPT_SPECSIZE] != DTRACEOPT_UNSET && opt[DTRACEOPT_SPECSIZE] != 0) { if (!state->dts_speculates) { /* * We're not going to create speculation buffers * because we don't have any ECBs that actually * speculate -- set the speculation size to 0. */ opt[DTRACEOPT_SPECSIZE] = 0; } } /* * The bare minimum size for any buffer that we're actually going to * do anything to is sizeof (uint64_t). */ sz = sizeof (uint64_t); if ((state->dts_needed != 0 && opt[DTRACEOPT_BUFSIZE] < sz) || (state->dts_speculates && opt[DTRACEOPT_SPECSIZE] < sz) || (state->dts_aggregations != NULL && opt[DTRACEOPT_AGGSIZE] < sz)) { /* * A buffer size has been explicitly set to 0 (or to a size * that will be adjusted to 0) and we need the space -- we * need to return failure. We return ENOSPC to differentiate * it from failing to allocate a buffer due to failure to meet * the reserve (for which we return E2BIG). */ rval = ENOSPC; goto out; } if ((rval = dtrace_state_buffers(state)) != 0) goto err; if ((sz = opt[DTRACEOPT_DYNVARSIZE]) == DTRACEOPT_UNSET) sz = dtrace_dstate_defsize; do { rval = dtrace_dstate_init(&state->dts_vstate.dtvs_dynvars, sz); if (rval == 0) break; if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL) goto err; } while (sz >>= 1); opt[DTRACEOPT_DYNVARSIZE] = sz; if (rval != 0) goto err; if (opt[DTRACEOPT_STATUSRATE] > dtrace_statusrate_max) opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_max; if (opt[DTRACEOPT_CLEANRATE] == 0) opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max; if (opt[DTRACEOPT_CLEANRATE] < dtrace_cleanrate_min) opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_min; if (opt[DTRACEOPT_CLEANRATE] > dtrace_cleanrate_max) opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max; state->dts_alive = state->dts_laststatus = dtrace_gethrtime(); #ifdef illumos hdlr.cyh_func = (cyc_func_t)dtrace_state_clean; hdlr.cyh_arg = state; hdlr.cyh_level = CY_LOW_LEVEL; when.cyt_when = 0; when.cyt_interval = opt[DTRACEOPT_CLEANRATE]; state->dts_cleaner = cyclic_add(&hdlr, &when); hdlr.cyh_func = (cyc_func_t)dtrace_state_deadman; hdlr.cyh_arg = state; hdlr.cyh_level = CY_LOW_LEVEL; when.cyt_when = 0; when.cyt_interval = dtrace_deadman_interval; state->dts_deadman = cyclic_add(&hdlr, &when); #else callout_reset(&state->dts_cleaner, hz * opt[DTRACEOPT_CLEANRATE] / NANOSEC, dtrace_state_clean, state); callout_reset(&state->dts_deadman, hz * dtrace_deadman_interval / NANOSEC, dtrace_state_deadman, state); #endif state->dts_activity = DTRACE_ACTIVITY_WARMUP; #ifdef illumos if (state->dts_getf != 0 && !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) { /* * We don't have kernel privs but we have at least one call * to getf(); we need to bump our zone's count, and (if * this is the first enabling to have an unprivileged call * to getf()) we need to hook into closef(). */ state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf++; if (dtrace_getf++ == 0) { ASSERT(dtrace_closef == NULL); dtrace_closef = dtrace_getf_barrier; } } #endif /* * Now it's time to actually fire the BEGIN probe. We need to disable * interrupts here both to record the CPU on which we fired the BEGIN * probe (the data from this CPU will be processed first at user * level) and to manually activate the buffer for this CPU. */ cookie = dtrace_interrupt_disable(); *cpu = curcpu; ASSERT(state->dts_buffer[*cpu].dtb_flags & DTRACEBUF_INACTIVE); state->dts_buffer[*cpu].dtb_flags &= ~DTRACEBUF_INACTIVE; dtrace_probe(dtrace_probeid_begin, (uint64_t)(uintptr_t)state, 0, 0, 0, 0); dtrace_interrupt_enable(cookie); /* * We may have had an exit action from a BEGIN probe; only change our * state to ACTIVE if we're still in WARMUP. */ ASSERT(state->dts_activity == DTRACE_ACTIVITY_WARMUP || state->dts_activity == DTRACE_ACTIVITY_DRAINING); if (state->dts_activity == DTRACE_ACTIVITY_WARMUP) state->dts_activity = DTRACE_ACTIVITY_ACTIVE; /* * Regardless of whether or not now we're in ACTIVE or DRAINING, we * want each CPU to transition its principal buffer out of the * INACTIVE state. Doing this assures that no CPU will suddenly begin * processing an ECB halfway down a probe's ECB chain; all CPUs will * atomically transition from processing none of a state's ECBs to * processing all of them. */ dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_buffer_activate, state); goto out; err: dtrace_buffer_free(state->dts_buffer); dtrace_buffer_free(state->dts_aggbuffer); if ((nspec = state->dts_nspeculations) == 0) { ASSERT(state->dts_speculations == NULL); goto out; } spec = state->dts_speculations; ASSERT(spec != NULL); for (i = 0; i < state->dts_nspeculations; i++) { if ((buf = spec[i].dtsp_buffer) == NULL) break; dtrace_buffer_free(buf); kmem_free(buf, bufsize); } kmem_free(spec, nspec * sizeof (dtrace_speculation_t)); state->dts_nspeculations = 0; state->dts_speculations = NULL; out: mutex_exit(&dtrace_lock); mutex_exit(&cpu_lock); return (rval); } static int dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu) { dtrace_icookie_t cookie; ASSERT(MUTEX_HELD(&dtrace_lock)); if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE && state->dts_activity != DTRACE_ACTIVITY_DRAINING) return (EINVAL); /* * We'll set the activity to DTRACE_ACTIVITY_DRAINING, and issue a sync * to be sure that every CPU has seen it. See below for the details * on why this is done. */ state->dts_activity = DTRACE_ACTIVITY_DRAINING; dtrace_sync(); /* * By this point, it is impossible for any CPU to be still processing * with DTRACE_ACTIVITY_ACTIVE. We can thus set our activity to * DTRACE_ACTIVITY_COOLDOWN and know that we're not racing with any * other CPU in dtrace_buffer_reserve(). This allows dtrace_probe() * and callees to know that the activity is DTRACE_ACTIVITY_COOLDOWN * iff we're in the END probe. */ state->dts_activity = DTRACE_ACTIVITY_COOLDOWN; dtrace_sync(); ASSERT(state->dts_activity == DTRACE_ACTIVITY_COOLDOWN); /* * Finally, we can release the reserve and call the END probe. We * disable interrupts across calling the END probe to allow us to * return the CPU on which we actually called the END probe. This * allows user-land to be sure that this CPU's principal buffer is * processed last. */ state->dts_reserve = 0; cookie = dtrace_interrupt_disable(); *cpu = curcpu; dtrace_probe(dtrace_probeid_end, (uint64_t)(uintptr_t)state, 0, 0, 0, 0); dtrace_interrupt_enable(cookie); state->dts_activity = DTRACE_ACTIVITY_STOPPED; dtrace_sync(); #ifdef illumos if (state->dts_getf != 0 && !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) { /* * We don't have kernel privs but we have at least one call * to getf(); we need to lower our zone's count, and (if * this is the last enabling to have an unprivileged call * to getf()) we need to clear the closef() hook. */ ASSERT(state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf > 0); ASSERT(dtrace_closef == dtrace_getf_barrier); ASSERT(dtrace_getf > 0); state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf--; if (--dtrace_getf == 0) dtrace_closef = NULL; } #endif return (0); } static int dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option, dtrace_optval_t val) { ASSERT(MUTEX_HELD(&dtrace_lock)); if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) return (EBUSY); if (option >= DTRACEOPT_MAX) return (EINVAL); if (option != DTRACEOPT_CPU && val < 0) return (EINVAL); switch (option) { case DTRACEOPT_DESTRUCTIVE: if (dtrace_destructive_disallow) return (EACCES); state->dts_cred.dcr_destructive = 1; break; case DTRACEOPT_BUFSIZE: case DTRACEOPT_DYNVARSIZE: case DTRACEOPT_AGGSIZE: case DTRACEOPT_SPECSIZE: case DTRACEOPT_STRSIZE: if (val < 0) return (EINVAL); if (val >= LONG_MAX) { /* * If this is an otherwise negative value, set it to * the highest multiple of 128m less than LONG_MAX. * Technically, we're adjusting the size without * regard to the buffer resizing policy, but in fact, * this has no effect -- if we set the buffer size to * ~LONG_MAX and the buffer policy is ultimately set to * be "manual", the buffer allocation is guaranteed to * fail, if only because the allocation requires two * buffers. (We set the the size to the highest * multiple of 128m because it ensures that the size * will remain a multiple of a megabyte when * repeatedly halved -- all the way down to 15m.) */ val = LONG_MAX - (1 << 27) + 1; } } state->dts_options[option] = val; return (0); } static void dtrace_state_destroy(dtrace_state_t *state) { dtrace_ecb_t *ecb; dtrace_vstate_t *vstate = &state->dts_vstate; #ifdef illumos minor_t minor = getminor(state->dts_dev); #endif int i, bufsize = NCPU * sizeof (dtrace_buffer_t); dtrace_speculation_t *spec = state->dts_speculations; int nspec = state->dts_nspeculations; uint32_t match; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(MUTEX_HELD(&cpu_lock)); /* * First, retract any retained enablings for this state. */ dtrace_enabling_retract(state); ASSERT(state->dts_nretained == 0); if (state->dts_activity == DTRACE_ACTIVITY_ACTIVE || state->dts_activity == DTRACE_ACTIVITY_DRAINING) { /* * We have managed to come into dtrace_state_destroy() on a * hot enabling -- almost certainly because of a disorderly * shutdown of a consumer. (That is, a consumer that is * exiting without having called dtrace_stop().) In this case, * we're going to set our activity to be KILLED, and then * issue a sync to be sure that everyone is out of probe * context before we start blowing away ECBs. */ state->dts_activity = DTRACE_ACTIVITY_KILLED; dtrace_sync(); } /* * Release the credential hold we took in dtrace_state_create(). */ if (state->dts_cred.dcr_cred != NULL) crfree(state->dts_cred.dcr_cred); /* * Now we can safely disable and destroy any enabled probes. Because * any DTRACE_PRIV_KERNEL probes may actually be slowing our progress * (especially if they're all enabled), we take two passes through the * ECBs: in the first, we disable just DTRACE_PRIV_KERNEL probes, and * in the second we disable whatever is left over. */ for (match = DTRACE_PRIV_KERNEL; ; match = 0) { for (i = 0; i < state->dts_necbs; i++) { if ((ecb = state->dts_ecbs[i]) == NULL) continue; if (match && ecb->dte_probe != NULL) { dtrace_probe_t *probe = ecb->dte_probe; dtrace_provider_t *prov = probe->dtpr_provider; if (!(prov->dtpv_priv.dtpp_flags & match)) continue; } dtrace_ecb_disable(ecb); dtrace_ecb_destroy(ecb); } if (!match) break; } /* * Before we free the buffers, perform one more sync to assure that * every CPU is out of probe context. */ dtrace_sync(); dtrace_buffer_free(state->dts_buffer); dtrace_buffer_free(state->dts_aggbuffer); for (i = 0; i < nspec; i++) dtrace_buffer_free(spec[i].dtsp_buffer); #ifdef illumos if (state->dts_cleaner != CYCLIC_NONE) cyclic_remove(state->dts_cleaner); if (state->dts_deadman != CYCLIC_NONE) cyclic_remove(state->dts_deadman); #else callout_stop(&state->dts_cleaner); callout_drain(&state->dts_cleaner); callout_stop(&state->dts_deadman); callout_drain(&state->dts_deadman); #endif dtrace_dstate_fini(&vstate->dtvs_dynvars); dtrace_vstate_fini(vstate); if (state->dts_ecbs != NULL) kmem_free(state->dts_ecbs, state->dts_necbs * sizeof (dtrace_ecb_t *)); if (state->dts_aggregations != NULL) { #ifdef DEBUG for (i = 0; i < state->dts_naggregations; i++) ASSERT(state->dts_aggregations[i] == NULL); #endif ASSERT(state->dts_naggregations > 0); kmem_free(state->dts_aggregations, state->dts_naggregations * sizeof (dtrace_aggregation_t *)); } kmem_free(state->dts_buffer, bufsize); kmem_free(state->dts_aggbuffer, bufsize); for (i = 0; i < nspec; i++) kmem_free(spec[i].dtsp_buffer, bufsize); if (spec != NULL) kmem_free(spec, nspec * sizeof (dtrace_speculation_t)); dtrace_format_destroy(state); if (state->dts_aggid_arena != NULL) { #ifdef illumos vmem_destroy(state->dts_aggid_arena); #else delete_unrhdr(state->dts_aggid_arena); #endif state->dts_aggid_arena = NULL; } #ifdef illumos ddi_soft_state_free(dtrace_softstate, minor); vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1); #endif } /* * DTrace Anonymous Enabling Functions */ static dtrace_state_t * dtrace_anon_grab(void) { dtrace_state_t *state; ASSERT(MUTEX_HELD(&dtrace_lock)); if ((state = dtrace_anon.dta_state) == NULL) { ASSERT(dtrace_anon.dta_enabling == NULL); return (NULL); } ASSERT(dtrace_anon.dta_enabling != NULL); ASSERT(dtrace_retained != NULL); dtrace_enabling_destroy(dtrace_anon.dta_enabling); dtrace_anon.dta_enabling = NULL; dtrace_anon.dta_state = NULL; return (state); } static void dtrace_anon_property(void) { int i, rv; dtrace_state_t *state; dof_hdr_t *dof; char c[32]; /* enough for "dof-data-" + digits */ ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(MUTEX_HELD(&cpu_lock)); for (i = 0; ; i++) { (void) snprintf(c, sizeof (c), "dof-data-%d", i); dtrace_err_verbose = 1; if ((dof = dtrace_dof_property(c)) == NULL) { dtrace_err_verbose = 0; break; } #ifdef illumos /* * We want to create anonymous state, so we need to transition * the kernel debugger to indicate that DTrace is active. If * this fails (e.g. because the debugger has modified text in * some way), we won't continue with the processing. */ if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) { cmn_err(CE_NOTE, "kernel debugger active; anonymous " "enabling ignored."); dtrace_dof_destroy(dof); break; } #endif /* * If we haven't allocated an anonymous state, we'll do so now. */ if ((state = dtrace_anon.dta_state) == NULL) { #ifdef illumos state = dtrace_state_create(NULL, NULL); #else state = dtrace_state_create(NULL); #endif dtrace_anon.dta_state = state; if (state == NULL) { /* * This basically shouldn't happen: the only * failure mode from dtrace_state_create() is a * failure of ddi_soft_state_zalloc() that * itself should never happen. Still, the * interface allows for a failure mode, and * we want to fail as gracefully as possible: * we'll emit an error message and cease * processing anonymous state in this case. */ cmn_err(CE_WARN, "failed to create " "anonymous state"); dtrace_dof_destroy(dof); break; } } rv = dtrace_dof_slurp(dof, &state->dts_vstate, CRED(), &dtrace_anon.dta_enabling, 0, B_TRUE); if (rv == 0) rv = dtrace_dof_options(dof, state); dtrace_err_verbose = 0; dtrace_dof_destroy(dof); if (rv != 0) { /* * This is malformed DOF; chuck any anonymous state * that we created. */ ASSERT(dtrace_anon.dta_enabling == NULL); dtrace_state_destroy(state); dtrace_anon.dta_state = NULL; break; } ASSERT(dtrace_anon.dta_enabling != NULL); } if (dtrace_anon.dta_enabling != NULL) { int rval; /* * dtrace_enabling_retain() can only fail because we are * trying to retain more enablings than are allowed -- but * we only have one anonymous enabling, and we are guaranteed * to be allowed at least one retained enabling; we assert * that dtrace_enabling_retain() returns success. */ rval = dtrace_enabling_retain(dtrace_anon.dta_enabling); ASSERT(rval == 0); dtrace_enabling_dump(dtrace_anon.dta_enabling); } } /* * DTrace Helper Functions */ static void dtrace_helper_trace(dtrace_helper_action_t *helper, dtrace_mstate_t *mstate, dtrace_vstate_t *vstate, int where) { uint32_t size, next, nnext, i; dtrace_helptrace_t *ent, *buffer; uint16_t flags = cpu_core[curcpu].cpuc_dtrace_flags; if ((buffer = dtrace_helptrace_buffer) == NULL) return; ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals); /* * What would a tracing framework be without its own tracing * framework? (Well, a hell of a lot simpler, for starters...) */ size = sizeof (dtrace_helptrace_t) + dtrace_helptrace_nlocals * sizeof (uint64_t) - sizeof (uint64_t); /* * Iterate until we can allocate a slot in the trace buffer. */ do { next = dtrace_helptrace_next; if (next + size < dtrace_helptrace_bufsize) { nnext = next + size; } else { nnext = size; } } while (dtrace_cas32(&dtrace_helptrace_next, next, nnext) != next); /* * We have our slot; fill it in. */ if (nnext == size) { dtrace_helptrace_wrapped++; next = 0; } ent = (dtrace_helptrace_t *)((uintptr_t)buffer + next); ent->dtht_helper = helper; ent->dtht_where = where; ent->dtht_nlocals = vstate->dtvs_nlocals; ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) ? mstate->dtms_fltoffs : -1; ent->dtht_fault = DTRACE_FLAGS2FLT(flags); ent->dtht_illval = cpu_core[curcpu].cpuc_dtrace_illval; for (i = 0; i < vstate->dtvs_nlocals; i++) { dtrace_statvar_t *svar; if ((svar = vstate->dtvs_locals[i]) == NULL) continue; ASSERT(svar->dtsv_size >= NCPU * sizeof (uint64_t)); ent->dtht_locals[i] = ((uint64_t *)(uintptr_t)svar->dtsv_data)[curcpu]; } } static uint64_t dtrace_helper(int which, dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t arg0, uint64_t arg1) { uint16_t *flags = &cpu_core[curcpu].cpuc_dtrace_flags; uint64_t sarg0 = mstate->dtms_arg[0]; uint64_t sarg1 = mstate->dtms_arg[1]; uint64_t rval = 0; dtrace_helpers_t *helpers = curproc->p_dtrace_helpers; dtrace_helper_action_t *helper; dtrace_vstate_t *vstate; dtrace_difo_t *pred; int i, trace = dtrace_helptrace_buffer != NULL; ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS); if (helpers == NULL) return (0); if ((helper = helpers->dthps_actions[which]) == NULL) return (0); vstate = &helpers->dthps_vstate; mstate->dtms_arg[0] = arg0; mstate->dtms_arg[1] = arg1; /* * Now iterate over each helper. If its predicate evaluates to 'true', * we'll call the corresponding actions. Note that the below calls * to dtrace_dif_emulate() may set faults in machine state. This is * okay: our caller (the outer dtrace_dif_emulate()) will simply plow * the stored DIF offset with its own (which is the desired behavior). * Also, note the calls to dtrace_dif_emulate() may allocate scratch * from machine state; this is okay, too. */ for (; helper != NULL; helper = helper->dtha_next) { if ((pred = helper->dtha_predicate) != NULL) { if (trace) dtrace_helper_trace(helper, mstate, vstate, 0); if (!dtrace_dif_emulate(pred, mstate, vstate, state)) goto next; if (*flags & CPU_DTRACE_FAULT) goto err; } for (i = 0; i < helper->dtha_nactions; i++) { if (trace) dtrace_helper_trace(helper, mstate, vstate, i + 1); rval = dtrace_dif_emulate(helper->dtha_actions[i], mstate, vstate, state); if (*flags & CPU_DTRACE_FAULT) goto err; } next: if (trace) dtrace_helper_trace(helper, mstate, vstate, DTRACE_HELPTRACE_NEXT); } if (trace) dtrace_helper_trace(helper, mstate, vstate, DTRACE_HELPTRACE_DONE); /* * Restore the arg0 that we saved upon entry. */ mstate->dtms_arg[0] = sarg0; mstate->dtms_arg[1] = sarg1; return (rval); err: if (trace) dtrace_helper_trace(helper, mstate, vstate, DTRACE_HELPTRACE_ERR); /* * Restore the arg0 that we saved upon entry. */ mstate->dtms_arg[0] = sarg0; mstate->dtms_arg[1] = sarg1; return (0); } static void dtrace_helper_action_destroy(dtrace_helper_action_t *helper, dtrace_vstate_t *vstate) { int i; if (helper->dtha_predicate != NULL) dtrace_difo_release(helper->dtha_predicate, vstate); for (i = 0; i < helper->dtha_nactions; i++) { ASSERT(helper->dtha_actions[i] != NULL); dtrace_difo_release(helper->dtha_actions[i], vstate); } kmem_free(helper->dtha_actions, helper->dtha_nactions * sizeof (dtrace_difo_t *)); kmem_free(helper, sizeof (dtrace_helper_action_t)); } static int dtrace_helper_destroygen(int gen) { proc_t *p = curproc; dtrace_helpers_t *help = p->p_dtrace_helpers; dtrace_vstate_t *vstate; int i; ASSERT(MUTEX_HELD(&dtrace_lock)); if (help == NULL || gen > help->dthps_generation) return (EINVAL); vstate = &help->dthps_vstate; for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) { dtrace_helper_action_t *last = NULL, *h, *next; for (h = help->dthps_actions[i]; h != NULL; h = next) { next = h->dtha_next; if (h->dtha_generation == gen) { if (last != NULL) { last->dtha_next = next; } else { help->dthps_actions[i] = next; } dtrace_helper_action_destroy(h, vstate); } else { last = h; } } } /* * Interate until we've cleared out all helper providers with the * given generation number. */ for (;;) { dtrace_helper_provider_t *prov; /* * Look for a helper provider with the right generation. We * have to start back at the beginning of the list each time * because we drop dtrace_lock. It's unlikely that we'll make * more than two passes. */ for (i = 0; i < help->dthps_nprovs; i++) { prov = help->dthps_provs[i]; if (prov->dthp_generation == gen) break; } /* * If there were no matches, we're done. */ if (i == help->dthps_nprovs) break; /* * Move the last helper provider into this slot. */ help->dthps_nprovs--; help->dthps_provs[i] = help->dthps_provs[help->dthps_nprovs]; help->dthps_provs[help->dthps_nprovs] = NULL; mutex_exit(&dtrace_lock); /* * If we have a meta provider, remove this helper provider. */ mutex_enter(&dtrace_meta_lock); if (dtrace_meta_pid != NULL) { ASSERT(dtrace_deferred_pid == NULL); dtrace_helper_provider_remove(&prov->dthp_prov, p->p_pid); } mutex_exit(&dtrace_meta_lock); dtrace_helper_provider_destroy(prov); mutex_enter(&dtrace_lock); } return (0); } static int dtrace_helper_validate(dtrace_helper_action_t *helper) { int err = 0, i; dtrace_difo_t *dp; if ((dp = helper->dtha_predicate) != NULL) err += dtrace_difo_validate_helper(dp); for (i = 0; i < helper->dtha_nactions; i++) err += dtrace_difo_validate_helper(helper->dtha_actions[i]); return (err == 0); } static int dtrace_helper_action_add(int which, dtrace_ecbdesc_t *ep) { dtrace_helpers_t *help; dtrace_helper_action_t *helper, *last; dtrace_actdesc_t *act; dtrace_vstate_t *vstate; dtrace_predicate_t *pred; int count = 0, nactions = 0, i; if (which < 0 || which >= DTRACE_NHELPER_ACTIONS) return (EINVAL); help = curproc->p_dtrace_helpers; last = help->dthps_actions[which]; vstate = &help->dthps_vstate; for (count = 0; last != NULL; last = last->dtha_next) { count++; if (last->dtha_next == NULL) break; } /* * If we already have dtrace_helper_actions_max helper actions for this * helper action type, we'll refuse to add a new one. */ if (count >= dtrace_helper_actions_max) return (ENOSPC); helper = kmem_zalloc(sizeof (dtrace_helper_action_t), KM_SLEEP); helper->dtha_generation = help->dthps_generation; if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) { ASSERT(pred->dtp_difo != NULL); dtrace_difo_hold(pred->dtp_difo); helper->dtha_predicate = pred->dtp_difo; } for (act = ep->dted_action; act != NULL; act = act->dtad_next) { if (act->dtad_kind != DTRACEACT_DIFEXPR) goto err; if (act->dtad_difo == NULL) goto err; nactions++; } helper->dtha_actions = kmem_zalloc(sizeof (dtrace_difo_t *) * (helper->dtha_nactions = nactions), KM_SLEEP); for (act = ep->dted_action, i = 0; act != NULL; act = act->dtad_next) { dtrace_difo_hold(act->dtad_difo); helper->dtha_actions[i++] = act->dtad_difo; } if (!dtrace_helper_validate(helper)) goto err; if (last == NULL) { help->dthps_actions[which] = helper; } else { last->dtha_next = helper; } if (vstate->dtvs_nlocals > dtrace_helptrace_nlocals) { dtrace_helptrace_nlocals = vstate->dtvs_nlocals; dtrace_helptrace_next = 0; } return (0); err: dtrace_helper_action_destroy(helper, vstate); return (EINVAL); } static void dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help, dof_helper_t *dofhp) { ASSERT(MUTEX_NOT_HELD(&dtrace_lock)); mutex_enter(&dtrace_meta_lock); mutex_enter(&dtrace_lock); if (!dtrace_attached() || dtrace_meta_pid == NULL) { /* * If the dtrace module is loaded but not attached, or if * there aren't isn't a meta provider registered to deal with * these provider descriptions, we need to postpone creating * the actual providers until later. */ if (help->dthps_next == NULL && help->dthps_prev == NULL && dtrace_deferred_pid != help) { help->dthps_deferred = 1; help->dthps_pid = p->p_pid; help->dthps_next = dtrace_deferred_pid; help->dthps_prev = NULL; if (dtrace_deferred_pid != NULL) dtrace_deferred_pid->dthps_prev = help; dtrace_deferred_pid = help; } mutex_exit(&dtrace_lock); } else if (dofhp != NULL) { /* * If the dtrace module is loaded and we have a particular * helper provider description, pass that off to the * meta provider. */ mutex_exit(&dtrace_lock); dtrace_helper_provide(dofhp, p->p_pid); } else { /* * Otherwise, just pass all the helper provider descriptions * off to the meta provider. */ int i; mutex_exit(&dtrace_lock); for (i = 0; i < help->dthps_nprovs; i++) { dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov, p->p_pid); } } mutex_exit(&dtrace_meta_lock); } static int dtrace_helper_provider_add(dof_helper_t *dofhp, int gen) { dtrace_helpers_t *help; dtrace_helper_provider_t *hprov, **tmp_provs; uint_t tmp_maxprovs, i; ASSERT(MUTEX_HELD(&dtrace_lock)); help = curproc->p_dtrace_helpers; ASSERT(help != NULL); /* * If we already have dtrace_helper_providers_max helper providers, * we're refuse to add a new one. */ if (help->dthps_nprovs >= dtrace_helper_providers_max) return (ENOSPC); /* * Check to make sure this isn't a duplicate. */ for (i = 0; i < help->dthps_nprovs; i++) { if (dofhp->dofhp_dof == help->dthps_provs[i]->dthp_prov.dofhp_dof) return (EALREADY); } hprov = kmem_zalloc(sizeof (dtrace_helper_provider_t), KM_SLEEP); hprov->dthp_prov = *dofhp; hprov->dthp_ref = 1; hprov->dthp_generation = gen; /* * Allocate a bigger table for helper providers if it's already full. */ if (help->dthps_maxprovs == help->dthps_nprovs) { tmp_maxprovs = help->dthps_maxprovs; tmp_provs = help->dthps_provs; if (help->dthps_maxprovs == 0) help->dthps_maxprovs = 2; else help->dthps_maxprovs *= 2; if (help->dthps_maxprovs > dtrace_helper_providers_max) help->dthps_maxprovs = dtrace_helper_providers_max; ASSERT(tmp_maxprovs < help->dthps_maxprovs); help->dthps_provs = kmem_zalloc(help->dthps_maxprovs * sizeof (dtrace_helper_provider_t *), KM_SLEEP); if (tmp_provs != NULL) { bcopy(tmp_provs, help->dthps_provs, tmp_maxprovs * sizeof (dtrace_helper_provider_t *)); kmem_free(tmp_provs, tmp_maxprovs * sizeof (dtrace_helper_provider_t *)); } } help->dthps_provs[help->dthps_nprovs] = hprov; help->dthps_nprovs++; return (0); } static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *hprov) { mutex_enter(&dtrace_lock); if (--hprov->dthp_ref == 0) { dof_hdr_t *dof; mutex_exit(&dtrace_lock); dof = (dof_hdr_t *)(uintptr_t)hprov->dthp_prov.dofhp_dof; dtrace_dof_destroy(dof); kmem_free(hprov, sizeof (dtrace_helper_provider_t)); } else { mutex_exit(&dtrace_lock); } } static int dtrace_helper_provider_validate(dof_hdr_t *dof, dof_sec_t *sec) { uintptr_t daddr = (uintptr_t)dof; dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec; dof_provider_t *provider; dof_probe_t *probe; uint8_t *arg; char *strtab, *typestr; dof_stridx_t typeidx; size_t typesz; uint_t nprobes, j, k; ASSERT(sec->dofs_type == DOF_SECT_PROVIDER); if (sec->dofs_offset & (sizeof (uint_t) - 1)) { dtrace_dof_error(dof, "misaligned section offset"); return (-1); } /* * The section needs to be large enough to contain the DOF provider * structure appropriate for the given version. */ if (sec->dofs_size < ((dof->dofh_ident[DOF_ID_VERSION] == DOF_VERSION_1) ? offsetof(dof_provider_t, dofpv_prenoffs) : sizeof (dof_provider_t))) { dtrace_dof_error(dof, "provider section too small"); return (-1); } provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset); str_sec = dtrace_dof_sect(dof, DOF_SECT_STRTAB, provider->dofpv_strtab); prb_sec = dtrace_dof_sect(dof, DOF_SECT_PROBES, provider->dofpv_probes); arg_sec = dtrace_dof_sect(dof, DOF_SECT_PRARGS, provider->dofpv_prargs); off_sec = dtrace_dof_sect(dof, DOF_SECT_PROFFS, provider->dofpv_proffs); if (str_sec == NULL || prb_sec == NULL || arg_sec == NULL || off_sec == NULL) return (-1); enoff_sec = NULL; if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 && provider->dofpv_prenoffs != DOF_SECT_NONE && (enoff_sec = dtrace_dof_sect(dof, DOF_SECT_PRENOFFS, provider->dofpv_prenoffs)) == NULL) return (-1); strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset); if (provider->dofpv_name >= str_sec->dofs_size || strlen(strtab + provider->dofpv_name) >= DTRACE_PROVNAMELEN) { dtrace_dof_error(dof, "invalid provider name"); return (-1); } if (prb_sec->dofs_entsize == 0 || prb_sec->dofs_entsize > prb_sec->dofs_size) { dtrace_dof_error(dof, "invalid entry size"); return (-1); } if (prb_sec->dofs_entsize & (sizeof (uintptr_t) - 1)) { dtrace_dof_error(dof, "misaligned entry size"); return (-1); } if (off_sec->dofs_entsize != sizeof (uint32_t)) { dtrace_dof_error(dof, "invalid entry size"); return (-1); } if (off_sec->dofs_offset & (sizeof (uint32_t) - 1)) { dtrace_dof_error(dof, "misaligned section offset"); return (-1); } if (arg_sec->dofs_entsize != sizeof (uint8_t)) { dtrace_dof_error(dof, "invalid entry size"); return (-1); } arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset); nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize; /* * Take a pass through the probes to check for errors. */ for (j = 0; j < nprobes; j++) { probe = (dof_probe_t *)(uintptr_t)(daddr + prb_sec->dofs_offset + j * prb_sec->dofs_entsize); if (probe->dofpr_func >= str_sec->dofs_size) { dtrace_dof_error(dof, "invalid function name"); return (-1); } if (strlen(strtab + probe->dofpr_func) >= DTRACE_FUNCNAMELEN) { dtrace_dof_error(dof, "function name too long"); return (-1); } if (probe->dofpr_name >= str_sec->dofs_size || strlen(strtab + probe->dofpr_name) >= DTRACE_NAMELEN) { dtrace_dof_error(dof, "invalid probe name"); return (-1); } /* * The offset count must not wrap the index, and the offsets * must also not overflow the section's data. */ if (probe->dofpr_offidx + probe->dofpr_noffs < probe->dofpr_offidx || (probe->dofpr_offidx + probe->dofpr_noffs) * off_sec->dofs_entsize > off_sec->dofs_size) { dtrace_dof_error(dof, "invalid probe offset"); return (-1); } if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1) { /* * If there's no is-enabled offset section, make sure * there aren't any is-enabled offsets. Otherwise * perform the same checks as for probe offsets * (immediately above). */ if (enoff_sec == NULL) { if (probe->dofpr_enoffidx != 0 || probe->dofpr_nenoffs != 0) { dtrace_dof_error(dof, "is-enabled " "offsets with null section"); return (-1); } } else if (probe->dofpr_enoffidx + probe->dofpr_nenoffs < probe->dofpr_enoffidx || (probe->dofpr_enoffidx + probe->dofpr_nenoffs) * enoff_sec->dofs_entsize > enoff_sec->dofs_size) { dtrace_dof_error(dof, "invalid is-enabled " "offset"); return (-1); } if (probe->dofpr_noffs + probe->dofpr_nenoffs == 0) { dtrace_dof_error(dof, "zero probe and " "is-enabled offsets"); return (-1); } } else if (probe->dofpr_noffs == 0) { dtrace_dof_error(dof, "zero probe offsets"); return (-1); } if (probe->dofpr_argidx + probe->dofpr_xargc < probe->dofpr_argidx || (probe->dofpr_argidx + probe->dofpr_xargc) * arg_sec->dofs_entsize > arg_sec->dofs_size) { dtrace_dof_error(dof, "invalid args"); return (-1); } typeidx = probe->dofpr_nargv; typestr = strtab + probe->dofpr_nargv; for (k = 0; k < probe->dofpr_nargc; k++) { if (typeidx >= str_sec->dofs_size) { dtrace_dof_error(dof, "bad " "native argument type"); return (-1); } typesz = strlen(typestr) + 1; if (typesz > DTRACE_ARGTYPELEN) { dtrace_dof_error(dof, "native " "argument type too long"); return (-1); } typeidx += typesz; typestr += typesz; } typeidx = probe->dofpr_xargv; typestr = strtab + probe->dofpr_xargv; for (k = 0; k < probe->dofpr_xargc; k++) { if (arg[probe->dofpr_argidx + k] > probe->dofpr_nargc) { dtrace_dof_error(dof, "bad " "native argument index"); return (-1); } if (typeidx >= str_sec->dofs_size) { dtrace_dof_error(dof, "bad " "translated argument type"); return (-1); } typesz = strlen(typestr) + 1; if (typesz > DTRACE_ARGTYPELEN) { dtrace_dof_error(dof, "translated argument " "type too long"); return (-1); } typeidx += typesz; typestr += typesz; } } return (0); } static int dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp) { dtrace_helpers_t *help; dtrace_vstate_t *vstate; dtrace_enabling_t *enab = NULL; int i, gen, rv, nhelpers = 0, nprovs = 0, destroy = 1; uintptr_t daddr = (uintptr_t)dof; ASSERT(MUTEX_HELD(&dtrace_lock)); if ((help = curproc->p_dtrace_helpers) == NULL) help = dtrace_helpers_create(curproc); vstate = &help->dthps_vstate; if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab, dhp != NULL ? dhp->dofhp_addr : 0, B_FALSE)) != 0) { dtrace_dof_destroy(dof); return (rv); } /* * Look for helper providers and validate their descriptions. */ if (dhp != NULL) { for (i = 0; i < dof->dofh_secnum; i++) { dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + i * dof->dofh_secsize); if (sec->dofs_type != DOF_SECT_PROVIDER) continue; if (dtrace_helper_provider_validate(dof, sec) != 0) { dtrace_enabling_destroy(enab); dtrace_dof_destroy(dof); return (-1); } nprovs++; } } /* * Now we need to walk through the ECB descriptions in the enabling. */ for (i = 0; i < enab->dten_ndesc; i++) { dtrace_ecbdesc_t *ep = enab->dten_desc[i]; dtrace_probedesc_t *desc = &ep->dted_probe; if (strcmp(desc->dtpd_provider, "dtrace") != 0) continue; if (strcmp(desc->dtpd_mod, "helper") != 0) continue; if (strcmp(desc->dtpd_func, "ustack") != 0) continue; if ((rv = dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK, ep)) != 0) { /* * Adding this helper action failed -- we are now going * to rip out the entire generation and return failure. */ (void) dtrace_helper_destroygen(help->dthps_generation); dtrace_enabling_destroy(enab); dtrace_dof_destroy(dof); return (-1); } nhelpers++; } if (nhelpers < enab->dten_ndesc) dtrace_dof_error(dof, "unmatched helpers"); gen = help->dthps_generation++; dtrace_enabling_destroy(enab); if (dhp != NULL && nprovs > 0) { dhp->dofhp_dof = (uint64_t)(uintptr_t)dof; if (dtrace_helper_provider_add(dhp, gen) == 0) { mutex_exit(&dtrace_lock); dtrace_helper_provider_register(curproc, help, dhp); mutex_enter(&dtrace_lock); destroy = 0; } } if (destroy) dtrace_dof_destroy(dof); return (gen); } static dtrace_helpers_t * dtrace_helpers_create(proc_t *p) { dtrace_helpers_t *help; ASSERT(MUTEX_HELD(&dtrace_lock)); ASSERT(p->p_dtrace_helpers == NULL); help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP); help->dthps_actions = kmem_zalloc(sizeof (dtrace_helper_action_t *) * DTRACE_NHELPER_ACTIONS, KM_SLEEP); p->p_dtrace_helpers = help; dtrace_helpers++; return (help); } #ifdef illumos static #endif void dtrace_helpers_destroy(proc_t *p) { dtrace_helpers_t *help; dtrace_vstate_t *vstate; #ifdef illumos proc_t *p = curproc; #endif int i; mutex_enter(&dtrace_lock); ASSERT(p->p_dtrace_helpers != NULL); ASSERT(dtrace_helpers > 0); help = p->p_dtrace_helpers; vstate = &help->dthps_vstate; /* * We're now going to lose the help from this process. */ p->p_dtrace_helpers = NULL; dtrace_sync(); /* * Destory the helper actions. */ for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) { dtrace_helper_action_t *h, *next; for (h = help->dthps_actions[i]; h != NULL; h = next) { next = h->dtha_next; dtrace_helper_action_destroy(h, vstate); h = next; } } mutex_exit(&dtrace_lock); /* * Destroy the helper providers. */ if (help->dthps_maxprovs > 0) { mutex_enter(&dtrace_meta_lock); if (dtrace_meta_pid != NULL) { ASSERT(dtrace_deferred_pid == NULL); for (i = 0; i < help->dthps_nprovs; i++) { dtrace_helper_provider_remove( &help->dthps_provs[i]->dthp_prov, p->p_pid); } } else { mutex_enter(&dtrace_lock); ASSERT(help->dthps_deferred == 0 || help->dthps_next != NULL || help->dthps_prev != NULL || help == dtrace_deferred_pid); /* * Remove the helper from the deferred list. */ if (help->dthps_next != NULL) help->dthps_next->dthps_prev = help->dthps_prev; if (help->dthps_prev != NULL) help->dthps_prev->dthps_next = help->dthps_next; if (dtrace_deferred_pid == help) { dtrace_deferred_pid = help->dthps_next; ASSERT(help->dthps_prev == NULL); } mutex_exit(&dtrace_lock); } mutex_exit(&dtrace_meta_lock); for (i = 0; i < help->dthps_nprovs; i++) { dtrace_helper_provider_destroy(help->dthps_provs[i]); } kmem_free(help->dthps_provs, help->dthps_maxprovs * sizeof (dtrace_helper_provider_t *)); } mutex_enter(&dtrace_lock); dtrace_vstate_fini(&help->dthps_vstate); kmem_free(help->dthps_actions, sizeof (dtrace_helper_action_t *) * DTRACE_NHELPER_ACTIONS); kmem_free(help, sizeof (dtrace_helpers_t)); --dtrace_helpers; mutex_exit(&dtrace_lock); } #ifdef illumos static #endif void dtrace_helpers_duplicate(proc_t *from, proc_t *to) { dtrace_helpers_t *help, *newhelp; dtrace_helper_action_t *helper, *new, *last; dtrace_difo_t *dp; dtrace_vstate_t *vstate; int i, j, sz, hasprovs = 0; mutex_enter(&dtrace_lock); ASSERT(from->p_dtrace_helpers != NULL); ASSERT(dtrace_helpers > 0); help = from->p_dtrace_helpers; newhelp = dtrace_helpers_create(to); ASSERT(to->p_dtrace_helpers != NULL); newhelp->dthps_generation = help->dthps_generation; vstate = &newhelp->dthps_vstate; /* * Duplicate the helper actions. */ for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) { if ((helper = help->dthps_actions[i]) == NULL) continue; for (last = NULL; helper != NULL; helper = helper->dtha_next) { new = kmem_zalloc(sizeof (dtrace_helper_action_t), KM_SLEEP); new->dtha_generation = helper->dtha_generation; if ((dp = helper->dtha_predicate) != NULL) { dp = dtrace_difo_duplicate(dp, vstate); new->dtha_predicate = dp; } new->dtha_nactions = helper->dtha_nactions; sz = sizeof (dtrace_difo_t *) * new->dtha_nactions; new->dtha_actions = kmem_alloc(sz, KM_SLEEP); for (j = 0; j < new->dtha_nactions; j++) { dtrace_difo_t *dp = helper->dtha_actions[j]; ASSERT(dp != NULL); dp = dtrace_difo_duplicate(dp, vstate); new->dtha_actions[j] = dp; } if (last != NULL) { last->dtha_next = new; } else { newhelp->dthps_actions[i] = new; } last = new; } } /* * Duplicate the helper providers and register them with the * DTrace framework. */ if (help->dthps_nprovs > 0) { newhelp->dthps_nprovs = help->dthps_nprovs; newhelp->dthps_maxprovs = help->dthps_nprovs; newhelp->dthps_provs = kmem_alloc(newhelp->dthps_nprovs * sizeof (dtrace_helper_provider_t *), KM_SLEEP); for (i = 0; i < newhelp->dthps_nprovs; i++) { newhelp->dthps_provs[i] = help->dthps_provs[i]; newhelp->dthps_provs[i]->dthp_ref++; } hasprovs = 1; } mutex_exit(&dtrace_lock); if (hasprovs) dtrace_helper_provider_register(to, newhelp, NULL); } /* * DTrace Hook Functions */ static void dtrace_module_loaded(modctl_t *ctl) { dtrace_provider_t *prv; mutex_enter(&dtrace_provider_lock); #ifdef illumos mutex_enter(&mod_lock); #endif #ifdef illumos ASSERT(ctl->mod_busy); #endif /* * We're going to call each providers per-module provide operation * specifying only this module. */ for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next) prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl); #ifdef illumos mutex_exit(&mod_lock); #endif mutex_exit(&dtrace_provider_lock); /* * If we have any retained enablings, we need to match against them. * Enabling probes requires that cpu_lock be held, and we cannot hold * cpu_lock here -- it is legal for cpu_lock to be held when loading a * module. (In particular, this happens when loading scheduling * classes.) So if we have any retained enablings, we need to dispatch * our task queue to do the match for us. */ mutex_enter(&dtrace_lock); if (dtrace_retained == NULL) { mutex_exit(&dtrace_lock); return; } (void) taskq_dispatch(dtrace_taskq, (task_func_t *)dtrace_enabling_matchall, NULL, TQ_SLEEP); mutex_exit(&dtrace_lock); /* * And now, for a little heuristic sleaze: in general, we want to * match modules as soon as they load. However, we cannot guarantee * this, because it would lead us to the lock ordering violation * outlined above. The common case, of course, is that cpu_lock is * _not_ held -- so we delay here for a clock tick, hoping that that's * long enough for the task queue to do its work. If it's not, it's * not a serious problem -- it just means that the module that we * just loaded may not be immediately instrumentable. */ delay(1); } static void #ifdef illumos dtrace_module_unloaded(modctl_t *ctl) #else dtrace_module_unloaded(modctl_t *ctl, int *error) #endif { dtrace_probe_t template, *probe, *first, *next; dtrace_provider_t *prov; #ifndef illumos char modname[DTRACE_MODNAMELEN]; size_t len; #endif #ifdef illumos template.dtpr_mod = ctl->mod_modname; #else /* Handle the fact that ctl->filename may end in ".ko". */ strlcpy(modname, ctl->filename, sizeof(modname)); len = strlen(ctl->filename); if (len > 3 && strcmp(modname + len - 3, ".ko") == 0) modname[len - 3] = '\0'; template.dtpr_mod = modname; #endif mutex_enter(&dtrace_provider_lock); #ifdef illumos mutex_enter(&mod_lock); #endif mutex_enter(&dtrace_lock); #ifndef illumos if (ctl->nenabled > 0) { /* Don't allow unloads if a probe is enabled. */ mutex_exit(&dtrace_provider_lock); mutex_exit(&dtrace_lock); *error = -1; printf( "kldunload: attempt to unload module that has DTrace probes enabled\n"); return; } #endif if (dtrace_bymod == NULL) { /* * The DTrace module is loaded (obviously) but not attached; * we don't have any work to do. */ mutex_exit(&dtrace_provider_lock); #ifdef illumos mutex_exit(&mod_lock); #endif mutex_exit(&dtrace_lock); return; } for (probe = first = dtrace_hash_lookup(dtrace_bymod, &template); probe != NULL; probe = probe->dtpr_nextmod) { if (probe->dtpr_ecb != NULL) { mutex_exit(&dtrace_provider_lock); #ifdef illumos mutex_exit(&mod_lock); #endif mutex_exit(&dtrace_lock); /* * This shouldn't _actually_ be possible -- we're * unloading a module that has an enabled probe in it. * (It's normally up to the provider to make sure that * this can't happen.) However, because dtps_enable() * doesn't have a failure mode, there can be an * enable/unload race. Upshot: we don't want to * assert, but we're not going to disable the * probe, either. */ if (dtrace_err_verbose) { #ifdef illumos cmn_err(CE_WARN, "unloaded module '%s' had " "enabled probes", ctl->mod_modname); #else cmn_err(CE_WARN, "unloaded module '%s' had " "enabled probes", modname); #endif } return; } } probe = first; for (first = NULL; probe != NULL; probe = next) { ASSERT(dtrace_probes[probe->dtpr_id - 1] == probe); dtrace_probes[probe->dtpr_id - 1] = NULL; next = probe->dtpr_nextmod; dtrace_hash_remove(dtrace_bymod, probe); dtrace_hash_remove(dtrace_byfunc, probe); dtrace_hash_remove(dtrace_byname, probe); if (first == NULL) { first = probe; probe->dtpr_nextmod = NULL; } else { probe->dtpr_nextmod = first; first = probe; } } /* * We've removed all of the module's probes from the hash chains and * from the probe array. Now issue a dtrace_sync() to be sure that * everyone has cleared out from any probe array processing. */ dtrace_sync(); for (probe = first; probe != NULL; probe = first) { first = probe->dtpr_nextmod; prov = probe->dtpr_provider; prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, probe->dtpr_id, probe->dtpr_arg); kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1); kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1); kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1); #ifdef illumos vmem_free(dtrace_arena, (void *)(uintptr_t)probe->dtpr_id, 1); #else free_unr(dtrace_arena, probe->dtpr_id); #endif kmem_free(probe, sizeof (dtrace_probe_t)); } mutex_exit(&dtrace_lock); #ifdef illumos mutex_exit(&mod_lock); #endif mutex_exit(&dtrace_provider_lock); } #ifndef illumos static void dtrace_kld_load(void *arg __unused, linker_file_t lf) { dtrace_module_loaded(lf); } static void dtrace_kld_unload_try(void *arg __unused, linker_file_t lf, int *error) { if (*error != 0) /* We already have an error, so don't do anything. */ return; dtrace_module_unloaded(lf, error); } #endif #ifdef illumos static void dtrace_suspend(void) { dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_suspend)); } static void dtrace_resume(void) { dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_resume)); } #endif static int dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu) { ASSERT(MUTEX_HELD(&cpu_lock)); mutex_enter(&dtrace_lock); switch (what) { case CPU_CONFIG: { dtrace_state_t *state; dtrace_optval_t *opt, rs, c; /* * For now, we only allocate a new buffer for anonymous state. */ if ((state = dtrace_anon.dta_state) == NULL) break; if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) break; opt = state->dts_options; c = opt[DTRACEOPT_CPU]; if (c != DTRACE_CPUALL && c != DTRACEOPT_UNSET && c != cpu) break; /* * Regardless of what the actual policy is, we're going to * temporarily set our resize policy to be manual. We're * also going to temporarily set our CPU option to denote * the newly configured CPU. */ rs = opt[DTRACEOPT_BUFRESIZE]; opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_MANUAL; opt[DTRACEOPT_CPU] = (dtrace_optval_t)cpu; (void) dtrace_state_buffers(state); opt[DTRACEOPT_BUFRESIZE] = rs; opt[DTRACEOPT_CPU] = c; break; } case CPU_UNCONFIG: /* * We don't free the buffer in the CPU_UNCONFIG case. (The * buffer will be freed when the consumer exits.) */ break; default: break; } mutex_exit(&dtrace_lock); return (0); } #ifdef illumos static void dtrace_cpu_setup_initial(processorid_t cpu) { (void) dtrace_cpu_setup(CPU_CONFIG, cpu); } #endif static void dtrace_toxrange_add(uintptr_t base, uintptr_t limit) { if (dtrace_toxranges >= dtrace_toxranges_max) { int osize, nsize; dtrace_toxrange_t *range; osize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t); if (osize == 0) { ASSERT(dtrace_toxrange == NULL); ASSERT(dtrace_toxranges_max == 0); dtrace_toxranges_max = 1; } else { dtrace_toxranges_max <<= 1; } nsize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t); range = kmem_zalloc(nsize, KM_SLEEP); if (dtrace_toxrange != NULL) { ASSERT(osize != 0); bcopy(dtrace_toxrange, range, osize); kmem_free(dtrace_toxrange, osize); } dtrace_toxrange = range; } ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_base == 0); ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_limit == 0); dtrace_toxrange[dtrace_toxranges].dtt_base = base; dtrace_toxrange[dtrace_toxranges].dtt_limit = limit; dtrace_toxranges++; } static void dtrace_getf_barrier() { #ifdef illumos /* * When we have unprivileged (that is, non-DTRACE_CRV_KERNEL) enablings * that contain calls to getf(), this routine will be called on every * closef() before either the underlying vnode is released or the * file_t itself is freed. By the time we are here, it is essential * that the file_t can no longer be accessed from a call to getf() * in probe context -- that assures that a dtrace_sync() can be used * to clear out any enablings referring to the old structures. */ if (curthread->t_procp->p_zone->zone_dtrace_getf != 0 || kcred->cr_zone->zone_dtrace_getf != 0) dtrace_sync(); #endif } /* * DTrace Driver Cookbook Functions */ #ifdef illumos /*ARGSUSED*/ static int dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) { dtrace_provider_id_t id; dtrace_state_t *state = NULL; dtrace_enabling_t *enab; mutex_enter(&cpu_lock); mutex_enter(&dtrace_provider_lock); mutex_enter(&dtrace_lock); if (ddi_soft_state_init(&dtrace_softstate, sizeof (dtrace_state_t), 0) != 0) { cmn_err(CE_NOTE, "/dev/dtrace failed to initialize soft state"); mutex_exit(&cpu_lock); mutex_exit(&dtrace_provider_lock); mutex_exit(&dtrace_lock); return (DDI_FAILURE); } if (ddi_create_minor_node(devi, DTRACEMNR_DTRACE, S_IFCHR, DTRACEMNRN_DTRACE, DDI_PSEUDO, NULL) == DDI_FAILURE || ddi_create_minor_node(devi, DTRACEMNR_HELPER, S_IFCHR, DTRACEMNRN_HELPER, DDI_PSEUDO, NULL) == DDI_FAILURE) { cmn_err(CE_NOTE, "/dev/dtrace couldn't create minor nodes"); ddi_remove_minor_node(devi, NULL); ddi_soft_state_fini(&dtrace_softstate); mutex_exit(&cpu_lock); mutex_exit(&dtrace_provider_lock); mutex_exit(&dtrace_lock); return (DDI_FAILURE); } ddi_report_dev(devi); dtrace_devi = devi; dtrace_modload = dtrace_module_loaded; dtrace_modunload = dtrace_module_unloaded; dtrace_cpu_init = dtrace_cpu_setup_initial; dtrace_helpers_cleanup = dtrace_helpers_destroy; dtrace_helpers_fork = dtrace_helpers_duplicate; dtrace_cpustart_init = dtrace_suspend; dtrace_cpustart_fini = dtrace_resume; dtrace_debugger_init = dtrace_suspend; dtrace_debugger_fini = dtrace_resume; register_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL); ASSERT(MUTEX_HELD(&cpu_lock)); dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1, NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); dtrace_minor = vmem_create("dtrace_minor", (void *)DTRACEMNRN_CLONE, UINT32_MAX - DTRACEMNRN_CLONE, 1, NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); dtrace_taskq = taskq_create("dtrace_taskq", 1, maxclsyspri, 1, INT_MAX, 0); dtrace_state_cache = kmem_cache_create("dtrace_state_cache", sizeof (dtrace_dstate_percpu_t) * NCPU, DTRACE_STATE_ALIGN, NULL, NULL, NULL, NULL, NULL, 0); ASSERT(MUTEX_HELD(&cpu_lock)); dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod), offsetof(dtrace_probe_t, dtpr_nextmod), offsetof(dtrace_probe_t, dtpr_prevmod)); dtrace_byfunc = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_func), offsetof(dtrace_probe_t, dtpr_nextfunc), offsetof(dtrace_probe_t, dtpr_prevfunc)); dtrace_byname = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_name), offsetof(dtrace_probe_t, dtpr_nextname), offsetof(dtrace_probe_t, dtpr_prevname)); if (dtrace_retain_max < 1) { cmn_err(CE_WARN, "illegal value (%lu) for dtrace_retain_max; " "setting to 1", dtrace_retain_max); dtrace_retain_max = 1; } /* * Now discover our toxic ranges. */ dtrace_toxic_ranges(dtrace_toxrange_add); /* * Before we register ourselves as a provider to our own framework, * we would like to assert that dtrace_provider is NULL -- but that's * not true if we were loaded as a dependency of a DTrace provider. * Once we've registered, we can assert that dtrace_provider is our * pseudo provider. */ (void) dtrace_register("dtrace", &dtrace_provider_attr, DTRACE_PRIV_NONE, 0, &dtrace_provider_ops, NULL, &id); ASSERT(dtrace_provider != NULL); ASSERT((dtrace_provider_id_t)dtrace_provider == id); dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t) dtrace_provider, NULL, NULL, "BEGIN", 0, NULL); dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t) dtrace_provider, NULL, NULL, "END", 0, NULL); dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t) dtrace_provider, NULL, NULL, "ERROR", 1, NULL); dtrace_anon_property(); mutex_exit(&cpu_lock); /* * If there are already providers, we must ask them to provide their * probes, and then match any anonymous enabling against them. Note * that there should be no other retained enablings at this time: * the only retained enablings at this time should be the anonymous * enabling. */ if (dtrace_anon.dta_enabling != NULL) { ASSERT(dtrace_retained == dtrace_anon.dta_enabling); dtrace_enabling_provide(NULL); state = dtrace_anon.dta_state; /* * We couldn't hold cpu_lock across the above call to * dtrace_enabling_provide(), but we must hold it to actually * enable the probes. We have to drop all of our locks, pick * up cpu_lock, and regain our locks before matching the * retained anonymous enabling. */ mutex_exit(&dtrace_lock); mutex_exit(&dtrace_provider_lock); mutex_enter(&cpu_lock); mutex_enter(&dtrace_provider_lock); mutex_enter(&dtrace_lock); if ((enab = dtrace_anon.dta_enabling) != NULL) (void) dtrace_enabling_match(enab, NULL); mutex_exit(&cpu_lock); } mutex_exit(&dtrace_lock); mutex_exit(&dtrace_provider_lock); if (state != NULL) { /* * If we created any anonymous state, set it going now. */ (void) dtrace_state_go(state, &dtrace_anon.dta_beganon); } return (DDI_SUCCESS); } #endif /* illumos */ #ifndef illumos static void dtrace_dtr(void *); #endif /*ARGSUSED*/ static int #ifdef illumos dtrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) #else dtrace_open(struct cdev *dev, int oflags, int devtype, struct thread *td) #endif { dtrace_state_t *state; uint32_t priv; uid_t uid; zoneid_t zoneid; #ifdef illumos if (getminor(*devp) == DTRACEMNRN_HELPER) return (0); /* * If this wasn't an open with the "helper" minor, then it must be * the "dtrace" minor. */ if (getminor(*devp) == DTRACEMNRN_DTRACE) return (ENXIO); #else cred_t *cred_p = NULL; cred_p = dev->si_cred; /* * If no DTRACE_PRIV_* bits are set in the credential, then the * caller lacks sufficient permission to do anything with DTrace. */ dtrace_cred2priv(cred_p, &priv, &uid, &zoneid); if (priv == DTRACE_PRIV_NONE) { #endif return (EACCES); } /* * Ask all providers to provide all their probes. */ mutex_enter(&dtrace_provider_lock); dtrace_probe_provide(NULL, NULL); mutex_exit(&dtrace_provider_lock); mutex_enter(&cpu_lock); mutex_enter(&dtrace_lock); dtrace_opens++; dtrace_membar_producer(); #ifdef illumos /* * If the kernel debugger is active (that is, if the kernel debugger * modified text in some way), we won't allow the open. */ if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) { dtrace_opens--; mutex_exit(&cpu_lock); mutex_exit(&dtrace_lock); return (EBUSY); } if (dtrace_helptrace_enable && dtrace_helptrace_buffer == NULL) { /* * If DTrace helper tracing is enabled, we need to allocate the * trace buffer and initialize the values. */ dtrace_helptrace_buffer = kmem_zalloc(dtrace_helptrace_bufsize, KM_SLEEP); dtrace_helptrace_next = 0; dtrace_helptrace_wrapped = 0; dtrace_helptrace_enable = 0; } state = dtrace_state_create(devp, cred_p); #else state = dtrace_state_create(dev); devfs_set_cdevpriv(state, dtrace_dtr); #endif mutex_exit(&cpu_lock); if (state == NULL) { #ifdef illumos if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL) (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE); #else --dtrace_opens; #endif mutex_exit(&dtrace_lock); return (EAGAIN); } mutex_exit(&dtrace_lock); return (0); } /*ARGSUSED*/ #ifdef illumos static int dtrace_close(dev_t dev, int flag, int otyp, cred_t *cred_p) #else static void dtrace_dtr(void *data) #endif { #ifdef illumos minor_t minor = getminor(dev); dtrace_state_t *state; #endif dtrace_helptrace_t *buf = NULL; #ifdef illumos if (minor == DTRACEMNRN_HELPER) return (0); state = ddi_get_soft_state(dtrace_softstate, minor); #else dtrace_state_t *state = data; #endif mutex_enter(&cpu_lock); mutex_enter(&dtrace_lock); #ifdef illumos if (state->dts_anon) #else if (state != NULL && state->dts_anon) #endif { /* * There is anonymous state. Destroy that first. */ ASSERT(dtrace_anon.dta_state == NULL); dtrace_state_destroy(state->dts_anon); } if (dtrace_helptrace_disable) { /* * If we have been told to disable helper tracing, set the * buffer to NULL before calling into dtrace_state_destroy(); * we take advantage of its dtrace_sync() to know that no * CPU is in probe context with enabled helper tracing * after it returns. */ buf = dtrace_helptrace_buffer; dtrace_helptrace_buffer = NULL; } #ifdef illumos dtrace_state_destroy(state); #else if (state != NULL) { dtrace_state_destroy(state); kmem_free(state, 0); } #endif ASSERT(dtrace_opens > 0); #ifdef illumos /* * Only relinquish control of the kernel debugger interface when there * are no consumers and no anonymous enablings. */ if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL) (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE); #else --dtrace_opens; #endif if (buf != NULL) { kmem_free(buf, dtrace_helptrace_bufsize); dtrace_helptrace_disable = 0; } mutex_exit(&dtrace_lock); mutex_exit(&cpu_lock); #ifdef illumos return (0); #endif } #ifdef illumos /*ARGSUSED*/ static int dtrace_ioctl_helper(int cmd, intptr_t arg, int *rv) { int rval; dof_helper_t help, *dhp = NULL; switch (cmd) { case DTRACEHIOC_ADDDOF: if (copyin((void *)arg, &help, sizeof (help)) != 0) { dtrace_dof_error(NULL, "failed to copyin DOF helper"); return (EFAULT); } dhp = &help; arg = (intptr_t)help.dofhp_dof; /*FALLTHROUGH*/ case DTRACEHIOC_ADD: { dof_hdr_t *dof = dtrace_dof_copyin(arg, &rval); if (dof == NULL) return (rval); mutex_enter(&dtrace_lock); /* * dtrace_helper_slurp() takes responsibility for the dof -- * it may free it now or it may save it and free it later. */ if ((rval = dtrace_helper_slurp(dof, dhp)) != -1) { *rv = rval; rval = 0; } else { rval = EINVAL; } mutex_exit(&dtrace_lock); return (rval); } case DTRACEHIOC_REMOVE: { mutex_enter(&dtrace_lock); rval = dtrace_helper_destroygen(arg); mutex_exit(&dtrace_lock); return (rval); } default: break; } return (ENOTTY); } /*ARGSUSED*/ static int dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv) { minor_t minor = getminor(dev); dtrace_state_t *state; int rval; if (minor == DTRACEMNRN_HELPER) return (dtrace_ioctl_helper(cmd, arg, rv)); state = ddi_get_soft_state(dtrace_softstate, minor); if (state->dts_anon) { ASSERT(dtrace_anon.dta_state == NULL); state = state->dts_anon; } switch (cmd) { case DTRACEIOC_PROVIDER: { dtrace_providerdesc_t pvd; dtrace_provider_t *pvp; if (copyin((void *)arg, &pvd, sizeof (pvd)) != 0) return (EFAULT); pvd.dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0'; mutex_enter(&dtrace_provider_lock); for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) { if (strcmp(pvp->dtpv_name, pvd.dtvd_name) == 0) break; } mutex_exit(&dtrace_provider_lock); if (pvp == NULL) return (ESRCH); bcopy(&pvp->dtpv_priv, &pvd.dtvd_priv, sizeof (dtrace_ppriv_t)); bcopy(&pvp->dtpv_attr, &pvd.dtvd_attr, sizeof (dtrace_pattr_t)); if (copyout(&pvd, (void *)arg, sizeof (pvd)) != 0) return (EFAULT); return (0); } case DTRACEIOC_EPROBE: { dtrace_eprobedesc_t epdesc; dtrace_ecb_t *ecb; dtrace_action_t *act; void *buf; size_t size; uintptr_t dest; int nrecs; if (copyin((void *)arg, &epdesc, sizeof (epdesc)) != 0) return (EFAULT); mutex_enter(&dtrace_lock); if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) { mutex_exit(&dtrace_lock); return (EINVAL); } if (ecb->dte_probe == NULL) { mutex_exit(&dtrace_lock); return (EINVAL); } epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id; epdesc.dtepd_uarg = ecb->dte_uarg; epdesc.dtepd_size = ecb->dte_size; nrecs = epdesc.dtepd_nrecs; epdesc.dtepd_nrecs = 0; for (act = ecb->dte_action; act != NULL; act = act->dta_next) { if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple) continue; epdesc.dtepd_nrecs++; } /* * Now that we have the size, we need to allocate a temporary * buffer in which to store the complete description. We need * the temporary buffer to be able to drop dtrace_lock() * across the copyout(), below. */ size = sizeof (dtrace_eprobedesc_t) + (epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t)); buf = kmem_alloc(size, KM_SLEEP); dest = (uintptr_t)buf; bcopy(&epdesc, (void *)dest, sizeof (epdesc)); dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]); for (act = ecb->dte_action; act != NULL; act = act->dta_next) { if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple) continue; if (nrecs-- == 0) break; bcopy(&act->dta_rec, (void *)dest, sizeof (dtrace_recdesc_t)); dest += sizeof (dtrace_recdesc_t); } mutex_exit(&dtrace_lock); if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) { kmem_free(buf, size); return (EFAULT); } kmem_free(buf, size); return (0); } case DTRACEIOC_AGGDESC: { dtrace_aggdesc_t aggdesc; dtrace_action_t *act; dtrace_aggregation_t *agg; int nrecs; uint32_t offs; dtrace_recdesc_t *lrec; void *buf; size_t size; uintptr_t dest; if (copyin((void *)arg, &aggdesc, sizeof (aggdesc)) != 0) return (EFAULT); mutex_enter(&dtrace_lock); if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) { mutex_exit(&dtrace_lock); return (EINVAL); } aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid; nrecs = aggdesc.dtagd_nrecs; aggdesc.dtagd_nrecs = 0; offs = agg->dtag_base; lrec = &agg->dtag_action.dta_rec; aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs; for (act = agg->dtag_first; ; act = act->dta_next) { ASSERT(act->dta_intuple || DTRACEACT_ISAGG(act->dta_kind)); /* * If this action has a record size of zero, it * denotes an argument to the aggregating action. * Because the presence of this record doesn't (or * shouldn't) affect the way the data is interpreted, * we don't copy it out to save user-level the * confusion of dealing with a zero-length record. */ if (act->dta_rec.dtrd_size == 0) { ASSERT(agg->dtag_hasarg); continue; } aggdesc.dtagd_nrecs++; if (act == &agg->dtag_action) break; } /* * Now that we have the size, we need to allocate a temporary * buffer in which to store the complete description. We need * the temporary buffer to be able to drop dtrace_lock() * across the copyout(), below. */ size = sizeof (dtrace_aggdesc_t) + (aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t)); buf = kmem_alloc(size, KM_SLEEP); dest = (uintptr_t)buf; bcopy(&aggdesc, (void *)dest, sizeof (aggdesc)); dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]); for (act = agg->dtag_first; ; act = act->dta_next) { dtrace_recdesc_t rec = act->dta_rec; /* * See the comment in the above loop for why we pass * over zero-length records. */ if (rec.dtrd_size == 0) { ASSERT(agg->dtag_hasarg); continue; } if (nrecs-- == 0) break; rec.dtrd_offset -= offs; bcopy(&rec, (void *)dest, sizeof (rec)); dest += sizeof (dtrace_recdesc_t); if (act == &agg->dtag_action) break; } mutex_exit(&dtrace_lock); if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) { kmem_free(buf, size); return (EFAULT); } kmem_free(buf, size); return (0); } case DTRACEIOC_ENABLE: { dof_hdr_t *dof; dtrace_enabling_t *enab = NULL; dtrace_vstate_t *vstate; int err = 0; *rv = 0; /* * If a NULL argument has been passed, we take this as our * cue to reevaluate our enablings. */ if (arg == NULL) { dtrace_enabling_matchall(); return (0); } if ((dof = dtrace_dof_copyin(arg, &rval)) == NULL) return (rval); mutex_enter(&cpu_lock); mutex_enter(&dtrace_lock); vstate = &state->dts_vstate; if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) { mutex_exit(&dtrace_lock); mutex_exit(&cpu_lock); dtrace_dof_destroy(dof); return (EBUSY); } if (dtrace_dof_slurp(dof, vstate, cr, &enab, 0, B_TRUE) != 0) { mutex_exit(&dtrace_lock); mutex_exit(&cpu_lock); dtrace_dof_destroy(dof); return (EINVAL); } if ((rval = dtrace_dof_options(dof, state)) != 0) { dtrace_enabling_destroy(enab); mutex_exit(&dtrace_lock); mutex_exit(&cpu_lock); dtrace_dof_destroy(dof); return (rval); } if ((err = dtrace_enabling_match(enab, rv)) == 0) { err = dtrace_enabling_retain(enab); } else { dtrace_enabling_destroy(enab); } mutex_exit(&cpu_lock); mutex_exit(&dtrace_lock); dtrace_dof_destroy(dof); return (err); } case DTRACEIOC_REPLICATE: { dtrace_repldesc_t desc; dtrace_probedesc_t *match = &desc.dtrpd_match; dtrace_probedesc_t *create = &desc.dtrpd_create; int err; if (copyin((void *)arg, &desc, sizeof (desc)) != 0) return (EFAULT); match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0'; match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0'; match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0'; match->dtpd_name[DTRACE_NAMELEN - 1] = '\0'; create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0'; create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0'; create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0'; create->dtpd_name[DTRACE_NAMELEN - 1] = '\0'; mutex_enter(&dtrace_lock); err = dtrace_enabling_replicate(state, match, create); mutex_exit(&dtrace_lock); return (err); } case DTRACEIOC_PROBEMATCH: case DTRACEIOC_PROBES: { dtrace_probe_t *probe = NULL; dtrace_probedesc_t desc; dtrace_probekey_t pkey; dtrace_id_t i; int m = 0; uint32_t priv; uid_t uid; zoneid_t zoneid; if (copyin((void *)arg, &desc, sizeof (desc)) != 0) return (EFAULT); desc.dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0'; desc.dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0'; desc.dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0'; desc.dtpd_name[DTRACE_NAMELEN - 1] = '\0'; /* * Before we attempt to match this probe, we want to give * all providers the opportunity to provide it. */ if (desc.dtpd_id == DTRACE_IDNONE) { mutex_enter(&dtrace_provider_lock); dtrace_probe_provide(&desc, NULL); mutex_exit(&dtrace_provider_lock); desc.dtpd_id++; } if (cmd == DTRACEIOC_PROBEMATCH) { dtrace_probekey(&desc, &pkey); pkey.dtpk_id = DTRACE_IDNONE; } dtrace_cred2priv(cr, &priv, &uid, &zoneid); mutex_enter(&dtrace_lock); if (cmd == DTRACEIOC_PROBEMATCH) { for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) { if ((probe = dtrace_probes[i - 1]) != NULL && (m = dtrace_match_probe(probe, &pkey, priv, uid, zoneid)) != 0) break; } if (m < 0) { mutex_exit(&dtrace_lock); return (EINVAL); } } else { for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) { if ((probe = dtrace_probes[i - 1]) != NULL && dtrace_match_priv(probe, priv, uid, zoneid)) break; } } if (probe == NULL) { mutex_exit(&dtrace_lock); return (ESRCH); } dtrace_probe_description(probe, &desc); mutex_exit(&dtrace_lock); if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) return (EFAULT); return (0); } case DTRACEIOC_PROBEARG: { dtrace_argdesc_t desc; dtrace_probe_t *probe; dtrace_provider_t *prov; if (copyin((void *)arg, &desc, sizeof (desc)) != 0) return (EFAULT); if (desc.dtargd_id == DTRACE_IDNONE) return (EINVAL); if (desc.dtargd_ndx == DTRACE_ARGNONE) return (EINVAL); mutex_enter(&dtrace_provider_lock); mutex_enter(&mod_lock); mutex_enter(&dtrace_lock); if (desc.dtargd_id > dtrace_nprobes) { mutex_exit(&dtrace_lock); mutex_exit(&mod_lock); mutex_exit(&dtrace_provider_lock); return (EINVAL); } if ((probe = dtrace_probes[desc.dtargd_id - 1]) == NULL) { mutex_exit(&dtrace_lock); mutex_exit(&mod_lock); mutex_exit(&dtrace_provider_lock); return (EINVAL); } mutex_exit(&dtrace_lock); prov = probe->dtpr_provider; if (prov->dtpv_pops.dtps_getargdesc == NULL) { /* * There isn't any typed information for this probe. * Set the argument number to DTRACE_ARGNONE. */ desc.dtargd_ndx = DTRACE_ARGNONE; } else { desc.dtargd_native[0] = '\0'; desc.dtargd_xlate[0] = '\0'; desc.dtargd_mapping = desc.dtargd_ndx; prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg, probe->dtpr_id, probe->dtpr_arg, &desc); } mutex_exit(&mod_lock); mutex_exit(&dtrace_provider_lock); if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) return (EFAULT); return (0); } case DTRACEIOC_GO: { processorid_t cpuid; rval = dtrace_state_go(state, &cpuid); if (rval != 0) return (rval); if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0) return (EFAULT); return (0); } case DTRACEIOC_STOP: { processorid_t cpuid; mutex_enter(&dtrace_lock); rval = dtrace_state_stop(state, &cpuid); mutex_exit(&dtrace_lock); if (rval != 0) return (rval); if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0) return (EFAULT); return (0); } case DTRACEIOC_DOFGET: { dof_hdr_t hdr, *dof; uint64_t len; if (copyin((void *)arg, &hdr, sizeof (hdr)) != 0) return (EFAULT); mutex_enter(&dtrace_lock); dof = dtrace_dof_create(state); mutex_exit(&dtrace_lock); len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz); rval = copyout(dof, (void *)arg, len); dtrace_dof_destroy(dof); return (rval == 0 ? 0 : EFAULT); } case DTRACEIOC_AGGSNAP: case DTRACEIOC_BUFSNAP: { dtrace_bufdesc_t desc; caddr_t cached; dtrace_buffer_t *buf; if (copyin((void *)arg, &desc, sizeof (desc)) != 0) return (EFAULT); if (desc.dtbd_cpu < 0 || desc.dtbd_cpu >= NCPU) return (EINVAL); mutex_enter(&dtrace_lock); if (cmd == DTRACEIOC_BUFSNAP) { buf = &state->dts_buffer[desc.dtbd_cpu]; } else { buf = &state->dts_aggbuffer[desc.dtbd_cpu]; } if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) { size_t sz = buf->dtb_offset; if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) { mutex_exit(&dtrace_lock); return (EBUSY); } /* * If this buffer has already been consumed, we're * going to indicate that there's nothing left here * to consume. */ if (buf->dtb_flags & DTRACEBUF_CONSUMED) { mutex_exit(&dtrace_lock); desc.dtbd_size = 0; desc.dtbd_drops = 0; desc.dtbd_errors = 0; desc.dtbd_oldest = 0; sz = sizeof (desc); if (copyout(&desc, (void *)arg, sz) != 0) return (EFAULT); return (0); } /* * If this is a ring buffer that has wrapped, we want * to copy the whole thing out. */ if (buf->dtb_flags & DTRACEBUF_WRAPPED) { dtrace_buffer_polish(buf); sz = buf->dtb_size; } if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) { mutex_exit(&dtrace_lock); return (EFAULT); } desc.dtbd_size = sz; desc.dtbd_drops = buf->dtb_drops; desc.dtbd_errors = buf->dtb_errors; desc.dtbd_oldest = buf->dtb_xamot_offset; desc.dtbd_timestamp = dtrace_gethrtime(); mutex_exit(&dtrace_lock); if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) return (EFAULT); buf->dtb_flags |= DTRACEBUF_CONSUMED; return (0); } if (buf->dtb_tomax == NULL) { ASSERT(buf->dtb_xamot == NULL); mutex_exit(&dtrace_lock); return (ENOENT); } cached = buf->dtb_tomax; ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH)); dtrace_xcall(desc.dtbd_cpu, (dtrace_xcall_t)dtrace_buffer_switch, buf); state->dts_errors += buf->dtb_xamot_errors; /* * If the buffers did not actually switch, then the cross call * did not take place -- presumably because the given CPU is * not in the ready set. If this is the case, we'll return * ENOENT. */ if (buf->dtb_tomax == cached) { ASSERT(buf->dtb_xamot != cached); mutex_exit(&dtrace_lock); return (ENOENT); } ASSERT(cached == buf->dtb_xamot); /* * We have our snapshot; now copy it out. */ if (copyout(buf->dtb_xamot, desc.dtbd_data, buf->dtb_xamot_offset) != 0) { mutex_exit(&dtrace_lock); return (EFAULT); } desc.dtbd_size = buf->dtb_xamot_offset; desc.dtbd_drops = buf->dtb_xamot_drops; desc.dtbd_errors = buf->dtb_xamot_errors; desc.dtbd_oldest = 0; desc.dtbd_timestamp = buf->dtb_switched; mutex_exit(&dtrace_lock); /* * Finally, copy out the buffer description. */ if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) return (EFAULT); return (0); } case DTRACEIOC_CONF: { dtrace_conf_t conf; bzero(&conf, sizeof (conf)); conf.dtc_difversion = DIF_VERSION; conf.dtc_difintregs = DIF_DIR_NREGS; conf.dtc_diftupregs = DIF_DTR_NREGS; conf.dtc_ctfmodel = CTF_MODEL_NATIVE; if (copyout(&conf, (void *)arg, sizeof (conf)) != 0) return (EFAULT); return (0); } case DTRACEIOC_STATUS: { dtrace_status_t stat; dtrace_dstate_t *dstate; int i, j; uint64_t nerrs; /* * See the comment in dtrace_state_deadman() for the reason * for setting dts_laststatus to INT64_MAX before setting * it to the correct value. */ state->dts_laststatus = INT64_MAX; dtrace_membar_producer(); state->dts_laststatus = dtrace_gethrtime(); bzero(&stat, sizeof (stat)); mutex_enter(&dtrace_lock); if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) { mutex_exit(&dtrace_lock); return (ENOENT); } if (state->dts_activity == DTRACE_ACTIVITY_DRAINING) stat.dtst_exiting = 1; nerrs = state->dts_errors; dstate = &state->dts_vstate.dtvs_dynvars; for (i = 0; i < NCPU; i++) { dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i]; stat.dtst_dyndrops += dcpu->dtdsc_drops; stat.dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops; stat.dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops; if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL) stat.dtst_filled++; nerrs += state->dts_buffer[i].dtb_errors; for (j = 0; j < state->dts_nspeculations; j++) { dtrace_speculation_t *spec; dtrace_buffer_t *buf; spec = &state->dts_speculations[j]; buf = &spec->dtsp_buffer[i]; stat.dtst_specdrops += buf->dtb_xamot_drops; } } stat.dtst_specdrops_busy = state->dts_speculations_busy; stat.dtst_specdrops_unavail = state->dts_speculations_unavail; stat.dtst_stkstroverflows = state->dts_stkstroverflows; stat.dtst_dblerrors = state->dts_dblerrors; stat.dtst_killed = (state->dts_activity == DTRACE_ACTIVITY_KILLED); stat.dtst_errors = nerrs; mutex_exit(&dtrace_lock); if (copyout(&stat, (void *)arg, sizeof (stat)) != 0) return (EFAULT); return (0); } case DTRACEIOC_FORMAT: { dtrace_fmtdesc_t fmt; char *str; int len; if (copyin((void *)arg, &fmt, sizeof (fmt)) != 0) return (EFAULT); mutex_enter(&dtrace_lock); if (fmt.dtfd_format == 0 || fmt.dtfd_format > state->dts_nformats) { mutex_exit(&dtrace_lock); return (EINVAL); } /* * Format strings are allocated contiguously and they are * never freed; if a format index is less than the number * of formats, we can assert that the format map is non-NULL * and that the format for the specified index is non-NULL. */ ASSERT(state->dts_formats != NULL); str = state->dts_formats[fmt.dtfd_format - 1]; ASSERT(str != NULL); len = strlen(str) + 1; if (len > fmt.dtfd_length) { fmt.dtfd_length = len; if (copyout(&fmt, (void *)arg, sizeof (fmt)) != 0) { mutex_exit(&dtrace_lock); return (EINVAL); } } else { if (copyout(str, fmt.dtfd_string, len) != 0) { mutex_exit(&dtrace_lock); return (EINVAL); } } mutex_exit(&dtrace_lock); return (0); } default: break; } return (ENOTTY); } /*ARGSUSED*/ static int dtrace_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { dtrace_state_t *state; switch (cmd) { case DDI_DETACH: break; case DDI_SUSPEND: return (DDI_SUCCESS); default: return (DDI_FAILURE); } mutex_enter(&cpu_lock); mutex_enter(&dtrace_provider_lock); mutex_enter(&dtrace_lock); ASSERT(dtrace_opens == 0); if (dtrace_helpers > 0) { mutex_exit(&dtrace_provider_lock); mutex_exit(&dtrace_lock); mutex_exit(&cpu_lock); return (DDI_FAILURE); } if (dtrace_unregister((dtrace_provider_id_t)dtrace_provider) != 0) { mutex_exit(&dtrace_provider_lock); mutex_exit(&dtrace_lock); mutex_exit(&cpu_lock); return (DDI_FAILURE); } dtrace_provider = NULL; if ((state = dtrace_anon_grab()) != NULL) { /* * If there were ECBs on this state, the provider should * have not been allowed to detach; assert that there is * none. */ ASSERT(state->dts_necbs == 0); dtrace_state_destroy(state); /* * If we're being detached with anonymous state, we need to * indicate to the kernel debugger that DTrace is now inactive. */ (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE); } bzero(&dtrace_anon, sizeof (dtrace_anon_t)); unregister_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL); dtrace_cpu_init = NULL; dtrace_helpers_cleanup = NULL; dtrace_helpers_fork = NULL; dtrace_cpustart_init = NULL; dtrace_cpustart_fini = NULL; dtrace_debugger_init = NULL; dtrace_debugger_fini = NULL; dtrace_modload = NULL; dtrace_modunload = NULL; ASSERT(dtrace_getf == 0); ASSERT(dtrace_closef == NULL); mutex_exit(&cpu_lock); kmem_free(dtrace_probes, dtrace_nprobes * sizeof (dtrace_probe_t *)); dtrace_probes = NULL; dtrace_nprobes = 0; dtrace_hash_destroy(dtrace_bymod); dtrace_hash_destroy(dtrace_byfunc); dtrace_hash_destroy(dtrace_byname); dtrace_bymod = NULL; dtrace_byfunc = NULL; dtrace_byname = NULL; kmem_cache_destroy(dtrace_state_cache); vmem_destroy(dtrace_minor); vmem_destroy(dtrace_arena); if (dtrace_toxrange != NULL) { kmem_free(dtrace_toxrange, dtrace_toxranges_max * sizeof (dtrace_toxrange_t)); dtrace_toxrange = NULL; dtrace_toxranges = 0; dtrace_toxranges_max = 0; } ddi_remove_minor_node(dtrace_devi, NULL); dtrace_devi = NULL; ddi_soft_state_fini(&dtrace_softstate); ASSERT(dtrace_vtime_references == 0); ASSERT(dtrace_opens == 0); ASSERT(dtrace_retained == NULL); mutex_exit(&dtrace_lock); mutex_exit(&dtrace_provider_lock); /* * We don't destroy the task queue until after we have dropped our * locks (taskq_destroy() may block on running tasks). To prevent * attempting to do work after we have effectively detached but before * the task queue has been destroyed, all tasks dispatched via the * task queue must check that DTrace is still attached before * performing any operation. */ taskq_destroy(dtrace_taskq); dtrace_taskq = NULL; return (DDI_SUCCESS); } #endif #ifdef illumos /*ARGSUSED*/ static int dtrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) { int error; switch (infocmd) { case DDI_INFO_DEVT2DEVINFO: *result = (void *)dtrace_devi; error = DDI_SUCCESS; break; case DDI_INFO_DEVT2INSTANCE: *result = (void *)0; error = DDI_SUCCESS; break; default: error = DDI_FAILURE; } return (error); } #endif #ifdef illumos static struct cb_ops dtrace_cb_ops = { dtrace_open, /* open */ dtrace_close, /* close */ nulldev, /* strategy */ nulldev, /* print */ nodev, /* dump */ nodev, /* read */ nodev, /* write */ dtrace_ioctl, /* ioctl */ nodev, /* devmap */ nodev, /* mmap */ nodev, /* segmap */ nochpoll, /* poll */ ddi_prop_op, /* cb_prop_op */ 0, /* streamtab */ D_NEW | D_MP /* Driver compatibility flag */ }; static struct dev_ops dtrace_ops = { DEVO_REV, /* devo_rev */ 0, /* refcnt */ dtrace_info, /* get_dev_info */ nulldev, /* identify */ nulldev, /* probe */ dtrace_attach, /* attach */ dtrace_detach, /* detach */ nodev, /* reset */ &dtrace_cb_ops, /* driver operations */ NULL, /* bus operations */ nodev /* dev power */ }; static struct modldrv modldrv = { &mod_driverops, /* module type (this is a pseudo driver) */ "Dynamic Tracing", /* name of module */ &dtrace_ops, /* driver ops */ }; static struct modlinkage modlinkage = { MODREV_1, (void *)&modldrv, NULL }; int _init(void) { return (mod_install(&modlinkage)); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } int _fini(void) { return (mod_remove(&modlinkage)); } #else static d_ioctl_t dtrace_ioctl; static d_ioctl_t dtrace_ioctl_helper; static void dtrace_load(void *); static int dtrace_unload(void); static struct cdev *dtrace_dev; static struct cdev *helper_dev; void dtrace_invop_init(void); void dtrace_invop_uninit(void); static struct cdevsw dtrace_cdevsw = { .d_version = D_VERSION, .d_ioctl = dtrace_ioctl, .d_open = dtrace_open, .d_name = "dtrace", }; static struct cdevsw helper_cdevsw = { .d_version = D_VERSION, .d_ioctl = dtrace_ioctl_helper, .d_name = "helper", }; #include #include #include #include #include #include #include #include #include SYSINIT(dtrace_load, SI_SUB_DTRACE, SI_ORDER_FIRST, dtrace_load, NULL); SYSUNINIT(dtrace_unload, SI_SUB_DTRACE, SI_ORDER_FIRST, dtrace_unload, NULL); SYSINIT(dtrace_anon_init, SI_SUB_DTRACE_ANON, SI_ORDER_FIRST, dtrace_anon_init, NULL); DEV_MODULE(dtrace, dtrace_modevent, NULL); MODULE_VERSION(dtrace, 1); MODULE_DEPEND(dtrace, opensolaris, 1, 1, 1); #endif Index: head/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace.h =================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace.h (revision 278528) +++ head/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace.h (revision 278529) @@ -1,2443 +1,2450 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright (c) 2013 by Delphix. All rights reserved. */ #ifndef _SYS_DTRACE_H #define _SYS_DTRACE_H #pragma ident "%Z%%M% %I% %E% SMI" #ifdef __cplusplus extern "C" { #endif /* * DTrace Dynamic Tracing Software: Kernel Interfaces * * Note: The contents of this file are private to the implementation of the * Solaris system and DTrace subsystem and are subject to change at any time * without notice. Applications and drivers using these interfaces will fail * to run on future releases. These interfaces should not be used for any * purpose except those expressly outlined in dtrace(7D) and libdtrace(3LIB). * Please refer to the "Solaris Dynamic Tracing Guide" for more information. */ #ifndef _ASM #include #include #include #ifdef illumos #include #else #include #include #include #include #include typedef int model_t; #endif #include #ifdef illumos #include #include #else #include #endif /* * DTrace Universal Constants and Typedefs */ #define DTRACE_CPUALL -1 /* all CPUs */ #define DTRACE_IDNONE 0 /* invalid probe identifier */ #define DTRACE_EPIDNONE 0 /* invalid enabled probe identifier */ #define DTRACE_AGGIDNONE 0 /* invalid aggregation identifier */ #define DTRACE_AGGVARIDNONE 0 /* invalid aggregation variable ID */ #define DTRACE_CACHEIDNONE 0 /* invalid predicate cache */ #define DTRACE_PROVNONE 0 /* invalid provider identifier */ #define DTRACE_METAPROVNONE 0 /* invalid meta-provider identifier */ #define DTRACE_ARGNONE -1 /* invalid argument index */ #define DTRACE_PROVNAMELEN 64 #define DTRACE_MODNAMELEN 64 #define DTRACE_FUNCNAMELEN 128 #define DTRACE_NAMELEN 64 #define DTRACE_FULLNAMELEN (DTRACE_PROVNAMELEN + DTRACE_MODNAMELEN + \ DTRACE_FUNCNAMELEN + DTRACE_NAMELEN + 4) #define DTRACE_ARGTYPELEN 128 typedef uint32_t dtrace_id_t; /* probe identifier */ typedef uint32_t dtrace_epid_t; /* enabled probe identifier */ typedef uint32_t dtrace_aggid_t; /* aggregation identifier */ typedef int64_t dtrace_aggvarid_t; /* aggregation variable identifier */ typedef uint16_t dtrace_actkind_t; /* action kind */ typedef int64_t dtrace_optval_t; /* option value */ typedef uint32_t dtrace_cacheid_t; /* predicate cache identifier */ typedef enum dtrace_probespec { DTRACE_PROBESPEC_NONE = -1, DTRACE_PROBESPEC_PROVIDER = 0, DTRACE_PROBESPEC_MOD, DTRACE_PROBESPEC_FUNC, DTRACE_PROBESPEC_NAME } dtrace_probespec_t; /* * DTrace Intermediate Format (DIF) * * The following definitions describe the DTrace Intermediate Format (DIF), a * a RISC-like instruction set and program encoding used to represent * predicates and actions that can be bound to DTrace probes. The constants * below defining the number of available registers are suggested minimums; the * compiler should use DTRACEIOC_CONF to dynamically obtain the number of * registers provided by the current DTrace implementation. */ #define DIF_VERSION_1 1 /* DIF version 1: Solaris 10 Beta */ #define DIF_VERSION_2 2 /* DIF version 2: Solaris 10 FCS */ #define DIF_VERSION DIF_VERSION_2 /* latest DIF instruction set version */ #define DIF_DIR_NREGS 8 /* number of DIF integer registers */ #define DIF_DTR_NREGS 8 /* number of DIF tuple registers */ #define DIF_OP_OR 1 /* or r1, r2, rd */ #define DIF_OP_XOR 2 /* xor r1, r2, rd */ #define DIF_OP_AND 3 /* and r1, r2, rd */ #define DIF_OP_SLL 4 /* sll r1, r2, rd */ #define DIF_OP_SRL 5 /* srl r1, r2, rd */ #define DIF_OP_SUB 6 /* sub r1, r2, rd */ #define DIF_OP_ADD 7 /* add r1, r2, rd */ #define DIF_OP_MUL 8 /* mul r1, r2, rd */ #define DIF_OP_SDIV 9 /* sdiv r1, r2, rd */ #define DIF_OP_UDIV 10 /* udiv r1, r2, rd */ #define DIF_OP_SREM 11 /* srem r1, r2, rd */ #define DIF_OP_UREM 12 /* urem r1, r2, rd */ #define DIF_OP_NOT 13 /* not r1, rd */ #define DIF_OP_MOV 14 /* mov r1, rd */ #define DIF_OP_CMP 15 /* cmp r1, r2 */ #define DIF_OP_TST 16 /* tst r1 */ #define DIF_OP_BA 17 /* ba label */ #define DIF_OP_BE 18 /* be label */ #define DIF_OP_BNE 19 /* bne label */ #define DIF_OP_BG 20 /* bg label */ #define DIF_OP_BGU 21 /* bgu label */ #define DIF_OP_BGE 22 /* bge label */ #define DIF_OP_BGEU 23 /* bgeu label */ #define DIF_OP_BL 24 /* bl label */ #define DIF_OP_BLU 25 /* blu label */ #define DIF_OP_BLE 26 /* ble label */ #define DIF_OP_BLEU 27 /* bleu label */ #define DIF_OP_LDSB 28 /* ldsb [r1], rd */ #define DIF_OP_LDSH 29 /* ldsh [r1], rd */ #define DIF_OP_LDSW 30 /* ldsw [r1], rd */ #define DIF_OP_LDUB 31 /* ldub [r1], rd */ #define DIF_OP_LDUH 32 /* lduh [r1], rd */ #define DIF_OP_LDUW 33 /* lduw [r1], rd */ #define DIF_OP_LDX 34 /* ldx [r1], rd */ #define DIF_OP_RET 35 /* ret rd */ #define DIF_OP_NOP 36 /* nop */ #define DIF_OP_SETX 37 /* setx intindex, rd */ #define DIF_OP_SETS 38 /* sets strindex, rd */ #define DIF_OP_SCMP 39 /* scmp r1, r2 */ #define DIF_OP_LDGA 40 /* ldga var, ri, rd */ #define DIF_OP_LDGS 41 /* ldgs var, rd */ #define DIF_OP_STGS 42 /* stgs var, rs */ #define DIF_OP_LDTA 43 /* ldta var, ri, rd */ #define DIF_OP_LDTS 44 /* ldts var, rd */ #define DIF_OP_STTS 45 /* stts var, rs */ #define DIF_OP_SRA 46 /* sra r1, r2, rd */ #define DIF_OP_CALL 47 /* call subr, rd */ #define DIF_OP_PUSHTR 48 /* pushtr type, rs, rr */ #define DIF_OP_PUSHTV 49 /* pushtv type, rs, rv */ #define DIF_OP_POPTS 50 /* popts */ #define DIF_OP_FLUSHTS 51 /* flushts */ #define DIF_OP_LDGAA 52 /* ldgaa var, rd */ #define DIF_OP_LDTAA 53 /* ldtaa var, rd */ #define DIF_OP_STGAA 54 /* stgaa var, rs */ #define DIF_OP_STTAA 55 /* sttaa var, rs */ #define DIF_OP_LDLS 56 /* ldls var, rd */ #define DIF_OP_STLS 57 /* stls var, rs */ #define DIF_OP_ALLOCS 58 /* allocs r1, rd */ #define DIF_OP_COPYS 59 /* copys r1, r2, rd */ #define DIF_OP_STB 60 /* stb r1, [rd] */ #define DIF_OP_STH 61 /* sth r1, [rd] */ #define DIF_OP_STW 62 /* stw r1, [rd] */ #define DIF_OP_STX 63 /* stx r1, [rd] */ #define DIF_OP_ULDSB 64 /* uldsb [r1], rd */ #define DIF_OP_ULDSH 65 /* uldsh [r1], rd */ #define DIF_OP_ULDSW 66 /* uldsw [r1], rd */ #define DIF_OP_ULDUB 67 /* uldub [r1], rd */ #define DIF_OP_ULDUH 68 /* ulduh [r1], rd */ #define DIF_OP_ULDUW 69 /* ulduw [r1], rd */ #define DIF_OP_ULDX 70 /* uldx [r1], rd */ #define DIF_OP_RLDSB 71 /* rldsb [r1], rd */ #define DIF_OP_RLDSH 72 /* rldsh [r1], rd */ #define DIF_OP_RLDSW 73 /* rldsw [r1], rd */ #define DIF_OP_RLDUB 74 /* rldub [r1], rd */ #define DIF_OP_RLDUH 75 /* rlduh [r1], rd */ #define DIF_OP_RLDUW 76 /* rlduw [r1], rd */ #define DIF_OP_RLDX 77 /* rldx [r1], rd */ #define DIF_OP_XLATE 78 /* xlate xlrindex, rd */ #define DIF_OP_XLARG 79 /* xlarg xlrindex, rd */ #define DIF_INTOFF_MAX 0xffff /* highest integer table offset */ #define DIF_STROFF_MAX 0xffff /* highest string table offset */ #define DIF_REGISTER_MAX 0xff /* highest register number */ #define DIF_VARIABLE_MAX 0xffff /* highest variable identifier */ #define DIF_SUBROUTINE_MAX 0xffff /* highest subroutine code */ #define DIF_VAR_ARRAY_MIN 0x0000 /* lowest numbered array variable */ #define DIF_VAR_ARRAY_UBASE 0x0080 /* lowest user-defined array */ #define DIF_VAR_ARRAY_MAX 0x00ff /* highest numbered array variable */ #define DIF_VAR_OTHER_MIN 0x0100 /* lowest numbered scalar or assc */ #define DIF_VAR_OTHER_UBASE 0x0500 /* lowest user-defined scalar or assc */ #define DIF_VAR_OTHER_MAX 0xffff /* highest numbered scalar or assc */ #define DIF_VAR_ARGS 0x0000 /* arguments array */ #define DIF_VAR_REGS 0x0001 /* registers array */ #define DIF_VAR_UREGS 0x0002 /* user registers array */ #define DIF_VAR_CURTHREAD 0x0100 /* thread pointer */ #define DIF_VAR_TIMESTAMP 0x0101 /* timestamp */ #define DIF_VAR_VTIMESTAMP 0x0102 /* virtual timestamp */ #define DIF_VAR_IPL 0x0103 /* interrupt priority level */ #define DIF_VAR_EPID 0x0104 /* enabled probe ID */ #define DIF_VAR_ID 0x0105 /* probe ID */ #define DIF_VAR_ARG0 0x0106 /* first argument */ #define DIF_VAR_ARG1 0x0107 /* second argument */ #define DIF_VAR_ARG2 0x0108 /* third argument */ #define DIF_VAR_ARG3 0x0109 /* fourth argument */ #define DIF_VAR_ARG4 0x010a /* fifth argument */ #define DIF_VAR_ARG5 0x010b /* sixth argument */ #define DIF_VAR_ARG6 0x010c /* seventh argument */ #define DIF_VAR_ARG7 0x010d /* eighth argument */ #define DIF_VAR_ARG8 0x010e /* ninth argument */ #define DIF_VAR_ARG9 0x010f /* tenth argument */ #define DIF_VAR_STACKDEPTH 0x0110 /* stack depth */ #define DIF_VAR_CALLER 0x0111 /* caller */ #define DIF_VAR_PROBEPROV 0x0112 /* probe provider */ #define DIF_VAR_PROBEMOD 0x0113 /* probe module */ #define DIF_VAR_PROBEFUNC 0x0114 /* probe function */ #define DIF_VAR_PROBENAME 0x0115 /* probe name */ #define DIF_VAR_PID 0x0116 /* process ID */ #define DIF_VAR_TID 0x0117 /* (per-process) thread ID */ #define DIF_VAR_EXECNAME 0x0118 /* name of executable */ #define DIF_VAR_ZONENAME 0x0119 /* zone name associated with process */ #define DIF_VAR_WALLTIMESTAMP 0x011a /* wall-clock timestamp */ #define DIF_VAR_USTACKDEPTH 0x011b /* user-land stack depth */ #define DIF_VAR_UCALLER 0x011c /* user-level caller */ #define DIF_VAR_PPID 0x011d /* parent process ID */ #define DIF_VAR_UID 0x011e /* process user ID */ #define DIF_VAR_GID 0x011f /* process group ID */ #define DIF_VAR_ERRNO 0x0120 /* thread errno */ #define DIF_VAR_EXECARGS 0x0121 /* process arguments */ #ifndef illumos #define DIF_VAR_CPU 0x0200 #endif #define DIF_SUBR_RAND 0 #define DIF_SUBR_MUTEX_OWNED 1 #define DIF_SUBR_MUTEX_OWNER 2 #define DIF_SUBR_MUTEX_TYPE_ADAPTIVE 3 #define DIF_SUBR_MUTEX_TYPE_SPIN 4 #define DIF_SUBR_RW_READ_HELD 5 #define DIF_SUBR_RW_WRITE_HELD 6 #define DIF_SUBR_RW_ISWRITER 7 #define DIF_SUBR_COPYIN 8 #define DIF_SUBR_COPYINSTR 9 #define DIF_SUBR_SPECULATION 10 #define DIF_SUBR_PROGENYOF 11 #define DIF_SUBR_STRLEN 12 #define DIF_SUBR_COPYOUT 13 #define DIF_SUBR_COPYOUTSTR 14 #define DIF_SUBR_ALLOCA 15 #define DIF_SUBR_BCOPY 16 #define DIF_SUBR_COPYINTO 17 #define DIF_SUBR_MSGDSIZE 18 #define DIF_SUBR_MSGSIZE 19 #define DIF_SUBR_GETMAJOR 20 #define DIF_SUBR_GETMINOR 21 #define DIF_SUBR_DDI_PATHNAME 22 #define DIF_SUBR_STRJOIN 23 #define DIF_SUBR_LLTOSTR 24 #define DIF_SUBR_BASENAME 25 #define DIF_SUBR_DIRNAME 26 #define DIF_SUBR_CLEANPATH 27 #define DIF_SUBR_STRCHR 28 #define DIF_SUBR_STRRCHR 29 #define DIF_SUBR_STRSTR 30 #define DIF_SUBR_STRTOK 31 #define DIF_SUBR_SUBSTR 32 #define DIF_SUBR_INDEX 33 #define DIF_SUBR_RINDEX 34 #define DIF_SUBR_HTONS 35 #define DIF_SUBR_HTONL 36 #define DIF_SUBR_HTONLL 37 #define DIF_SUBR_NTOHS 38 #define DIF_SUBR_NTOHL 39 #define DIF_SUBR_NTOHLL 40 #define DIF_SUBR_INET_NTOP 41 #define DIF_SUBR_INET_NTOA 42 #define DIF_SUBR_INET_NTOA6 43 #define DIF_SUBR_TOUPPER 44 #define DIF_SUBR_TOLOWER 45 #define DIF_SUBR_MEMREF 46 #define DIF_SUBR_TYPEREF 47 #define DIF_SUBR_SX_SHARED_HELD 48 #define DIF_SUBR_SX_EXCLUSIVE_HELD 49 #define DIF_SUBR_SX_ISEXCLUSIVE 50 #define DIF_SUBR_MEMSTR 51 #define DIF_SUBR_GETF 52 #define DIF_SUBR_JSON 53 #define DIF_SUBR_STRTOLL 54 #define DIF_SUBR_MAX 54 /* max subroutine value */ typedef uint32_t dif_instr_t; #define DIF_INSTR_OP(i) (((i) >> 24) & 0xff) #define DIF_INSTR_R1(i) (((i) >> 16) & 0xff) #define DIF_INSTR_R2(i) (((i) >> 8) & 0xff) #define DIF_INSTR_RD(i) ((i) & 0xff) #define DIF_INSTR_RS(i) ((i) & 0xff) #define DIF_INSTR_LABEL(i) ((i) & 0xffffff) #define DIF_INSTR_VAR(i) (((i) >> 8) & 0xffff) #define DIF_INSTR_INTEGER(i) (((i) >> 8) & 0xffff) #define DIF_INSTR_STRING(i) (((i) >> 8) & 0xffff) #define DIF_INSTR_SUBR(i) (((i) >> 8) & 0xffff) #define DIF_INSTR_TYPE(i) (((i) >> 16) & 0xff) #define DIF_INSTR_XLREF(i) (((i) >> 8) & 0xffff) #define DIF_INSTR_FMT(op, r1, r2, d) \ (((op) << 24) | ((r1) << 16) | ((r2) << 8) | (d)) #define DIF_INSTR_NOT(r1, d) (DIF_INSTR_FMT(DIF_OP_NOT, r1, 0, d)) #define DIF_INSTR_MOV(r1, d) (DIF_INSTR_FMT(DIF_OP_MOV, r1, 0, d)) #define DIF_INSTR_CMP(op, r1, r2) (DIF_INSTR_FMT(op, r1, r2, 0)) #define DIF_INSTR_TST(r1) (DIF_INSTR_FMT(DIF_OP_TST, r1, 0, 0)) #define DIF_INSTR_BRANCH(op, label) (((op) << 24) | (label)) #define DIF_INSTR_LOAD(op, r1, d) (DIF_INSTR_FMT(op, r1, 0, d)) #define DIF_INSTR_STORE(op, r1, d) (DIF_INSTR_FMT(op, r1, 0, d)) #define DIF_INSTR_SETX(i, d) ((DIF_OP_SETX << 24) | ((i) << 8) | (d)) #define DIF_INSTR_SETS(s, d) ((DIF_OP_SETS << 24) | ((s) << 8) | (d)) #define DIF_INSTR_RET(d) (DIF_INSTR_FMT(DIF_OP_RET, 0, 0, d)) #define DIF_INSTR_NOP (DIF_OP_NOP << 24) #define DIF_INSTR_LDA(op, v, r, d) (DIF_INSTR_FMT(op, v, r, d)) #define DIF_INSTR_LDV(op, v, d) (((op) << 24) | ((v) << 8) | (d)) #define DIF_INSTR_STV(op, v, rs) (((op) << 24) | ((v) << 8) | (rs)) #define DIF_INSTR_CALL(s, d) ((DIF_OP_CALL << 24) | ((s) << 8) | (d)) #define DIF_INSTR_PUSHTS(op, t, r2, rs) (DIF_INSTR_FMT(op, t, r2, rs)) #define DIF_INSTR_POPTS (DIF_OP_POPTS << 24) #define DIF_INSTR_FLUSHTS (DIF_OP_FLUSHTS << 24) #define DIF_INSTR_ALLOCS(r1, d) (DIF_INSTR_FMT(DIF_OP_ALLOCS, r1, 0, d)) #define DIF_INSTR_COPYS(r1, r2, d) (DIF_INSTR_FMT(DIF_OP_COPYS, r1, r2, d)) #define DIF_INSTR_XLATE(op, r, d) (((op) << 24) | ((r) << 8) | (d)) #define DIF_REG_R0 0 /* %r0 is always set to zero */ /* * A DTrace Intermediate Format Type (DIF Type) is used to represent the types * of variables, function and associative array arguments, and the return type * for each DIF object (shown below). It contains a description of the type, * its size in bytes, and a module identifier. */ typedef struct dtrace_diftype { uint8_t dtdt_kind; /* type kind (see below) */ uint8_t dtdt_ckind; /* type kind in CTF */ uint8_t dtdt_flags; /* type flags (see below) */ uint8_t dtdt_pad; /* reserved for future use */ uint32_t dtdt_size; /* type size in bytes (unless string) */ } dtrace_diftype_t; #define DIF_TYPE_CTF 0 /* type is a CTF type */ #define DIF_TYPE_STRING 1 /* type is a D string */ #define DIF_TF_BYREF 0x1 /* type is passed by reference */ #define DIF_TF_BYUREF 0x2 /* user type is passed by reference */ /* * A DTrace Intermediate Format variable record is used to describe each of the * variables referenced by a given DIF object. It contains an integer variable * identifier along with variable scope and properties, as shown below. The * size of this structure must be sizeof (int) aligned. */ typedef struct dtrace_difv { uint32_t dtdv_name; /* variable name index in dtdo_strtab */ uint32_t dtdv_id; /* variable reference identifier */ uint8_t dtdv_kind; /* variable kind (see below) */ uint8_t dtdv_scope; /* variable scope (see below) */ uint16_t dtdv_flags; /* variable flags (see below) */ dtrace_diftype_t dtdv_type; /* variable type (see above) */ } dtrace_difv_t; #define DIFV_KIND_ARRAY 0 /* variable is an array of quantities */ #define DIFV_KIND_SCALAR 1 /* variable is a scalar quantity */ #define DIFV_SCOPE_GLOBAL 0 /* variable has global scope */ #define DIFV_SCOPE_THREAD 1 /* variable has thread scope */ #define DIFV_SCOPE_LOCAL 2 /* variable has local scope */ #define DIFV_F_REF 0x1 /* variable is referenced by DIFO */ #define DIFV_F_MOD 0x2 /* variable is written by DIFO */ /* * DTrace Actions * * The upper byte determines the class of the action; the low bytes determines * the specific action within that class. The classes of actions are as * follows: * * [ no class ] <= May record process- or kernel-related data * DTRACEACT_PROC <= Only records process-related data * DTRACEACT_PROC_DESTRUCTIVE <= Potentially destructive to processes * DTRACEACT_KERNEL <= Only records kernel-related data * DTRACEACT_KERNEL_DESTRUCTIVE <= Potentially destructive to the kernel * DTRACEACT_SPECULATIVE <= Speculation-related action * DTRACEACT_AGGREGATION <= Aggregating action */ #define DTRACEACT_NONE 0 /* no action */ #define DTRACEACT_DIFEXPR 1 /* action is DIF expression */ #define DTRACEACT_EXIT 2 /* exit() action */ #define DTRACEACT_PRINTF 3 /* printf() action */ #define DTRACEACT_PRINTA 4 /* printa() action */ #define DTRACEACT_LIBACT 5 /* library-controlled action */ #define DTRACEACT_TRACEMEM 6 /* tracemem() action */ #define DTRACEACT_TRACEMEM_DYNSIZE 7 /* dynamic tracemem() size */ #define DTRACEACT_PRINTM 8 /* printm() action (BSD) */ #define DTRACEACT_PRINTT 9 /* printt() action (BSD) */ #define DTRACEACT_PROC 0x0100 #define DTRACEACT_USTACK (DTRACEACT_PROC + 1) #define DTRACEACT_JSTACK (DTRACEACT_PROC + 2) #define DTRACEACT_USYM (DTRACEACT_PROC + 3) #define DTRACEACT_UMOD (DTRACEACT_PROC + 4) #define DTRACEACT_UADDR (DTRACEACT_PROC + 5) #define DTRACEACT_PROC_DESTRUCTIVE 0x0200 #define DTRACEACT_STOP (DTRACEACT_PROC_DESTRUCTIVE + 1) #define DTRACEACT_RAISE (DTRACEACT_PROC_DESTRUCTIVE + 2) #define DTRACEACT_SYSTEM (DTRACEACT_PROC_DESTRUCTIVE + 3) #define DTRACEACT_FREOPEN (DTRACEACT_PROC_DESTRUCTIVE + 4) #define DTRACEACT_PROC_CONTROL 0x0300 #define DTRACEACT_KERNEL 0x0400 #define DTRACEACT_STACK (DTRACEACT_KERNEL + 1) #define DTRACEACT_SYM (DTRACEACT_KERNEL + 2) #define DTRACEACT_MOD (DTRACEACT_KERNEL + 3) #define DTRACEACT_KERNEL_DESTRUCTIVE 0x0500 #define DTRACEACT_BREAKPOINT (DTRACEACT_KERNEL_DESTRUCTIVE + 1) #define DTRACEACT_PANIC (DTRACEACT_KERNEL_DESTRUCTIVE + 2) #define DTRACEACT_CHILL (DTRACEACT_KERNEL_DESTRUCTIVE + 3) #define DTRACEACT_SPECULATIVE 0x0600 #define DTRACEACT_SPECULATE (DTRACEACT_SPECULATIVE + 1) #define DTRACEACT_COMMIT (DTRACEACT_SPECULATIVE + 2) #define DTRACEACT_DISCARD (DTRACEACT_SPECULATIVE + 3) #define DTRACEACT_CLASS(x) ((x) & 0xff00) #define DTRACEACT_ISDESTRUCTIVE(x) \ (DTRACEACT_CLASS(x) == DTRACEACT_PROC_DESTRUCTIVE || \ DTRACEACT_CLASS(x) == DTRACEACT_KERNEL_DESTRUCTIVE) #define DTRACEACT_ISSPECULATIVE(x) \ (DTRACEACT_CLASS(x) == DTRACEACT_SPECULATIVE) #define DTRACEACT_ISPRINTFLIKE(x) \ ((x) == DTRACEACT_PRINTF || (x) == DTRACEACT_PRINTA || \ (x) == DTRACEACT_SYSTEM || (x) == DTRACEACT_FREOPEN) /* * DTrace Aggregating Actions * * These are functions f(x) for which the following is true: * * f(f(x_0) U f(x_1) U ... U f(x_n)) = f(x_0 U x_1 U ... U x_n) * * where x_n is a set of arbitrary data. Aggregating actions are in their own * DTrace action class, DTTRACEACT_AGGREGATION. The macros provided here allow * for easier processing of the aggregation argument and data payload for a few * aggregating actions (notably: quantize(), lquantize(), and ustack()). */ #define DTRACEACT_AGGREGATION 0x0700 #define DTRACEAGG_COUNT (DTRACEACT_AGGREGATION + 1) #define DTRACEAGG_MIN (DTRACEACT_AGGREGATION + 2) #define DTRACEAGG_MAX (DTRACEACT_AGGREGATION + 3) #define DTRACEAGG_AVG (DTRACEACT_AGGREGATION + 4) #define DTRACEAGG_SUM (DTRACEACT_AGGREGATION + 5) #define DTRACEAGG_STDDEV (DTRACEACT_AGGREGATION + 6) #define DTRACEAGG_QUANTIZE (DTRACEACT_AGGREGATION + 7) #define DTRACEAGG_LQUANTIZE (DTRACEACT_AGGREGATION + 8) #define DTRACEAGG_LLQUANTIZE (DTRACEACT_AGGREGATION + 9) #define DTRACEACT_ISAGG(x) \ (DTRACEACT_CLASS(x) == DTRACEACT_AGGREGATION) #define DTRACE_QUANTIZE_NBUCKETS \ (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1) #define DTRACE_QUANTIZE_ZEROBUCKET ((sizeof (uint64_t) * NBBY) - 1) #define DTRACE_QUANTIZE_BUCKETVAL(buck) \ (int64_t)((buck) < DTRACE_QUANTIZE_ZEROBUCKET ? \ -(1LL << (DTRACE_QUANTIZE_ZEROBUCKET - 1 - (buck))) : \ (buck) == DTRACE_QUANTIZE_ZEROBUCKET ? 0 : \ 1LL << ((buck) - DTRACE_QUANTIZE_ZEROBUCKET - 1)) #define DTRACE_LQUANTIZE_STEPSHIFT 48 #define DTRACE_LQUANTIZE_STEPMASK ((uint64_t)UINT16_MAX << 48) #define DTRACE_LQUANTIZE_LEVELSHIFT 32 #define DTRACE_LQUANTIZE_LEVELMASK ((uint64_t)UINT16_MAX << 32) #define DTRACE_LQUANTIZE_BASESHIFT 0 #define DTRACE_LQUANTIZE_BASEMASK UINT32_MAX #define DTRACE_LQUANTIZE_STEP(x) \ (uint16_t)(((x) & DTRACE_LQUANTIZE_STEPMASK) >> \ DTRACE_LQUANTIZE_STEPSHIFT) #define DTRACE_LQUANTIZE_LEVELS(x) \ (uint16_t)(((x) & DTRACE_LQUANTIZE_LEVELMASK) >> \ DTRACE_LQUANTIZE_LEVELSHIFT) #define DTRACE_LQUANTIZE_BASE(x) \ (int32_t)(((x) & DTRACE_LQUANTIZE_BASEMASK) >> \ DTRACE_LQUANTIZE_BASESHIFT) #define DTRACE_LLQUANTIZE_FACTORSHIFT 48 #define DTRACE_LLQUANTIZE_FACTORMASK ((uint64_t)UINT16_MAX << 48) #define DTRACE_LLQUANTIZE_LOWSHIFT 32 #define DTRACE_LLQUANTIZE_LOWMASK ((uint64_t)UINT16_MAX << 32) #define DTRACE_LLQUANTIZE_HIGHSHIFT 16 #define DTRACE_LLQUANTIZE_HIGHMASK ((uint64_t)UINT16_MAX << 16) #define DTRACE_LLQUANTIZE_NSTEPSHIFT 0 #define DTRACE_LLQUANTIZE_NSTEPMASK UINT16_MAX #define DTRACE_LLQUANTIZE_FACTOR(x) \ (uint16_t)(((x) & DTRACE_LLQUANTIZE_FACTORMASK) >> \ DTRACE_LLQUANTIZE_FACTORSHIFT) #define DTRACE_LLQUANTIZE_LOW(x) \ (uint16_t)(((x) & DTRACE_LLQUANTIZE_LOWMASK) >> \ DTRACE_LLQUANTIZE_LOWSHIFT) #define DTRACE_LLQUANTIZE_HIGH(x) \ (uint16_t)(((x) & DTRACE_LLQUANTIZE_HIGHMASK) >> \ DTRACE_LLQUANTIZE_HIGHSHIFT) #define DTRACE_LLQUANTIZE_NSTEP(x) \ (uint16_t)(((x) & DTRACE_LLQUANTIZE_NSTEPMASK) >> \ DTRACE_LLQUANTIZE_NSTEPSHIFT) #define DTRACE_USTACK_NFRAMES(x) (uint32_t)((x) & UINT32_MAX) #define DTRACE_USTACK_STRSIZE(x) (uint32_t)((x) >> 32) #define DTRACE_USTACK_ARG(x, y) \ ((((uint64_t)(y)) << 32) | ((x) & UINT32_MAX)) #ifndef _LP64 #if BYTE_ORDER == _BIG_ENDIAN #define DTRACE_PTR(type, name) uint32_t name##pad; type *name #else #define DTRACE_PTR(type, name) type *name; uint32_t name##pad #endif #else #define DTRACE_PTR(type, name) type *name #endif /* * DTrace Object Format (DOF) * * DTrace programs can be persistently encoded in the DOF format so that they * may be embedded in other programs (for example, in an ELF file) or in the * dtrace driver configuration file for use in anonymous tracing. The DOF * format is versioned and extensible so that it can be revised and so that * internal data structures can be modified or extended compatibly. All DOF * structures use fixed-size types, so the 32-bit and 64-bit representations * are identical and consumers can use either data model transparently. * * The file layout is structured as follows: * * +---------------+-------------------+----- ... ----+---- ... ------+ * | dof_hdr_t | dof_sec_t[ ... ] | loadable | non-loadable | * | (file header) | (section headers) | section data | section data | * +---------------+-------------------+----- ... ----+---- ... ------+ * |<------------ dof_hdr.dofh_loadsz --------------->| | * |<------------ dof_hdr.dofh_filesz ------------------------------->| * * The file header stores meta-data including a magic number, data model for * the instrumentation, data encoding, and properties of the DIF code within. * The header describes its own size and the size of the section headers. By * convention, an array of section headers follows the file header, and then * the data for all loadable sections and unloadable sections. This permits * consumer code to easily download the headers and all loadable data into the * DTrace driver in one contiguous chunk, omitting other extraneous sections. * * The section headers describe the size, offset, alignment, and section type * for each section. Sections are described using a set of #defines that tell * the consumer what kind of data is expected. Sections can contain links to * other sections by storing a dof_secidx_t, an index into the section header * array, inside of the section data structures. The section header includes * an entry size so that sections with data arrays can grow their structures. * * The DOF data itself can contain many snippets of DIF (i.e. >1 DIFOs), which * are represented themselves as a collection of related DOF sections. This * permits us to change the set of sections associated with a DIFO over time, * and also permits us to encode DIFOs that contain different sets of sections. * When a DOF section wants to refer to a DIFO, it stores the dof_secidx_t of a * section of type DOF_SECT_DIFOHDR. This section's data is then an array of * dof_secidx_t's which in turn denote the sections associated with this DIFO. * * This loose coupling of the file structure (header and sections) to the * structure of the DTrace program itself (ECB descriptions, action * descriptions, and DIFOs) permits activities such as relocation processing * to occur in a single pass without having to understand D program structure. * * Finally, strings are always stored in ELF-style string tables along with a * string table section index and string table offset. Therefore strings in * DOF are always arbitrary-length and not bound to the current implementation. */ #define DOF_ID_SIZE 16 /* total size of dofh_ident[] in bytes */ typedef struct dof_hdr { uint8_t dofh_ident[DOF_ID_SIZE]; /* identification bytes (see below) */ uint32_t dofh_flags; /* file attribute flags (if any) */ uint32_t dofh_hdrsize; /* size of file header in bytes */ uint32_t dofh_secsize; /* size of section header in bytes */ uint32_t dofh_secnum; /* number of section headers */ uint64_t dofh_secoff; /* file offset of section headers */ uint64_t dofh_loadsz; /* file size of loadable portion */ uint64_t dofh_filesz; /* file size of entire DOF file */ uint64_t dofh_pad; /* reserved for future use */ } dof_hdr_t; #define DOF_ID_MAG0 0 /* first byte of magic number */ #define DOF_ID_MAG1 1 /* second byte of magic number */ #define DOF_ID_MAG2 2 /* third byte of magic number */ #define DOF_ID_MAG3 3 /* fourth byte of magic number */ #define DOF_ID_MODEL 4 /* DOF data model (see below) */ #define DOF_ID_ENCODING 5 /* DOF data encoding (see below) */ #define DOF_ID_VERSION 6 /* DOF file format major version (see below) */ #define DOF_ID_DIFVERS 7 /* DIF instruction set version */ #define DOF_ID_DIFIREG 8 /* DIF integer registers used by compiler */ #define DOF_ID_DIFTREG 9 /* DIF tuple registers used by compiler */ #define DOF_ID_PAD 10 /* start of padding bytes (all zeroes) */ #define DOF_MAG_MAG0 0x7F /* DOF_ID_MAG[0-3] */ #define DOF_MAG_MAG1 'D' #define DOF_MAG_MAG2 'O' #define DOF_MAG_MAG3 'F' #define DOF_MAG_STRING "\177DOF" #define DOF_MAG_STRLEN 4 #define DOF_MODEL_NONE 0 /* DOF_ID_MODEL */ #define DOF_MODEL_ILP32 1 #define DOF_MODEL_LP64 2 #ifdef _LP64 #define DOF_MODEL_NATIVE DOF_MODEL_LP64 #else #define DOF_MODEL_NATIVE DOF_MODEL_ILP32 #endif #define DOF_ENCODE_NONE 0 /* DOF_ID_ENCODING */ #define DOF_ENCODE_LSB 1 #define DOF_ENCODE_MSB 2 #if BYTE_ORDER == _BIG_ENDIAN #define DOF_ENCODE_NATIVE DOF_ENCODE_MSB #else #define DOF_ENCODE_NATIVE DOF_ENCODE_LSB #endif #define DOF_VERSION_1 1 /* DOF version 1: Solaris 10 FCS */ #define DOF_VERSION_2 2 /* DOF version 2: Solaris Express 6/06 */ #define DOF_VERSION DOF_VERSION_2 /* Latest DOF version */ #define DOF_FL_VALID 0 /* mask of all valid dofh_flags bits */ typedef uint32_t dof_secidx_t; /* section header table index type */ typedef uint32_t dof_stridx_t; /* string table index type */ #define DOF_SECIDX_NONE (-1U) /* null value for section indices */ #define DOF_STRIDX_NONE (-1U) /* null value for string indices */ typedef struct dof_sec { uint32_t dofs_type; /* section type (see below) */ uint32_t dofs_align; /* section data memory alignment */ uint32_t dofs_flags; /* section flags (if any) */ uint32_t dofs_entsize; /* size of section entry (if table) */ uint64_t dofs_offset; /* offset of section data within file */ uint64_t dofs_size; /* size of section data in bytes */ } dof_sec_t; #define DOF_SECT_NONE 0 /* null section */ #define DOF_SECT_COMMENTS 1 /* compiler comments */ #define DOF_SECT_SOURCE 2 /* D program source code */ #define DOF_SECT_ECBDESC 3 /* dof_ecbdesc_t */ #define DOF_SECT_PROBEDESC 4 /* dof_probedesc_t */ #define DOF_SECT_ACTDESC 5 /* dof_actdesc_t array */ #define DOF_SECT_DIFOHDR 6 /* dof_difohdr_t (variable length) */ #define DOF_SECT_DIF 7 /* uint32_t array of byte code */ #define DOF_SECT_STRTAB 8 /* string table */ #define DOF_SECT_VARTAB 9 /* dtrace_difv_t array */ #define DOF_SECT_RELTAB 10 /* dof_relodesc_t array */ #define DOF_SECT_TYPTAB 11 /* dtrace_diftype_t array */ #define DOF_SECT_URELHDR 12 /* dof_relohdr_t (user relocations) */ #define DOF_SECT_KRELHDR 13 /* dof_relohdr_t (kernel relocations) */ #define DOF_SECT_OPTDESC 14 /* dof_optdesc_t array */ #define DOF_SECT_PROVIDER 15 /* dof_provider_t */ #define DOF_SECT_PROBES 16 /* dof_probe_t array */ #define DOF_SECT_PRARGS 17 /* uint8_t array (probe arg mappings) */ #define DOF_SECT_PROFFS 18 /* uint32_t array (probe arg offsets) */ #define DOF_SECT_INTTAB 19 /* uint64_t array */ #define DOF_SECT_UTSNAME 20 /* struct utsname */ #define DOF_SECT_XLTAB 21 /* dof_xlref_t array */ #define DOF_SECT_XLMEMBERS 22 /* dof_xlmember_t array */ #define DOF_SECT_XLIMPORT 23 /* dof_xlator_t */ #define DOF_SECT_XLEXPORT 24 /* dof_xlator_t */ #define DOF_SECT_PREXPORT 25 /* dof_secidx_t array (exported objs) */ #define DOF_SECT_PRENOFFS 26 /* uint32_t array (enabled offsets) */ #define DOF_SECF_LOAD 1 /* section should be loaded */ #define DOF_SEC_ISLOADABLE(x) \ (((x) == DOF_SECT_ECBDESC) || ((x) == DOF_SECT_PROBEDESC) || \ ((x) == DOF_SECT_ACTDESC) || ((x) == DOF_SECT_DIFOHDR) || \ ((x) == DOF_SECT_DIF) || ((x) == DOF_SECT_STRTAB) || \ ((x) == DOF_SECT_VARTAB) || ((x) == DOF_SECT_RELTAB) || \ ((x) == DOF_SECT_TYPTAB) || ((x) == DOF_SECT_URELHDR) || \ ((x) == DOF_SECT_KRELHDR) || ((x) == DOF_SECT_OPTDESC) || \ ((x) == DOF_SECT_PROVIDER) || ((x) == DOF_SECT_PROBES) || \ ((x) == DOF_SECT_PRARGS) || ((x) == DOF_SECT_PROFFS) || \ ((x) == DOF_SECT_INTTAB) || ((x) == DOF_SECT_XLTAB) || \ ((x) == DOF_SECT_XLMEMBERS) || ((x) == DOF_SECT_XLIMPORT) || \ ((x) == DOF_SECT_XLIMPORT) || ((x) == DOF_SECT_XLEXPORT) || \ ((x) == DOF_SECT_PREXPORT) || ((x) == DOF_SECT_PRENOFFS)) typedef struct dof_ecbdesc { dof_secidx_t dofe_probes; /* link to DOF_SECT_PROBEDESC */ dof_secidx_t dofe_pred; /* link to DOF_SECT_DIFOHDR */ dof_secidx_t dofe_actions; /* link to DOF_SECT_ACTDESC */ uint32_t dofe_pad; /* reserved for future use */ uint64_t dofe_uarg; /* user-supplied library argument */ } dof_ecbdesc_t; typedef struct dof_probedesc { dof_secidx_t dofp_strtab; /* link to DOF_SECT_STRTAB section */ dof_stridx_t dofp_provider; /* provider string */ dof_stridx_t dofp_mod; /* module string */ dof_stridx_t dofp_func; /* function string */ dof_stridx_t dofp_name; /* name string */ uint32_t dofp_id; /* probe identifier (or zero) */ } dof_probedesc_t; typedef struct dof_actdesc { dof_secidx_t dofa_difo; /* link to DOF_SECT_DIFOHDR */ dof_secidx_t dofa_strtab; /* link to DOF_SECT_STRTAB section */ uint32_t dofa_kind; /* action kind (DTRACEACT_* constant) */ uint32_t dofa_ntuple; /* number of subsequent tuple actions */ uint64_t dofa_arg; /* kind-specific argument */ uint64_t dofa_uarg; /* user-supplied argument */ } dof_actdesc_t; typedef struct dof_difohdr { dtrace_diftype_t dofd_rtype; /* return type for this fragment */ dof_secidx_t dofd_links[1]; /* variable length array of indices */ } dof_difohdr_t; typedef struct dof_relohdr { dof_secidx_t dofr_strtab; /* link to DOF_SECT_STRTAB for names */ dof_secidx_t dofr_relsec; /* link to DOF_SECT_RELTAB for relos */ dof_secidx_t dofr_tgtsec; /* link to section we are relocating */ } dof_relohdr_t; typedef struct dof_relodesc { dof_stridx_t dofr_name; /* string name of relocation symbol */ uint32_t dofr_type; /* relo type (DOF_RELO_* constant) */ uint64_t dofr_offset; /* byte offset for relocation */ uint64_t dofr_data; /* additional type-specific data */ } dof_relodesc_t; #define DOF_RELO_NONE 0 /* empty relocation entry */ #define DOF_RELO_SETX 1 /* relocate setx value */ typedef struct dof_optdesc { uint32_t dofo_option; /* option identifier */ dof_secidx_t dofo_strtab; /* string table, if string option */ uint64_t dofo_value; /* option value or string index */ } dof_optdesc_t; typedef uint32_t dof_attr_t; /* encoded stability attributes */ #define DOF_ATTR(n, d, c) (((n) << 24) | ((d) << 16) | ((c) << 8)) #define DOF_ATTR_NAME(a) (((a) >> 24) & 0xff) #define DOF_ATTR_DATA(a) (((a) >> 16) & 0xff) #define DOF_ATTR_CLASS(a) (((a) >> 8) & 0xff) typedef struct dof_provider { dof_secidx_t dofpv_strtab; /* link to DOF_SECT_STRTAB section */ dof_secidx_t dofpv_probes; /* link to DOF_SECT_PROBES section */ dof_secidx_t dofpv_prargs; /* link to DOF_SECT_PRARGS section */ dof_secidx_t dofpv_proffs; /* link to DOF_SECT_PROFFS section */ dof_stridx_t dofpv_name; /* provider name string */ dof_attr_t dofpv_provattr; /* provider attributes */ dof_attr_t dofpv_modattr; /* module attributes */ dof_attr_t dofpv_funcattr; /* function attributes */ dof_attr_t dofpv_nameattr; /* name attributes */ dof_attr_t dofpv_argsattr; /* args attributes */ dof_secidx_t dofpv_prenoffs; /* link to DOF_SECT_PRENOFFS section */ } dof_provider_t; typedef struct dof_probe { uint64_t dofpr_addr; /* probe base address or offset */ dof_stridx_t dofpr_func; /* probe function string */ dof_stridx_t dofpr_name; /* probe name string */ dof_stridx_t dofpr_nargv; /* native argument type strings */ dof_stridx_t dofpr_xargv; /* translated argument type strings */ uint32_t dofpr_argidx; /* index of first argument mapping */ uint32_t dofpr_offidx; /* index of first offset entry */ uint8_t dofpr_nargc; /* native argument count */ uint8_t dofpr_xargc; /* translated argument count */ uint16_t dofpr_noffs; /* number of offset entries for probe */ uint32_t dofpr_enoffidx; /* index of first is-enabled offset */ uint16_t dofpr_nenoffs; /* number of is-enabled offsets */ uint16_t dofpr_pad1; /* reserved for future use */ uint32_t dofpr_pad2; /* reserved for future use */ } dof_probe_t; typedef struct dof_xlator { dof_secidx_t dofxl_members; /* link to DOF_SECT_XLMEMBERS section */ dof_secidx_t dofxl_strtab; /* link to DOF_SECT_STRTAB section */ dof_stridx_t dofxl_argv; /* input parameter type strings */ uint32_t dofxl_argc; /* input parameter list length */ dof_stridx_t dofxl_type; /* output type string name */ dof_attr_t dofxl_attr; /* output stability attributes */ } dof_xlator_t; typedef struct dof_xlmember { dof_secidx_t dofxm_difo; /* member link to DOF_SECT_DIFOHDR */ dof_stridx_t dofxm_name; /* member name */ dtrace_diftype_t dofxm_type; /* member type */ } dof_xlmember_t; typedef struct dof_xlref { dof_secidx_t dofxr_xlator; /* link to DOF_SECT_XLATORS section */ uint32_t dofxr_member; /* index of referenced dof_xlmember */ uint32_t dofxr_argn; /* index of argument for DIF_OP_XLARG */ } dof_xlref_t; /* * DTrace Intermediate Format Object (DIFO) * * A DIFO is used to store the compiled DIF for a D expression, its return * type, and its string and variable tables. The string table is a single * buffer of character data into which sets instructions and variable * references can reference strings using a byte offset. The variable table * is an array of dtrace_difv_t structures that describe the name and type of * each variable and the id used in the DIF code. This structure is described * above in the DIF section of this header file. The DIFO is used at both * user-level (in the library) and in the kernel, but the structure is never * passed between the two: the DOF structures form the only interface. As a * result, the definition can change depending on the presence of _KERNEL. */ typedef struct dtrace_difo { dif_instr_t *dtdo_buf; /* instruction buffer */ uint64_t *dtdo_inttab; /* integer table (optional) */ char *dtdo_strtab; /* string table (optional) */ dtrace_difv_t *dtdo_vartab; /* variable table (optional) */ uint_t dtdo_len; /* length of instruction buffer */ uint_t dtdo_intlen; /* length of integer table */ uint_t dtdo_strlen; /* length of string table */ uint_t dtdo_varlen; /* length of variable table */ dtrace_diftype_t dtdo_rtype; /* return type */ uint_t dtdo_refcnt; /* owner reference count */ uint_t dtdo_destructive; /* invokes destructive subroutines */ #ifndef _KERNEL dof_relodesc_t *dtdo_kreltab; /* kernel relocations */ dof_relodesc_t *dtdo_ureltab; /* user relocations */ struct dt_node **dtdo_xlmtab; /* translator references */ uint_t dtdo_krelen; /* length of krelo table */ uint_t dtdo_urelen; /* length of urelo table */ uint_t dtdo_xlmlen; /* length of translator table */ #endif } dtrace_difo_t; /* * DTrace Enabling Description Structures * * When DTrace is tracking the description of a DTrace enabling entity (probe, * predicate, action, ECB, record, etc.), it does so in a description * structure. These structures all end in "desc", and are used at both * user-level and in the kernel -- but (with the exception of * dtrace_probedesc_t) they are never passed between them. Typically, * user-level will use the description structures when assembling an enabling. * It will then distill those description structures into a DOF object (see * above), and send it into the kernel. The kernel will again use the * description structures to create a description of the enabling as it reads * the DOF. When the description is complete, the enabling will be actually * created -- turning it into the structures that represent the enabling * instead of merely describing it. Not surprisingly, the description * structures bear a strong resemblance to the DOF structures that act as their * conduit. */ struct dtrace_predicate; typedef struct dtrace_probedesc { dtrace_id_t dtpd_id; /* probe identifier */ char dtpd_provider[DTRACE_PROVNAMELEN]; /* probe provider name */ char dtpd_mod[DTRACE_MODNAMELEN]; /* probe module name */ char dtpd_func[DTRACE_FUNCNAMELEN]; /* probe function name */ char dtpd_name[DTRACE_NAMELEN]; /* probe name */ } dtrace_probedesc_t; typedef struct dtrace_repldesc { dtrace_probedesc_t dtrpd_match; /* probe descr. to match */ dtrace_probedesc_t dtrpd_create; /* probe descr. to create */ } dtrace_repldesc_t; typedef struct dtrace_preddesc { dtrace_difo_t *dtpdd_difo; /* pointer to DIF object */ struct dtrace_predicate *dtpdd_predicate; /* pointer to predicate */ } dtrace_preddesc_t; typedef struct dtrace_actdesc { dtrace_difo_t *dtad_difo; /* pointer to DIF object */ struct dtrace_actdesc *dtad_next; /* next action */ dtrace_actkind_t dtad_kind; /* kind of action */ uint32_t dtad_ntuple; /* number in tuple */ uint64_t dtad_arg; /* action argument */ uint64_t dtad_uarg; /* user argument */ int dtad_refcnt; /* reference count */ } dtrace_actdesc_t; typedef struct dtrace_ecbdesc { dtrace_actdesc_t *dted_action; /* action description(s) */ dtrace_preddesc_t dted_pred; /* predicate description */ dtrace_probedesc_t dted_probe; /* probe description */ uint64_t dted_uarg; /* library argument */ int dted_refcnt; /* reference count */ } dtrace_ecbdesc_t; /* * DTrace Metadata Description Structures * * DTrace separates the trace data stream from the metadata stream. The only * metadata tokens placed in the data stream are the dtrace_rechdr_t (EPID + * timestamp) or (in the case of aggregations) aggregation identifiers. To * determine the structure of the data, DTrace consumers pass the token to the * kernel, and receive in return a corresponding description of the enabled * probe (via the dtrace_eprobedesc structure) or the aggregation (via the * dtrace_aggdesc structure). Both of these structures are expressed in terms * of record descriptions (via the dtrace_recdesc structure) that describe the * exact structure of the data. Some record descriptions may also contain a * format identifier; this additional bit of metadata can be retrieved from the * kernel, for which a format description is returned via the dtrace_fmtdesc * structure. Note that all four of these structures must be bitness-neutral * to allow for a 32-bit DTrace consumer on a 64-bit kernel. */ typedef struct dtrace_recdesc { dtrace_actkind_t dtrd_action; /* kind of action */ uint32_t dtrd_size; /* size of record */ uint32_t dtrd_offset; /* offset in ECB's data */ uint16_t dtrd_alignment; /* required alignment */ uint16_t dtrd_format; /* format, if any */ uint64_t dtrd_arg; /* action argument */ uint64_t dtrd_uarg; /* user argument */ } dtrace_recdesc_t; typedef struct dtrace_eprobedesc { dtrace_epid_t dtepd_epid; /* enabled probe ID */ dtrace_id_t dtepd_probeid; /* probe ID */ uint64_t dtepd_uarg; /* library argument */ uint32_t dtepd_size; /* total size */ int dtepd_nrecs; /* number of records */ dtrace_recdesc_t dtepd_rec[1]; /* records themselves */ } dtrace_eprobedesc_t; typedef struct dtrace_aggdesc { DTRACE_PTR(char, dtagd_name); /* not filled in by kernel */ dtrace_aggvarid_t dtagd_varid; /* not filled in by kernel */ int dtagd_flags; /* not filled in by kernel */ dtrace_aggid_t dtagd_id; /* aggregation ID */ dtrace_epid_t dtagd_epid; /* enabled probe ID */ uint32_t dtagd_size; /* size in bytes */ int dtagd_nrecs; /* number of records */ uint32_t dtagd_pad; /* explicit padding */ dtrace_recdesc_t dtagd_rec[1]; /* record descriptions */ } dtrace_aggdesc_t; typedef struct dtrace_fmtdesc { DTRACE_PTR(char, dtfd_string); /* format string */ int dtfd_length; /* length of format string */ uint16_t dtfd_format; /* format identifier */ } dtrace_fmtdesc_t; #define DTRACE_SIZEOF_EPROBEDESC(desc) \ (sizeof (dtrace_eprobedesc_t) + ((desc)->dtepd_nrecs ? \ (((desc)->dtepd_nrecs - 1) * sizeof (dtrace_recdesc_t)) : 0)) #define DTRACE_SIZEOF_AGGDESC(desc) \ (sizeof (dtrace_aggdesc_t) + ((desc)->dtagd_nrecs ? \ (((desc)->dtagd_nrecs - 1) * sizeof (dtrace_recdesc_t)) : 0)) /* * DTrace Option Interface * * Run-time DTrace options are set and retrieved via DOF_SECT_OPTDESC sections * in a DOF image. The dof_optdesc structure contains an option identifier and * an option value. The valid option identifiers are found below; the mapping * between option identifiers and option identifying strings is maintained at * user-level. Note that the value of DTRACEOPT_UNSET is such that all of the * following are potentially valid option values: all positive integers, zero * and negative one. Some options (notably "bufpolicy" and "bufresize") take * predefined tokens as their values; these are defined with * DTRACEOPT_{option}_{token}. */ #define DTRACEOPT_BUFSIZE 0 /* buffer size */ #define DTRACEOPT_BUFPOLICY 1 /* buffer policy */ #define DTRACEOPT_DYNVARSIZE 2 /* dynamic variable size */ #define DTRACEOPT_AGGSIZE 3 /* aggregation size */ #define DTRACEOPT_SPECSIZE 4 /* speculation size */ #define DTRACEOPT_NSPEC 5 /* number of speculations */ #define DTRACEOPT_STRSIZE 6 /* string size */ #define DTRACEOPT_CLEANRATE 7 /* dynvar cleaning rate */ #define DTRACEOPT_CPU 8 /* CPU to trace */ #define DTRACEOPT_BUFRESIZE 9 /* buffer resizing policy */ #define DTRACEOPT_GRABANON 10 /* grab anonymous state, if any */ #define DTRACEOPT_FLOWINDENT 11 /* indent function entry/return */ #define DTRACEOPT_QUIET 12 /* only output explicitly traced data */ #define DTRACEOPT_STACKFRAMES 13 /* number of stack frames */ #define DTRACEOPT_USTACKFRAMES 14 /* number of user stack frames */ #define DTRACEOPT_AGGRATE 15 /* aggregation snapshot rate */ #define DTRACEOPT_SWITCHRATE 16 /* buffer switching rate */ #define DTRACEOPT_STATUSRATE 17 /* status rate */ #define DTRACEOPT_DESTRUCTIVE 18 /* destructive actions allowed */ #define DTRACEOPT_STACKINDENT 19 /* output indent for stack traces */ #define DTRACEOPT_RAWBYTES 20 /* always print bytes in raw form */ #define DTRACEOPT_JSTACKFRAMES 21 /* number of jstack() frames */ #define DTRACEOPT_JSTACKSTRSIZE 22 /* size of jstack() string table */ #define DTRACEOPT_AGGSORTKEY 23 /* sort aggregations by key */ #define DTRACEOPT_AGGSORTREV 24 /* reverse-sort aggregations */ #define DTRACEOPT_AGGSORTPOS 25 /* agg. position to sort on */ #define DTRACEOPT_AGGSORTKEYPOS 26 /* agg. key position to sort on */ #define DTRACEOPT_TEMPORAL 27 /* temporally ordered output */ #define DTRACEOPT_AGGHIST 28 /* histogram aggregation output */ #define DTRACEOPT_AGGPACK 29 /* packed aggregation output */ #define DTRACEOPT_AGGZOOM 30 /* zoomed aggregation scaling */ #define DTRACEOPT_ZONE 31 /* zone in which to enable probes */ #define DTRACEOPT_MAX 32 /* number of options */ #define DTRACEOPT_UNSET (dtrace_optval_t)-2 /* unset option */ #define DTRACEOPT_BUFPOLICY_RING 0 /* ring buffer */ #define DTRACEOPT_BUFPOLICY_FILL 1 /* fill buffer, then stop */ #define DTRACEOPT_BUFPOLICY_SWITCH 2 /* switch buffers */ #define DTRACEOPT_BUFRESIZE_AUTO 0 /* automatic resizing */ #define DTRACEOPT_BUFRESIZE_MANUAL 1 /* manual resizing */ /* * DTrace Buffer Interface * * In order to get a snapshot of the principal or aggregation buffer, * user-level passes a buffer description to the kernel with the dtrace_bufdesc * structure. This describes which CPU user-level is interested in, and * where user-level wishes the kernel to snapshot the buffer to (the * dtbd_data field). The kernel uses the same structure to pass back some * information regarding the buffer: the size of data actually copied out, the * number of drops, the number of errors, the offset of the oldest record, * and the time of the snapshot. * * If the buffer policy is a "switch" policy, taking a snapshot of the * principal buffer has the additional effect of switching the active and * inactive buffers. Taking a snapshot of the aggregation buffer _always_ has * the additional effect of switching the active and inactive buffers. */ typedef struct dtrace_bufdesc { uint64_t dtbd_size; /* size of buffer */ uint32_t dtbd_cpu; /* CPU or DTRACE_CPUALL */ uint32_t dtbd_errors; /* number of errors */ uint64_t dtbd_drops; /* number of drops */ DTRACE_PTR(char, dtbd_data); /* data */ uint64_t dtbd_oldest; /* offset of oldest record */ uint64_t dtbd_timestamp; /* hrtime of snapshot */ } dtrace_bufdesc_t; /* * Each record in the buffer (dtbd_data) begins with a header that includes * the epid and a timestamp. The timestamp is split into two 4-byte parts * so that we do not require 8-byte alignment. */ typedef struct dtrace_rechdr { dtrace_epid_t dtrh_epid; /* enabled probe id */ uint32_t dtrh_timestamp_hi; /* high bits of hrtime_t */ uint32_t dtrh_timestamp_lo; /* low bits of hrtime_t */ } dtrace_rechdr_t; #define DTRACE_RECORD_LOAD_TIMESTAMP(dtrh) \ ((dtrh)->dtrh_timestamp_lo + \ ((uint64_t)(dtrh)->dtrh_timestamp_hi << 32)) #define DTRACE_RECORD_STORE_TIMESTAMP(dtrh, hrtime) { \ (dtrh)->dtrh_timestamp_lo = (uint32_t)hrtime; \ (dtrh)->dtrh_timestamp_hi = hrtime >> 32; \ } /* * DTrace Status * * The status of DTrace is relayed via the dtrace_status structure. This * structure contains members to count drops other than the capacity drops * available via the buffer interface (see above). This consists of dynamic * drops (including capacity dynamic drops, rinsing drops and dirty drops), and * speculative drops (including capacity speculative drops, drops due to busy * speculative buffers and drops due to unavailable speculative buffers). * Additionally, the status structure contains a field to indicate the number * of "fill"-policy buffers have been filled and a boolean field to indicate * that exit() has been called. If the dtst_exiting field is non-zero, no * further data will be generated until tracing is stopped (at which time any * enablings of the END action will be processed); if user-level sees that * this field is non-zero, tracing should be stopped as soon as possible. */ typedef struct dtrace_status { uint64_t dtst_dyndrops; /* dynamic drops */ uint64_t dtst_dyndrops_rinsing; /* dyn drops due to rinsing */ uint64_t dtst_dyndrops_dirty; /* dyn drops due to dirty */ uint64_t dtst_specdrops; /* speculative drops */ uint64_t dtst_specdrops_busy; /* spec drops due to busy */ uint64_t dtst_specdrops_unavail; /* spec drops due to unavail */ uint64_t dtst_errors; /* total errors */ uint64_t dtst_filled; /* number of filled bufs */ uint64_t dtst_stkstroverflows; /* stack string tab overflows */ uint64_t dtst_dblerrors; /* errors in ERROR probes */ char dtst_killed; /* non-zero if killed */ char dtst_exiting; /* non-zero if exit() called */ char dtst_pad[6]; /* pad out to 64-bit align */ } dtrace_status_t; /* * DTrace Configuration * * User-level may need to understand some elements of the kernel DTrace * configuration in order to generate correct DIF. This information is * conveyed via the dtrace_conf structure. */ typedef struct dtrace_conf { uint_t dtc_difversion; /* supported DIF version */ uint_t dtc_difintregs; /* # of DIF integer registers */ uint_t dtc_diftupregs; /* # of DIF tuple registers */ uint_t dtc_ctfmodel; /* CTF data model */ uint_t dtc_pad[8]; /* reserved for future use */ } dtrace_conf_t; /* * DTrace Faults * * The constants below DTRACEFLT_LIBRARY indicate probe processing faults; * constants at or above DTRACEFLT_LIBRARY indicate faults in probe * postprocessing at user-level. Probe processing faults induce an ERROR * probe and are replicated in unistd.d to allow users' ERROR probes to decode * the error condition using thse symbolic labels. */ #define DTRACEFLT_UNKNOWN 0 /* Unknown fault */ #define DTRACEFLT_BADADDR 1 /* Bad address */ #define DTRACEFLT_BADALIGN 2 /* Bad alignment */ #define DTRACEFLT_ILLOP 3 /* Illegal operation */ #define DTRACEFLT_DIVZERO 4 /* Divide-by-zero */ #define DTRACEFLT_NOSCRATCH 5 /* Out of scratch space */ #define DTRACEFLT_KPRIV 6 /* Illegal kernel access */ #define DTRACEFLT_UPRIV 7 /* Illegal user access */ #define DTRACEFLT_TUPOFLOW 8 /* Tuple stack overflow */ #define DTRACEFLT_BADSTACK 9 /* Bad stack */ #define DTRACEFLT_LIBRARY 1000 /* Library-level fault */ /* * DTrace Argument Types * * Because it would waste both space and time, argument types do not reside * with the probe. In order to determine argument types for args[X] * variables, the D compiler queries for argument types on a probe-by-probe * basis. (This optimizes for the common case that arguments are either not * used or used in an untyped fashion.) Typed arguments are specified with a * string of the type name in the dtragd_native member of the argument * description structure. Typed arguments may be further translated to types * of greater stability; the provider indicates such a translated argument by * filling in the dtargd_xlate member with the string of the translated type. * Finally, the provider may indicate which argument value a given argument * maps to by setting the dtargd_mapping member -- allowing a single argument * to map to multiple args[X] variables. */ typedef struct dtrace_argdesc { dtrace_id_t dtargd_id; /* probe identifier */ int dtargd_ndx; /* arg number (-1 iff none) */ int dtargd_mapping; /* value mapping */ char dtargd_native[DTRACE_ARGTYPELEN]; /* native type name */ char dtargd_xlate[DTRACE_ARGTYPELEN]; /* translated type name */ } dtrace_argdesc_t; /* * DTrace Stability Attributes * * Each DTrace provider advertises the name and data stability of each of its * probe description components, as well as its architectural dependencies. * The D compiler can query the provider attributes (dtrace_pattr_t below) in * order to compute the properties of an input program and report them. */ typedef uint8_t dtrace_stability_t; /* stability code (see attributes(5)) */ typedef uint8_t dtrace_class_t; /* architectural dependency class */ #define DTRACE_STABILITY_INTERNAL 0 /* private to DTrace itself */ #define DTRACE_STABILITY_PRIVATE 1 /* private to Sun (see docs) */ #define DTRACE_STABILITY_OBSOLETE 2 /* scheduled for removal */ #define DTRACE_STABILITY_EXTERNAL 3 /* not controlled by Sun */ #define DTRACE_STABILITY_UNSTABLE 4 /* new or rapidly changing */ #define DTRACE_STABILITY_EVOLVING 5 /* less rapidly changing */ #define DTRACE_STABILITY_STABLE 6 /* mature interface from Sun */ #define DTRACE_STABILITY_STANDARD 7 /* industry standard */ #define DTRACE_STABILITY_MAX 7 /* maximum valid stability */ #define DTRACE_CLASS_UNKNOWN 0 /* unknown architectural dependency */ #define DTRACE_CLASS_CPU 1 /* CPU-module-specific */ #define DTRACE_CLASS_PLATFORM 2 /* platform-specific (uname -i) */ #define DTRACE_CLASS_GROUP 3 /* hardware-group-specific (uname -m) */ #define DTRACE_CLASS_ISA 4 /* ISA-specific (uname -p) */ #define DTRACE_CLASS_COMMON 5 /* common to all systems */ #define DTRACE_CLASS_MAX 5 /* maximum valid class */ #define DTRACE_PRIV_NONE 0x0000 #define DTRACE_PRIV_KERNEL 0x0001 #define DTRACE_PRIV_USER 0x0002 #define DTRACE_PRIV_PROC 0x0004 #define DTRACE_PRIV_OWNER 0x0008 #define DTRACE_PRIV_ZONEOWNER 0x0010 #define DTRACE_PRIV_ALL \ (DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER | \ DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER) typedef struct dtrace_ppriv { uint32_t dtpp_flags; /* privilege flags */ uid_t dtpp_uid; /* user ID */ zoneid_t dtpp_zoneid; /* zone ID */ } dtrace_ppriv_t; typedef struct dtrace_attribute { dtrace_stability_t dtat_name; /* entity name stability */ dtrace_stability_t dtat_data; /* entity data stability */ dtrace_class_t dtat_class; /* entity data dependency */ } dtrace_attribute_t; typedef struct dtrace_pattr { dtrace_attribute_t dtpa_provider; /* provider attributes */ dtrace_attribute_t dtpa_mod; /* module attributes */ dtrace_attribute_t dtpa_func; /* function attributes */ dtrace_attribute_t dtpa_name; /* name attributes */ dtrace_attribute_t dtpa_args; /* args[] attributes */ } dtrace_pattr_t; typedef struct dtrace_providerdesc { char dtvd_name[DTRACE_PROVNAMELEN]; /* provider name */ dtrace_pattr_t dtvd_attr; /* stability attributes */ dtrace_ppriv_t dtvd_priv; /* privileges required */ } dtrace_providerdesc_t; /* * DTrace Pseudodevice Interface * * DTrace is controlled through ioctl(2)'s to the in-kernel dtrace:dtrace * pseudodevice driver. These ioctls comprise the user-kernel interface to * DTrace. */ #ifdef illumos #define DTRACEIOC (('d' << 24) | ('t' << 16) | ('r' << 8)) #define DTRACEIOC_PROVIDER (DTRACEIOC | 1) /* provider query */ #define DTRACEIOC_PROBES (DTRACEIOC | 2) /* probe query */ #define DTRACEIOC_BUFSNAP (DTRACEIOC | 4) /* snapshot buffer */ #define DTRACEIOC_PROBEMATCH (DTRACEIOC | 5) /* match probes */ #define DTRACEIOC_ENABLE (DTRACEIOC | 6) /* enable probes */ #define DTRACEIOC_AGGSNAP (DTRACEIOC | 7) /* snapshot agg. */ #define DTRACEIOC_EPROBE (DTRACEIOC | 8) /* get eprobe desc. */ #define DTRACEIOC_PROBEARG (DTRACEIOC | 9) /* get probe arg */ #define DTRACEIOC_CONF (DTRACEIOC | 10) /* get config. */ #define DTRACEIOC_STATUS (DTRACEIOC | 11) /* get status */ #define DTRACEIOC_GO (DTRACEIOC | 12) /* start tracing */ #define DTRACEIOC_STOP (DTRACEIOC | 13) /* stop tracing */ #define DTRACEIOC_AGGDESC (DTRACEIOC | 15) /* get agg. desc. */ #define DTRACEIOC_FORMAT (DTRACEIOC | 16) /* get format str */ #define DTRACEIOC_DOFGET (DTRACEIOC | 17) /* get DOF */ #define DTRACEIOC_REPLICATE (DTRACEIOC | 18) /* replicate enab */ #else #define DTRACEIOC_PROVIDER _IOWR('x',1,dtrace_providerdesc_t) /* provider query */ #define DTRACEIOC_PROBES _IOWR('x',2,dtrace_probedesc_t) /* probe query */ #define DTRACEIOC_BUFSNAP _IOW('x',4,dtrace_bufdesc_t *) /* snapshot buffer */ #define DTRACEIOC_PROBEMATCH _IOWR('x',5,dtrace_probedesc_t) /* match probes */ typedef struct { void *dof; /* DOF userland address written to driver. */ int n_matched; /* # matches returned by driver. */ } dtrace_enable_io_t; #define DTRACEIOC_ENABLE _IOWR('x',6,dtrace_enable_io_t) /* enable probes */ #define DTRACEIOC_AGGSNAP _IOW('x',7,dtrace_bufdesc_t *) /* snapshot agg. */ #define DTRACEIOC_EPROBE _IOW('x',8,dtrace_eprobedesc_t) /* get eprobe desc. */ #define DTRACEIOC_PROBEARG _IOWR('x',9,dtrace_argdesc_t) /* get probe arg */ #define DTRACEIOC_CONF _IOR('x',10,dtrace_conf_t) /* get config. */ #define DTRACEIOC_STATUS _IOR('x',11,dtrace_status_t) /* get status */ #define DTRACEIOC_GO _IOR('x',12,processorid_t) /* start tracing */ #define DTRACEIOC_STOP _IOWR('x',13,processorid_t) /* stop tracing */ #define DTRACEIOC_AGGDESC _IOW('x',15,dtrace_aggdesc_t *) /* get agg. desc. */ #define DTRACEIOC_FORMAT _IOWR('x',16,dtrace_fmtdesc_t) /* get format str */ #define DTRACEIOC_DOFGET _IOW('x',17,dof_hdr_t *) /* get DOF */ #define DTRACEIOC_REPLICATE _IOW('x',18,dtrace_repldesc_t) /* replicate enab */ #endif /* * DTrace Helpers * * In general, DTrace establishes probes in processes and takes actions on * processes without knowing their specific user-level structures. Instead of * existing in the framework, process-specific knowledge is contained by the * enabling D program -- which can apply process-specific knowledge by making * appropriate use of DTrace primitives like copyin() and copyinstr() to * operate on user-level data. However, there may exist some specific probes * of particular semantic relevance that the application developer may wish to * explicitly export. For example, an application may wish to export a probe * at the point that it begins and ends certain well-defined transactions. In * addition to providing probes, programs may wish to offer assistance for * certain actions. For example, in highly dynamic environments (e.g., Java), * it may be difficult to obtain a stack trace in terms of meaningful symbol * names (the translation from instruction addresses to corresponding symbol * names may only be possible in situ); these environments may wish to define * a series of actions to be applied in situ to obtain a meaningful stack * trace. * * These two mechanisms -- user-level statically defined tracing and assisting * DTrace actions -- are provided via DTrace _helpers_. Helpers are specified * via DOF, but unlike enabling DOF, helper DOF may contain definitions of * providers, probes and their arguments. If a helper wishes to provide * action assistance, probe descriptions and corresponding DIF actions may be * specified in the helper DOF. For such helper actions, however, the probe * description describes the specific helper: all DTrace helpers have the * provider name "dtrace" and the module name "helper", and the name of the * helper is contained in the function name (for example, the ustack() helper * is named "ustack"). Any helper-specific name may be contained in the name * (for example, if a helper were to have a constructor, it might be named * "dtrace:helper::init"). Helper actions are only called when the * action that they are helping is taken. Helper actions may only return DIF * expressions, and may only call the following subroutines: * * alloca() <= Allocates memory out of the consumer's scratch space * bcopy() <= Copies memory to scratch space * copyin() <= Copies memory from user-level into consumer's scratch * copyinto() <= Copies memory into a specific location in scratch * copyinstr() <= Copies a string into a specific location in scratch * * Helper actions may only access the following built-in variables: * * curthread <= Current kthread_t pointer * tid <= Current thread identifier * pid <= Current process identifier * ppid <= Parent process identifier * uid <= Current user ID * gid <= Current group ID * execname <= Current executable name * zonename <= Current zone name * * Helper actions may not manipulate or allocate dynamic variables, but they * may have clause-local and statically-allocated global variables. The * helper action variable state is specific to the helper action -- variables * used by the helper action may not be accessed outside of the helper * action, and the helper action may not access variables that like outside * of it. Helper actions may not load from kernel memory at-large; they are * restricting to loading current user state (via copyin() and variants) and * scratch space. As with probe enablings, helper actions are executed in * program order. The result of the helper action is the result of the last * executing helper expression. * * Helpers -- composed of either providers/probes or probes/actions (or both) * -- are added by opening the "helper" minor node, and issuing an ioctl(2) * (DTRACEHIOC_ADDDOF) that specifies the dof_helper_t structure. This * encapsulates the name and base address of the user-level library or * executable publishing the helpers and probes as well as the DOF that * contains the definitions of those helpers and probes. * * The DTRACEHIOC_ADD and DTRACEHIOC_REMOVE are left in place for legacy * helpers and should no longer be used. No other ioctls are valid on the * helper minor node. */ #ifdef illumos #define DTRACEHIOC (('d' << 24) | ('t' << 16) | ('h' << 8)) #define DTRACEHIOC_ADD (DTRACEHIOC | 1) /* add helper */ #define DTRACEHIOC_REMOVE (DTRACEHIOC | 2) /* remove helper */ #define DTRACEHIOC_ADDDOF (DTRACEHIOC | 3) /* add helper DOF */ #else #define DTRACEHIOC_ADD _IOWR('z', 1, dof_hdr_t)/* add helper */ #define DTRACEHIOC_REMOVE _IOW('z', 2, int) /* remove helper */ #define DTRACEHIOC_ADDDOF _IOWR('z', 3, dof_helper_t)/* add helper DOF */ #endif typedef struct dof_helper { char dofhp_mod[DTRACE_MODNAMELEN]; /* executable or library name */ uint64_t dofhp_addr; /* base address of object */ uint64_t dofhp_dof; /* address of helper DOF */ #ifndef illumos int gen; #endif } dof_helper_t; #define DTRACEMNR_DTRACE "dtrace" /* node for DTrace ops */ #define DTRACEMNR_HELPER "helper" /* node for helpers */ #define DTRACEMNRN_DTRACE 0 /* minor for DTrace ops */ #define DTRACEMNRN_HELPER 1 /* minor for helpers */ #define DTRACEMNRN_CLONE 2 /* first clone minor */ #ifdef _KERNEL /* * DTrace Provider API * * The following functions are implemented by the DTrace framework and are * used to implement separate in-kernel DTrace providers. Common functions * are provided in uts/common/os/dtrace.c. ISA-dependent subroutines are * defined in uts//dtrace/dtrace_asm.s or uts//dtrace/dtrace_isa.c. * * The provider API has two halves: the API that the providers consume from * DTrace, and the API that providers make available to DTrace. * * 1 Framework-to-Provider API * * 1.1 Overview * * The Framework-to-Provider API is represented by the dtrace_pops structure * that the provider passes to the framework when registering itself. This * structure consists of the following members: * * dtps_provide() <-- Provide all probes, all modules * dtps_provide_module() <-- Provide all probes in specified module * dtps_enable() <-- Enable specified probe * dtps_disable() <-- Disable specified probe * dtps_suspend() <-- Suspend specified probe * dtps_resume() <-- Resume specified probe * dtps_getargdesc() <-- Get the argument description for args[X] * dtps_getargval() <-- Get the value for an argX or args[X] variable * dtps_usermode() <-- Find out if the probe was fired in user mode * dtps_destroy() <-- Destroy all state associated with this probe * * 1.2 void dtps_provide(void *arg, const dtrace_probedesc_t *spec) * * 1.2.1 Overview * * Called to indicate that the provider should provide all probes. If the * specified description is non-NULL, dtps_provide() is being called because * no probe matched a specified probe -- if the provider has the ability to * create custom probes, it may wish to create a probe that matches the * specified description. * * 1.2.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_register(). The * second argument is a pointer to a probe description that the provider may * wish to consider when creating custom probes. The provider is expected to * call back into the DTrace framework via dtrace_probe_create() to create * any necessary probes. dtps_provide() may be called even if the provider * has made available all probes; the provider should check the return value * of dtrace_probe_create() to handle this case. Note that the provider need * not implement both dtps_provide() and dtps_provide_module(); see * "Arguments and Notes" for dtrace_register(), below. * * 1.2.3 Return value * * None. * * 1.2.4 Caller's context * * dtps_provide() is typically called from open() or ioctl() context, but may * be called from other contexts as well. The DTrace framework is locked in * such a way that providers may not register or unregister. This means that * the provider may not call any DTrace API that affects its registration with * the framework, including dtrace_register(), dtrace_unregister(), * dtrace_invalidate(), and dtrace_condense(). However, the context is such * that the provider may (and indeed, is expected to) call probe-related * DTrace routines, including dtrace_probe_create(), dtrace_probe_lookup(), * and dtrace_probe_arg(). * * 1.3 void dtps_provide_module(void *arg, modctl_t *mp) * * 1.3.1 Overview * * Called to indicate that the provider should provide all probes in the * specified module. * * 1.3.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_register(). The * second argument is a pointer to a modctl structure that indicates the * module for which probes should be created. * * 1.3.3 Return value * * None. * * 1.3.4 Caller's context * * dtps_provide_module() may be called from open() or ioctl() context, but * may also be called from a module loading context. mod_lock is held, and * the DTrace framework is locked in such a way that providers may not * register or unregister. This means that the provider may not call any * DTrace API that affects its registration with the framework, including * dtrace_register(), dtrace_unregister(), dtrace_invalidate(), and * dtrace_condense(). However, the context is such that the provider may (and * indeed, is expected to) call probe-related DTrace routines, including * dtrace_probe_create(), dtrace_probe_lookup(), and dtrace_probe_arg(). Note * that the provider need not implement both dtps_provide() and * dtps_provide_module(); see "Arguments and Notes" for dtrace_register(), * below. * * 1.4 void dtps_enable(void *arg, dtrace_id_t id, void *parg) * * 1.4.1 Overview * * Called to enable the specified probe. * * 1.4.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_register(). The * second argument is the identifier of the probe to be enabled. The third * argument is the probe argument as passed to dtrace_probe_create(). * dtps_enable() will be called when a probe transitions from not being * enabled at all to having one or more ECB. The number of ECBs associated * with the probe may change without subsequent calls into the provider. * When the number of ECBs drops to zero, the provider will be explicitly * told to disable the probe via dtps_disable(). dtrace_probe() should never * be called for a probe identifier that hasn't been explicitly enabled via * dtps_enable(). * * 1.4.3 Return value * * None. * * 1.4.4 Caller's context * * The DTrace framework is locked in such a way that it may not be called * back into at all. cpu_lock is held. mod_lock is not held and may not * be acquired. * * 1.5 void dtps_disable(void *arg, dtrace_id_t id, void *parg) * * 1.5.1 Overview * * Called to disable the specified probe. * * 1.5.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_register(). The * second argument is the identifier of the probe to be disabled. The third * argument is the probe argument as passed to dtrace_probe_create(). * dtps_disable() will be called when a probe transitions from being enabled * to having zero ECBs. dtrace_probe() should never be called for a probe * identifier that has been explicitly enabled via dtps_disable(). * * 1.5.3 Return value * * None. * * 1.5.4 Caller's context * * The DTrace framework is locked in such a way that it may not be called * back into at all. cpu_lock is held. mod_lock is not held and may not * be acquired. * * 1.6 void dtps_suspend(void *arg, dtrace_id_t id, void *parg) * * 1.6.1 Overview * * Called to suspend the specified enabled probe. This entry point is for * providers that may need to suspend some or all of their probes when CPUs * are being powered on or when the boot monitor is being entered for a * prolonged period of time. * * 1.6.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_register(). The * second argument is the identifier of the probe to be suspended. The * third argument is the probe argument as passed to dtrace_probe_create(). * dtps_suspend will only be called on an enabled probe. Providers that * provide a dtps_suspend entry point will want to take roughly the action * that it takes for dtps_disable. * * 1.6.3 Return value * * None. * * 1.6.4 Caller's context * * Interrupts are disabled. The DTrace framework is in a state such that the * specified probe cannot be disabled or destroyed for the duration of * dtps_suspend(). As interrupts are disabled, the provider is afforded * little latitude; the provider is expected to do no more than a store to * memory. * * 1.7 void dtps_resume(void *arg, dtrace_id_t id, void *parg) * * 1.7.1 Overview * * Called to resume the specified enabled probe. This entry point is for * providers that may need to resume some or all of their probes after the * completion of an event that induced a call to dtps_suspend(). * * 1.7.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_register(). The * second argument is the identifier of the probe to be resumed. The * third argument is the probe argument as passed to dtrace_probe_create(). * dtps_resume will only be called on an enabled probe. Providers that * provide a dtps_resume entry point will want to take roughly the action * that it takes for dtps_enable. * * 1.7.3 Return value * * None. * * 1.7.4 Caller's context * * Interrupts are disabled. The DTrace framework is in a state such that the * specified probe cannot be disabled or destroyed for the duration of * dtps_resume(). As interrupts are disabled, the provider is afforded * little latitude; the provider is expected to do no more than a store to * memory. * * 1.8 void dtps_getargdesc(void *arg, dtrace_id_t id, void *parg, * dtrace_argdesc_t *desc) * * 1.8.1 Overview * * Called to retrieve the argument description for an args[X] variable. * * 1.8.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_register(). The * second argument is the identifier of the current probe. The third * argument is the probe argument as passed to dtrace_probe_create(). The * fourth argument is a pointer to the argument description. This * description is both an input and output parameter: it contains the * index of the desired argument in the dtargd_ndx field, and expects * the other fields to be filled in upon return. If there is no argument * corresponding to the specified index, the dtargd_ndx field should be set * to DTRACE_ARGNONE. * * 1.8.3 Return value * * None. The dtargd_ndx, dtargd_native, dtargd_xlate and dtargd_mapping * members of the dtrace_argdesc_t structure are all output values. * * 1.8.4 Caller's context * * dtps_getargdesc() is called from ioctl() context. mod_lock is held, and * the DTrace framework is locked in such a way that providers may not * register or unregister. This means that the provider may not call any * DTrace API that affects its registration with the framework, including * dtrace_register(), dtrace_unregister(), dtrace_invalidate(), and * dtrace_condense(). * * 1.9 uint64_t dtps_getargval(void *arg, dtrace_id_t id, void *parg, * int argno, int aframes) * * 1.9.1 Overview * * Called to retrieve a value for an argX or args[X] variable. * * 1.9.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_register(). The * second argument is the identifier of the current probe. The third * argument is the probe argument as passed to dtrace_probe_create(). The * fourth argument is the number of the argument (the X in the example in * 1.9.1). The fifth argument is the number of stack frames that were used * to get from the actual place in the code that fired the probe to * dtrace_probe() itself, the so-called artificial frames. This argument may * be used to descend an appropriate number of frames to find the correct * values. If this entry point is left NULL, the dtrace_getarg() built-in * function is used. * * 1.9.3 Return value * * The value of the argument. * * 1.9.4 Caller's context * * This is called from within dtrace_probe() meaning that interrupts * are disabled. No locks should be taken within this entry point. * * 1.10 int dtps_usermode(void *arg, dtrace_id_t id, void *parg) * * 1.10.1 Overview * * Called to determine if the probe was fired in a user context. * * 1.10.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_register(). The * second argument is the identifier of the current probe. The third * argument is the probe argument as passed to dtrace_probe_create(). This * entry point must not be left NULL for providers whose probes allow for * mixed mode tracing, that is to say those probes that can fire during * kernel- _or_ user-mode execution * * 1.10.3 Return value * * A bitwise OR that encapsulates both the mode (either DTRACE_MODE_KERNEL * or DTRACE_MODE_USER) and the policy when the privilege of the enabling * is insufficient for that mode (a combination of DTRACE_MODE_NOPRIV_DROP, * DTRACE_MODE_NOPRIV_RESTRICT, and DTRACE_MODE_LIMITEDPRIV_RESTRICT). If * DTRACE_MODE_NOPRIV_DROP bit is set, insufficient privilege will result * in the probe firing being silently ignored for the enabling; if the * DTRACE_NODE_NOPRIV_RESTRICT bit is set, insufficient privilege will not * prevent probe processing for the enabling, but restrictions will be in * place that induce a UPRIV fault upon attempt to examine probe arguments * or current process state. If the DTRACE_MODE_LIMITEDPRIV_RESTRICT bit * is set, similar restrictions will be placed upon operation if the * privilege is sufficient to process the enabling, but does not otherwise * entitle the enabling to all zones. The DTRACE_MODE_NOPRIV_DROP and * DTRACE_MODE_NOPRIV_RESTRICT are mutually exclusive (and one of these * two policies must be specified), but either may be combined (or not) * with DTRACE_MODE_LIMITEDPRIV_RESTRICT. * * 1.10.4 Caller's context * * This is called from within dtrace_probe() meaning that interrupts * are disabled. No locks should be taken within this entry point. * * 1.11 void dtps_destroy(void *arg, dtrace_id_t id, void *parg) * * 1.11.1 Overview * * Called to destroy the specified probe. * * 1.11.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_register(). The * second argument is the identifier of the probe to be destroyed. The third * argument is the probe argument as passed to dtrace_probe_create(). The * provider should free all state associated with the probe. The framework * guarantees that dtps_destroy() is only called for probes that have either * been disabled via dtps_disable() or were never enabled via dtps_enable(). * Once dtps_disable() has been called for a probe, no further call will be * made specifying the probe. * * 1.11.3 Return value * * None. * * 1.11.4 Caller's context * * The DTrace framework is locked in such a way that it may not be called * back into at all. mod_lock is held. cpu_lock is not held, and may not be * acquired. * * * 2 Provider-to-Framework API * * 2.1 Overview * * The Provider-to-Framework API provides the mechanism for the provider to * register itself with the DTrace framework, to create probes, to lookup * probes and (most importantly) to fire probes. The Provider-to-Framework * consists of: * * dtrace_register() <-- Register a provider with the DTrace framework * dtrace_unregister() <-- Remove a provider's DTrace registration * dtrace_invalidate() <-- Invalidate the specified provider * dtrace_condense() <-- Remove a provider's unenabled probes * dtrace_attached() <-- Indicates whether or not DTrace has attached * dtrace_probe_create() <-- Create a DTrace probe * dtrace_probe_lookup() <-- Lookup a DTrace probe based on its name * dtrace_probe_arg() <-- Return the probe argument for a specific probe * dtrace_probe() <-- Fire the specified probe * * 2.2 int dtrace_register(const char *name, const dtrace_pattr_t *pap, * uint32_t priv, cred_t *cr, const dtrace_pops_t *pops, void *arg, * dtrace_provider_id_t *idp) * * 2.2.1 Overview * * dtrace_register() registers the calling provider with the DTrace * framework. It should generally be called by DTrace providers in their * attach(9E) entry point. * * 2.2.2 Arguments and Notes * * The first argument is the name of the provider. The second argument is a * pointer to the stability attributes for the provider. The third argument * is the privilege flags for the provider, and must be some combination of: * * DTRACE_PRIV_NONE <= All users may enable probes from this provider * * DTRACE_PRIV_PROC <= Any user with privilege of PRIV_DTRACE_PROC may * enable probes from this provider * * DTRACE_PRIV_USER <= Any user with privilege of PRIV_DTRACE_USER may * enable probes from this provider * * DTRACE_PRIV_KERNEL <= Any user with privilege of PRIV_DTRACE_KERNEL * may enable probes from this provider * * DTRACE_PRIV_OWNER <= This flag places an additional constraint on * the privilege requirements above. These probes * require either (a) a user ID matching the user * ID of the cred passed in the fourth argument * or (b) the PRIV_PROC_OWNER privilege. * * DTRACE_PRIV_ZONEOWNER<= This flag places an additional constraint on * the privilege requirements above. These probes * require either (a) a zone ID matching the zone * ID of the cred passed in the fourth argument * or (b) the PRIV_PROC_ZONE privilege. * * Note that these flags designate the _visibility_ of the probes, not * the conditions under which they may or may not fire. * * The fourth argument is the credential that is associated with the * provider. This argument should be NULL if the privilege flags don't * include DTRACE_PRIV_OWNER or DTRACE_PRIV_ZONEOWNER. If non-NULL, the * framework stashes the uid and zoneid represented by this credential * for use at probe-time, in implicit predicates. These limit visibility * of the probes to users and/or zones which have sufficient privilege to * access them. * * The fifth argument is a DTrace provider operations vector, which provides * the implementation for the Framework-to-Provider API. (See Section 1, * above.) This must be non-NULL, and each member must be non-NULL. The * exceptions to this are (1) the dtps_provide() and dtps_provide_module() * members (if the provider so desires, _one_ of these members may be left * NULL -- denoting that the provider only implements the other) and (2) * the dtps_suspend() and dtps_resume() members, which must either both be * NULL or both be non-NULL. * * The sixth argument is a cookie to be specified as the first argument for * each function in the Framework-to-Provider API. This argument may have * any value. * * The final argument is a pointer to dtrace_provider_id_t. If * dtrace_register() successfully completes, the provider identifier will be * stored in the memory pointed to be this argument. This argument must be * non-NULL. * * 2.2.3 Return value * * On success, dtrace_register() returns 0 and stores the new provider's * identifier into the memory pointed to by the idp argument. On failure, * dtrace_register() returns an errno: * * EINVAL The arguments passed to dtrace_register() were somehow invalid. * This may because a parameter that must be non-NULL was NULL, * because the name was invalid (either empty or an illegal * provider name) or because the attributes were invalid. * * No other failure code is returned. * * 2.2.4 Caller's context * * dtrace_register() may induce calls to dtrace_provide(); the provider must * hold no locks across dtrace_register() that may also be acquired by * dtrace_provide(). cpu_lock and mod_lock must not be held. * * 2.3 int dtrace_unregister(dtrace_provider_t id) * * 2.3.1 Overview * * Unregisters the specified provider from the DTrace framework. It should * generally be called by DTrace providers in their detach(9E) entry point. * * 2.3.2 Arguments and Notes * * The only argument is the provider identifier, as returned from a * successful call to dtrace_register(). As a result of calling * dtrace_unregister(), the DTrace framework will call back into the provider * via the dtps_destroy() entry point. Once dtrace_unregister() successfully * completes, however, the DTrace framework will no longer make calls through * the Framework-to-Provider API. * * 2.3.3 Return value * * On success, dtrace_unregister returns 0. On failure, dtrace_unregister() * returns an errno: * * EBUSY There are currently processes that have the DTrace pseudodevice * open, or there exists an anonymous enabling that hasn't yet * been claimed. * * No other failure code is returned. * * 2.3.4 Caller's context * * Because a call to dtrace_unregister() may induce calls through the * Framework-to-Provider API, the caller may not hold any lock across * dtrace_register() that is also acquired in any of the Framework-to- * Provider API functions. Additionally, mod_lock may not be held. * * 2.4 void dtrace_invalidate(dtrace_provider_id_t id) * * 2.4.1 Overview * * Invalidates the specified provider. All subsequent probe lookups for the * specified provider will fail, but its probes will not be removed. * * 2.4.2 Arguments and note * * The only argument is the provider identifier, as returned from a * successful call to dtrace_register(). In general, a provider's probes * always remain valid; dtrace_invalidate() is a mechanism for invalidating * an entire provider, regardless of whether or not probes are enabled or * not. Note that dtrace_invalidate() will _not_ prevent already enabled * probes from firing -- it will merely prevent any new enablings of the * provider's probes. * * 2.5 int dtrace_condense(dtrace_provider_id_t id) * * 2.5.1 Overview * * Removes all the unenabled probes for the given provider. This function is * not unlike dtrace_unregister(), except that it doesn't remove the * provider just as many of its associated probes as it can. * * 2.5.2 Arguments and Notes * * As with dtrace_unregister(), the sole argument is the provider identifier * as returned from a successful call to dtrace_register(). As a result of * calling dtrace_condense(), the DTrace framework will call back into the * given provider's dtps_destroy() entry point for each of the provider's * unenabled probes. * * 2.5.3 Return value * * Currently, dtrace_condense() always returns 0. However, consumers of this * function should check the return value as appropriate; its behavior may * change in the future. * * 2.5.4 Caller's context * * As with dtrace_unregister(), the caller may not hold any lock across * dtrace_condense() that is also acquired in the provider's entry points. * Also, mod_lock may not be held. * * 2.6 int dtrace_attached() * * 2.6.1 Overview * * Indicates whether or not DTrace has attached. * * 2.6.2 Arguments and Notes * * For most providers, DTrace makes initial contact beyond registration. * That is, once a provider has registered with DTrace, it waits to hear * from DTrace to create probes. However, some providers may wish to * proactively create probes without first being told by DTrace to do so. * If providers wish to do this, they must first call dtrace_attached() to * determine if DTrace itself has attached. If dtrace_attached() returns 0, * the provider must not make any other Provider-to-Framework API call. * * 2.6.3 Return value * * dtrace_attached() returns 1 if DTrace has attached, 0 otherwise. * * 2.7 int dtrace_probe_create(dtrace_provider_t id, const char *mod, * const char *func, const char *name, int aframes, void *arg) * * 2.7.1 Overview * * Creates a probe with specified module name, function name, and name. * * 2.7.2 Arguments and Notes * * The first argument is the provider identifier, as returned from a * successful call to dtrace_register(). The second, third, and fourth * arguments are the module name, function name, and probe name, * respectively. Of these, module name and function name may both be NULL * (in which case the probe is considered to be unanchored), or they may both * be non-NULL. The name must be non-NULL, and must point to a non-empty * string. * * The fifth argument is the number of artificial stack frames that will be * found on the stack when dtrace_probe() is called for the new probe. These * artificial frames will be automatically be pruned should the stack() or * stackdepth() functions be called as part of one of the probe's ECBs. If * the parameter doesn't add an artificial frame, this parameter should be * zero. * * The final argument is a probe argument that will be passed back to the * provider when a probe-specific operation is called. (e.g., via * dtps_enable(), dtps_disable(), etc.) * * Note that it is up to the provider to be sure that the probe that it * creates does not already exist -- if the provider is unsure of the probe's * existence, it should assure its absence with dtrace_probe_lookup() before * calling dtrace_probe_create(). * * 2.7.3 Return value * * dtrace_probe_create() always succeeds, and always returns the identifier * of the newly-created probe. * * 2.7.4 Caller's context * * While dtrace_probe_create() is generally expected to be called from * dtps_provide() and/or dtps_provide_module(), it may be called from other * non-DTrace contexts. Neither cpu_lock nor mod_lock may be held. * * 2.8 dtrace_id_t dtrace_probe_lookup(dtrace_provider_t id, const char *mod, * const char *func, const char *name) * * 2.8.1 Overview * * Looks up a probe based on provdider and one or more of module name, * function name and probe name. * * 2.8.2 Arguments and Notes * * The first argument is the provider identifier, as returned from a * successful call to dtrace_register(). The second, third, and fourth * arguments are the module name, function name, and probe name, * respectively. Any of these may be NULL; dtrace_probe_lookup() will return * the identifier of the first probe that is provided by the specified * provider and matches all of the non-NULL matching criteria. * dtrace_probe_lookup() is generally used by a provider to be check the * existence of a probe before creating it with dtrace_probe_create(). * * 2.8.3 Return value * * If the probe exists, returns its identifier. If the probe does not exist, * return DTRACE_IDNONE. * * 2.8.4 Caller's context * * While dtrace_probe_lookup() is generally expected to be called from * dtps_provide() and/or dtps_provide_module(), it may also be called from * other non-DTrace contexts. Neither cpu_lock nor mod_lock may be held. * * 2.9 void *dtrace_probe_arg(dtrace_provider_t id, dtrace_id_t probe) * * 2.9.1 Overview * * Returns the probe argument associated with the specified probe. * * 2.9.2 Arguments and Notes * * The first argument is the provider identifier, as returned from a * successful call to dtrace_register(). The second argument is a probe * identifier, as returned from dtrace_probe_lookup() or * dtrace_probe_create(). This is useful if a probe has multiple * provider-specific components to it: the provider can create the probe * once with provider-specific state, and then add to the state by looking * up the probe based on probe identifier. * * 2.9.3 Return value * * Returns the argument associated with the specified probe. If the * specified probe does not exist, or if the specified probe is not provided * by the specified provider, NULL is returned. * * 2.9.4 Caller's context * * While dtrace_probe_arg() is generally expected to be called from * dtps_provide() and/or dtps_provide_module(), it may also be called from * other non-DTrace contexts. Neither cpu_lock nor mod_lock may be held. * * 2.10 void dtrace_probe(dtrace_id_t probe, uintptr_t arg0, uintptr_t arg1, * uintptr_t arg2, uintptr_t arg3, uintptr_t arg4) * * 2.10.1 Overview * * The epicenter of DTrace: fires the specified probes with the specified * arguments. * * 2.10.2 Arguments and Notes * * The first argument is a probe identifier as returned by * dtrace_probe_create() or dtrace_probe_lookup(). The second through sixth * arguments are the values to which the D variables "arg0" through "arg4" * will be mapped. * * dtrace_probe() should be called whenever the specified probe has fired -- * however the provider defines it. * * 2.10.3 Return value * * None. * * 2.10.4 Caller's context * * dtrace_probe() may be called in virtually any context: kernel, user, * interrupt, high-level interrupt, with arbitrary adaptive locks held, with * dispatcher locks held, with interrupts disabled, etc. The only latitude * that must be afforded to DTrace is the ability to make calls within * itself (and to its in-kernel subroutines) and the ability to access * arbitrary (but mapped) memory. On some platforms, this constrains * context. For example, on UltraSPARC, dtrace_probe() cannot be called * from any context in which TL is greater than zero. dtrace_probe() may * also not be called from any routine which may be called by dtrace_probe() * -- which includes functions in the DTrace framework and some in-kernel * DTrace subroutines. All such functions "dtrace_"; providers that * instrument the kernel arbitrarily should be sure to not instrument these * routines. */ typedef struct dtrace_pops { void (*dtps_provide)(void *arg, dtrace_probedesc_t *spec); void (*dtps_provide_module)(void *arg, modctl_t *mp); void (*dtps_enable)(void *arg, dtrace_id_t id, void *parg); void (*dtps_disable)(void *arg, dtrace_id_t id, void *parg); void (*dtps_suspend)(void *arg, dtrace_id_t id, void *parg); void (*dtps_resume)(void *arg, dtrace_id_t id, void *parg); void (*dtps_getargdesc)(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc); uint64_t (*dtps_getargval)(void *arg, dtrace_id_t id, void *parg, int argno, int aframes); int (*dtps_usermode)(void *arg, dtrace_id_t id, void *parg); void (*dtps_destroy)(void *arg, dtrace_id_t id, void *parg); } dtrace_pops_t; #define DTRACE_MODE_KERNEL 0x01 #define DTRACE_MODE_USER 0x02 #define DTRACE_MODE_NOPRIV_DROP 0x10 #define DTRACE_MODE_NOPRIV_RESTRICT 0x20 #define DTRACE_MODE_LIMITEDPRIV_RESTRICT 0x40 typedef uintptr_t dtrace_provider_id_t; extern int dtrace_register(const char *, const dtrace_pattr_t *, uint32_t, cred_t *, const dtrace_pops_t *, void *, dtrace_provider_id_t *); extern int dtrace_unregister(dtrace_provider_id_t); extern int dtrace_condense(dtrace_provider_id_t); extern void dtrace_invalidate(dtrace_provider_id_t); extern dtrace_id_t dtrace_probe_lookup(dtrace_provider_id_t, char *, char *, char *); extern dtrace_id_t dtrace_probe_create(dtrace_provider_id_t, const char *, const char *, const char *, int, void *); extern void *dtrace_probe_arg(dtrace_provider_id_t, dtrace_id_t); extern void dtrace_probe(dtrace_id_t, uintptr_t arg0, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4); /* * DTrace Meta Provider API * * The following functions are implemented by the DTrace framework and are * used to implement meta providers. Meta providers plug into the DTrace * framework and are used to instantiate new providers on the fly. At * present, there is only one type of meta provider and only one meta * provider may be registered with the DTrace framework at a time. The * sole meta provider type provides user-land static tracing facilities * by taking meta probe descriptions and adding a corresponding provider * into the DTrace framework. * * 1 Framework-to-Provider * * 1.1 Overview * * The Framework-to-Provider API is represented by the dtrace_mops structure * that the meta provider passes to the framework when registering itself as * a meta provider. This structure consists of the following members: * * dtms_create_probe() <-- Add a new probe to a created provider * dtms_provide_pid() <-- Create a new provider for a given process * dtms_remove_pid() <-- Remove a previously created provider * * 1.2 void dtms_create_probe(void *arg, void *parg, * dtrace_helper_probedesc_t *probedesc); * * 1.2.1 Overview * * Called by the DTrace framework to create a new probe in a provider * created by this meta provider. * * 1.2.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_meta_register(). * The second argument is the provider cookie for the associated provider; * this is obtained from the return value of dtms_provide_pid(). The third * argument is the helper probe description. * * 1.2.3 Return value * * None * * 1.2.4 Caller's context * * dtms_create_probe() is called from either ioctl() or module load context. * The DTrace framework is locked in such a way that meta providers may not * register or unregister. This means that the meta provider cannot call * dtrace_meta_register() or dtrace_meta_unregister(). However, the context is * such that the provider may (and is expected to) call provider-related * DTrace provider APIs including dtrace_probe_create(). * * 1.3 void *dtms_provide_pid(void *arg, dtrace_meta_provider_t *mprov, * pid_t pid) * * 1.3.1 Overview * * Called by the DTrace framework to instantiate a new provider given the * description of the provider and probes in the mprov argument. The * meta provider should call dtrace_register() to insert the new provider * into the DTrace framework. * * 1.3.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_meta_register(). * The second argument is a pointer to a structure describing the new * helper provider. The third argument is the process identifier for * process associated with this new provider. Note that the name of the * provider as passed to dtrace_register() should be the contatenation of * the dtmpb_provname member of the mprov argument and the processs * identifier as a string. * * 1.3.3 Return value * * The cookie for the provider that the meta provider creates. This is * the same value that it passed to dtrace_register(). * * 1.3.4 Caller's context * * dtms_provide_pid() is called from either ioctl() or module load context. * The DTrace framework is locked in such a way that meta providers may not * register or unregister. This means that the meta provider cannot call * dtrace_meta_register() or dtrace_meta_unregister(). However, the context * is such that the provider may -- and is expected to -- call * provider-related DTrace provider APIs including dtrace_register(). * * 1.4 void dtms_remove_pid(void *arg, dtrace_meta_provider_t *mprov, * pid_t pid) * * 1.4.1 Overview * * Called by the DTrace framework to remove a provider that had previously * been instantiated via the dtms_provide_pid() entry point. The meta * provider need not remove the provider immediately, but this entry * point indicates that the provider should be removed as soon as possible * using the dtrace_unregister() API. * * 1.4.2 Arguments and notes * * The first argument is the cookie as passed to dtrace_meta_register(). * The second argument is a pointer to a structure describing the helper * provider. The third argument is the process identifier for process * associated with this new provider. * * 1.4.3 Return value * * None * * 1.4.4 Caller's context * * dtms_remove_pid() is called from either ioctl() or exit() context. * The DTrace framework is locked in such a way that meta providers may not * register or unregister. This means that the meta provider cannot call * dtrace_meta_register() or dtrace_meta_unregister(). However, the context * is such that the provider may -- and is expected to -- call * provider-related DTrace provider APIs including dtrace_unregister(). */ typedef struct dtrace_helper_probedesc { char *dthpb_mod; /* probe module */ char *dthpb_func; /* probe function */ char *dthpb_name; /* probe name */ uint64_t dthpb_base; /* base address */ uint32_t *dthpb_offs; /* offsets array */ uint32_t *dthpb_enoffs; /* is-enabled offsets array */ uint32_t dthpb_noffs; /* offsets count */ uint32_t dthpb_nenoffs; /* is-enabled offsets count */ uint8_t *dthpb_args; /* argument mapping array */ uint8_t dthpb_xargc; /* translated argument count */ uint8_t dthpb_nargc; /* native argument count */ char *dthpb_xtypes; /* translated types strings */ char *dthpb_ntypes; /* native types strings */ } dtrace_helper_probedesc_t; typedef struct dtrace_helper_provdesc { char *dthpv_provname; /* provider name */ dtrace_pattr_t dthpv_pattr; /* stability attributes */ } dtrace_helper_provdesc_t; typedef struct dtrace_mops { void (*dtms_create_probe)(void *, void *, dtrace_helper_probedesc_t *); void *(*dtms_provide_pid)(void *, dtrace_helper_provdesc_t *, pid_t); void (*dtms_remove_pid)(void *, dtrace_helper_provdesc_t *, pid_t); } dtrace_mops_t; typedef uintptr_t dtrace_meta_provider_id_t; extern int dtrace_meta_register(const char *, const dtrace_mops_t *, void *, dtrace_meta_provider_id_t *); extern int dtrace_meta_unregister(dtrace_meta_provider_id_t); /* * DTrace Kernel Hooks * * The following functions are implemented by the base kernel and form a set of * hooks used by the DTrace framework. DTrace hooks are implemented in either * uts/common/os/dtrace_subr.c, an ISA-specific assembly file, or in a * uts//os/dtrace_subr.c corresponding to each hardware platform. */ typedef enum dtrace_vtime_state { DTRACE_VTIME_INACTIVE = 0, /* No DTrace, no TNF */ DTRACE_VTIME_ACTIVE, /* DTrace virtual time, no TNF */ DTRACE_VTIME_INACTIVE_TNF, /* No DTrace, TNF active */ DTRACE_VTIME_ACTIVE_TNF /* DTrace virtual time _and_ TNF */ } dtrace_vtime_state_t; #ifdef illumos extern dtrace_vtime_state_t dtrace_vtime_active; #endif extern void dtrace_vtime_switch(kthread_t *next); extern void dtrace_vtime_enable_tnf(void); extern void dtrace_vtime_disable_tnf(void); extern void dtrace_vtime_enable(void); extern void dtrace_vtime_disable(void); struct regs; struct reg; #ifdef illumos extern int (*dtrace_pid_probe_ptr)(struct reg *); extern int (*dtrace_return_probe_ptr)(struct reg *); extern void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *); extern void (*dtrace_fasttrap_exec_ptr)(proc_t *); extern void (*dtrace_fasttrap_exit_ptr)(proc_t *); extern void dtrace_fasttrap_fork(proc_t *, proc_t *); #endif typedef uintptr_t dtrace_icookie_t; typedef void (*dtrace_xcall_t)(void *); extern dtrace_icookie_t dtrace_interrupt_disable(void); extern void dtrace_interrupt_enable(dtrace_icookie_t); extern void dtrace_membar_producer(void); extern void dtrace_membar_consumer(void); extern void (*dtrace_cpu_init)(processorid_t); #ifdef illumos extern void (*dtrace_modload)(modctl_t *); extern void (*dtrace_modunload)(modctl_t *); #endif extern void (*dtrace_helpers_cleanup)(void); extern void (*dtrace_helpers_fork)(proc_t *parent, proc_t *child); extern void (*dtrace_cpustart_init)(void); extern void (*dtrace_cpustart_fini)(void); extern void (*dtrace_closef)(void); extern void (*dtrace_debugger_init)(void); extern void (*dtrace_debugger_fini)(void); extern dtrace_cacheid_t dtrace_predcache_id; #ifdef illumos extern hrtime_t dtrace_gethrtime(void); #else void dtrace_debug_printf(const char *, ...) __printflike(1, 2); #endif extern void dtrace_sync(void); extern void dtrace_toxic_ranges(void (*)(uintptr_t, uintptr_t)); extern void dtrace_xcall(processorid_t, dtrace_xcall_t, void *); extern void dtrace_vpanic(const char *, __va_list); extern void dtrace_panic(const char *, ...); extern int dtrace_safe_defer_signal(void); extern void dtrace_safe_synchronous_signal(void); extern int dtrace_mach_aframes(void); #if defined(__i386) || defined(__amd64) extern int dtrace_instr_size(uchar_t *instr); extern int dtrace_instr_size_isa(uchar_t *, model_t, int *); extern void dtrace_invop_callsite(void); #endif extern void dtrace_invop_add(int (*)(uintptr_t, uintptr_t *, uintptr_t)); extern void dtrace_invop_remove(int (*)(uintptr_t, uintptr_t *, uintptr_t)); #ifdef __sparc extern int dtrace_blksuword32(uintptr_t, uint32_t *, int); extern void dtrace_getfsr(uint64_t *); #endif #ifndef illumos extern void dtrace_helpers_duplicate(proc_t *, proc_t *); extern void dtrace_helpers_destroy(proc_t *); #endif #define DTRACE_CPUFLAG_ISSET(flag) \ (cpu_core[curcpu].cpuc_dtrace_flags & (flag)) #define DTRACE_CPUFLAG_SET(flag) \ (cpu_core[curcpu].cpuc_dtrace_flags |= (flag)) #define DTRACE_CPUFLAG_CLEAR(flag) \ (cpu_core[curcpu].cpuc_dtrace_flags &= ~(flag)) #endif /* _KERNEL */ #endif /* _ASM */ #if defined(__i386) || defined(__amd64) #define DTRACE_INVOP_PUSHL_EBP 1 #define DTRACE_INVOP_POPL_EBP 2 #define DTRACE_INVOP_LEAVE 3 #define DTRACE_INVOP_NOP 4 #define DTRACE_INVOP_RET 5 #elif defined(__powerpc__) #define DTRACE_INVOP_RET 1 #define DTRACE_INVOP_BCTR 2 #define DTRACE_INVOP_BLR 3 #define DTRACE_INVOP_JUMP 4 #define DTRACE_INVOP_MFLR_R0 5 #define DTRACE_INVOP_NOP 6 +#elif defined(__arm__) + +#define DTRACE_INVOP_PUSHM 1 +#define DTRACE_INVOP_POPM 2 +#define DTRACE_INVOP_B 3 + + #endif #ifdef __cplusplus } #endif #endif /* _SYS_DTRACE_H */ Index: head/sys/cddl/dev/dtrace/arm/dtrace_asm.S =================================================================== --- head/sys/cddl/dev/dtrace/arm/dtrace_asm.S (nonexistent) +++ head/sys/cddl/dev/dtrace/arm/dtrace_asm.S (revision 278529) @@ -0,0 +1,197 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#define _ASM +#define _LOCORE +#define LOCORE + +#include +#include + +#include +#include + +#include "assym.s" + +/* +void dtrace_membar_producer(void) +*/ +ENTRY(dtrace_membar_producer) + RET +END(dtrace_membar_producer) + +/* +void dtrace_membar_consumer(void) +*/ +ENTRY(dtrace_membar_consumer) + RET +END(dtrace_membar_consumer) + +/* +dtrace_icookie_t dtrace_interrupt_disable(void) +*/ +ENTRY(dtrace_interrupt_disable) + mrs r0, cpsr + mov r1, r0 + orr r1, r1, #(PSR_I | PSR_F) + msr cpsr_c, r1 + RET +END(dtrace_interrupt_disable) + +/* +void dtrace_interrupt_enable(dtrace_icookie_t cookie) +*/ +ENTRY(dtrace_interrupt_enable) + and r0, r0, #(PSR_I | PSR_F) + mrs r1, cpsr + bic r1, r1, #(PSR_I | PSR_F) + orr r1, r1, r0 + msr cpsr_c, r1 + RET +END(dtrace_interrupt_enable) +/* +uint8_t +dtrace_fuword8_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword8_nocheck) + ldrb r3, [r0] + mov r0, r3 + RET +END(dtrace_fuword8_nocheck) + +/* +uint16_t +dtrace_fuword16_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword16_nocheck) + ldrh r3, [r0] + mov r0, r3 + RET +END(dtrace_fuword16_nocheck) + +/* +uint32_t +dtrace_fuword32_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword32_nocheck) + ldr r3, [r0] + mov r0, r3 + RET +END(dtrace_fuword32_nocheck) + +/* +uint64_t +dtrace_fuword64_nocheck(void *addr) +*/ +ENTRY(dtrace_fuword64_nocheck) + ldm r0, {r2, r3} + + mov r0, r2 + mov r1, r3 +#if defined(__BIG_ENDIAN__) +/* big endian */ + mov r0, r3 + mov r1, r2 +#else +/* little endian */ + mov r0, r2 + mov r1, r3 + +#endif + RET +END(dtrace_fuword64_nocheck) + +/* +void +dtrace_copy(uintptr_t uaddr, uintptr_t kaddr, size_t size) +*/ +ENTRY(dtrace_copy) + stmfd sp!, {r4-r5} /* stack is 8 byte aligned */ + teq r2, #0x00000000 + mov r5, #0x00000000 + beq 2f + +1: ldrb r4, [r0], #0x0001 + add r5, r5, #0x00000001 + strb r4, [r1], #0x0001 + teqne r5, r2 + bne 1b + +2: ldmfd sp!, {r4-r5} /* stack is 8 byte aligned */ + RET +END(dtrace_copy) + +/* +void +dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +XXX: Check for flags? +*/ +ENTRY(dtrace_copystr) + stmfd sp!, {r4-r5} /* stack is 8 byte aligned */ + teq r2, #0x00000000 + mov r5, #0x00000000 + beq 2f + +1: ldrb r4, [r0], #0x0001 + add r5, r5, #0x00000001 + teq r4, #0x00000000 + strb r4, [r1], #0x0001 + teqne r5, r2 + bne 1b + +2: ldmfd sp!, {r4-r5} /* stack is 8 byte aligned */ + RET +END(dtrace_copystr) + +/* +void +vpanic(const char *format, va_list alist) +*/ +ENTRY(vpanic) /* Initial stack layout: */ +vpanic_common: + RET +END(vpanic) + +/* +void +dtrace_vpanic(const char *format, va_list alist) +*/ +ENTRY(dtrace_vpanic) /* Initial stack layout: */ + b vpanic + RET +END(dtrace_vpanic) /* Initial stack layout: */ + +/* +uintptr_t +dtrace_caller(int aframes) +*/ +ENTRY(dtrace_caller) + mov r0, #-1 + RET +END(dtrace_caller) Property changes on: head/sys/cddl/dev/dtrace/arm/dtrace_asm.S ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/cddl/dev/dtrace/arm/dtrace_isa.c =================================================================== --- head/sys/cddl/dev/dtrace/arm/dtrace_isa.c (nonexistent) +++ head/sys/cddl/dev/dtrace/arm/dtrace_isa.c (revision 278529) @@ -0,0 +1,356 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "regset.h" + +/* + * Wee need some reasonable default to prevent backtrace code + * from wandering too far + */ +#define MAX_FUNCTION_SIZE 0x10000 +#define MAX_PROLOGUE_SIZE 0x100 + + +uint8_t dtrace_fuword8_nocheck(void *); +uint16_t dtrace_fuword16_nocheck(void *); +uint32_t dtrace_fuword32_nocheck(void *); +uint64_t dtrace_fuword64_nocheck(void *); + +void +dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes, + uint32_t *intrpc) +{ + u_int32_t *frame, *lastframe; + int scp_offset; + int depth = 0; + pc_t caller = (pc_t) solaris_cpu[curcpu].cpu_dtrace_caller; + + if (intrpc != 0) + pcstack[depth++] = (pc_t) intrpc; + + aframes++; + + frame = (u_int32_t *)__builtin_frame_address(0);; + lastframe = NULL; + scp_offset = -(get_pc_str_offset() >> 2); + + while ((frame != NULL) && (depth < pcstack_limit)) { + db_addr_t scp; +#if 0 + u_int32_t savecode; + int r; + u_int32_t *rp; +#endif + + /* + * In theory, the SCP isn't guaranteed to be in the function + * that generated the stack frame. We hope for the best. + */ + scp = frame[FR_SCP]; + + if (aframes > 0) { + aframes--; + if ((aframes == 0) && (caller != 0)) { + pcstack[depth++] = caller; + } + } + else { + pcstack[depth++] = scp; + } + +#if 0 + savecode = ((u_int32_t *)scp)[scp_offset]; + if ((savecode & 0x0e100000) == 0x08000000) { + /* Looks like an STM */ + rp = frame - 4; + for (r = 10; r >= 0; r--) { + if (savecode & (1 << r)) { + /* register r == *rp-- */ + } + } + } +#endif + + /* + * Switch to next frame up + */ + if (frame[FR_RFP] == 0) + break; /* Top of stack */ + + lastframe = frame; + frame = (u_int32_t *)(frame[FR_RFP]); + + if (INKERNEL((int)frame)) { + /* staying in kernel */ + if (frame <= lastframe) { + /* bad frame pointer */ + break; + } + } + else + break; + } + + for (; depth < pcstack_limit; depth++) { + pcstack[depth] = 0; + } +} + +void +dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit) +{ + printf("IMPLEMENT ME: %s\n", __func__); +} + +int +dtrace_getustackdepth(void) +{ + printf("IMPLEMENT ME: %s\n", __func__); + return (0); +} + +void +dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit) +{ + printf("IMPLEMENT ME: %s\n", __func__); +} + +/*ARGSUSED*/ +uint64_t +dtrace_getarg(int arg, int aframes) +{ +/* struct arm_frame *fp = (struct arm_frame *)dtrace_getfp();*/ + + return (0); +} + +int +dtrace_getstackdepth(int aframes) +{ + u_int32_t *frame, *lastframe; + int scp_offset; + int depth = 1; + + frame = (u_int32_t *)__builtin_frame_address(0);; + lastframe = NULL; + scp_offset = -(get_pc_str_offset() >> 2); + + while (frame != NULL) { + db_addr_t scp; +#if 0 + u_int32_t savecode; + int r; + u_int32_t *rp; +#endif + + /* + * In theory, the SCP isn't guaranteed to be in the function + * that generated the stack frame. We hope for the best. + */ + scp = frame[FR_SCP]; + + depth++; + + /* + * Switch to next frame up + */ + if (frame[FR_RFP] == 0) + break; /* Top of stack */ + + lastframe = frame; + frame = (u_int32_t *)(frame[FR_RFP]); + + if (INKERNEL((int)frame)) { + /* staying in kernel */ + if (frame <= lastframe) { + /* bad frame pointer */ + break; + } + } + else + break; + } + + if (depth < aframes) + return 0; + else + return depth - aframes; + +} + +ulong_t +dtrace_getreg(struct trapframe *rp, uint_t reg) +{ + printf("IMPLEMENT ME: %s\n", __func__); + + return (0); +} + +static int +dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size) +{ + + if (uaddr + size > VM_MAXUSER_ADDRESS || uaddr + size < uaddr) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = uaddr; + return (0); + } + + return (1); +} + +void +dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(uaddr, kaddr, size); +} + +void +dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copy(kaddr, uaddr, size); +} + +void +dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(uaddr, kaddr, size, flags); +} + +void +dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size, + volatile uint16_t *flags) +{ + if (dtrace_copycheck(uaddr, kaddr, size)) + dtrace_copystr(kaddr, uaddr, size, flags); +} + +uint8_t +dtrace_fuword8(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword8_nocheck(uaddr)); +} + +uint16_t +dtrace_fuword16(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword16_nocheck(uaddr)); +} + +uint32_t +dtrace_fuword32(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword32_nocheck(uaddr)); +} + +uint64_t +dtrace_fuword64(void *uaddr) +{ + if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) { + DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); + cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr; + return (0); + } + return (dtrace_fuword64_nocheck(uaddr)); +} + +#define __with_interrupts_disabled(expr) \ + do { \ + u_int cpsr_save, tmp; \ + \ + __asm __volatile( \ + "mrs %0, cpsr;" \ + "orr %1, %0, %2;" \ + "msr cpsr_fsxc, %1;" \ + : "=r" (cpsr_save), "=r" (tmp) \ + : "I" (PSR_I | PSR_F) \ + : "cc" ); \ + (expr); \ + __asm __volatile( \ + "msr cpsr_fsxc, %0" \ + : /* no output */ \ + : "r" (cpsr_save) \ + : "cc" ); \ + } while(0) + +uint32_t dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) +{ + return atomic_cmpset_32((uint32_t*)target, (uint32_t)cmp, (uint32_t)new); + +} + +void * dtrace_casptr(volatile void *target, volatile void *cmp, volatile void *new) +{ + return (void*)dtrace_cas32((uint32_t*)target, (uint32_t)cmp, (uint32_t)new); +} + Property changes on: head/sys/cddl/dev/dtrace/arm/dtrace_isa.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/cddl/dev/dtrace/arm/dtrace_subr.c =================================================================== --- head/sys/cddl/dev/dtrace/arm/dtrace_subr.c (nonexistent) +++ head/sys/cddl/dev/dtrace/arm/dtrace_subr.c (revision 278529) @@ -0,0 +1,261 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DELAYBRANCH(x) ((int)(x) < 0) + +extern uintptr_t dtrace_in_probe_addr; +extern int dtrace_in_probe; +extern dtrace_id_t dtrace_probeid_error; +extern int (*dtrace_invop_jump_addr)(struct trapframe *); + +int dtrace_invop(uintptr_t, uintptr_t *, uintptr_t); +void dtrace_invop_init(void); +void dtrace_invop_uninit(void); + +typedef struct dtrace_invop_hdlr { + int (*dtih_func)(uintptr_t, uintptr_t *, uintptr_t); + struct dtrace_invop_hdlr *dtih_next; +} dtrace_invop_hdlr_t; + +dtrace_invop_hdlr_t *dtrace_invop_hdlr; + +int +dtrace_invop(uintptr_t addr, uintptr_t *stack, uintptr_t eax) +{ + dtrace_invop_hdlr_t *hdlr; + int rval; + + for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) + if ((rval = hdlr->dtih_func(addr, stack, eax)) != 0) + return (rval); + + return (0); +} + + +void +dtrace_invop_add(int (*func)(uintptr_t, uintptr_t *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr; + + hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP); + hdlr->dtih_func = func; + hdlr->dtih_next = dtrace_invop_hdlr; + dtrace_invop_hdlr = hdlr; +} + +void +dtrace_invop_remove(int (*func)(uintptr_t, uintptr_t *, uintptr_t)) +{ + dtrace_invop_hdlr_t *hdlr = dtrace_invop_hdlr, *prev = NULL; + + for (;;) { + if (hdlr == NULL) + panic("attempt to remove non-existent invop handler"); + + if (hdlr->dtih_func == func) + break; + + prev = hdlr; + hdlr = hdlr->dtih_next; + } + + if (prev == NULL) { + ASSERT(dtrace_invop_hdlr == hdlr); + dtrace_invop_hdlr = hdlr->dtih_next; + } else { + ASSERT(dtrace_invop_hdlr != hdlr); + prev->dtih_next = hdlr->dtih_next; + } + + kmem_free(hdlr, 0); +} + + +/*ARGSUSED*/ +void +dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) +{ + printf("IMPLEMENT ME: dtrace_toxic_ranges\n"); +} + +void +dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg) +{ + cpuset_t cpus; + + if (cpu == DTRACE_CPUALL) + cpus = all_cpus; + else + CPU_SETOF(cpu, &cpus); + + smp_rendezvous_cpus(cpus, smp_no_rendevous_barrier, func, + smp_no_rendevous_barrier, arg); +} + +static void +dtrace_sync_func(void) +{ +} + +void +dtrace_sync(void) +{ + dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL); +} + +/* + * DTrace needs a high resolution time function which can + * be called from a probe context and guaranteed not to have + * instrumented with probes itself. + * + * Returns nanoseconds since boot. + */ +uint64_t +dtrace_gethrtime() +{ + struct timespec curtime; + + nanouptime(&curtime); + + return (curtime.tv_sec * 1000000000UL + curtime.tv_nsec); + +} + +uint64_t +dtrace_gethrestime(void) +{ + struct timespec curtime; + + getnanotime(&curtime); + + return (curtime.tv_sec * 1000000000UL + curtime.tv_nsec); +} + +/* Function to handle DTrace traps during probes. See amd64/amd64/trap.c */ +int +dtrace_trap(struct trapframe *frame, u_int type) +{ + /* + * A trap can occur while DTrace executes a probe. Before + * executing the probe, DTrace blocks re-scheduling and sets + * a flag in it's per-cpu flags to indicate that it doesn't + * want to fault. On returning from the probe, the no-fault + * flag is cleared and finally re-scheduling is enabled. + * + * Check if DTrace has enabled 'no-fault' mode: + * + */ + if ((cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT) != 0) { + /* + * There are only a couple of trap types that are expected. + * All the rest will be handled in the usual way. + */ + switch (type) { + /* Page fault. */ + case FAULT_ALIGN: + /* Flag a bad address. */ + cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR; + cpu_core[curcpu].cpuc_dtrace_illval = 0; + + /* + * Offset the instruction pointer to the instruction + * following the one causing the fault. + */ + frame->tf_pc += sizeof(int); + return (1); + default: + /* Handle all other traps in the usual way. */ + break; + } + } + + /* Handle the trap in the usual way. */ + return (0); +} + +void +dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, + int fault, int fltoffs, uintptr_t illval) +{ + + dtrace_probe(dtrace_probeid_error, (uint64_t)(uintptr_t)state, + (uintptr_t)epid, + (uintptr_t)which, (uintptr_t)fault, (uintptr_t)fltoffs); +} + +static int +dtrace_invop_start(struct trapframe *frame) +{ + printf("IMPLEMENT ME: %s\n", __func__); + switch (dtrace_invop(frame->tf_pc, (uintptr_t *)frame, frame->tf_pc)) { + case DTRACE_INVOP_PUSHM: + // TODO: + break; + case DTRACE_INVOP_POPM: + // TODO: + break; + case DTRACE_INVOP_B: + // TODO + break; + default: + return (-1); + break; + } + + return (0); +} + +void dtrace_invop_init(void) +{ + dtrace_invop_jump_addr = dtrace_invop_start; +} + +void dtrace_invop_uninit(void) +{ + dtrace_invop_jump_addr = 0; +} Property changes on: head/sys/cddl/dev/dtrace/arm/dtrace_subr.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/cddl/dev/dtrace/arm/regset.h =================================================================== --- head/sys/cddl/dev/dtrace/arm/regset.h (nonexistent) +++ head/sys/cddl/dev/dtrace/arm/regset.h (revision 278529) @@ -0,0 +1,57 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ + +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + +#ifndef _REGSET_H +#define _REGSET_H + +/* + * #pragma ident "@(#)regset.h 1.11 05/06/08 SMI" + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#if 0 +#define REG_LINK R14 +#define REG_SP R12 +#define REG_PS R0 +#define REG_R0 R0 +#define REG_R1 R1 +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _REGSET_H */ Property changes on: head/sys/cddl/dev/dtrace/arm/regset.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/cddl/dev/fbt/arm/fbt_isa.c =================================================================== --- head/sys/cddl/dev/fbt/arm/fbt_isa.c (nonexistent) +++ head/sys/cddl/dev/fbt/arm/fbt_isa.c (revision 278529) @@ -0,0 +1,1303 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * Portions Copyright 2006-2008 John Birrell jb@freebsd.org + * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org + * Portions Copyright 2013 Howard Su howardsu@freebsd.org + * + * $FreeBSD$ + * + */ + +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static MALLOC_DEFINE(M_FBT, "fbt", "Function Boundary Tracing"); + +#define FBT_PATCHVAL 0xe06a0cfe // illegal instruction + +#define FBT_PUSHM 0xe92d0000 +#define FBT_POPM 0xe8bd0000 +#define FBT_JUMP 0xea000000 + +static d_open_t fbt_open; +static int fbt_unload(void); +static void fbt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *); +static void fbt_provide_module(void *, modctl_t *); +static void fbt_destroy(void *, dtrace_id_t, void *); +static void fbt_enable(void *, dtrace_id_t, void *); +static void fbt_disable(void *, dtrace_id_t, void *); +static void fbt_load(void *); +static void fbt_suspend(void *, dtrace_id_t, void *); +static void fbt_resume(void *, dtrace_id_t, void *); + +#define FBT_ENTRY "entry" +#define FBT_RETURN "return" +#define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask) +#define FBT_PROBETAB_SIZE 0x8000 /* 32k entries -- 128K total */ + +static struct cdevsw fbt_cdevsw = { + .d_version = D_VERSION, + .d_open = fbt_open, + .d_name = "fbt", +}; + +static dtrace_pattr_t fbt_attr = { +{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, +{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, +{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, +{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, +{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, +}; + +static dtrace_pops_t fbt_pops = { + NULL, + fbt_provide_module, + fbt_enable, + fbt_disable, + fbt_suspend, + fbt_resume, + fbt_getargdesc, + NULL, + NULL, + fbt_destroy +}; + +typedef struct fbt_probe { + struct fbt_probe *fbtp_hashnext; + uint32_t *fbtp_patchpoint; + int8_t fbtp_rval; + uint32_t fbtp_patchval; + uint32_t fbtp_savedval; + uintptr_t fbtp_roffset; + dtrace_id_t fbtp_id; + const char *fbtp_name; + modctl_t *fbtp_ctl; + int fbtp_loadcnt; + int fbtp_primary; + int fbtp_invop_cnt; + int fbtp_symindx; + struct fbt_probe *fbtp_next; +} fbt_probe_t; + +static struct cdev *fbt_cdev; +static dtrace_provider_id_t fbt_id; +static fbt_probe_t **fbt_probetab; +static int fbt_probetab_size; +static int fbt_probetab_mask; +static int fbt_verbose = 0; + +static int +fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval) +{ + struct trapframe *frame = (struct trapframe *)stack; + solaris_cpu_t *cpu = &solaris_cpu[curcpu]; + fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; + + for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { + if ((uintptr_t)fbt->fbtp_patchpoint == addr) { + fbt->fbtp_invop_cnt++; + cpu->cpu_dtrace_caller = addr; + + dtrace_probe(fbt->fbtp_id, frame->tf_r0, + frame->tf_r1, frame->tf_r2, + frame->tf_r3, 0); // TODO: Need 5th parameter from stack + + cpu->cpu_dtrace_caller = 0; + + return (fbt->fbtp_rval); + } + } + + return (0); +} + +static int +fbt_provide_module_function(linker_file_t lf, int symindx, + linker_symval_t *symval, void *opaque) +{ + char *modname = opaque; + const char *name = symval->name; + fbt_probe_t *fbt, *retfbt; + int popm; + u_int32_t *instr, *limit; + + if (strncmp(name, "dtrace_", 7) == 0 && + strncmp(name, "dtrace_safe_", 12) != 0) { + /* + * Anything beginning with "dtrace_" may be called + * from probe context unless it explicitly indicates + * that it won't be called from probe context by + * using the prefix "dtrace_safe_". + */ + return (0); + } + + if (name[0] == '_' && name[1] == '_') + return (0); + + instr = (u_int32_t *) symval->value; + limit = (u_int32_t *)(symval->value + symval->size); + + for (; instr < limit; instr++) + if ((*instr & 0xffff0000) == FBT_PUSHM && (*instr & 0x4000) != 0) + break; + + if (instr >= limit) + return (0); + + fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); + fbt->fbtp_name = name; + fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, + name, FBT_ENTRY, 3, fbt); + fbt->fbtp_patchpoint = instr; + fbt->fbtp_ctl = lf; + fbt->fbtp_loadcnt = lf->loadcnt; + fbt->fbtp_savedval = *instr; + fbt->fbtp_patchval = FBT_PATCHVAL; + fbt->fbtp_rval = DTRACE_INVOP_PUSHM; + fbt->fbtp_symindx = symindx; + + fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; + fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; + + lf->fbt_nentries++; + + popm = FBT_POPM | ((*instr) & 0x3FFF) | 0x8000; + + + retfbt = NULL; +again: + for(; instr < limit; instr++) + { + if (*instr == popm) + break; + else if ((*instr & 0xff000000) == FBT_JUMP) + { + int offset; + u_int32_t *target, *start; + offset = (*instr & 0xffffff); + offset <<= 8; + offset /= 64; + target = instr + (2 + offset); + start = (u_int32_t *) symval->value; + if (target >= limit || target < start) + break; + instr++; //skip delay slot + } + } + + if (instr >= limit) + return (0); + + /* + * We have a winner! + */ + fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); + fbt->fbtp_name = name; + if (retfbt == NULL) { + fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, + name, FBT_RETURN, 5, fbt); + } else { + retfbt->fbtp_next = fbt; + fbt->fbtp_id = retfbt->fbtp_id; + } + retfbt = fbt; + + fbt->fbtp_patchpoint = instr; + fbt->fbtp_ctl = lf; + fbt->fbtp_loadcnt = lf->loadcnt; + fbt->fbtp_symindx = symindx; + if ((*instr & 0xff000000) == FBT_JUMP) + fbt->fbtp_rval = DTRACE_INVOP_B; + else + fbt->fbtp_rval = DTRACE_INVOP_POPM; + fbt->fbtp_savedval = *instr; + fbt->fbtp_patchval = FBT_PATCHVAL; + fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; + fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; + + lf->fbt_nentries++; + + instr++; + goto again; +} + +static void +fbt_provide_module(void *arg, modctl_t *lf) +{ + char modname[MAXPATHLEN]; + int i; + size_t len; + + strlcpy(modname, lf->filename, sizeof(modname)); + len = strlen(modname); + if (len > 3 && strcmp(modname + len - 3, ".ko") == 0) + modname[len - 3] = '\0'; + + /* + * Employees of dtrace and their families are ineligible. Void + * where prohibited. + */ + if (strcmp(modname, "dtrace") == 0) + return; + + /* + * The cyclic timer subsystem can be built as a module and DTrace + * depends on that, so it is ineligible too. + */ + if (strcmp(modname, "cyclic") == 0) + return; + + /* + * To register with DTrace, a module must list 'dtrace' as a + * dependency in order for the kernel linker to resolve + * symbols like dtrace_register(). All modules with such a + * dependency are ineligible for FBT tracing. + */ + for (i = 0; i < lf->ndeps; i++) + if (strncmp(lf->deps[i]->filename, "dtrace", 6) == 0) + return; + + if (lf->fbt_nentries) { + /* + * This module has some FBT entries allocated; we're afraid + * to screw with it. + */ + return; + } + + /* + * List the functions in the module and the symbol values. + */ + (void) linker_file_function_listall(lf, fbt_provide_module_function, modname); +} + +static void +fbt_destroy(void *arg, dtrace_id_t id, void *parg) +{ + fbt_probe_t *fbt = parg, *next, *hash, *last; + modctl_t *ctl; + int ndx; + + do { + ctl = fbt->fbtp_ctl; + + ctl->fbt_nentries--; + + /* + * Now we need to remove this probe from the fbt_probetab. + */ + ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint); + last = NULL; + hash = fbt_probetab[ndx]; + + while (hash != fbt) { + ASSERT(hash != NULL); + last = hash; + hash = hash->fbtp_hashnext; + } + + if (last != NULL) { + last->fbtp_hashnext = fbt->fbtp_hashnext; + } else { + fbt_probetab[ndx] = fbt->fbtp_hashnext; + } + + next = fbt->fbtp_next; + free(fbt, M_FBT); + + fbt = next; + } while (fbt != NULL); +} + +static void +fbt_enable(void *arg, dtrace_id_t id, void *parg) +{ + fbt_probe_t *fbt = parg; + modctl_t *ctl = fbt->fbtp_ctl; + + ctl->nenabled++; + + /* + * Now check that our modctl has the expected load count. If it + * doesn't, this module must have been unloaded and reloaded -- and + * we're not going to touch it. + */ + if (ctl->loadcnt != fbt->fbtp_loadcnt) { + if (fbt_verbose) { + printf("fbt is failing for probe %s " + "(module %s reloaded)", + fbt->fbtp_name, ctl->filename); + } + + return; + } + + for (; fbt != NULL; fbt = fbt->fbtp_next) { + *fbt->fbtp_patchpoint = fbt->fbtp_patchval; + cpu_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); + } +} + +static void +fbt_disable(void *arg, dtrace_id_t id, void *parg) +{ + fbt_probe_t *fbt = parg; + modctl_t *ctl = fbt->fbtp_ctl; + + ASSERT(ctl->nenabled > 0); + ctl->nenabled--; + + if ((ctl->loadcnt != fbt->fbtp_loadcnt)) + return; + + for (; fbt != NULL; fbt = fbt->fbtp_next) { + *fbt->fbtp_patchpoint = fbt->fbtp_savedval; + cpu_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); + } +} + +static void +fbt_suspend(void *arg, dtrace_id_t id, void *parg) +{ + fbt_probe_t *fbt = parg; + modctl_t *ctl = fbt->fbtp_ctl; + + ASSERT(ctl->nenabled > 0); + + if ((ctl->loadcnt != fbt->fbtp_loadcnt)) + return; + + for (; fbt != NULL; fbt = fbt->fbtp_next) { + *fbt->fbtp_patchpoint = fbt->fbtp_savedval; + cpu_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); + } +} + +static void +fbt_resume(void *arg, dtrace_id_t id, void *parg) +{ + fbt_probe_t *fbt = parg; + modctl_t *ctl = fbt->fbtp_ctl; + + ASSERT(ctl->nenabled > 0); + + if ((ctl->loadcnt != fbt->fbtp_loadcnt)) + return; + + for (; fbt != NULL; fbt = fbt->fbtp_next) { + *fbt->fbtp_patchpoint = fbt->fbtp_patchval; + cpu_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); + } +} + +static int +fbt_ctfoff_init(modctl_t *lf, linker_ctf_t *lc) +{ + const Elf_Sym *symp = lc->symtab;; + const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; + const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t); + int i; + uint32_t *ctfoff; + uint32_t objtoff = hp->cth_objtoff; + uint32_t funcoff = hp->cth_funcoff; + ushort_t info; + ushort_t vlen; + + /* Sanity check. */ + if (hp->cth_magic != CTF_MAGIC) { + printf("Bad magic value in CTF data of '%s'\n",lf->pathname); + return (EINVAL); + } + + if (lc->symtab == NULL) { + printf("No symbol table in '%s'\n",lf->pathname); + return (EINVAL); + } + + if ((ctfoff = malloc(sizeof(uint32_t) * lc->nsym, M_LINKER, M_WAITOK)) == NULL) + return (ENOMEM); + + *lc->ctfoffp = ctfoff; + + for (i = 0; i < lc->nsym; i++, ctfoff++, symp++) { + if (symp->st_name == 0 || symp->st_shndx == SHN_UNDEF) { + *ctfoff = 0xffffffff; + continue; + } + + switch (ELF_ST_TYPE(symp->st_info)) { + case STT_OBJECT: + if (objtoff >= hp->cth_funcoff || + (symp->st_shndx == SHN_ABS && symp->st_value == 0)) { + *ctfoff = 0xffffffff; + break; + } + + *ctfoff = objtoff; + objtoff += sizeof (ushort_t); + break; + + case STT_FUNC: + if (funcoff >= hp->cth_typeoff) { + *ctfoff = 0xffffffff; + break; + } + + *ctfoff = funcoff; + + info = *((const ushort_t *)(ctfdata + funcoff)); + vlen = CTF_INFO_VLEN(info); + + /* + * If we encounter a zero pad at the end, just skip it. + * Otherwise skip over the function and its return type + * (+2) and the argument list (vlen). + */ + if (CTF_INFO_KIND(info) == CTF_K_UNKNOWN && vlen == 0) + funcoff += sizeof (ushort_t); /* skip pad */ + else + funcoff += sizeof (ushort_t) * (vlen + 2); + break; + + default: + *ctfoff = 0xffffffff; + break; + } + } + + return (0); +} + +static ssize_t +fbt_get_ctt_size(uint8_t version, const ctf_type_t *tp, ssize_t *sizep, + ssize_t *incrementp) +{ + ssize_t size, increment; + + if (version > CTF_VERSION_1 && + tp->ctt_size == CTF_LSIZE_SENT) { + size = CTF_TYPE_LSIZE(tp); + increment = sizeof (ctf_type_t); + } else { + size = tp->ctt_size; + increment = sizeof (ctf_stype_t); + } + + if (sizep) + *sizep = size; + if (incrementp) + *incrementp = increment; + + return (size); +} + +static int +fbt_typoff_init(linker_ctf_t *lc) +{ + const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; + const ctf_type_t *tbuf; + const ctf_type_t *tend; + const ctf_type_t *tp; + const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t); + int ctf_typemax = 0; + uint32_t *xp; + ulong_t pop[CTF_K_MAX + 1] = { 0 }; + + + /* Sanity check. */ + if (hp->cth_magic != CTF_MAGIC) + return (EINVAL); + + tbuf = (const ctf_type_t *) (ctfdata + hp->cth_typeoff); + tend = (const ctf_type_t *) (ctfdata + hp->cth_stroff); + + int child = hp->cth_parname != 0; + + /* + * We make two passes through the entire type section. In this first + * pass, we count the number of each type and the total number of types. + */ + for (tp = tbuf; tp < tend; ctf_typemax++) { + ushort_t kind = CTF_INFO_KIND(tp->ctt_info); + ulong_t vlen = CTF_INFO_VLEN(tp->ctt_info); + ssize_t size, increment; + + size_t vbytes; + uint_t n; + + (void) fbt_get_ctt_size(hp->cth_version, tp, &size, &increment); + + switch (kind) { + case CTF_K_INTEGER: + case CTF_K_FLOAT: + vbytes = sizeof (uint_t); + break; + case CTF_K_ARRAY: + vbytes = sizeof (ctf_array_t); + break; + case CTF_K_FUNCTION: + vbytes = sizeof (ushort_t) * (vlen + (vlen & 1)); + break; + case CTF_K_STRUCT: + case CTF_K_UNION: + if (size < CTF_LSTRUCT_THRESH) { + ctf_member_t *mp = (ctf_member_t *) + ((uintptr_t)tp + increment); + + vbytes = sizeof (ctf_member_t) * vlen; + for (n = vlen; n != 0; n--, mp++) + child |= CTF_TYPE_ISCHILD(mp->ctm_type); + } else { + ctf_lmember_t *lmp = (ctf_lmember_t *) + ((uintptr_t)tp + increment); + + vbytes = sizeof (ctf_lmember_t) * vlen; + for (n = vlen; n != 0; n--, lmp++) + child |= + CTF_TYPE_ISCHILD(lmp->ctlm_type); + } + break; + case CTF_K_ENUM: + vbytes = sizeof (ctf_enum_t) * vlen; + break; + case CTF_K_FORWARD: + /* + * For forward declarations, ctt_type is the CTF_K_* + * kind for the tag, so bump that population count too. + * If ctt_type is unknown, treat the tag as a struct. + */ + if (tp->ctt_type == CTF_K_UNKNOWN || + tp->ctt_type >= CTF_K_MAX) + pop[CTF_K_STRUCT]++; + else + pop[tp->ctt_type]++; + /*FALLTHRU*/ + case CTF_K_UNKNOWN: + vbytes = 0; + break; + case CTF_K_POINTER: + case CTF_K_TYPEDEF: + case CTF_K_VOLATILE: + case CTF_K_CONST: + case CTF_K_RESTRICT: + child |= CTF_TYPE_ISCHILD(tp->ctt_type); + vbytes = 0; + break; + default: + printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind); + return (EIO); + } + tp = (ctf_type_t *)((uintptr_t)tp + increment + vbytes); + pop[kind]++; + } + + /* account for a sentinel value below */ + ctf_typemax++; + *lc->typlenp = ctf_typemax; + + if ((xp = malloc(sizeof(uint32_t) * ctf_typemax, M_LINKER, M_ZERO | M_WAITOK)) == NULL) + return (ENOMEM); + + *lc->typoffp = xp; + + /* type id 0 is used as a sentinel value */ + *xp++ = 0; + + /* + * In the second pass, fill in the type offset. + */ + for (tp = tbuf; tp < tend; xp++) { + ushort_t kind = CTF_INFO_KIND(tp->ctt_info); + ulong_t vlen = CTF_INFO_VLEN(tp->ctt_info); + ssize_t size, increment; + + size_t vbytes; + uint_t n; + + (void) fbt_get_ctt_size(hp->cth_version, tp, &size, &increment); + + switch (kind) { + case CTF_K_INTEGER: + case CTF_K_FLOAT: + vbytes = sizeof (uint_t); + break; + case CTF_K_ARRAY: + vbytes = sizeof (ctf_array_t); + break; + case CTF_K_FUNCTION: + vbytes = sizeof (ushort_t) * (vlen + (vlen & 1)); + break; + case CTF_K_STRUCT: + case CTF_K_UNION: + if (size < CTF_LSTRUCT_THRESH) { + ctf_member_t *mp = (ctf_member_t *) + ((uintptr_t)tp + increment); + + vbytes = sizeof (ctf_member_t) * vlen; + for (n = vlen; n != 0; n--, mp++) + child |= CTF_TYPE_ISCHILD(mp->ctm_type); + } else { + ctf_lmember_t *lmp = (ctf_lmember_t *) + ((uintptr_t)tp + increment); + + vbytes = sizeof (ctf_lmember_t) * vlen; + for (n = vlen; n != 0; n--, lmp++) + child |= + CTF_TYPE_ISCHILD(lmp->ctlm_type); + } + break; + case CTF_K_ENUM: + vbytes = sizeof (ctf_enum_t) * vlen; + break; + case CTF_K_FORWARD: + case CTF_K_UNKNOWN: + vbytes = 0; + break; + case CTF_K_POINTER: + case CTF_K_TYPEDEF: + case CTF_K_VOLATILE: + case CTF_K_CONST: + case CTF_K_RESTRICT: + vbytes = 0; + break; + default: + printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind); + return (EIO); + } + *xp = (uint32_t)((uintptr_t) tp - (uintptr_t) ctfdata); + tp = (ctf_type_t *)((uintptr_t)tp + increment + vbytes); + } + + return (0); +} + +/* + * CTF Declaration Stack + * + * In order to implement ctf_type_name(), we must convert a type graph back + * into a C type declaration. Unfortunately, a type graph represents a storage + * class ordering of the type whereas a type declaration must obey the C rules + * for operator precedence, and the two orderings are frequently in conflict. + * For example, consider these CTF type graphs and their C declarations: + * + * CTF_K_POINTER -> CTF_K_FUNCTION -> CTF_K_INTEGER : int (*)() + * CTF_K_POINTER -> CTF_K_ARRAY -> CTF_K_INTEGER : int (*)[] + * + * In each case, parentheses are used to raise operator * to higher lexical + * precedence, so the string form of the C declaration cannot be constructed by + * walking the type graph links and forming the string from left to right. + * + * The functions in this file build a set of stacks from the type graph nodes + * corresponding to the C operator precedence levels in the appropriate order. + * The code in ctf_type_name() can then iterate over the levels and nodes in + * lexical precedence order and construct the final C declaration string. + */ +typedef struct ctf_list { + struct ctf_list *l_prev; /* previous pointer or tail pointer */ + struct ctf_list *l_next; /* next pointer or head pointer */ +} ctf_list_t; + +#define ctf_list_prev(elem) ((void *)(((ctf_list_t *)(elem))->l_prev)) +#define ctf_list_next(elem) ((void *)(((ctf_list_t *)(elem))->l_next)) + +typedef enum { + CTF_PREC_BASE, + CTF_PREC_POINTER, + CTF_PREC_ARRAY, + CTF_PREC_FUNCTION, + CTF_PREC_MAX +} ctf_decl_prec_t; + +typedef struct ctf_decl_node { + ctf_list_t cd_list; /* linked list pointers */ + ctf_id_t cd_type; /* type identifier */ + uint_t cd_kind; /* type kind */ + uint_t cd_n; /* type dimension if array */ +} ctf_decl_node_t; + +typedef struct ctf_decl { + ctf_list_t cd_nodes[CTF_PREC_MAX]; /* declaration node stacks */ + int cd_order[CTF_PREC_MAX]; /* storage order of decls */ + ctf_decl_prec_t cd_qualp; /* qualifier precision */ + ctf_decl_prec_t cd_ordp; /* ordered precision */ + char *cd_buf; /* buffer for output */ + char *cd_ptr; /* buffer location */ + char *cd_end; /* buffer limit */ + size_t cd_len; /* buffer space required */ + int cd_err; /* saved error value */ +} ctf_decl_t; + +/* + * Simple doubly-linked list append routine. This implementation assumes that + * each list element contains an embedded ctf_list_t as the first member. + * An additional ctf_list_t is used to store the head (l_next) and tail + * (l_prev) pointers. The current head and tail list elements have their + * previous and next pointers set to NULL, respectively. + */ +static void +ctf_list_append(ctf_list_t *lp, void *new) +{ + ctf_list_t *p = lp->l_prev; /* p = tail list element */ + ctf_list_t *q = new; /* q = new list element */ + + lp->l_prev = q; + q->l_prev = p; + q->l_next = NULL; + + if (p != NULL) + p->l_next = q; + else + lp->l_next = q; +} + +/* + * Prepend the specified existing element to the given ctf_list_t. The + * existing pointer should be pointing at a struct with embedded ctf_list_t. + */ +static void +ctf_list_prepend(ctf_list_t *lp, void *new) +{ + ctf_list_t *p = new; /* p = new list element */ + ctf_list_t *q = lp->l_next; /* q = head list element */ + + lp->l_next = p; + p->l_prev = NULL; + p->l_next = q; + + if (q != NULL) + q->l_prev = p; + else + lp->l_prev = p; +} + +static void +ctf_decl_init(ctf_decl_t *cd, char *buf, size_t len) +{ + int i; + + bzero(cd, sizeof (ctf_decl_t)); + + for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) + cd->cd_order[i] = CTF_PREC_BASE - 1; + + cd->cd_qualp = CTF_PREC_BASE; + cd->cd_ordp = CTF_PREC_BASE; + + cd->cd_buf = buf; + cd->cd_ptr = buf; + cd->cd_end = buf + len; +} + +static void +ctf_decl_fini(ctf_decl_t *cd) +{ + ctf_decl_node_t *cdp, *ndp; + int i; + + for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) { + for (cdp = ctf_list_next(&cd->cd_nodes[i]); + cdp != NULL; cdp = ndp) { + ndp = ctf_list_next(cdp); + free(cdp, M_FBT); + } + } +} + +static const ctf_type_t * +ctf_lookup_by_id(linker_ctf_t *lc, ctf_id_t type) +{ + const ctf_type_t *tp; + uint32_t offset; + uint32_t *typoff = *lc->typoffp; + + if (type >= *lc->typlenp) { + printf("%s(%d): type %d exceeds max %ld\n",__func__,__LINE__,(int) type,*lc->typlenp); + return(NULL); + } + + /* Check if the type isn't cross-referenced. */ + if ((offset = typoff[type]) == 0) { + printf("%s(%d): type %d isn't cross referenced\n",__func__,__LINE__, (int) type); + return(NULL); + } + + tp = (const ctf_type_t *)(lc->ctftab + offset + sizeof(ctf_header_t)); + + return (tp); +} + +static void +fbt_array_info(linker_ctf_t *lc, ctf_id_t type, ctf_arinfo_t *arp) +{ + const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab; + const ctf_type_t *tp; + const ctf_array_t *ap; + ssize_t increment; + + bzero(arp, sizeof(*arp)); + + if ((tp = ctf_lookup_by_id(lc, type)) == NULL) + return; + + if (CTF_INFO_KIND(tp->ctt_info) != CTF_K_ARRAY) + return; + + (void) fbt_get_ctt_size(hp->cth_version, tp, NULL, &increment); + + ap = (const ctf_array_t *)((uintptr_t)tp + increment); + arp->ctr_contents = ap->cta_contents; + arp->ctr_index = ap->cta_index; + arp->ctr_nelems = ap->cta_nelems; +} + +static const char * +ctf_strptr(linker_ctf_t *lc, int name) +{ + const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;; + const char *strp = ""; + + if (name < 0 || name >= hp->cth_strlen) + return (strp); + + strp = (const char *)(lc->ctftab + hp->cth_stroff + name + sizeof(ctf_header_t)); + + return (strp); +} + +static void +ctf_decl_push(ctf_decl_t *cd, linker_ctf_t *lc, ctf_id_t type) +{ + ctf_decl_node_t *cdp; + ctf_decl_prec_t prec; + uint_t kind, n = 1; + int is_qual = 0; + + const ctf_type_t *tp; + ctf_arinfo_t ar; + + if ((tp = ctf_lookup_by_id(lc, type)) == NULL) { + cd->cd_err = ENOENT; + return; + } + + switch (kind = CTF_INFO_KIND(tp->ctt_info)) { + case CTF_K_ARRAY: + fbt_array_info(lc, type, &ar); + ctf_decl_push(cd, lc, ar.ctr_contents); + n = ar.ctr_nelems; + prec = CTF_PREC_ARRAY; + break; + + case CTF_K_TYPEDEF: + if (ctf_strptr(lc, tp->ctt_name)[0] == '\0') { + ctf_decl_push(cd, lc, tp->ctt_type); + return; + } + prec = CTF_PREC_BASE; + break; + + case CTF_K_FUNCTION: + ctf_decl_push(cd, lc, tp->ctt_type); + prec = CTF_PREC_FUNCTION; + break; + + case CTF_K_POINTER: + ctf_decl_push(cd, lc, tp->ctt_type); + prec = CTF_PREC_POINTER; + break; + + case CTF_K_VOLATILE: + case CTF_K_CONST: + case CTF_K_RESTRICT: + ctf_decl_push(cd, lc, tp->ctt_type); + prec = cd->cd_qualp; + is_qual++; + break; + + default: + prec = CTF_PREC_BASE; + } + + if ((cdp = malloc(sizeof (ctf_decl_node_t), M_FBT, M_WAITOK)) == NULL) { + cd->cd_err = EAGAIN; + return; + } + + cdp->cd_type = type; + cdp->cd_kind = kind; + cdp->cd_n = n; + + if (ctf_list_next(&cd->cd_nodes[prec]) == NULL) + cd->cd_order[prec] = cd->cd_ordp++; + + /* + * Reset cd_qualp to the highest precedence level that we've seen so + * far that can be qualified (CTF_PREC_BASE or CTF_PREC_POINTER). + */ + if (prec > cd->cd_qualp && prec < CTF_PREC_ARRAY) + cd->cd_qualp = prec; + + /* + * C array declarators are ordered inside out so prepend them. Also by + * convention qualifiers of base types precede the type specifier (e.g. + * const int vs. int const) even though the two forms are equivalent. + */ + if (kind == CTF_K_ARRAY || (is_qual && prec == CTF_PREC_BASE)) + ctf_list_prepend(&cd->cd_nodes[prec], cdp); + else + ctf_list_append(&cd->cd_nodes[prec], cdp); +} + +static void +ctf_decl_sprintf(ctf_decl_t *cd, const char *format, ...) +{ + size_t len = (size_t)(cd->cd_end - cd->cd_ptr); + va_list ap; + size_t n; + + va_start(ap, format); + n = vsnprintf(cd->cd_ptr, len, format, ap); + va_end(ap); + + cd->cd_ptr += MIN(n, len); + cd->cd_len += n; +} + +static ssize_t +fbt_type_name(linker_ctf_t *lc, ctf_id_t type, char *buf, size_t len) +{ + ctf_decl_t cd; + ctf_decl_node_t *cdp; + ctf_decl_prec_t prec, lp, rp; + int ptr, arr; + uint_t k; + + if (lc == NULL && type == CTF_ERR) + return (-1); /* simplify caller code by permitting CTF_ERR */ + + ctf_decl_init(&cd, buf, len); + ctf_decl_push(&cd, lc, type); + + if (cd.cd_err != 0) { + ctf_decl_fini(&cd); + return (-1); + } + + /* + * If the type graph's order conflicts with lexical precedence order + * for pointers or arrays, then we need to surround the declarations at + * the corresponding lexical precedence with parentheses. This can + * result in either a parenthesized pointer (*) as in int (*)() or + * int (*)[], or in a parenthesized pointer and array as in int (*[])(). + */ + ptr = cd.cd_order[CTF_PREC_POINTER] > CTF_PREC_POINTER; + arr = cd.cd_order[CTF_PREC_ARRAY] > CTF_PREC_ARRAY; + + rp = arr ? CTF_PREC_ARRAY : ptr ? CTF_PREC_POINTER : -1; + lp = ptr ? CTF_PREC_POINTER : arr ? CTF_PREC_ARRAY : -1; + + k = CTF_K_POINTER; /* avoid leading whitespace (see below) */ + + for (prec = CTF_PREC_BASE; prec < CTF_PREC_MAX; prec++) { + for (cdp = ctf_list_next(&cd.cd_nodes[prec]); + cdp != NULL; cdp = ctf_list_next(cdp)) { + + const ctf_type_t *tp = + ctf_lookup_by_id(lc, cdp->cd_type); + const char *name = ctf_strptr(lc, tp->ctt_name); + + if (k != CTF_K_POINTER && k != CTF_K_ARRAY) + ctf_decl_sprintf(&cd, " "); + + if (lp == prec) { + ctf_decl_sprintf(&cd, "("); + lp = -1; + } + + switch (cdp->cd_kind) { + case CTF_K_INTEGER: + case CTF_K_FLOAT: + case CTF_K_TYPEDEF: + ctf_decl_sprintf(&cd, "%s", name); + break; + case CTF_K_POINTER: + ctf_decl_sprintf(&cd, "*"); + break; + case CTF_K_ARRAY: + ctf_decl_sprintf(&cd, "[%u]", cdp->cd_n); + break; + case CTF_K_FUNCTION: + ctf_decl_sprintf(&cd, "()"); + break; + case CTF_K_STRUCT: + case CTF_K_FORWARD: + ctf_decl_sprintf(&cd, "struct %s", name); + break; + case CTF_K_UNION: + ctf_decl_sprintf(&cd, "union %s", name); + break; + case CTF_K_ENUM: + ctf_decl_sprintf(&cd, "enum %s", name); + break; + case CTF_K_VOLATILE: + ctf_decl_sprintf(&cd, "volatile"); + break; + case CTF_K_CONST: + ctf_decl_sprintf(&cd, "const"); + break; + case CTF_K_RESTRICT: + ctf_decl_sprintf(&cd, "restrict"); + break; + } + + k = cdp->cd_kind; + } + + if (rp == prec) + ctf_decl_sprintf(&cd, ")"); + } + + ctf_decl_fini(&cd); + return (cd.cd_len); +} + +static void +fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_argdesc_t *desc) +{ + const ushort_t *dp; + fbt_probe_t *fbt = parg; + linker_ctf_t lc; + modctl_t *ctl = fbt->fbtp_ctl; + int ndx = desc->dtargd_ndx; + int symindx = fbt->fbtp_symindx; + uint32_t *ctfoff; + uint32_t offset; + ushort_t info, kind, n; + + if (fbt->fbtp_roffset != 0 && desc->dtargd_ndx == 0) { + (void) strcpy(desc->dtargd_native, "int"); + return; + } + + desc->dtargd_ndx = DTRACE_ARGNONE; + + /* Get a pointer to the CTF data and it's length. */ + if (linker_ctf_get(ctl, &lc) != 0) + /* No CTF data? Something wrong? *shrug* */ + return; + + /* Check if this module hasn't been initialised yet. */ + if (*lc.ctfoffp == NULL) { + /* + * Initialise the CTF object and function symindx to + * byte offset array. + */ + if (fbt_ctfoff_init(ctl, &lc) != 0) + return; + + /* Initialise the CTF type to byte offset array. */ + if (fbt_typoff_init(&lc) != 0) + return; + } + + ctfoff = *lc.ctfoffp; + + if (ctfoff == NULL || *lc.typoffp == NULL) + return; + + /* Check if the symbol index is out of range. */ + if (symindx >= lc.nsym) + return; + + /* Check if the symbol isn't cross-referenced. */ + if ((offset = ctfoff[symindx]) == 0xffffffff) + return; + + dp = (const ushort_t *)(lc.ctftab + offset + sizeof(ctf_header_t)); + + info = *dp++; + kind = CTF_INFO_KIND(info); + n = CTF_INFO_VLEN(info); + + if (kind == CTF_K_UNKNOWN && n == 0) { + printf("%s(%d): Unknown function!\n",__func__,__LINE__); + return; + } + + if (kind != CTF_K_FUNCTION) { + printf("%s(%d): Expected a function!\n",__func__,__LINE__); + return; + } + + if (fbt->fbtp_roffset != 0) { + /* Only return type is available for args[1] in return probe. */ + if (ndx > 1) + return; + ASSERT(ndx == 1); + } else { + /* Check if the requested argument doesn't exist. */ + if (ndx >= n) + return; + + /* Skip the return type and arguments up to the one requested. */ + dp += ndx + 1; + } + + if (fbt_type_name(&lc, *dp, desc->dtargd_native, sizeof(desc->dtargd_native)) > 0) + desc->dtargd_ndx = ndx; + + return; +} + +static int +fbt_linker_file_cb(linker_file_t lf, void *arg) +{ + + fbt_provide_module(arg, lf); + + return (0); +} + +static void +fbt_load(void *dummy) +{ + /* Create the /dev/dtrace/fbt entry. */ + fbt_cdev = make_dev(&fbt_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, + "dtrace/fbt"); + + /* Default the probe table size if not specified. */ + if (fbt_probetab_size == 0) + fbt_probetab_size = FBT_PROBETAB_SIZE; + + /* Choose the hash mask for the probe table. */ + fbt_probetab_mask = fbt_probetab_size - 1; + + /* Allocate memory for the probe table. */ + fbt_probetab = + malloc(fbt_probetab_size * sizeof (fbt_probe_t *), M_FBT, M_WAITOK | M_ZERO); + + dtrace_invop_add(fbt_invop); + + if (dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_USER, + NULL, &fbt_pops, NULL, &fbt_id) != 0) + return; + + /* Create probes for the kernel and already-loaded modules. */ + linker_file_foreach(fbt_linker_file_cb, NULL); +} + + +static int +fbt_unload() +{ + int error = 0; + + /* De-register the invalid opcode handler. */ + dtrace_invop_remove(fbt_invop); + + /* De-register this DTrace provider. */ + if ((error = dtrace_unregister(fbt_id)) != 0) + return (error); + + /* Free the probe table. */ + free(fbt_probetab, M_FBT); + fbt_probetab = NULL; + fbt_probetab_mask = 0; + + destroy_dev(fbt_cdev); + + return (error); +} + +static int +fbt_modevent(module_t mod __unused, int type, void *data __unused) +{ + int error = 0; + + switch (type) { + case MOD_LOAD: + break; + + case MOD_UNLOAD: + break; + + case MOD_SHUTDOWN: + break; + + default: + error = EOPNOTSUPP; + break; + + } + + return (error); +} + +static int +fbt_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused) +{ + return (0); +} + +SYSINIT(fbt_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_load, NULL); +SYSUNINIT(fbt_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_unload, NULL); + +DEV_MODULE(fbt, fbt_modevent, NULL); +MODULE_VERSION(fbt, 1); +MODULE_DEPEND(fbt, dtrace, 1, 1, 1); +MODULE_DEPEND(fbt, opensolaris, 1, 1, 1); Property changes on: head/sys/cddl/dev/fbt/arm/fbt_isa.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/cddl/dev/fbt/arm/fbt_isa.h =================================================================== --- head/sys/cddl/dev/fbt/arm/fbt_isa.h (nonexistent) +++ head/sys/cddl/dev/fbt/arm/fbt_isa.h (revision 278529) @@ -0,0 +1,30 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + * + * $FreeBSD$ + * + */ + +#ifndef _FBT_ISA_H_ +#define _FBT_ISA_H_ + +typedef uint32_t fbt_patchval_t; + +#endif Property changes on: head/sys/cddl/dev/fbt/arm/fbt_isa.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/cddl/dev/lockstat/lockstat.c =================================================================== --- head/sys/cddl/dev/lockstat/lockstat.c (revision 278528) +++ head/sys/cddl/dev/lockstat/lockstat.c (revision 278529) @@ -1,326 +1,327 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright (c) 2008-2009 Stacey Son * * $FreeBSD$ * */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__i386__) || defined(__amd64__) || \ - defined(__mips__) || defined(__powerpc__) + defined(__mips__) || defined(__powerpc__) || \ + defined(__arm__) #define LOCKSTAT_AFRAMES 1 #else #error "architecture not supported" #endif static d_open_t lockstat_open; static void lockstat_provide(void *, dtrace_probedesc_t *); static void lockstat_destroy(void *, dtrace_id_t, void *); static void lockstat_enable(void *, dtrace_id_t, void *); static void lockstat_disable(void *, dtrace_id_t, void *); static void lockstat_load(void *); static int lockstat_unload(void); typedef struct lockstat_probe { char *lsp_func; char *lsp_name; int lsp_probe; dtrace_id_t lsp_id; #ifdef __FreeBSD__ int lsp_frame; #endif } lockstat_probe_t; #ifdef __FreeBSD__ lockstat_probe_t lockstat_probes[] = { /* Spin Locks */ { LS_MTX_SPIN_LOCK, LSS_ACQUIRE, LS_MTX_SPIN_LOCK_ACQUIRE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_MTX_SPIN_LOCK, LSS_SPIN, LS_MTX_SPIN_LOCK_SPIN, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_MTX_SPIN_UNLOCK, LSS_RELEASE, LS_MTX_SPIN_UNLOCK_RELEASE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, /* Adaptive Locks */ { LS_MTX_LOCK, LSA_ACQUIRE, LS_MTX_LOCK_ACQUIRE, DTRACE_IDNONE, (LOCKSTAT_AFRAMES + 1) }, { LS_MTX_LOCK, LSA_BLOCK, LS_MTX_LOCK_BLOCK, DTRACE_IDNONE, (LOCKSTAT_AFRAMES + 1) }, { LS_MTX_LOCK, LSA_SPIN, LS_MTX_LOCK_SPIN, DTRACE_IDNONE, (LOCKSTAT_AFRAMES + 1) }, { LS_MTX_UNLOCK, LSA_RELEASE, LS_MTX_UNLOCK_RELEASE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_MTX_TRYLOCK, LSA_ACQUIRE, LS_MTX_TRYLOCK_ACQUIRE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, /* Reader/Writer Locks */ { LS_RW_RLOCK, LSR_ACQUIRE, LS_RW_RLOCK_ACQUIRE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_RW_RLOCK, LSR_BLOCK, LS_RW_RLOCK_BLOCK, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_RW_RLOCK, LSR_SPIN, LS_RW_RLOCK_SPIN, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_RW_RUNLOCK, LSR_RELEASE, LS_RW_RUNLOCK_RELEASE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_RW_WLOCK, LSR_ACQUIRE, LS_RW_WLOCK_ACQUIRE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_RW_WLOCK, LSR_BLOCK, LS_RW_WLOCK_BLOCK, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_RW_WLOCK, LSR_SPIN, LS_RW_WLOCK_SPIN, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_RW_WUNLOCK, LSR_RELEASE, LS_RW_WUNLOCK_RELEASE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_RW_TRYUPGRADE, LSR_UPGRADE, LS_RW_TRYUPGRADE_UPGRADE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_RW_DOWNGRADE, LSR_DOWNGRADE, LS_RW_DOWNGRADE_DOWNGRADE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, /* Shared/Exclusive Locks */ { LS_SX_SLOCK, LSX_ACQUIRE, LS_SX_SLOCK_ACQUIRE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_SX_SLOCK, LSX_BLOCK, LS_SX_SLOCK_BLOCK, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_SX_SLOCK, LSX_SPIN, LS_SX_SLOCK_SPIN, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_SX_SUNLOCK, LSX_RELEASE, LS_SX_SUNLOCK_RELEASE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_SX_XLOCK, LSX_ACQUIRE, LS_SX_XLOCK_ACQUIRE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_SX_XLOCK, LSX_BLOCK, LS_SX_XLOCK_BLOCK, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_SX_XLOCK, LSX_SPIN, LS_SX_XLOCK_SPIN, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_SX_XUNLOCK, LSX_RELEASE, LS_SX_XUNLOCK_RELEASE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_SX_TRYUPGRADE, LSX_UPGRADE, LS_SX_TRYUPGRADE_UPGRADE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { LS_SX_DOWNGRADE, LSX_DOWNGRADE, LS_SX_DOWNGRADE_DOWNGRADE, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, /* Thread Locks */ { LS_THREAD_LOCK, LST_SPIN, LS_THREAD_LOCK_SPIN, DTRACE_IDNONE, LOCKSTAT_AFRAMES }, { NULL } }; #else #error "OS not supported" #endif static struct cdevsw lockstat_cdevsw = { .d_version = D_VERSION, .d_open = lockstat_open, .d_name = "lockstat", }; static struct cdev *lockstat_cdev; static dtrace_provider_id_t lockstat_id; /*ARGSUSED*/ static void lockstat_enable(void *arg, dtrace_id_t id, void *parg) { lockstat_probe_t *probe = parg; ASSERT(!lockstat_probemap[probe->lsp_probe]); lockstat_probemap[probe->lsp_probe] = id; #ifdef DOODAD membar_producer(); #endif lockstat_probe_func = dtrace_probe; #ifdef DOODAD membar_producer(); lockstat_hot_patch(); membar_producer(); #endif } /*ARGSUSED*/ static void lockstat_disable(void *arg, dtrace_id_t id, void *parg) { lockstat_probe_t *probe = parg; int i; ASSERT(lockstat_probemap[probe->lsp_probe]); lockstat_probemap[probe->lsp_probe] = 0; #ifdef DOODAD lockstat_hot_patch(); membar_producer(); #endif /* * See if we have any probes left enabled. */ for (i = 0; i < LS_NPROBES; i++) { if (lockstat_probemap[i]) { /* * This probe is still enabled. We don't need to deal * with waiting for all threads to be out of the * lockstat critical sections; just return. */ return; } } } /*ARGSUSED*/ static int lockstat_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused) { return (0); } /*ARGSUSED*/ static void lockstat_provide(void *arg, dtrace_probedesc_t *desc) { int i = 0; for (i = 0; lockstat_probes[i].lsp_func != NULL; i++) { lockstat_probe_t *probe = &lockstat_probes[i]; if (dtrace_probe_lookup(lockstat_id, "kernel", probe->lsp_func, probe->lsp_name) != 0) continue; ASSERT(!probe->lsp_id); #ifdef __FreeBSD__ probe->lsp_id = dtrace_probe_create(lockstat_id, "kernel", probe->lsp_func, probe->lsp_name, probe->lsp_frame, probe); #else probe->lsp_id = dtrace_probe_create(lockstat_id, "kernel", probe->lsp_func, probe->lsp_name, LOCKSTAT_AFRAMES, probe); #endif } } /*ARGSUSED*/ static void lockstat_destroy(void *arg, dtrace_id_t id, void *parg) { lockstat_probe_t *probe = parg; ASSERT(!lockstat_probemap[probe->lsp_probe]); probe->lsp_id = 0; } static dtrace_pattr_t lockstat_attr = { { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, }; static dtrace_pops_t lockstat_pops = { lockstat_provide, NULL, lockstat_enable, lockstat_disable, NULL, NULL, NULL, NULL, NULL, lockstat_destroy }; static void lockstat_load(void *dummy) { /* Create the /dev/dtrace/lockstat entry. */ lockstat_cdev = make_dev(&lockstat_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "dtrace/lockstat"); if (dtrace_register("lockstat", &lockstat_attr, DTRACE_PRIV_USER, NULL, &lockstat_pops, NULL, &lockstat_id) != 0) return; } static int lockstat_unload() { int error = 0; if ((error = dtrace_unregister(lockstat_id)) != 0) return (error); destroy_dev(lockstat_cdev); return (error); } /* ARGSUSED */ static int lockstat_modevent(module_t mod __unused, int type, void *data __unused) { int error = 0; switch (type) { case MOD_LOAD: break; case MOD_UNLOAD: break; case MOD_SHUTDOWN: break; default: error = EOPNOTSUPP; break; } return (error); } SYSINIT(lockstat_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, lockstat_load, NULL); SYSUNINIT(lockstat_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, lockstat_unload, NULL); DEV_MODULE(lockstat, lockstat_modevent, NULL); MODULE_VERSION(lockstat, 1); MODULE_DEPEND(lockstat, dtrace, 1, 1, 1); MODULE_DEPEND(lockstat, opensolaris, 1, 1, 1); Index: head/sys/cddl/dev/profile/profile.c =================================================================== --- head/sys/cddl/dev/profile/profile.c (revision 278528) +++ head/sys/cddl/dev/profile/profile.c (revision 278529) @@ -1,704 +1,714 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * * $FreeBSD$ * */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define PROF_NAMELEN 15 #define PROF_PROFILE 0 #define PROF_TICK 1 #define PROF_PREFIX_PROFILE "profile-" #define PROF_PREFIX_TICK "tick-" /* * Regardless of platform, there are five artificial frames in the case of the * profile provider: * * profile_fire * cyclic_expire * cyclic_fire * [ cbe ] * [ locore ] * * On amd64, there are two frames associated with locore: one in locore, and * another in common interrupt dispatch code. (i386 has not been modified to * use this common layer.) Further, on i386, the interrupted instruction * appears as its own stack frame. All of this means that we need to add one * frame for amd64, and then take one away for both amd64 and i386. * * On SPARC, the picture is further complicated because the compiler * optimizes away tail-calls -- so the following frames are optimized away: * * profile_fire * cyclic_expire * * This gives three frames. However, on DEBUG kernels, the cyclic_expire * frame cannot be tail-call eliminated, yielding four frames in this case. * * All of the above constraints lead to the mess below. Yes, the profile * provider should ideally figure this out on-the-fly by hiting one of its own * probes and then walking its own stack trace. This is complicated, however, * and the static definition doesn't seem to be overly brittle. Still, we * allow for a manual override in case we get it completely wrong. */ #ifdef __amd64 #define PROF_ARTIFICIAL_FRAMES 10 #else #ifdef __i386 #define PROF_ARTIFICIAL_FRAMES 6 #else #ifdef __sparc #ifdef DEBUG #define PROF_ARTIFICIAL_FRAMES 4 #else #define PROF_ARTIFICIAL_FRAMES 3 #endif #endif #endif #endif #ifdef __mips /* * This value is bogus just to make module compilable on mips */ #define PROF_ARTIFICIAL_FRAMES 3 #endif #ifdef __powerpc__ /* * This value is bogus just to make module compilable on powerpc */ #define PROF_ARTIFICIAL_FRAMES 3 #endif struct profile_probe_percpu; +#ifdef __mips +/* bogus */ +#define PROF_ARTIFICIAL_FRAMES 3 +#endif + +#ifdef __arm__ +/* bogus */ +#define PROF_ARTIFICIAL_FRAMES 3 +#endif + typedef struct profile_probe { char prof_name[PROF_NAMELEN]; dtrace_id_t prof_id; int prof_kind; #ifdef illumos hrtime_t prof_interval; cyclic_id_t prof_cyclic; #else sbintime_t prof_interval; struct callout prof_cyclic; sbintime_t prof_expected; struct profile_probe_percpu **prof_pcpus; #endif } profile_probe_t; typedef struct profile_probe_percpu { hrtime_t profc_expected; hrtime_t profc_interval; profile_probe_t *profc_probe; #ifdef __FreeBSD__ struct callout profc_cyclic; #endif } profile_probe_percpu_t; static d_open_t profile_open; static int profile_unload(void); static void profile_create(hrtime_t, char *, int); static void profile_destroy(void *, dtrace_id_t, void *); static void profile_enable(void *, dtrace_id_t, void *); static void profile_disable(void *, dtrace_id_t, void *); static void profile_load(void *); static void profile_provide(void *, dtrace_probedesc_t *); static int profile_rates[] = { 97, 199, 499, 997, 1999, 4001, 4999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; static int profile_ticks[] = { 1, 10, 100, 500, 1000, 5000, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; /* * profile_max defines the upper bound on the number of profile probes that * can exist (this is to prevent malicious or clumsy users from exhausing * system resources by creating a slew of profile probes). At mod load time, * this gets its value from PROFILE_MAX_DEFAULT or profile-max-probes if it's * present in the profile.conf file. */ #define PROFILE_MAX_DEFAULT 1000 /* default max. number of probes */ static uint32_t profile_max = PROFILE_MAX_DEFAULT; /* maximum number of profile probes */ static uint32_t profile_total; /* current number of profile probes */ static struct cdevsw profile_cdevsw = { .d_version = D_VERSION, .d_open = profile_open, .d_name = "profile", }; static dtrace_pattr_t profile_attr = { { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, }; static dtrace_pops_t profile_pops = { profile_provide, NULL, profile_enable, profile_disable, NULL, NULL, NULL, NULL, NULL, profile_destroy }; static struct cdev *profile_cdev; static dtrace_provider_id_t profile_id; static hrtime_t profile_interval_min = NANOSEC / 5000; /* 5000 hz */ static int profile_aframes = 0; /* override */ static sbintime_t nsec_to_sbt(hrtime_t nsec) { time_t sec; /* * We need to calculate nsec * 2^32 / 10^9 * Seconds and nanoseconds are split to avoid overflow. */ sec = nsec / NANOSEC; nsec = nsec % NANOSEC; return (((sbintime_t)sec << 32) | ((sbintime_t)nsec << 32) / NANOSEC); } static hrtime_t sbt_to_nsec(sbintime_t sbt) { return ((sbt >> 32) * NANOSEC + (((uint32_t)sbt * (hrtime_t)NANOSEC) >> 32)); } static void profile_fire(void *arg) { profile_probe_percpu_t *pcpu = arg; profile_probe_t *prof = pcpu->profc_probe; hrtime_t late; struct trapframe *frame; uintfptr_t pc, upc; #ifdef illumos late = gethrtime() - pcpu->profc_expected; #else late = sbt_to_nsec(sbinuptime() - pcpu->profc_expected); #endif pc = 0; upc = 0; /* * td_intr_frame can be unset if this is a catch up event * after waking up from idle sleep. * This can only happen on a CPU idle thread. */ frame = curthread->td_intr_frame; if (frame != NULL) { if (TRAPF_USERMODE(frame)) upc = TRAPF_PC(frame); else pc = TRAPF_PC(frame); } dtrace_probe(prof->prof_id, pc, upc, late, 0, 0); pcpu->profc_expected += pcpu->profc_interval; callout_schedule_sbt_curcpu(&pcpu->profc_cyclic, pcpu->profc_expected, 0, C_DIRECT_EXEC | C_ABSOLUTE); } static void profile_tick(void *arg) { profile_probe_t *prof = arg; struct trapframe *frame; uintfptr_t pc, upc; pc = 0; upc = 0; /* * td_intr_frame can be unset if this is a catch up event * after waking up from idle sleep. * This can only happen on a CPU idle thread. */ frame = curthread->td_intr_frame; if (frame != NULL) { if (TRAPF_USERMODE(frame)) upc = TRAPF_PC(frame); else pc = TRAPF_PC(frame); } dtrace_probe(prof->prof_id, pc, upc, 0, 0, 0); prof->prof_expected += prof->prof_interval; callout_schedule_sbt(&prof->prof_cyclic, prof->prof_expected, 0, C_DIRECT_EXEC | C_ABSOLUTE); } static void profile_create(hrtime_t interval, char *name, int kind) { profile_probe_t *prof; if (interval < profile_interval_min) return; if (dtrace_probe_lookup(profile_id, NULL, NULL, name) != 0) return; atomic_add_32(&profile_total, 1); if (profile_total > profile_max) { atomic_add_32(&profile_total, -1); return; } prof = kmem_zalloc(sizeof (profile_probe_t), KM_SLEEP); (void) strcpy(prof->prof_name, name); #ifdef illumos prof->prof_interval = interval; prof->prof_cyclic = CYCLIC_NONE; #else prof->prof_interval = nsec_to_sbt(interval); callout_init(&prof->prof_cyclic, CALLOUT_MPSAFE); #endif prof->prof_kind = kind; prof->prof_id = dtrace_probe_create(profile_id, NULL, NULL, name, profile_aframes ? profile_aframes : PROF_ARTIFICIAL_FRAMES, prof); } /*ARGSUSED*/ static void profile_provide(void *arg, dtrace_probedesc_t *desc) { int i, j, rate, kind; hrtime_t val = 0, mult = 1, len = 0; char *name, *suffix = NULL; const struct { char *prefix; int kind; } types[] = { { PROF_PREFIX_PROFILE, PROF_PROFILE }, { PROF_PREFIX_TICK, PROF_TICK }, { 0, 0 } }; const struct { char *name; hrtime_t mult; } suffixes[] = { { "ns", NANOSEC / NANOSEC }, { "nsec", NANOSEC / NANOSEC }, { "us", NANOSEC / MICROSEC }, { "usec", NANOSEC / MICROSEC }, { "ms", NANOSEC / MILLISEC }, { "msec", NANOSEC / MILLISEC }, { "s", NANOSEC / SEC }, { "sec", NANOSEC / SEC }, { "m", NANOSEC * (hrtime_t)60 }, { "min", NANOSEC * (hrtime_t)60 }, { "h", NANOSEC * (hrtime_t)(60 * 60) }, { "hour", NANOSEC * (hrtime_t)(60 * 60) }, { "d", NANOSEC * (hrtime_t)(24 * 60 * 60) }, { "day", NANOSEC * (hrtime_t)(24 * 60 * 60) }, { "hz", 0 }, { NULL } }; if (desc == NULL) { char n[PROF_NAMELEN]; /* * If no description was provided, provide all of our probes. */ for (i = 0; i < sizeof (profile_rates) / sizeof (int); i++) { if ((rate = profile_rates[i]) == 0) continue; (void) snprintf(n, PROF_NAMELEN, "%s%d", PROF_PREFIX_PROFILE, rate); profile_create(NANOSEC / rate, n, PROF_PROFILE); } for (i = 0; i < sizeof (profile_ticks) / sizeof (int); i++) { if ((rate = profile_ticks[i]) == 0) continue; (void) snprintf(n, PROF_NAMELEN, "%s%d", PROF_PREFIX_TICK, rate); profile_create(NANOSEC / rate, n, PROF_TICK); } return; } name = desc->dtpd_name; for (i = 0; types[i].prefix != NULL; i++) { len = strlen(types[i].prefix); if (strncmp(name, types[i].prefix, len) != 0) continue; break; } if (types[i].prefix == NULL) return; kind = types[i].kind; j = strlen(name) - len; /* * We need to start before any time suffix. */ for (j = strlen(name); j >= len; j--) { if (name[j] >= '0' && name[j] <= '9') break; suffix = &name[j]; } ASSERT(suffix != NULL); /* * Now determine the numerical value present in the probe name. */ for (; j >= len; j--) { if (name[j] < '0' || name[j] > '9') return; val += (name[j] - '0') * mult; mult *= (hrtime_t)10; } if (val == 0) return; /* * Look-up the suffix to determine the multiplier. */ for (i = 0, mult = 0; suffixes[i].name != NULL; i++) { if (strcasecmp(suffixes[i].name, suffix) == 0) { mult = suffixes[i].mult; break; } } if (suffixes[i].name == NULL && *suffix != '\0') return; if (mult == 0) { /* * The default is frequency-per-second. */ val = NANOSEC / val; } else { val *= mult; } profile_create(val, name, kind); } /* ARGSUSED */ static void profile_destroy(void *arg, dtrace_id_t id, void *parg) { profile_probe_t *prof = parg; #ifdef illumos ASSERT(prof->prof_cyclic == CYCLIC_NONE); #else ASSERT(!callout_active(&prof->prof_cyclic) && prof->prof_pcpus == NULL); #endif kmem_free(prof, sizeof (profile_probe_t)); ASSERT(profile_total >= 1); atomic_add_32(&profile_total, -1); } #ifdef illumos /*ARGSUSED*/ static void profile_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when) { profile_probe_t *prof = arg; profile_probe_percpu_t *pcpu; pcpu = kmem_zalloc(sizeof (profile_probe_percpu_t), KM_SLEEP); pcpu->profc_probe = prof; hdlr->cyh_func = profile_fire; hdlr->cyh_arg = pcpu; when->cyt_interval = prof->prof_interval; when->cyt_when = gethrtime() + when->cyt_interval; pcpu->profc_expected = when->cyt_when; pcpu->profc_interval = when->cyt_interval; } /*ARGSUSED*/ static void profile_offline(void *arg, cpu_t *cpu, void *oarg) { profile_probe_percpu_t *pcpu = oarg; ASSERT(pcpu->profc_probe == arg); kmem_free(pcpu, sizeof (profile_probe_percpu_t)); } /* ARGSUSED */ static void profile_enable(void *arg, dtrace_id_t id, void *parg) { profile_probe_t *prof = parg; cyc_omni_handler_t omni; cyc_handler_t hdlr; cyc_time_t when; ASSERT(prof->prof_interval != 0); ASSERT(MUTEX_HELD(&cpu_lock)); if (prof->prof_kind == PROF_TICK) { hdlr.cyh_func = profile_tick; hdlr.cyh_arg = prof; when.cyt_interval = prof->prof_interval; when.cyt_when = gethrtime() + when.cyt_interval; } else { ASSERT(prof->prof_kind == PROF_PROFILE); omni.cyo_online = profile_online; omni.cyo_offline = profile_offline; omni.cyo_arg = prof; } if (prof->prof_kind == PROF_TICK) { prof->prof_cyclic = cyclic_add(&hdlr, &when); } else { prof->prof_cyclic = cyclic_add_omni(&omni); } } /* ARGSUSED */ static void profile_disable(void *arg, dtrace_id_t id, void *parg) { profile_probe_t *prof = parg; ASSERT(prof->prof_cyclic != CYCLIC_NONE); ASSERT(MUTEX_HELD(&cpu_lock)); cyclic_remove(prof->prof_cyclic); prof->prof_cyclic = CYCLIC_NONE; } #else static void profile_enable_omni(profile_probe_t *prof) { profile_probe_percpu_t *pcpu; int cpu; prof->prof_pcpus = kmem_zalloc((mp_maxid + 1) * sizeof(pcpu), KM_SLEEP); CPU_FOREACH(cpu) { pcpu = kmem_zalloc(sizeof(profile_probe_percpu_t), KM_SLEEP); prof->prof_pcpus[cpu] = pcpu; pcpu->profc_probe = prof; pcpu->profc_expected = sbinuptime() + prof->prof_interval; pcpu->profc_interval = prof->prof_interval; callout_init(&pcpu->profc_cyclic, CALLOUT_MPSAFE); callout_reset_sbt_on(&pcpu->profc_cyclic, pcpu->profc_expected, 0, profile_fire, pcpu, cpu, C_DIRECT_EXEC | C_ABSOLUTE); } } static void profile_disable_omni(profile_probe_t *prof) { profile_probe_percpu_t *pcpu; int cpu; ASSERT(prof->prof_pcpus != NULL); CPU_FOREACH(cpu) { pcpu = prof->prof_pcpus[cpu]; ASSERT(pcpu->profc_probe == prof); ASSERT(callout_active(&pcpu->profc_cyclic)); callout_stop(&pcpu->profc_cyclic); callout_drain(&pcpu->profc_cyclic); kmem_free(pcpu, sizeof(profile_probe_percpu_t)); } kmem_free(prof->prof_pcpus, (mp_maxid + 1) * sizeof(pcpu)); prof->prof_pcpus = NULL; } /* ARGSUSED */ static void profile_enable(void *arg, dtrace_id_t id, void *parg) { profile_probe_t *prof = parg; if (prof->prof_kind == PROF_TICK) { prof->prof_expected = sbinuptime() + prof->prof_interval; callout_reset_sbt(&prof->prof_cyclic, prof->prof_expected, 0, profile_tick, prof, C_DIRECT_EXEC | C_ABSOLUTE); } else { ASSERT(prof->prof_kind == PROF_PROFILE); profile_enable_omni(prof); } } /* ARGSUSED */ static void profile_disable(void *arg, dtrace_id_t id, void *parg) { profile_probe_t *prof = parg; if (prof->prof_kind == PROF_TICK) { ASSERT(callout_active(&prof->prof_cyclic)); callout_stop(&prof->prof_cyclic); callout_drain(&prof->prof_cyclic); } else { ASSERT(prof->prof_kind == PROF_PROFILE); profile_disable_omni(prof); } } #endif static void profile_load(void *dummy) { /* Create the /dev/dtrace/profile entry. */ profile_cdev = make_dev(&profile_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "dtrace/profile"); if (dtrace_register("profile", &profile_attr, DTRACE_PRIV_USER, NULL, &profile_pops, NULL, &profile_id) != 0) return; } static int profile_unload() { int error = 0; if ((error = dtrace_unregister(profile_id)) != 0) return (error); destroy_dev(profile_cdev); return (error); } /* ARGSUSED */ static int profile_modevent(module_t mod __unused, int type, void *data __unused) { int error = 0; switch (type) { case MOD_LOAD: break; case MOD_UNLOAD: break; case MOD_SHUTDOWN: break; default: error = EOPNOTSUPP; break; } return (error); } /* ARGSUSED */ static int profile_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused) { return (0); } SYSINIT(profile_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, profile_load, NULL); SYSUNINIT(profile_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, profile_unload, NULL); DEV_MODULE(profile, profile_modevent, NULL); MODULE_VERSION(profile, 1); MODULE_DEPEND(profile, dtrace, 1, 1, 1); MODULE_DEPEND(profile, opensolaris, 1, 1, 1); Index: head/sys/modules/dtrace/Makefile =================================================================== --- head/sys/modules/dtrace/Makefile (revision 278528) +++ head/sys/modules/dtrace/Makefile (revision 278529) @@ -1,26 +1,28 @@ # $FreeBSD$ .include "Makefile.inc" SUBDIR= dtmalloc \ dtnfscl \ dtrace \ dtraceall \ dtrace_test \ lockstat \ profile \ prototype \ sdt \ systrace .if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386" SUBDIR+= fasttrap fbt systrace_linux32 .endif .if ${MACHINE_CPUARCH} == "powerpc" SUBDIR+= fbt fasttrap .endif .if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_ARCH} == "powerpc64" SUBDIR+= systrace_freebsd32 .endif - +.if ${MACHINE_CPUARCH} == "arm" +SUBDIR+= fbt +.endif .include Index: head/sys/modules/dtrace/dtrace/Makefile =================================================================== --- head/sys/modules/dtrace/dtrace/Makefile (revision 278528) +++ head/sys/modules/dtrace/dtrace/Makefile (revision 278529) @@ -1,63 +1,68 @@ # $FreeBSD$ SYSDIR?= ${.CURDIR}/../../.. ARCHDIR= ${MACHINE_CPUARCH} .PATH: ${SYSDIR}/cddl/contrib/opensolaris/uts/common/dtrace .PATH: ${SYSDIR}/cddl/compat/opensolaris/kern .PATH: ${SYSDIR}/cddl/kern .PATH: ${SYSDIR}/cddl/dev/dtrace .PATH: ${SYSDIR}/cddl/dev/dtrace/${ARCHDIR} KMOD= dtrace SRCS= dtrace.c \ dtrace_asm.S \ dtrace_subr.c .if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386" .PATH: ${SYSDIR}/cddl/dev/dtrace/x86 SRCS+= dis_tables.c \ instr_size.c CFLAGS+= -I${SYSDIR}/cddl/contrib/opensolaris/uts/intel \ -I${SYSDIR}/cddl/dev/dtrace/x86 .endif SRCS+= bus_if.h device_if.h vnode_if.h # Needed for dtrace_asm.S SRCS+= assym.s # These are needed for assym.s SRCS+= opt_compat.h opt_kstack_pages.h opt_nfs.h opt_hwpmc_hooks.h #This is needed for dtrace.c SRCS += opensolaris_taskq.c .if ${MACHINE_CPUARCH} == "i386" SRCS+= opt_apic.h .endif CFLAGS+= -I${SYSDIR}/cddl/compat/opensolaris \ -I${SYSDIR}/cddl/dev/dtrace \ -I${SYSDIR}/cddl/dev/dtrace/${ARCHDIR} \ -I${SYSDIR}/cddl/contrib/opensolaris/uts/common \ -I${SYSDIR}/cddl/contrib/opensolaris/common/util \ -I${SYSDIR} -DDIS_MEM CFLAGS+= -DSMP EXPORT_SYMS= dtrace_register \ dtrace_unregister \ dtrace_probe_lookup dtrace_asm.o: assym.s +.if ${MACHINE_CPUARCH} == "arm" +assym.o: assym.s + ${AS} -meabi=5 -o assym.o assym.s +.endif + .include CFLAGS+= -include ${SYSDIR}/cddl/compat/opensolaris/sys/debug_compat.h CWARNFLAGS+= -Wno-parentheses CWARNFLAGS+= -Wno-uninitialized CWARNFLAGS+= -Wno-cast-qual CWARNFLAGS+= -Wno-unused