Differential D25477 Diff 73725 sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c

Changeset View

Standalone View

View Options

sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c

This file is larger than 256 KB, so syntax highlighting is disabled by default.

Show First 20 Lines • Show All 65 Lines • ▼ Show 20 Lines

* [Group] Functions", allowing one to find each block by searching forward

* on capital-f functions.

#include <sys/errno.h>

#ifndef illumos

#include <sys/time.h>

#endif

#include <sys/stat.h>

#include <sys/modctl.h>

#include <sys/conf.h>

#include <sys/systm.h>

#ifdef illumos

#include <sys/ddi.h>

#include <sys/sunddi.h>

#endif

#include <sys/cpuvar.h>

#include <sys/kmem.h>

#ifdef illumos

#include <sys/strsubr.h>

#endif

#include <sys/sysmacros.h>

#include <sys/dtrace_impl.h>

#include <sys/atomic.h>

#include <sys/cmn_err.h>

#ifdef illumos

#include <sys/mutex_impl.h>

#include <sys/rwlock_impl.h>

#endif

#include <sys/ctf_api.h>

#ifdef illumos

#include <sys/panic.h>

#include <sys/priv_impl.h>

#endif

#include <sys/policy.h>

#ifdef illumos

#include <sys/cred_impl.h>

#include <sys/procfs_isa.h>

#endif

#include <sys/taskq.h>

#ifdef illumos

#include <sys/mkdev.h>

#include <sys/kdi.h>

#endif

#include <sys/zone.h>

#include <sys/socket.h>

#include <netinet/in.h>

#include "strtolctype.h"

/* FreeBSD includes: */

#ifndef illumos

#include <sys/callout.h>

#include <sys/ctype.h>

#include <sys/eventhandler.h>

#include <sys/limits.h>

#include <sys/linker.h>

#include <sys/kdb.h>

#include <sys/jail.h>

#include <sys/kernel.h>

#include <sys/malloc.h>

#include <sys/lock.h>

#include <sys/mutex.h>

#include <sys/ptrace.h>

#include <sys/random.h>

#include <sys/rwlock.h>

#include <sys/sx.h>

#include <sys/sysctl.h>

#include <sys/mount.h>

#undef AT_UID

#undef AT_GID

#include <sys/vnode.h>

#include <sys/cred.h>

impAuthorUnsubmitted

Not Done

extra newline here.

imp: extra newline here.

#include <sys/dtrace_bsd.h>

#include <netinet/in.h>

#include "dtrace_cddl.h"

#include "dtrace_debug.c"

#endif

▲ Show 20 Lines • Show All 154 Lines • ▼ Show 20 Lines

#ifndef illumos

/* XXX FreeBSD hacks. */

#define cr_suid cr_svuid

#define cr_sgid cr_svgid

#define ipaddr_t in_addr_t

#define mod_modname pathname

#define vuprintf vprintf

#define ttoproc(_a) ((_a)->td_proc)

#ifndef crgetzoneid

#define crgetzoneid(_a) 0

#endif

#define ttoproc(_a) ((_a)->td_proc)

#define SNOCD 0

#define CPU_ON_INTR(_a) 0

#define PRIV_EFFECTIVE (1 << 0)

#define PRIV_DTRACE_KERNEL (1 << 1)

#define PRIV_DTRACE_PROC (1 << 2)

#define PRIV_DTRACE_USER (1 << 3)

#define PRIV_PROC_OWNER (1 << 4)

▲ Show 20 Lines • Show All 2,463 Lines • ▼ Show 20 Lines

* state and transitions it into the ACTIVE state. If there is no speculation

* in the INACTIVE state, 0 is returned. In this case, no error counter is

* incremented -- it is up to the caller to take appropriate action.

static int

dtrace_speculation(dtrace_state_t *state)

{

int i = 0;

dtrace_speculation_state_t current;

dtrace_speculation_state_t curstate;

uint32_t *stat = &state->dts_speculations_unavail, count;

while (i < state->dts_nspeculations) {

dtrace_speculation_t *spec = &state->dts_speculations[i];

current = spec->dtsp_state;

curstate = spec->dtsp_state;

if (current != DTRACESPEC_INACTIVE) {

if (curstate != DTRACESPEC_INACTIVE) {

if (current == DTRACESPEC_COMMITTINGMANY ||

if (curstate == DTRACESPEC_COMMITTINGMANY ||

current == DTRACESPEC_COMMITTING ||

curstate == DTRACESPEC_COMMITTING ||

current == DTRACESPEC_DISCARDING)

curstate == DTRACESPEC_DISCARDING)

stat = &state->dts_speculations_busy;

i++;

continue;

}

if (dtrace_cas32((uint32_t *)&spec->dtsp_state,

current, DTRACESPEC_ACTIVE) == current)

curstate, DTRACESPEC_ACTIVE) == curstate)

return (i + 1);

}

* We couldn't find a speculation. If we found as much as a single

* busy speculation buffer, we'll attribute this failure as "busy"

* instead of "unavail".

Show All 12 Lines

static void

dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu,

dtrace_specid_t which)

{

dtrace_speculation_t *spec;

dtrace_buffer_t *src, *dest;

uintptr_t daddr, saddr, dlimit, slimit;

dtrace_speculation_state_t current, new = 0;

dtrace_speculation_state_t curstate, new = 0;

intptr_t offs;

uint64_t timestamp;

if (which == 0)

return;

if (which > state->dts_nspeculations) {

cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;

return;

}

spec = &state->dts_speculations[which - 1];

src = &spec->dtsp_buffer[cpu];

dest = &state->dts_buffer[cpu];

do {

current = spec->dtsp_state;

curstate = spec->dtsp_state;

if (current == DTRACESPEC_COMMITTINGMANY)

if (curstate == DTRACESPEC_COMMITTINGMANY)

break;

switch (current) {

switch (curstate) {

case DTRACESPEC_INACTIVE:

case DTRACESPEC_DISCARDING:

return;

case DTRACESPEC_COMMITTING:

* This is only possible if we are (a) commit()'ing

* without having done a prior speculate() on this CPU

Show All 25 Lines

do {

case DTRACESPEC_ACTIVEMANY:

new = DTRACESPEC_COMMITTINGMANY;

break;

default:

ASSERT(0);

}

} while (dtrace_cas32((uint32_t *)&spec->dtsp_state,

current, new) != current);

curstate, new) != curstate);

* We have set the state to indicate that we are committing this

* speculation. Now reserve the necessary space in the destination

* buffer.

if ((offs = dtrace_buffer_reserve(dest, src->dtb_offset,

sizeof (uint64_t), state, NULL)) < 0) {

▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines

dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu,

dest->dtb_offset = offs + src->dtb_offset;

out:

* If we're lucky enough to be the only active CPU on this speculation

* buffer, we can just set the state back to DTRACESPEC_INACTIVE.

if (current == DTRACESPEC_ACTIVE ||

if (curstate == DTRACESPEC_ACTIVE ||

(current == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) {

(curstate == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) {

uint32_t rval = dtrace_cas32((uint32_t *)&spec->dtsp_state,

DTRACESPEC_COMMITTING, DTRACESPEC_INACTIVE);

ASSERT(rval == DTRACESPEC_COMMITTING);

}

src->dtb_offset = 0;

src->dtb_xamot_drops += src->dtb_drops;

src->dtb_drops = 0;

}

* This routine discards an active speculation. If the specified speculation

* is not in a valid state to perform a discard(), this routine will silently

* do nothing. The state of the specified speculation is transitioned

* according to the state transition diagram outlined in <sys/dtrace_impl.h>

static void

dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu,

dtrace_specid_t which)

{

dtrace_speculation_t *spec;

dtrace_speculation_state_t current, new = 0;

dtrace_speculation_state_t curstate, new = 0;

dtrace_buffer_t *buf;

if (which == 0)

return;

if (which > state->dts_nspeculations) {

cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;

return;

}

spec = &state->dts_speculations[which - 1];

buf = &spec->dtsp_buffer[cpu];

do {

current = spec->dtsp_state;

curstate = spec->dtsp_state;

switch (current) {

switch (curstate) {

case DTRACESPEC_INACTIVE:

case DTRACESPEC_COMMITTINGMANY:

case DTRACESPEC_COMMITTING:

case DTRACESPEC_DISCARDING:

return;

case DTRACESPEC_ACTIVE:

case DTRACESPEC_ACTIVEMANY:

new = DTRACESPEC_DISCARDING;

break;

case DTRACESPEC_ACTIVEONE:

if (buf->dtb_offset != 0) {

new = DTRACESPEC_INACTIVE;

} else {

new = DTRACESPEC_DISCARDING;

}

break;

default:

ASSERT(0);

}

} while (dtrace_cas32((uint32_t *)&spec->dtsp_state,

current, new) != current);

curstate, new) != curstate);

buf->dtb_offset = 0;

buf->dtb_drops = 0;

}

* Note: not called from probe context. This function is called

* asynchronously from cross call context to clean any speculations that are

▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Lines

dtrace_speculation_clean(dtrace_state_t *state)

* We now know that all CPUs have committed or discarded their

* speculation buffers, as appropriate. We can now set the state

* to inactive.

for (i = 0; i < state->dts_nspeculations; i++) {

dtrace_speculation_t *spec = &state->dts_speculations[i];

dtrace_speculation_state_t current, new;

dtrace_speculation_state_t curstate, new;

if (!spec->dtsp_cleaning)

continue;

current = spec->dtsp_state;

curstate = spec->dtsp_state;

ASSERT(current == DTRACESPEC_DISCARDING ||

ASSERT(curstate == DTRACESPEC_DISCARDING ||

current == DTRACESPEC_COMMITTINGMANY);

curstate == DTRACESPEC_COMMITTINGMANY);

new = DTRACESPEC_INACTIVE;

rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new);

rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, curstate, new);

ASSERT(rv == current);

ASSERT(rv == curstate);

spec->dtsp_cleaning = 0;

}

* Called as part of a speculate() to get the speculative buffer associated

* with a given speculation. Returns NULL if the specified speculation is not

* in an ACTIVE state. If the speculation is in the ACTIVEONE state -- and

* the active CPU is not the specified CPU -- the speculation will be

* atomically transitioned into the ACTIVEMANY state.

static dtrace_buffer_t *

dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid,

dtrace_specid_t which)

{

dtrace_speculation_t *spec;

dtrace_speculation_state_t current, new = 0;

dtrace_speculation_state_t curstate, new = 0;

dtrace_buffer_t *buf;

if (which == 0)

return (NULL);

if (which > state->dts_nspeculations) {

cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;

return (NULL);

}

spec = &state->dts_speculations[which - 1];

buf = &spec->dtsp_buffer[cpuid];

do {

current = spec->dtsp_state;

curstate = spec->dtsp_state;

switch (current) {

switch (curstate) {

case DTRACESPEC_INACTIVE:

case DTRACESPEC_COMMITTINGMANY:

case DTRACESPEC_DISCARDING:

return (NULL);

case DTRACESPEC_COMMITTING:

ASSERT(buf->dtb_offset == 0);

return (NULL);

case DTRACESPEC_ACTIVEONE:

* This speculation is currently active on one CPU.

delphijUnsubmitted

Done

This doesn't seem right...

delphij: This doesn't seem right...

delphijUnsubmitted

Done

case DTRACESPEC_ACTIVEONE:

- * This speculation is curstately active on one CPU.

+ * This speculation is currently active on one CPU.

* Check the offset in the buffer; if it's non-zero,

delphij:

* Check the offset in the buffer; if it's non-zero,

* that CPU must be us (and we leave the state alone).

* If it's zero, assume that we're starting on a new

* CPU -- and change the state to indicate that the

* speculation is active on more than one CPU.

if (buf->dtb_offset != 0)

return (buf);

new = DTRACESPEC_ACTIVEMANY;

break;

case DTRACESPEC_ACTIVEMANY:

return (buf);

case DTRACESPEC_ACTIVE:

new = DTRACESPEC_ACTIVEONE;

break;

default:

ASSERT(0);

}

} while (dtrace_cas32((uint32_t *)&spec->dtsp_state,

current, new) != current);

curstate, new) != curstate);

ASSERT(new == DTRACESPEC_ACTIVEONE || new == DTRACESPEC_ACTIVEMANY);

return (buf);

}

* Return a string. In the event that the user lacks the privilege to access

* arbitrary kernel memory, we copy the string out to scratch memory so that we

▲ Show 20 Lines • Show All 272 Lines • ▼ Show 20 Lines

case DIF_VAR_UCALLER:

if (!dtrace_priv_proc(state))

return (0);

if (!(mstate->dtms_present & DTRACE_MSTATE_UCALLER)) {

uint64_t ustack[3];

* dtrace_getupcstack() fills in the first uint64_t

* with the current PID. The second uint64_t will

delphijUnsubmitted

Not Done

This should be current

delphij: This should be current

delphijUnsubmitted

Not Done

* dtrace_getupcstack() fills in the first uint64_t

- * with the curstate PID. The second uint64_t will

+ * with the current PID. The second uint64_t will

* be the program counter at user-level. The third

delphij:

* be the program counter at user-level. The third

* uint64_t will contain the caller, which is what

* we're after.

ustack[2] = 0;

DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);

dtrace_getupcstack(ustack, 3);

DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);

▲ Show 20 Lines • Show All 2,600 Lines • ▼ Show 20 Lines

dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate,

uint8_t cc_n = 0, cc_z = 0, cc_v = 0, cc_c = 0;

int64_t cc_r;

uint_t pc = 0, id, opc = 0;

uint8_t ttop = 0;

dif_instr_t instr;

uint_t r1, r2, rd;

* We stash the current DIF object into the machine state: we need it

delphijUnsubmitted

Done

Ditto.

delphij: Ditto.

delphijUnsubmitted

Done

uint_t r1, r2, rd;

- * We stash the curstate DIF object into the machine state: we need it

+ * We stash the current DIF object into the machine state: we need it

* for subsequent access checking.

delphij:

* for subsequent access checking.

mstate->dtms_difo = difo;

regs[DIF_REG_R0] = 0; /* %r0 is fixed at zero */

while (pc < textlen && !(*flags & CPU_DTRACE_FAULT)) {

opc = pc;

▲ Show 20 Lines • Show All 918 Lines • ▼ Show 20 Lines

if (dtrace_destructive_disallow)

return;

flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags;

now = dtrace_gethrtime();

if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) {

* We need to advance the mark to the current time.

delphijUnsubmitted

Not Done

Ditto.

delphij: Ditto.

delphijUnsubmitted

Not Done

if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) {

- * We need to advance the mark to the curstate time.

+ * We need to advance the mark to the current time.

cpu->cpu_dtrace_chillmark = now;

delphij:

cpu->cpu_dtrace_chillmark = now;

cpu->cpu_dtrace_chilled = 0;

}

* Now check to see if the requested chill time would take us over

* the maximum amount of time allowed in the chill interval. (Or

▲ Show 20 Lines • Show All 256 Lines • ▼ Show 20 Lines

dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,

if (panicstr != NULL)

return;

#ifdef illumos

* Kick out immediately if this CPU is still being born (in which case

* curthread will be set to -1) or the current thread can't allow

* probes in its current context.

delphijUnsubmitted

Not Done

* curthread will be set to -1) or the curstate thread can't allow

- * probes in its curstate context.

+ * probes in its current context.

if (((uintptr_t)curthread & 1) || (curthread->t_flag & T_DONTDTRACE))

Ditto

delphij: Ditto

if (((uintptr_t)curthread & 1) || (curthread->t_flag & T_DONTDTRACE))

return;

#endif

cookie = dtrace_probe_enter(id);

probe = dtrace_probes[id - 1];

cpuid = curcpu;

onintr = CPU_ON_INTR(CPU);

▲ Show 20 Lines • Show All 171 Lines • ▼ Show 20 Lines

if (now - state->dts_alive > dtrace_deadman_timeout) {

* not been disabled, we're going to transition into

* the KILLED state, from which no further processing

* on this state will be performed.

if (!dtrace_priv_kernel_destructive(state) ||

!state->dts_cred.dcr_destructive ||

dtrace_destructive_disallow) {

void *activity = &state->dts_activity;

dtrace_activity_t current;

dtrace_activity_t curstate;

impAuthorUnsubmitted

Not Done

Can the current-> curstate stuff be done as one commit? That would make this review much shorter

imp: Can the current-> curstate stuff be done as one commit? That would make this review much shorter

do {

current = state->dts_activity;

curstate = state->dts_activity;

} while (dtrace_cas32(activity, current,

} while (dtrace_cas32(activity, curstate,

DTRACE_ACTIVITY_KILLED) != current);

DTRACE_ACTIVITY_KILLED) != curstate);

continue;

}

if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed,

ecb->dte_alignment, state, &mstate)) < 0)

continue;

▲ Show 20 Lines • Show All 318 Lines • ▼ Show 20 Lines

#endif

* can be successfully recorded at most once

* when we're in the ACTIVE state. If we're

* encountering the exit() action while in

* COOLDOWN, however, we want to honor the new

* status code. (We know that we're the only

* thread in COOLDOWN, so there is no race.)

void *activity = &state->dts_activity;

dtrace_activity_t current = state->dts_activity;

dtrace_activity_t curstate = state->dts_activity;

if (current == DTRACE_ACTIVITY_COOLDOWN)

if (curstate == DTRACE_ACTIVITY_COOLDOWN)

break;

if (current != DTRACE_ACTIVITY_WARMUP)

if (curstate != DTRACE_ACTIVITY_WARMUP)

current = DTRACE_ACTIVITY_ACTIVE;

curstate = DTRACE_ACTIVITY_ACTIVE;

if (dtrace_cas32(activity, current,

if (dtrace_cas32(activity, curstate,

DTRACE_ACTIVITY_DRAINING) != current) {

DTRACE_ACTIVITY_DRAINING) != curstate) {

*flags |= CPU_DTRACE_DROP;

continue;

}

break;

}

default:

▲ Show 20 Lines • Show All 10,553 Lines • Show Last 20 Lines