diff --git a/sys/cddl/dev/dtrace/riscv/dtrace_isa.c b/sys/cddl/dev/dtrace/riscv/dtrace_isa.c
index 1f585aa6fa3c..bc8426a752df 100644
--- a/sys/cddl/dev/dtrace/riscv/dtrace_isa.c
+++ b/sys/cddl/dev/dtrace/riscv/dtrace_isa.c
@@ -1,389 +1,397 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*
* Portions Copyright 2016 Ruslan Bukin
*
* $FreeBSD$
*/
/*
* Copyright 2005 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include "regset.h"
#define MAX_USTACK_DEPTH 2048
uint8_t dtrace_fuword8_nocheck(void *);
uint16_t dtrace_fuword16_nocheck(void *);
uint32_t dtrace_fuword32_nocheck(void *);
uint64_t dtrace_fuword64_nocheck(void *);
void
dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes,
uint32_t *intrpc)
{
struct unwind_state state;
- int scp_offset;
+ uintptr_t caller;
register_t sp;
+ int scp_offset;
int depth;
depth = 0;
+ caller = solaris_cpu[curcpu].cpu_dtrace_caller;
if (intrpc != 0) {
- pcstack[depth++] = (pc_t) intrpc;
+ pcstack[depth++] = (pc_t)intrpc;
}
- aframes++;
-
+ /*
+ * Construct the unwind state, starting from this function. This frame,
+ * and 'aframes' others will be skipped.
+ */
__asm __volatile("mv %0, sp" : "=&r" (sp));
state.fp = (uintptr_t)__builtin_frame_address(0);
- state.sp = sp;
+ state.sp = (uintptr_t)sp;
state.pc = (uintptr_t)dtrace_getpcstack;
while (depth < pcstack_limit) {
if (!unwind_frame(curthread, &state))
break;
- if (!INKERNEL(state.pc) || !INKERNEL(state.fp))
+ if (!INKERNEL(state.pc) || !kstack_contains(curthread,
+ (vm_offset_t)state.fp, sizeof(uintptr_t)))
break;
- /*
- * NB: Unlike some other architectures, we don't need to
- * explicitly insert cpu_dtrace_caller as it appears in the
- * normal kernel stack trace rather than a special trap frame.
- */
if (aframes > 0) {
aframes--;
+
+ /*
+ * fbt_invop() records the return address at the time
+ * the FBT probe fires. We need to insert this into the
+ * backtrace manually, since the stack frame state at
+ * the time of the probe does not capture it.
+ */
+ if (aframes == 0 && caller != 0)
+ pcstack[depth++] = caller;
} else {
pcstack[depth++] = state.pc;
}
-
}
for (; depth < pcstack_limit; depth++) {
pcstack[depth] = 0;
}
}
static int
dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, uintptr_t pc,
uintptr_t fp)
{
volatile uint16_t *flags;
uintptr_t oldfp;
int ret;
oldfp = fp;
ret = 0;
flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags;
ASSERT(pcstack == NULL || pcstack_limit > 0);
while (pc != 0) {
/*
* We limit the number of times we can go around this
* loop to account for a circular stack.
*/
if (ret++ >= MAX_USTACK_DEPTH) {
*flags |= CPU_DTRACE_BADSTACK;
cpu_core[curcpu].cpuc_dtrace_illval = fp;
break;
}
if (pcstack != NULL) {
*pcstack++ = (uint64_t)pc;
pcstack_limit--;
if (pcstack_limit <= 0)
break;
}
if (fp == 0)
break;
pc = dtrace_fuword64((void *)(fp +
offsetof(struct riscv_frame, f_retaddr)));
fp = dtrace_fuword64((void *)fp);
if (fp == oldfp) {
*flags |= CPU_DTRACE_BADSTACK;
cpu_core[curcpu].cpuc_dtrace_illval = fp;
break;
}
oldfp = fp;
}
return (ret);
}
void
dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit)
{
volatile uint16_t *flags;
struct trapframe *tf;
uintptr_t pc, sp, fp;
proc_t *p;
int n;
p = curproc;
flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags;
if (*flags & CPU_DTRACE_FAULT)
return;
if (pcstack_limit <= 0)
return;
/*
* If there's no user context we still need to zero the stack.
*/
if (p == NULL || (tf = curthread->td_frame) == NULL)
goto zero;
*pcstack++ = (uint64_t)p->p_pid;
pcstack_limit--;
if (pcstack_limit <= 0)
return;
pc = tf->tf_sepc;
sp = tf->tf_sp;
fp = tf->tf_s[0];
if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
/*
* In an entry probe. The frame pointer has not yet been
* pushed (that happens in the function prologue). The
* best approach is to add the current pc as a missing top
* of stack and back the pc up to the caller, which is stored
* at the current stack pointer address since the call
* instruction puts it there right before the branch.
*/
*pcstack++ = (uint64_t)pc;
pcstack_limit--;
if (pcstack_limit <= 0)
return;
pc = tf->tf_ra;
}
n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp);
ASSERT(n >= 0);
ASSERT(n <= pcstack_limit);
pcstack += n;
pcstack_limit -= n;
zero:
while (pcstack_limit-- > 0)
*pcstack++ = 0;
}
int
dtrace_getustackdepth(void)
{
printf("IMPLEMENT ME: %s\n", __func__);
return (0);
}
void
dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit)
{
printf("IMPLEMENT ME: %s\n", __func__);
}
/*ARGSUSED*/
uint64_t
dtrace_getarg(int arg, int aframes)
{
printf("IMPLEMENT ME: %s\n", __func__);
return (0);
}
int
dtrace_getstackdepth(int aframes)
{
struct unwind_state state;
int scp_offset;
register_t sp;
int depth;
bool done;
depth = 1;
done = false;
__asm __volatile("mv %0, sp" : "=&r" (sp));
state.fp = (uintptr_t)__builtin_frame_address(0);
state.sp = sp;
state.pc = (uintptr_t)dtrace_getstackdepth;
do {
done = !unwind_frame(curthread, &state);
if (!INKERNEL(state.pc) || !INKERNEL(state.fp))
break;
depth++;
} while (!done);
if (depth < aframes)
return (0);
else
return (depth - aframes);
}
ulong_t
dtrace_getreg(struct trapframe *rp, uint_t reg)
{
printf("IMPLEMENT ME: %s\n", __func__);
return (0);
}
static int
dtrace_copycheck(uintptr_t uaddr, uintptr_t kaddr, size_t size)
{
if (uaddr + size > VM_MAXUSER_ADDRESS || uaddr + size < uaddr) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
cpu_core[curcpu].cpuc_dtrace_illval = uaddr;
return (0);
}
return (1);
}
void
dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size,
volatile uint16_t *flags)
{
if (dtrace_copycheck(uaddr, kaddr, size))
dtrace_copy(uaddr, kaddr, size);
}
void
dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size,
volatile uint16_t *flags)
{
if (dtrace_copycheck(uaddr, kaddr, size))
dtrace_copy(kaddr, uaddr, size);
}
void
dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size,
volatile uint16_t *flags)
{
if (dtrace_copycheck(uaddr, kaddr, size))
dtrace_copystr(uaddr, kaddr, size, flags);
}
void
dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size,
volatile uint16_t *flags)
{
if (dtrace_copycheck(uaddr, kaddr, size))
dtrace_copystr(kaddr, uaddr, size, flags);
}
uint8_t
dtrace_fuword8(void *uaddr)
{
if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
return (0);
}
return (dtrace_fuword8_nocheck(uaddr));
}
uint16_t
dtrace_fuword16(void *uaddr)
{
if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
return (0);
}
return (dtrace_fuword16_nocheck(uaddr));
}
uint32_t
dtrace_fuword32(void *uaddr)
{
if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
return (0);
}
return (dtrace_fuword32_nocheck(uaddr));
}
uint64_t
dtrace_fuword64(void *uaddr)
{
if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) {
DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
return (0);
}
return (dtrace_fuword64_nocheck(uaddr));
}
diff --git a/sys/cddl/dev/fbt/riscv/fbt_isa.c b/sys/cddl/dev/fbt/riscv/fbt_isa.c
index 659a9d44c81c..a67e83811d9e 100644
--- a/sys/cddl/dev/fbt/riscv/fbt_isa.c
+++ b/sys/cddl/dev/fbt/riscv/fbt_isa.c
@@ -1,264 +1,264 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*
* Portions Copyright 2006-2008 John Birrell jb@freebsd.org
* Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org
* Portions Copyright 2013 Howard Su howardsu@freebsd.org
* Portions Copyright 2016-2018 Ruslan Bukin
*
* $FreeBSD$
*/
/*
* Copyright 2006 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include
#include
#include
#include
#include
#include "fbt.h"
#define FBT_C_PATCHVAL MATCH_C_EBREAK
#define FBT_PATCHVAL MATCH_EBREAK
#define FBT_ENTRY "entry"
#define FBT_RETURN "return"
int
fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval)
{
solaris_cpu_t *cpu;
fbt_probe_t *fbt;
cpu = &solaris_cpu[curcpu];
fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
if ((uintptr_t)fbt->fbtp_patchpoint == addr) {
- cpu->cpu_dtrace_caller = addr;
+ cpu->cpu_dtrace_caller = frame->tf_ra - INSN_SIZE;
if (fbt->fbtp_roffset == 0) {
dtrace_probe(fbt->fbtp_id, frame->tf_a[0],
frame->tf_a[1], frame->tf_a[2],
frame->tf_a[3], frame->tf_a[4]);
} else {
dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset,
frame->tf_a[0], frame->tf_a[1], 0, 0);
}
cpu->cpu_dtrace_caller = 0;
return (fbt->fbtp_savedval);
}
}
return (0);
}
void
fbt_patch_tracepoint(fbt_probe_t *fbt, fbt_patchval_t val)
{
switch(fbt->fbtp_patchval) {
case FBT_C_PATCHVAL:
*(uint16_t *)fbt->fbtp_patchpoint = (uint16_t)val;
fence_i();
break;
case FBT_PATCHVAL:
*fbt->fbtp_patchpoint = val;
fence_i();
break;
};
}
static int
match_opcode(uint32_t insn, int match, int mask)
{
if (((insn ^ match) & mask) == 0)
return (1);
return (0);
}
static int
check_c_ret(uint32_t **instr)
{
uint16_t *instr1;
int i;
for (i = 0; i < 2; i++) {
instr1 = (uint16_t *)(*instr) + i;
if (match_opcode(*instr1, (MATCH_C_JR | (X_RA << RD_SHIFT)),
(MASK_C_JR | RD_MASK))) {
*instr = (uint32_t *)instr1;
return (1);
}
}
return (0);
}
static int
check_c_sdsp(uint32_t **instr)
{
uint16_t *instr1;
int i;
for (i = 0; i < 2; i++) {
instr1 = (uint16_t *)(*instr) + i;
if (match_opcode(*instr1, (MATCH_C_SDSP | RS2_C_RA),
(MASK_C_SDSP | RS2_C_MASK))) {
*instr = (uint32_t *)instr1;
return (1);
}
}
return (0);
}
int
fbt_provide_module_function(linker_file_t lf, int symindx,
linker_symval_t *symval, void *opaque)
{
fbt_probe_t *fbt, *retfbt;
uint32_t *instr, *limit;
const char *name;
char *modname;
int patchval;
int rval;
modname = opaque;
name = symval->name;
/* Check if function is excluded from instrumentation */
if (fbt_excluded(name))
return (0);
/*
* Some assembly-language exception handlers are not suitable for
* instrumentation.
*/
if (strcmp(name, "cpu_exception_handler") == 0)
return (0);
if (strcmp(name, "cpu_exception_handler_user") == 0)
return (0);
if (strcmp(name, "cpu_exception_handler_supervisor") == 0)
return (0);
if (strcmp(name, "do_trap_supervisor") == 0)
return (0);
instr = (uint32_t *)(symval->value);
limit = (uint32_t *)(symval->value + symval->size);
/* Look for sd operation */
for (; instr < limit; instr++) {
/* Look for a non-compressed store of ra to sp */
if (match_opcode(*instr, (MATCH_SD | RS2_RA | RS1_SP),
(MASK_SD | RS2_MASK | RS1_MASK))) {
rval = DTRACE_INVOP_SD;
patchval = FBT_PATCHVAL;
break;
}
/* Look for a 'C'-compressed store of ra to sp. */
if (check_c_sdsp(&instr)) {
rval = DTRACE_INVOP_C_SDSP;
patchval = FBT_C_PATCHVAL;
break;
}
}
if (instr >= limit)
return (0);
fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
fbt->fbtp_name = name;
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_ENTRY, 3, fbt);
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = lf;
fbt->fbtp_loadcnt = lf->loadcnt;
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = patchval;
fbt->fbtp_rval = rval;
fbt->fbtp_symindx = symindx;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
lf->fbt_nentries++;
retfbt = NULL;
again:
for (; instr < limit; instr++) {
/* Look for non-compressed return */
if (match_opcode(*instr, (MATCH_JALR | (X_RA << RS1_SHIFT)),
(MASK_JALR | RD_MASK | RS1_MASK | IMM_MASK))) {
rval = DTRACE_INVOP_RET;
patchval = FBT_PATCHVAL;
break;
}
/* Look for 'C'-compressed return */
if (check_c_ret(&instr)) {
rval = DTRACE_INVOP_C_RET;
patchval = FBT_C_PATCHVAL;
break;
}
}
if (instr >= limit)
return (0);
/*
* We have a winner!
*/
fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
fbt->fbtp_name = name;
if (retfbt == NULL) {
fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
name, FBT_RETURN, 3, fbt);
} else {
retfbt->fbtp_probenext = fbt;
fbt->fbtp_id = retfbt->fbtp_id;
}
retfbt = fbt;
fbt->fbtp_patchpoint = instr;
fbt->fbtp_ctl = lf;
fbt->fbtp_loadcnt = lf->loadcnt;
fbt->fbtp_symindx = symindx;
fbt->fbtp_rval = rval;
fbt->fbtp_roffset = (uintptr_t)instr - (uintptr_t)symval->value;
fbt->fbtp_savedval = *instr;
fbt->fbtp_patchval = patchval;
fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
lf->fbt_nentries++;
instr++;
goto again;
}