Page MenuHomeFreeBSD
Files F104164750

D40337.id124157.diff
No OneTemporary
Actions

D40337.id124157.diff
View Options

diff --git a/sys/cddl/dev/kinst/aarch64/kinst_isa.h b/sys/cddl/dev/kinst/aarch64/kinst_isa.h
new file mode 100644
--- /dev/null
+++ b/sys/cddl/dev/kinst/aarch64/kinst_isa.h
@@ -0,0 +1,42 @@
+/*-
+ * Copyright (c) 2023 The FreeBSD Foundation
+ *
+ * This software was developed by Christos Margiolis <christos@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE
+ */
+
+#ifndef _KINST_ISA_H_
+#define _KINST_ISA_H_
+
+#define KINST_PATCHVAL DTRACE_PATCHVAL
+
+#define KINST_TRAMP_SIZE 8
+
+typedef uint32_t kinst_patchval_t;
+
+struct kinst_probe_md {
+ bool emulate; /* emulate in sw */
+};
+
+#endif /* _KINST_ISA_H_ */
diff --git a/sys/cddl/dev/kinst/aarch64/kinst_isa.c b/sys/cddl/dev/kinst/aarch64/kinst_isa.c
new file mode 100644
--- /dev/null
+++ b/sys/cddl/dev/kinst/aarch64/kinst_isa.c
@@ -0,0 +1,490 @@
+/*-
+ * Copyright (c) 2023 The FreeBSD Foundation
+ *
+ * This software was developed by Christos Margiolis <christos@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE
+ */
+
+#include <sys/param.h>
+
+#include <sys/dtrace.h>
+#include <cddl/dev/dtrace/dtrace_cddl.h>
+
+#include "kinst.h"
+
+/*
+ * Per-CPU trampolines used when the interrupted thread is executing with
+ * interrupts disabled. If an interrupt is raised while executing a trampoline,
+ * the interrupt thread cannot safely overwrite its trampoline if it hits a
+ * kinst probe while executing the interrupt handler.
+ */
+DPCPU_DEFINE_STATIC(uint8_t *, intr_tramp);
+
+/*
+ * The double-breakpoint mechanism needs to save the current probe for the next
+ * call to kinst_invop(). As with per-CPU trampolines, this also has to be done
+ * per-CPU when interrupts are disabled.
+ */
+DPCPU_DEFINE_STATIC(struct kinst_probe *, intr_probe);
+
+static int
+kinst_emulate(struct trapframe *frame, struct kinst_probe *kp)
+{
+ kinst_patchval_t instr = kp->kp_savedval;
+ uint64_t imm;
+ uint8_t cond, reg, bitpos;
+ bool res;
+
+ if (((instr >> 24) & 0x1f) == 0b10000) {
+ /* adr/adrp */
+ reg = instr & 0x1f;
+ imm = (instr >> 29) & 0x3;
+ imm |= ((instr >> 5) & 0x0007ffff) << 2;
+ if (((instr >> 31) & 0x1) == 0) {
+ /* adr */
+ if (imm & 0x0000000000100000)
+ imm |= 0xfffffffffff00000;
+ frame->tf_x[reg] = frame->tf_elr + imm;
+ } else {
+ /* adrp */
+ imm = (imm << 12) & ~0xfff;
+ if (imm & 0x0000000100000000)
+ imm |= 0xffffffff00000000;
+ frame->tf_x[reg] = (frame->tf_elr & ~0xfff) + imm;
+ }
+ frame->tf_elr += INSN_SIZE;
+ } else if (((instr >> 26) & 0x3f) == 0b000101) {
+ /* b */
+ imm = instr & 0x03ffffff;
+ if (imm & 0x0000000002000000)
+ imm |= 0xfffffffffe000000;
+ frame->tf_elr += imm << 2;
+ } else if (((instr >> 24) & 0xff) == 0b01010100) {
+ /* b.cond */
+ imm = (instr >> 5) & 0x0007ffff;
+ if (imm & 0x0000000000040000)
+ imm |= 0xfffffffffffc0000;
+ cond = instr & 0xf;
+ switch ((cond >> 1) & 0x7) {
+ case 0b000: /* eq/ne */
+ res = (frame->tf_spsr & PSR_Z) != 0;
+ break;
+ case 0b001: /* cs/cc */
+ res = (frame->tf_spsr & PSR_C) != 0;
+ break;
+ case 0b010: /* mi/pl */
+ res = (frame->tf_spsr & PSR_N) != 0;
+ break;
+ case 0b011: /* vs/vc */
+ res = (frame->tf_spsr & PSR_V) != 0;
+ break;
+ case 0b100: /* hi/ls */
+ res = ((frame->tf_spsr & PSR_C) != 0) &&
+ ((frame->tf_spsr & PSR_Z) == 0);
+ break;
+ case 0b101: /* ge/lt */
+ res = ((frame->tf_spsr & PSR_N) != 0) ==
+ ((frame->tf_spsr & PSR_V) != 0);
+ break;
+ case 0b110: /* gt/le */
+ res = ((frame->tf_spsr & PSR_Z) == 0) &&
+ (((frame->tf_spsr & PSR_N) != 0) ==
+ ((frame->tf_spsr & PSR_V) != 0));
+ break;
+ case 0b111: /* al */
+ res = 1;
+ break;
+ }
+ if ((cond & 0x1) && cond != 0b1111)
+ res = !res;
+ if (res)
+ frame->tf_elr += imm << 2;
+ else
+ frame->tf_elr += INSN_SIZE;
+ } else if (((instr >> 26) & 0x3f) == 0b100101) {
+ /* bl */
+ imm = instr & 0x03ffffff;
+ if (imm & 0x0000000002000000)
+ imm |= 0xfffffffffe000000;
+ frame->tf_lr = frame->tf_elr + INSN_SIZE;
+ frame->tf_elr += imm << 2;
+ } else if (((instr >> 25) & 0x3f) == 0b011010) {
+ /* cbnz/cbz */
+ cond = (instr >> 24) & 0x1;
+ reg = instr & 0x1f;
+ imm = (instr >> 5) & 0x0007ffff;
+ if (imm & 0x0000000000040000)
+ imm |= 0xfffffffffffc0000;
+ if (cond == 1 && frame->tf_x[reg] != 0)
+ /* cbnz */
+ frame->tf_elr += imm << 2;
+ else if (cond == 0 && frame->tf_x[reg] == 0)
+ /* cbz */
+ frame->tf_elr += imm << 2;
+ else
+ frame->tf_elr += INSN_SIZE;
+ } else if (((instr >> 25) & 0x3f) == 0b011011) {
+ /* tbnz/tbz */
+ cond = (instr >> 24) & 0x1;
+ reg = instr & 0x1f;
+ bitpos = (instr >> 19) & 0x1f;
+ bitpos |= ((instr >> 31) & 0x1) << 5;
+ imm = (instr >> 5) & 0x3fff;
+ if (imm & 0x0000000000002000)
+ imm |= 0xffffffffffffe000;
+ if (cond == 1 && (frame->tf_x[reg] & (1 << bitpos)) != 0)
+ /* tbnz */
+ frame->tf_elr += imm << 2;
+ else if (cond == 0 && (frame->tf_x[reg] & (1 << bitpos)) == 0)
+ /* tbz */
+ frame->tf_elr += imm << 2;
+ else
+ frame->tf_elr += INSN_SIZE;
+ }
+
+ return (0);
+}
+
+static int
+kinst_jump_next_instr(struct trapframe *frame, struct kinst_probe *kp)
+{
+ frame->tf_elr = (register_t)((uint8_t *)kp->kp_patchpoint + INSN_SIZE);
+
+ return (0);
+}
+
+static void
+kinst_trampoline_populate(struct kinst_probe *kp, uint8_t *tramp)
+{
+ static uint32_t bpt = KINST_PATCHVAL;
+
+ kinst_memcpy(tramp, &kp->kp_savedval, INSN_SIZE);
+ kinst_memcpy(&tramp[INSN_SIZE], &bpt, INSN_SIZE);
+
+ cpu_icache_sync_range((vm_offset_t)kp->kp_patchpoint,
+ (vm_size_t)INSN_SIZE);
+ cpu_icache_sync_range((vm_offset_t)tramp, (vm_size_t)KINST_TRAMP_SIZE);
+}
+
+/*
+ * There are two ways by which an instruction is traced:
+ *
+ * - By using the trampoline.
+ * - By emulating it in software (see kinst_emulate()).
+ *
+ * The trampoline is used for instructions that can be copied and executed
+ * as-is without additional modification. However, instructions that use
+ * PC-relative addressing have to be emulated, because ARM64 doesn't allow
+ * encoding of large displacements in a single instruction, and since we cannot
+ * clobber a register in order to encode the two-instruction sequence needed to
+ * create large displacements, we cannot use the trampoline at all.
+ * Fortunately, the instructions are simple enough to be emulated in just a few
+ * lines of code.
+ *
+ * The problem discussed above also means that, unlike amd64, we cannot encode
+ * a far-jump back from the trampoline to the next instruction. The mechanism
+ * employed to achieve this functionality, is to use a breakpoint instead of a
+ * jump after the copied instruction. This breakpoint is detected and handled
+ * by kinst_invop(), which performs the jump back to the next instruction
+ * manually (see kinst_jump_next_instr()).
+ */
+int
+kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch)
+{
+ solaris_cpu_t *cpu;
+ struct kinst_probe *kp;
+ uint8_t *tramp;
+
+ /*
+ * Use per-CPU trampolines and probes if the thread executing the
+ * instruction was executing with interrupts disabled.
+ */
+ if ((frame->tf_spsr & PSR_I) != 0) {
+ tramp = DPCPU_GET(intr_tramp);
+ kp = DPCPU_GET(intr_probe);
+ } else {
+ tramp = curthread->t_kinst_tramp;
+ kp = curthread->t_kinst_curprobe;
+ }
+
+ /*
+ * Detect if the breakpoint was triggered by the trampoline, and
+ * manually set the PC to the next instruction.
+ */
+ if (addr == (uintptr_t)(tramp + INSN_SIZE))
+ return (kinst_jump_next_instr(frame, kp));
+
+ LIST_FOREACH(kp, KINST_GETPROBE(addr), kp_hashnext) {
+ if ((uintptr_t)kp->kp_patchpoint == addr)
+ break;
+ }
+ if (kp == NULL)
+ return (0);
+
+ cpu = &solaris_cpu[curcpu];
+ cpu->cpu_dtrace_caller = addr;
+ dtrace_probe(kp->kp_id, 0, 0, 0, 0, 0);
+ cpu->cpu_dtrace_caller = 0;
+
+ if (kp->kp_md.emulate)
+ return (kinst_emulate(frame, kp));
+
+ if (tramp == NULL) {
+ /*
+ * A trampoline allocation failed, so this probe is
+ * effectively disabled. Restore the original
+ * instruction.
+ *
+ * We can't safely print anything here, but the
+ * trampoline allocator should have left a breadcrumb in
+ * the dmesg.
+ */
+ kinst_patch_tracepoint(kp, kp->kp_savedval);
+ frame->tf_elr = (register_t)kp->kp_patchpoint;
+ } else {
+ kinst_trampoline_populate(kp, tramp);
+ frame->tf_elr = (register_t)tramp;
+ if ((frame->tf_spsr & PSR_I) != 0)
+ DPCPU_SET(intr_probe, kp);
+ else
+ curthread->t_kinst_curprobe = kp;
+ }
+
+ return (0);
+}
+
+void
+kinst_patch_tracepoint(struct kinst_probe *kp, kinst_patchval_t val)
+{
+ vm_offset_t addr;
+
+ if (!arm64_get_writable_addr((vm_offset_t)kp->kp_patchpoint, &addr))
+ panic("%s: Unable to write new instruction", __func__);
+ *(kinst_patchval_t *)addr = val;
+ cpu_icache_sync_range((vm_offset_t)kp->kp_patchpoint,
+ (vm_size_t)INSN_SIZE);
+}
+
+static void
+kinst_instr_dissect(struct kinst_probe *kp)
+{
+ struct kinst_probe_md *kpmd;
+ kinst_patchval_t instr = kp->kp_savedval;
+
+ kpmd = &kp->kp_md;
+ kpmd->emulate = false;
+
+ if (((instr >> 24) & 0x1f) == 0b10000)
+ kpmd->emulate = true; /* adr/adrp */
+ else if (((instr >> 26) & 0x3f) == 0b000101)
+ kpmd->emulate = true; /* b */
+ else if (((instr >> 24) & 0xff) == 0b01010100)
+ kpmd->emulate = true; /* b.cond */
+ else if (((instr >> 26) & 0x3f) == 0b100101)
+ kpmd->emulate = true; /* bl */
+ else if (((instr >> 25) & 0x3f) == 0b011010)
+ kpmd->emulate = true; /* cbnz/cbz */
+ else if (((instr >> 25) & 0x3f) == 0b011011)
+ kpmd->emulate = true; /* tbnz/tbz */
+}
+
+static bool
+kinst_instr_ldx(kinst_patchval_t instr)
+{
+ if (((instr >> 22) & 0xff) == 0b00100001)
+ return (true);
+
+ return (false);
+}
+
+static bool
+kinst_instr_stx(kinst_patchval_t instr)
+{
+ if (((instr >> 22) & 0xff) == 0b00100000)
+ return (true);
+
+ return (false);
+}
+
+int
+kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval,
+ void *opaque)
+{
+ struct kinst_probe *kp;
+ dtrace_kinst_probedesc_t *pd;
+ const char *func;
+ kinst_patchval_t *instr, *limit, *tmp;
+ int n, off;
+ bool ldxstx_block, found;
+
+ pd = opaque;
+ func = symval->name;
+
+ if (kinst_excluded(func))
+ return (0);
+ if (strcmp(func, pd->kpd_func) != 0)
+ return (0);
+
+ instr = (kinst_patchval_t *)(symval->value);
+ limit = (kinst_patchval_t *)(symval->value + symval->size);
+ if (instr >= limit)
+ return (0);
+
+ tmp = instr;
+
+ /*
+ * Ignore any bti instruction at the start of the function
+ * we need to keep it there for any indirect branches calling
+ * the function on Armv8.5+
+ */
+ if ((*tmp & BTI_MASK) == BTI_INSTR)
+ tmp++;
+
+ /* Look for stp (pre-indexed) operation */
+ found = false;
+
+ /*
+ * If the first instruction is a nop it's a specially marked
+ * asm function. We only support a nop first as it's not a normal
+ * part of the function prologue.
+ */
+ if (*tmp == NOP_INSTR)
+ found = true;
+ for (; !found && tmp < limit; tmp++) {
+ /*
+ * Functions start with "stp xt1, xt2, [xn, <const>]!" or
+ * "sub sp, sp, <const>".
+ *
+ * Sometimes the compiler will have a sub instruction that is
+ * not of the above type so don't stop if we see one.
+ */
+ if ((*tmp & LDP_STP_MASK) == STP_64) {
+ /*
+ * Assume any other store of this type means we are
+ * past the function prolog.
+ */
+ if (((*tmp >> ADDR_SHIFT) & ADDR_MASK) == 31)
+ found = true;
+ } else if ((*tmp & SUB_MASK) == SUB_INSTR &&
+ ((*tmp >> SUB_RD_SHIFT) & SUB_R_MASK) == 31 &&
+ ((*tmp >> SUB_RN_SHIFT) & SUB_R_MASK) == 31)
+ found = true;
+ }
+
+ if (!found)
+ return (0);
+
+ ldxstx_block = false;
+ for (n = 0; instr < limit; instr++) {
+ off = (int)((uint8_t *)instr - (uint8_t *)symval->value);
+
+ /*
+ * Skip LDX/STX blocks that contain atomic operations. If a
+ * breakpoint is placed in a LDX/STX block, we violate the
+ * operation and the loop might fail.
+ */
+ if (kinst_instr_ldx(*instr))
+ ldxstx_block = true;
+ else if (kinst_instr_stx(*instr)) {
+ ldxstx_block = false;
+ continue;
+ }
+ if (ldxstx_block)
+ continue;
+
+ if (pd->kpd_off != -1 && off != pd->kpd_off)
+ continue;
+ /*
+ * Prevent separate dtrace(1) instances from creating copies of
+ * the same probe.
+ */
+ LIST_FOREACH(kp, KINST_GETPROBE(instr), kp_hashnext) {
+ if (strcmp(kp->kp_func, func) == 0 &&
+ strtol(kp->kp_name, NULL, 10) == off)
+ return (0);
+ }
+ if (++n > KINST_PROBETAB_MAX) {
+ KINST_LOG("probe list full: %d entries", n);
+ return (ENOMEM);
+ }
+ kp = malloc(sizeof(struct kinst_probe), M_KINST,
+ M_WAITOK | M_ZERO);
+ kp->kp_func = func;
+ snprintf(kp->kp_name, sizeof(kp->kp_name), "%d", off);
+ kp->kp_patchpoint = instr;
+ kp->kp_savedval = *instr;
+ kp->kp_patchval = KINST_PATCHVAL;
+
+ kinst_instr_dissect(kp);
+ kinst_probe_create(kp, lf);
+ }
+ if (ldxstx_block)
+ KINST_LOG("warning: unterminated LDX/STX block");
+
+ return (0);
+}
+
+int
+kinst_md_init(void)
+{
+ uint8_t *tramp;
+ int cpu;
+
+ CPU_FOREACH(cpu) {
+ tramp = kinst_trampoline_alloc(M_WAITOK);
+ if (tramp == NULL)
+ return (ENOMEM);
+ DPCPU_ID_SET(cpu, intr_tramp, tramp);
+ }
+
+ return (0);
+}
+
+void
+kinst_md_deinit(void)
+{
+ uint8_t *tramp;
+ int cpu;
+
+ CPU_FOREACH(cpu) {
+ tramp = DPCPU_ID_GET(cpu, intr_tramp);
+ if (tramp != NULL) {
+ kinst_trampoline_dealloc(tramp);
+ DPCPU_ID_SET(cpu, intr_tramp, NULL);
+ }
+ }
+}
+
+/*
+ * Exclude machine-dependent functions that are not safe-to-trace.
+ */
+bool
+kinst_md_excluded(const char *name)
+{
+ if (strcmp(name, "handle_el1h_sync") == 0 ||
+ strcmp(name, "do_el1h_sync") == 0)
+ return (true);
+
+ return (false);
+}
diff --git a/sys/modules/dtrace/Makefile b/sys/modules/dtrace/Makefile
--- a/sys/modules/dtrace/Makefile
+++ b/sys/modules/dtrace/Makefile
@@ -21,6 +21,9 @@
SUBDIR+= systrace_linux32
SUBDIR+= kinst
.endif
+.if ${MACHINE_CPUARCH} == "riscv" || ${MACHINE_CPUARCH} == "aarch64"
+SUBDIR+= kinst
+.endif
.if ${MACHINE_CPUARCH} == "powerpc"
SUBDIR+= fasttrap
.endif

File Metadata

Mime Type
text/plain
Expires
Thu, Dec 5, 6:40 AM (11 h, 28 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
15047583
Default Alt Text
D40337.id124157.diff (16 KB)

Event Timeline