Page MenuHomeFreeBSD
Files F157271068

D40337.id122620.diff
No OneTemporary
Actions

D40337.id122620.diff
View Options

diff --git a/sys/cddl/dev/kinst/aarch64/kinst_isa.h b/sys/cddl/dev/kinst/aarch64/kinst_isa.h
new file mode 100644
--- /dev/null
+++ b/sys/cddl/dev/kinst/aarch64/kinst_isa.h
@@ -0,0 +1,28 @@
+/*
+ * SPDX-License-Identifier: CDDL 1.0
+ *
+ * Copyright 2023 Christos Margiolis <christos@FreeBSD.org>
+ */
+
+#ifndef _KINST_ISA_H_
+#define _KINST_ISA_H_
+
+#define AARCH64_BRK 0xd4200000
+#define AARCH64_BRK_IMM16_SHIFT 5
+#define AARCH64_BRK_IMM16_VAL (0x40d << AARCH64_BRK_IMM16_SHIFT)
+#define KINST_PATCHVAL (AARCH64_BRK | AARCH64_BRK_IMM16_VAL)
+
+#define KINST_TRAMP_SIZE 8
+#define KINST_TRAMPCHUNK_SIZE PAGE_SIZE
+
+#define KINST_TRAMP_FILL_PATTERN ((uint32_t []){KINST_PATCHVAL})
+#define KINST_TRAMP_FILL_SIZE sizeof(uint32_t)
+
+typedef uint32_t kinst_patchval_t;
+
+struct kinst_probe_md {
+ int emulate; /* emulate in sw */
+ uint8_t template[INSN_SIZE]; /* copied into thread tramps */
+};
+
+#endif /* _KINST_ISA_H_ */
diff --git a/sys/cddl/dev/kinst/aarch64/kinst_isa.c b/sys/cddl/dev/kinst/aarch64/kinst_isa.c
new file mode 100644
--- /dev/null
+++ b/sys/cddl/dev/kinst/aarch64/kinst_isa.c
@@ -0,0 +1,421 @@
+/*
+ * SPDX-License-Identifier: CDDL 1.0
+ *
+ * Copyright 2023 Christos Margiolis <christos@FreeBSD.org>
+ */
+
+#include <sys/param.h>
+
+#include <sys/dtrace.h>
+#include <cddl/dev/dtrace/dtrace_cddl.h>
+
+#include "kinst.h"
+
+/*
+ * Per-CPU trampolines used when the interrupted thread is executing with
+ * interrupts disabled. If an interrupt is raised while executing a trampoline,
+ * the interrupt thread cannot safely overwrite its trampoline if it hits a
+ * kinst probe while executing the interrupt handler.
+ */
+DPCPU_DEFINE_STATIC(uint8_t *, intr_tramp);
+
+/*
+ * The double-breakpoint mechanism needs to save the current probe for the next
+ * call to kinst_invop(). As with per-CPU trampolines, this also has to be done
+ * per-CPU when interrupts are disabled.
+ */
+DPCPU_DEFINE_STATIC(struct kinst_probe *, intr_probe);
+
+static int
+kinst_emulate(struct trapframe *frame, struct kinst_probe *kp)
+{
+ kinst_patchval_t instr = kp->kp_savedval;
+ uint64_t imm;
+ uint8_t cond, reg, bitpos;
+ bool res;
+
+ if (((instr >> 24) & 0x1f) == 0b10000) {
+ /* adr/adrp */
+ reg = instr & 0x1f;
+ imm = (instr >> 29) & 0x3;
+ imm |= ((instr >> 5) & 0x0007ffff) << 2;
+ if (((instr >> 31) & 0x1) == 0) {
+ /* adr */
+ if (imm & 0x0000000000100000)
+ imm |= 0xfffffffffff00000;
+ frame->tf_x[reg] = frame->tf_elr + imm;
+ } else {
+ /* adrp */
+ imm = (imm << 12) & ~0xfff;
+ if (imm & 0x0000000100000000)
+ imm |= 0xffffffff00000000;
+ frame->tf_x[reg] = (frame->tf_elr & ~0xfff) + imm;
+ }
+ frame->tf_elr += INSN_SIZE;
+ } else if (((instr >> 26) & 0x3f) == 0b000101) {
+ /* b */
+ imm = instr & 0x03ffffff;
+ if (imm & 0x0000000002000000)
+ imm |= 0xfffffffffe000000;
+ frame->tf_elr += imm << 2;
+ } else if (((instr >> 24) & 0xff) == 0b01010100) {
+ /* b.cond */
+ imm = (instr >> 5) & 0x0007ffff;
+ if (imm & 0x0000000000040000)
+ imm |= 0xfffffffffffc0000;
+ cond = instr & 0xf;
+ switch ((cond >> 1) & 0x7) {
+ case 0b000: /* eq/ne */
+ res = (frame->tf_spsr & PSR_Z) != 0;
+ break;
+ case 0b001: /* cs/cc */
+ res = (frame->tf_spsr & PSR_C) != 0;
+ break;
+ case 0b010: /* mi/pl */
+ res = (frame->tf_spsr & PSR_N) != 0;
+ break;
+ case 0b011: /* vs/vc */
+ res = (frame->tf_spsr & PSR_V) != 0;
+ break;
+ case 0b100: /* hi/ls */
+ res = ((frame->tf_spsr & PSR_C) != 0) &&
+ ((frame->tf_spsr & PSR_Z) == 0);
+ break;
+ case 0b101: /* ge/lt */
+ res = ((frame->tf_spsr & PSR_N) != 0) ==
+ ((frame->tf_spsr & PSR_V) != 0);
+ break;
+ case 0b110: /* gt/le */
+ res = ((frame->tf_spsr & PSR_Z) == 0) &&
+ (((frame->tf_spsr & PSR_N) != 0) ==
+ ((frame->tf_spsr & PSR_V) != 0));
+ break;
+ case 0b111: /* al */
+ res = 1;
+ break;
+ }
+ if ((cond & 0x1) && cond != 0b1111)
+ res = !res;
+ if (res)
+ frame->tf_elr += imm << 2;
+ else
+ frame->tf_elr += INSN_SIZE;
+ } else if (((instr >> 26) & 0x3f) == 0b100101) {
+ /* bl */
+ imm = instr & 0x03ffffff;
+ if (imm & 0x0000000002000000)
+ imm |= 0xfffffffffe000000;
+ frame->tf_lr = frame->tf_elr + INSN_SIZE;
+ frame->tf_elr += imm << 2;
+ } else if (((instr >> 25) & 0x3f) == 0b011010) {
+ /* cbnz/cbz */
+ cond = (instr >> 24) & 0x1;
+ reg = instr & 0x1f;
+ imm = (instr >> 5) & 0x0007ffff;
+ if (imm & 0x0000000000040000)
+ imm |= 0xfffffffffffc0000;
+ if (cond == 1 && frame->tf_x[reg] != 0)
+ /* cbnz */
+ frame->tf_elr += imm << 2;
+ else if (cond == 0 && frame->tf_x[reg] == 0)
+ /* cbz */
+ frame->tf_elr += imm << 2;
+ else
+ frame->tf_elr += INSN_SIZE;
+ } else if (((instr >> 25) & 0x3f) == 0b011011) {
+ /* tbnz/tbz */
+ cond = (instr >> 24) & 0x1;
+ reg = instr & 0x1f;
+ bitpos = (instr >> 19) & 0x1f;
+ bitpos |= ((instr >> 31) & 0x1) << 5;
+ imm = (instr >> 5) & 0x3fff;
+ if (imm & 0x0000000000002000)
+ imm |= 0xffffffffffffe000;
+ if (cond == 1 && (frame->tf_x[reg] & (1 << bitpos)) != 0)
+ /* tbnz */
+ frame->tf_elr += imm << 2;
+ else if (cond == 0 && (frame->tf_x[reg] & (1 << bitpos)) == 0)
+ /* tbz */
+ frame->tf_elr += imm << 2;
+ else
+ frame->tf_elr += INSN_SIZE;
+ }
+
+ return (0);
+}
+
+static int
+kinst_jump_next_instr(struct trapframe *frame, struct kinst_probe *kp)
+{
+ frame->tf_elr = (register_t)((uint8_t *)kp->kp_patchpoint + INSN_SIZE);
+
+ return (0);
+}
+
+static void
+kinst_trampoline_populate(struct kinst_probe *kp, uint8_t *tramp)
+{
+ static uint32_t bpt = KINST_PATCHVAL;
+
+ kinst_memcpy(tramp, kp->kp_md.template, INSN_SIZE);
+ kinst_memcpy(&tramp[INSN_SIZE], &bpt, INSN_SIZE);
+}
+
+int
+kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch)
+{
+ solaris_cpu_t *cpu;
+ struct kinst_probe *kp;
+ uint8_t *tramp;
+
+ if ((frame->tf_spsr & PSR_I) != 0) {
+ tramp = DPCPU_GET(intr_tramp);
+ if (addr == (uintptr_t)(tramp + INSN_SIZE)) {
+ kp = DPCPU_GET(intr_probe);
+ return (kinst_jump_next_instr(frame, kp));
+ }
+ } else {
+ tramp = curthread->t_kinst;
+ if (addr == (uintptr_t)(tramp + INSN_SIZE)) {
+ kp = curthread->t_kinst_curprobe;
+ return (kinst_jump_next_instr(frame, kp));
+ }
+ }
+
+ LIST_FOREACH(kp, KINST_GETPROBE(addr), kp_hashnext) {
+ if ((uintptr_t)kp->kp_patchpoint == addr)
+ break;
+ }
+ if (kp == NULL)
+ return (0);
+
+ cpu = &solaris_cpu[curcpu];
+ cpu->cpu_dtrace_caller = addr;
+ dtrace_probe(kp->kp_id, 0, 0, 0, 0, 0);
+ cpu->cpu_dtrace_caller = 0;
+
+ if (kp->kp_md.emulate)
+ return (kinst_emulate(frame, kp));
+
+ if (tramp == NULL) {
+ /*
+ * A trampoline allocation failed, so this probe is
+ * effectively disabled. Restore the original
+ * instruction.
+ *
+ * We can't safely print anything here, but the
+ * trampoline allocator should have left a breadcrumb in
+ * the dmesg.
+ */
+ kinst_patch_tracepoint(kp, kp->kp_savedval);
+ frame->tf_elr = (register_t)kp->kp_patchpoint;
+ } else {
+ kinst_trampoline_populate(kp, tramp);
+ frame->tf_elr = (register_t)tramp;
+ if ((frame->tf_spsr & PSR_I) != 0)
+ DPCPU_SET(intr_probe, kp);
+ else
+ curthread->t_kinst_curprobe = kp;
+ }
+
+ return (0);
+}
+
+void
+kinst_patch_tracepoint(struct kinst_probe *kp, kinst_patchval_t val)
+{
+ vm_offset_t addr;
+
+ if (!arm64_get_writable_addr((vm_offset_t)kp->kp_patchpoint, &addr))
+ panic("%s: Unable to write new instruction", __func__);
+ *(kinst_patchval_t *)addr = val;
+ cpu_icache_sync_range((vm_offset_t)kp->kp_patchpoint, INSN_SIZE);
+}
+
+static void
+kinst_instr_dissect(struct kinst_probe *kp)
+{
+ struct kinst_probe_md *kpmd;
+ kinst_patchval_t instr = kp->kp_savedval;
+
+ kpmd = &kp->kp_md;
+ kpmd->emulate = 0;
+
+ if (((instr >> 24) & 0x1f) == 0b10000)
+ kpmd->emulate = 1; /* adr/adrp */
+ else if (((instr >> 26) & 0x3f) == 0b000101)
+ kpmd->emulate = 1; /* b */
+ else if (((instr >> 24) & 0xff) == 0b01010100)
+ kpmd->emulate = 1; /* b.cond */
+ else if (((instr >> 26) & 0x3f) == 0b100101)
+ kpmd->emulate = 1; /* bl */
+ else if (((instr >> 25) & 0x3f) == 0b011010)
+ kpmd->emulate = 1; /* cbnz/cbz */
+ else if (((instr >> 25) & 0x3f) == 0b011011)
+ kpmd->emulate = 1; /* tbnz/tbz */
+
+ if (!kpmd->emulate)
+ memcpy(kpmd->template, &instr, INSN_SIZE);
+}
+
+int
+kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval,
+ void *opaque)
+{
+ struct kinst_probe *kp;
+ dtrace_kinst_probedesc_t *pd;
+ const char *func;
+ kinst_patchval_t *instr, *limit;
+ int n, off;
+ bool atomic, found;
+
+ pd = opaque;
+ func = symval->name;
+
+ if (kinst_excluded(func))
+ return (0);
+ if (strcmp(func, pd->kpd_func) != 0)
+ return (0);
+
+ instr = (kinst_patchval_t *)(symval->value);
+ limit = (kinst_patchval_t *)(symval->value + symval->size);
+ if (instr >= limit)
+ return (0);
+
+ /*
+ * Ignore any bti instruction at the start of the function
+ * we need to keep it there for any indirect branches calling
+ * the function on Armv8.5+
+ */
+ if ((*instr & BTI_MASK) == BTI_INSTR)
+ instr++;
+
+ /* Look for stp (pre-indexed) operation */
+ found = false;
+
+ /*
+ * If the first instruction is a nop it's a specially marked
+ * asm function. We only support a nop first as it's not a normal
+ * part of the function prologue.
+ */
+ if (*instr == NOP_INSTR)
+ found = true;
+ if (!found) {
+ for (; instr < limit; instr++) {
+ /*
+ * Some functions start with
+ * "stp xt1, xt2, [xn, <const>]!"
+ */
+ if ((*instr & LDP_STP_MASK) == STP_64) {
+ /*
+ * Assume any other store of this type means we
+ * are past the function prolog.
+ */
+ if (((*instr >> ADDR_SHIFT) & ADDR_MASK) == 31)
+ found = true;
+ break;
+ }
+
+ /*
+ * Some functions start with a "sub sp, sp, <const>"
+ * Sometimes the compiler will have a sub instruction
+ * that is not of the above type so don't stop if we
+ * see one.
+ */
+ if ((*instr & SUB_MASK) == SUB_INSTR &&
+ ((*instr >> SUB_RD_SHIFT) & SUB_R_MASK) == 31 &&
+ ((*instr >> SUB_RN_SHIFT) & SUB_R_MASK) == 31) {
+ found = true;
+ break;
+ }
+ }
+ }
+
+ if (!found)
+ return (0);
+
+ atomic = false;
+ for (n = 0; instr < limit; instr++) {
+ off = (int)((uint8_t *)instr - (uint8_t *)symval->value);
+
+ /* Skip LDX/STX blocks that contain atomic operations. */
+ if (((*instr >> 22) & 0xff) == 0b00100001)
+ atomic = true;
+ else if (((*(instr - 1) >> 22) & 0xff) == 0b00100000 && atomic)
+ atomic = false;
+ if (atomic)
+ continue;
+
+ if (pd->kpd_off != -1 && off != pd->kpd_off)
+ continue;
+ /*
+ * Prevent separate dtrace(1) instances from creating copies of
+ * the same probe.
+ */
+ LIST_FOREACH(kp, KINST_GETPROBE(instr), kp_hashnext) {
+ if (strcmp(kp->kp_func, func) == 0 &&
+ strtol(kp->kp_name, NULL, 10) == off)
+ return (0);
+ }
+ if (++n > KINST_PROBETAB_MAX) {
+ KINST_LOG("probe list full: %d entries", n);
+ return (ENOMEM);
+ }
+ kp = malloc(sizeof(struct kinst_probe), M_KINST,
+ M_WAITOK | M_ZERO);
+ kp->kp_func = func;
+ snprintf(kp->kp_name, sizeof(kp->kp_name), "%d", off);
+ kp->kp_patchpoint = instr;
+ kp->kp_savedval = *instr;
+ kp->kp_patchval = KINST_PATCHVAL;
+
+ kinst_instr_dissect(kp);
+ kinst_probe_create(kp, lf);
+ }
+
+ return (0);
+}
+
+int
+kinst_md_init(void)
+{
+ uint8_t *tramp;
+ int cpu;
+
+ CPU_FOREACH(cpu) {
+ tramp = kinst_trampoline_alloc(M_WAITOK);
+ if (tramp == NULL)
+ return (ENOMEM);
+ DPCPU_ID_SET(cpu, intr_tramp, tramp);
+ }
+
+ return (0);
+}
+
+void
+kinst_md_deinit(void)
+{
+ uint8_t *tramp;
+ int cpu;
+
+ CPU_FOREACH(cpu) {
+ tramp = DPCPU_ID_GET(cpu, intr_tramp);
+ if (tramp != NULL) {
+ kinst_trampoline_dealloc(tramp);
+ DPCPU_ID_SET(cpu, intr_tramp, NULL);
+ }
+ }
+}
+
+/*
+ * Exclude machine-dependent functions that are not safe-to-trace.
+ */
+int
+kinst_md_excluded(const char *name)
+{
+ if (strcmp(name, "handle_el1h_sync") == 0 ||
+ strcmp(name, "do_el1h_sync") == 0)
+ return (1);
+
+ return (0);
+}
diff --git a/sys/modules/dtrace/Makefile b/sys/modules/dtrace/Makefile
--- a/sys/modules/dtrace/Makefile
+++ b/sys/modules/dtrace/Makefile
@@ -21,6 +21,9 @@
SUBDIR+= systrace_linux32
SUBDIR+= kinst
.endif
+.if ${MACHINE_CPUARCH} == "riscv" || ${MACHINE_CPUARCH} == "aarch64"
+SUBDIR+= kinst
+.endif
.if ${MACHINE_CPUARCH} == "powerpc"
SUBDIR+= fasttrap
.endif
diff --git a/sys/modules/dtrace/dtraceall/dtraceall.c b/sys/modules/dtrace/dtraceall/dtraceall.c
--- a/sys/modules/dtrace/dtraceall/dtraceall.c
+++ b/sys/modules/dtrace/dtraceall/dtraceall.c
@@ -76,9 +76,9 @@
#endif
#if defined(__amd64__) || defined(__i386__) || defined(__powerpc__)
MODULE_DEPEND(dtraceall, fasttrap, 1, 1, 1);
-#if defined(__amd64__)
-MODULE_DEPEND(dtraceall, kinst, 1, 1, 1);
#endif
+#if defined(__amd64__) || defined (__riscv) || defined(__aarch64__)
+MODULE_DEPEND(dtraceall, kinst, 1, 1, 1);
#endif
MODULE_DEPEND(dtraceall, sdt, 1, 1, 1);
MODULE_DEPEND(dtraceall, systrace, 1, 1, 1);

File Metadata

Mime Type
text/plain
Expires
Wed, May 20, 10:05 PM (20 h, 18 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
33358922
Default Alt Text
D40337.id122620.diff (12 KB)

Event Timeline