diff --git a/sys/amd64/amd64/zcond_machdep.c b/sys/amd64/amd64/zcond_machdep.c new file mode 100644 --- /dev/null +++ b/sys/amd64/amd64/zcond_machdep.c @@ -0,0 +1,109 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Marko Vlaić + * + * This code was developed as a Google Summer of Code 2024. project + * under the guidance of Bojan Novković . + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include +#include +#include + +static uint8_t insn[ZCOND_MAX_INSN_SIZE]; + +static uint8_t * +insn_nop(size_t size) +{ + if (size == ZCOND_INSN_SHORT_SIZE) { + return &nop_short_bytes[0]; + } + return &nop_long_bytes[0]; +} + +static uint8_t * +insn_jmp(size_t size, vm_offset_t patch_addr, vm_offset_t lbl_true_addr) +{ + int i; + vm_offset_t offset; + + offset = lbl_true_addr - patch_addr - size; + + if (size == ZCOND_INSN_SHORT_SIZE) { + insn[0] = ZCOND_JMP_SHORT_OPCODE; + insn[1] = offset; + } else { + insn[0] = ZCOND_JMP_LONG_OPCODE; + for (i = 0; i < ZCOND_INSN_LONG_SIZE - 1; i++) { + insn[i + 1] = (offset >> (i * 8)) & 0xFF; + } + } + + return &insn[0]; +} + +void +zcond_before_patch(struct zcond_md_ctxt *ctxt) +{ + ctxt->wp = disable_wp(); +} + +void +zcond_after_patch(struct zcond_md_ctxt *ctxt) +{ + restore_wp(ctxt->wp); +} + +uint8_t * +zcond_get_patch_insn(vm_offset_t patch_addr, vm_offset_t lbl_true_addr, + size_t *size) +{ + uint8_t *pa; + + pa = (uint8_t *)patch_addr; + if (*pa == nop_short_bytes[0]) { + /* two byte nop */ + *size = ZCOND_INSN_SHORT_SIZE; + return insn_jmp(*size, patch_addr, lbl_true_addr); + } else if (*pa == nop_long_bytes[0]) { + *size = ZCOND_INSN_LONG_SIZE; + return insn_jmp(*size, patch_addr, lbl_true_addr); + } else if (*pa == ZCOND_JMP_SHORT_OPCODE) { + /* two byte jump */ + *size = ZCOND_INSN_SHORT_SIZE; + return insn_nop(*size); + } else if (*pa == ZCOND_JMP_LONG_OPCODE) { + /* five byte jump */ + *size = ZCOND_INSN_LONG_SIZE; + return insn_nop(*size); + } else { + panic("%s: unexpected opcode: %02hhx", __func__, *pa); + } +} diff --git a/sys/amd64/include/sdt_machdep.h b/sys/amd64/include/sdt_machdep.h deleted file mode 100644 --- a/sys/amd64/include/sdt_machdep.h +++ /dev/null @@ -1,12 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2024 Mark Johnston - */ - -#ifndef _SYS_SDT_MACHDEP_H_ -#define _SYS_SDT_MACHDEP_H_ - -#define _SDT_ASM_PATCH_INSTR "nopw 0(%%rax,%%rax,1)" - -#endif diff --git a/sys/amd64/include/zcond.h b/sys/amd64/include/zcond.h new file mode 100644 --- /dev/null +++ b/sys/amd64/include/zcond.h @@ -0,0 +1,64 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Marko Vlaić + * + * This code was developed as a Google Summer of Code 2024. project + * under the guidance of Bojan Novković . + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifdef _KERNEL +#ifndef _MACHINE_ZCOND_H +#define _MACHINE_ZCOND_H + +// #include +#include + +/* from Intel® 64 and IA-32 Architectures Software Developer’s Manual, Volume 2B + * 4-165 */ +static uint8_t nop_short_bytes[] = { 0x66, 0x90 }; +static uint8_t nop_long_bytes[] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 }; + +#define ZCOND_NOP_ASM \ + ".byte 0x0f \n\t" \ + ".byte 0x1f \n\t" \ + ".byte 0x44 \n\t" \ + ".byte 0x00 \n\t" \ + ".byte 0x00 \n\t" + +#define ZCOND_JMP_ASM "jmp" + +#define ZCOND_JMP_SHORT_OPCODE 0xeb +#define ZCOND_JMP_LONG_OPCODE 0xe9 + +#define ZCOND_INSN_SHORT_SIZE 2 +#define ZCOND_INSN_LONG_SIZE 5 +#define ZCOND_MAX_INSN_SIZE 5 + +struct zcond_md_ctxt { + bool wp; +}; + +#endif /*_MACHINE_ZCOND_H*/ +#endif /* _KERNEL */ diff --git a/sys/arm/arm/sdt_machdep.c b/sys/arm/arm/sdt_machdep.c deleted file mode 100644 --- a/sys/arm/arm/sdt_machdep.c +++ /dev/null @@ -1,63 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2024 Mark Johnston - */ - -#include -#include - -#include - -/* - * Return true if we can overwrite a nop at "patchpoint" with a jump to the - * target address. - */ -bool -sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target) -{ - int32_t offset; - - if (patchpoint == target || - (patchpoint & (INSN_SIZE - 1)) != 0 || - (target & (INSN_SIZE - 1)) != 0 || - patchpoint + 2 * INSN_SIZE < patchpoint) - return (false); - offset = target - (patchpoint + 2 * INSN_SIZE); - if (offset < -(1 << 24) || offset > (1 >> 24)) - return (false); - return (true); -} - -/* - * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to - * the target address. - */ -void -sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target) -{ - uint32_t instr; - - KASSERT(sdt_tracepoint_valid(patchpoint, target), - ("%s: invalid tracepoint %#x -> %#x", - __func__, patchpoint, target)); - - instr = - (((target - (patchpoint + 2 * INSN_SIZE)) >> 2) & ((1 << 24) - 1)) | - 0xea000000; - memcpy((void *)patchpoint, &instr, sizeof(instr)); - icache_sync(patchpoint, sizeof(instr)); -} - -/* - * Overwrite the patchpoint with a nop instruction. - */ -void -sdt_tracepoint_restore(uintptr_t patchpoint) -{ - uint32_t instr; - - instr = 0xe320f000u; - memcpy((void *)patchpoint, &instr, sizeof(instr)); - icache_sync(patchpoint, sizeof(instr)); -} diff --git a/sys/arm/include/sdt_machdep.h b/sys/arm/include/sdt_machdep.h deleted file mode 100644 --- a/sys/arm/include/sdt_machdep.h +++ /dev/null @@ -1,12 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2024 Mark Johnston - */ - -#ifndef _SYS_SDT_MACHDEP_H_ -#define _SYS_SDT_MACHDEP_H_ - -#define _SDT_ASM_PATCH_INSTR "nop" - -#endif /* _SYS_SDT_MACHDEP_H_ */ diff --git a/sys/arm64/arm64/sdt_machdep.c b/sys/arm64/arm64/sdt_machdep.c deleted file mode 100644 --- a/sys/arm64/arm64/sdt_machdep.c +++ /dev/null @@ -1,77 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2024 Mark Johnston - */ - -#include -#include - -#include -#include - -#include -#include -#include - -/* - * Return true if we can overwrite a nop at "patchpoint" with a jump to the - * target address. - */ -bool -sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target) -{ - void *addr; - int64_t offset; - - if (!arm64_get_writable_addr((void *)patchpoint, &addr)) - return (false); - - if (patchpoint == target || - (patchpoint & (INSN_SIZE - 1)) != 0 || - (target & (INSN_SIZE - 1)) != 0) - return (false); - offset = target - patchpoint; - if (offset < -(1 << 26) || offset > (1 << 26)) - return (false); - return (true); -} - -/* - * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to the - * target address. - */ -void -sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target) -{ - void *addr; - uint32_t instr; - - KASSERT(sdt_tracepoint_valid(patchpoint, target), - ("%s: invalid tracepoint %#lx -> %#lx", - __func__, patchpoint, target)); - - if (!arm64_get_writable_addr((void *)patchpoint, &addr)) - panic("%s: Unable to write new instruction", __func__); - - instr = (((target - patchpoint) >> 2) & 0x3fffffful) | 0x14000000; - memcpy(addr, &instr, sizeof(instr)); - cpu_icache_sync_range((void *)patchpoint, INSN_SIZE); -} - -/* - * Overwrite the patchpoint with a nop instruction. - */ -void -sdt_tracepoint_restore(uintptr_t patchpoint) -{ - void *addr; - uint32_t instr; - - if (!arm64_get_writable_addr((void *)patchpoint, &addr)) - panic("%s: Unable to write new instruction", __func__); - - instr = 0xd503201f; - memcpy(addr, &instr, sizeof(instr)); - cpu_icache_sync_range((void *)patchpoint, INSN_SIZE); -} diff --git a/sys/arm64/include/sdt_machdep.h b/sys/arm64/include/sdt_machdep.h deleted file mode 100644 --- a/sys/arm64/include/sdt_machdep.h +++ /dev/null @@ -1,12 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2024 Mark Johnston - */ - -#ifndef _SYS_SDT_MACHDEP_H_ -#define _SYS_SDT_MACHDEP_H_ - -#define _SDT_ASM_PATCH_INSTR "nop" - -#endif /* _SYS_SDT_MACHDEP_H_ */ diff --git a/sys/cddl/dev/sdt/sdt.c b/sys/cddl/dev/sdt/sdt.c --- a/sys/cddl/dev/sdt/sdt.c +++ b/sys/cddl/dev/sdt/sdt.c @@ -54,6 +54,7 @@ #include #include #include +#include #include #include @@ -214,55 +215,6 @@ { } -struct sdt_enable_cb_arg { - struct sdt_probe *probe; - int cpu; - int arrived; - int done; - bool enable; -}; - -static void -sdt_probe_update_cb(void *_arg) -{ - struct sdt_enable_cb_arg *arg; - struct sdt_tracepoint *tp; - - arg = _arg; - if (arg->cpu != curcpu) { - atomic_add_rel_int(&arg->arrived, 1); - while (atomic_load_acq_int(&arg->done) == 0) - cpu_spinwait(); - return; - } else { - while (atomic_load_acq_int(&arg->arrived) != mp_ncpus - 1) - cpu_spinwait(); - } - - STAILQ_FOREACH(tp, &arg->probe->tracepoint_list, tracepoint_entry) { - if (arg->enable) - sdt_tracepoint_patch(tp->patchpoint, tp->target); - else - sdt_tracepoint_restore(tp->patchpoint); - } - - atomic_store_rel_int(&arg->done, 1); -} - -static void -sdt_probe_update(struct sdt_probe *probe, bool enable) -{ - struct sdt_enable_cb_arg cbarg; - - sched_pin(); - cbarg.probe = probe; - cbarg.cpu = curcpu; - atomic_store_rel_int(&cbarg.arrived, 0); - atomic_store_rel_int(&cbarg.done, 0); - cbarg.enable = enable; - smp_rendezvous(NULL, sdt_probe_update_cb, NULL, &cbarg); - sched_unpin(); -} static void sdt_enable(void *arg __unused, dtrace_id_t id, void *parg) @@ -282,7 +234,7 @@ if (sdt_probes_enabled_count == 1) sdt_probes_enabled = true; - sdt_probe_update(probe, true); + zcond_enable(probe->enabled); } static void @@ -293,7 +245,7 @@ probe = parg; KASSERT(probe->sdtp_lf->nenabled > 0, ("no probes enabled")); - sdt_probe_update(probe, false); + zcond_disable(probe->enabled); sdt_probes_enabled_count--; if (sdt_probes_enabled_count == 0) @@ -305,6 +257,7 @@ } probe->id = 0; probe->sdtp_lf->nenabled--; + } static void @@ -374,7 +327,6 @@ { struct sdt_probe **p_begin, **p_end; struct sdt_argtype **a_begin, **a_end; - struct sdt_tracepoint *tp_begin, *tp_end; if (linker_file_lookup_set(lf, "sdt_probes_set", &p_begin, &p_end, NULL) == 0) { @@ -383,7 +335,6 @@ (*probe)->sdtp_lf = lf; sdt_create_probe(*probe); TAILQ_INIT(&(*probe)->argtype_list); - STAILQ_INIT(&(*probe)->tracepoint_list); } } @@ -396,23 +347,6 @@ *argtype, argtype_entry); } } - - if (linker_file_lookup_set(lf, __XSTRING(_SDT_TRACEPOINT_SET), - &tp_begin, &tp_end, NULL) == 0) { - for (struct sdt_tracepoint *tp = tp_begin; tp < tp_end; tp++) { - if (!sdt_tracepoint_valid(tp->patchpoint, tp->target)) { - printf( - "invalid tracepoint %#jx->%#jx for %s:%s:%s:%s\n", - (uintmax_t)tp->patchpoint, - (uintmax_t)tp->target, - tp->probe->prov->name, tp->probe->mod, - tp->probe->func, tp->probe->name); - continue; - } - STAILQ_INSERT_TAIL(&tp->probe->tracepoint_list, tp, - tracepoint_entry); - } - } } /* @@ -494,7 +428,8 @@ TAILQ_INIT(&sdt_prov_list); - sdt_probe_func = sdt_dtrace_probe; + sdt_probe_func = dtrace_probe; + sdt_probe6_func = (sdt_probe6_func_t)sdt_dtrace_probe; sdt_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, sdt_kld_load, NULL, EVENTHANDLER_PRI_ANY); @@ -520,6 +455,7 @@ EVENTHANDLER_DEREGISTER(kld_unload_try, sdt_kld_unload_try_tag); sdt_probe_func = sdt_probe_stub; + sdt_probe6_func = (sdt_probe6_func_t)sdt_probe_stub; TAILQ_FOREACH_SAFE(prov, &sdt_prov_list, prov_entry, tmp) { ret = dtrace_unregister(prov->id); diff --git a/sys/conf/files b/sys/conf/files --- a/sys/conf/files +++ b/sys/conf/files @@ -3865,6 +3865,7 @@ kern/kern_uuid.c standard kern/kern_vnodedumper.c standard kern/kern_xxx.c standard +kern/kern_zcond.c standard kern/link_elf.c standard kern/linker_if.m standard kern/md4c.c optional netsmb diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -93,6 +93,7 @@ amd64/amd64/trap.c standard amd64/amd64/uio_machdep.c standard amd64/amd64/vm_machdep.c standard +amd64/amd64/zcond_machdep.c standard amd64/pci/pci_cfgreg.c optional pci cddl/dev/dtrace/amd64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" diff --git a/sys/conf/files.arm b/sys/conf/files.arm --- a/sys/conf/files.arm +++ b/sys/conf/files.arm @@ -58,7 +58,6 @@ arm/arm/pmu_fdt.c optional fdt pmu | fdt hwpmc arm/arm/ptrace_machdep.c standard arm/arm/sc_machdep.c optional sc -arm/arm/sdt_machdep.c optional kdtrace_hooks arm/arm/setcpsr.S standard arm/arm/setstack.S standard arm/arm/stack_machdep.c optional ddb | stack diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -68,7 +68,6 @@ compile-with "${NORMAL_C:N-mbranch-protection*} -mbranch-protection=bti" arm64/arm64/pmap.c standard arm64/arm64/ptrace_machdep.c standard -arm64/arm64/sdt_machdep.c optional kdtrace_hooks arm64/arm64/sigtramp.S standard arm64/arm64/stack_machdep.c optional ddb | stack arm64/arm64/strcmp.S standard @@ -84,6 +83,7 @@ compile-with "${NORMAL_C:N-fsanitize*:N-fno-sanitize*}" arm64/arm64/vfp.c standard arm64/arm64/vm_machdep.c standard +arm64/arm64/zcond_machdep.c standard arm64/coresight/coresight.c standard arm64/coresight/coresight_acpi.c optional acpi diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc --- a/sys/conf/files.powerpc +++ b/sys/conf/files.powerpc @@ -379,7 +379,6 @@ powerpc/powerpc/platform_if.m standard powerpc/powerpc/ptrace_machdep.c standard powerpc/powerpc/sc_machdep.c optional sc -powerpc/powerpc/sdt_machdep.c optional powerpc64 kdtrace_hooks powerpc/powerpc/setjmp.S standard powerpc/powerpc/sigcode32.S optional powerpc | powerpcspe | compat_freebsd32 powerpc/powerpc/sigcode64.S optional powerpc64 | powerpc64le diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -62,7 +62,6 @@ riscv/riscv/sigtramp.S standard riscv/riscv/sbi.c standard riscv/riscv/sbi_ipi.c optional smp -riscv/riscv/sdt_machdep.c optional kdtrace_hooks riscv/riscv/stack_machdep.c optional ddb | stack riscv/riscv/support.S standard riscv/riscv/swtch.S standard diff --git a/sys/conf/files.x86 b/sys/conf/files.x86 --- a/sys/conf/files.x86 +++ b/sys/conf/files.x86 @@ -377,7 +377,6 @@ x86/x86/mp_x86.c optional smp x86/x86/nexus.c standard x86/x86/pvclock.c optional kvm_clock | xenhvm -x86/x86/sdt_machdep.c optional kdtrace_hooks x86/x86/stack_machdep.c optional ddb | stack x86/x86/tsc.c standard x86/x86/ucode.c standard diff --git a/sys/i386/include/sdt_machdep.h b/sys/i386/include/sdt_machdep.h deleted file mode 100644 --- a/sys/i386/include/sdt_machdep.h +++ /dev/null @@ -1,19 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2024 Mark Johnston - */ - -#ifndef _SYS_SDT_MACHDEP_H_ -#define _SYS_SDT_MACHDEP_H_ - -#define _SDT_ASM_PATCH_INSTR "nop; nop; nop; nop; nop" - -/* - * Work around an apparent clang bug or limitation which prevents the use of the - * "i" (immediate) constraint with the probe structure. - */ -#define _SDT_ASM_PROBE_CONSTRAINT "Ws" -#define _SDT_ASM_PROBE_OPERAND "p" - -#endif diff --git a/sys/kern/kern_sdt.c b/sys/kern/kern_sdt.c --- a/sys/kern/kern_sdt.c +++ b/sys/kern/kern_sdt.c @@ -11,7 +11,7 @@ * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -38,6 +38,7 @@ * dtrace_probe() when it loads. */ sdt_probe_func_t sdt_probe_func = sdt_probe_stub; +sdt_probe6_func_t sdt_probe6_func = (sdt_probe6_func_t) sdt_probe_stub; volatile bool __read_frequently sdt_probes_enabled; /* @@ -48,7 +49,7 @@ void sdt_probe_stub(uint32_t id __unused, uintptr_t arg0 __unused, uintptr_t arg1 __unused, uintptr_t arg2 __unused, uintptr_t arg3 __unused, - uintptr_t arg4 __unused, uintptr_t arg5 __unused) + uintptr_t arg4 __unused) { printf("sdt_probe_stub: unexpectedly called\n"); kdb_backtrace(); @@ -58,12 +59,12 @@ sdt_probe(uint32_t id, uintptr_t arg0, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4) { - sdt_probe_func(id, arg0, arg1, arg2, arg3, arg4, 0); + sdt_probe_func(id, arg0, arg1, arg2, arg3, arg4); } void sdt_probe6(uint32_t id, uintptr_t arg0, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5) { - sdt_probe_func(id, arg0, arg1, arg2, arg3, arg4, arg5); + sdt_probe6_func(id, arg0, arg1, arg2, arg3, arg4, arg5); } diff --git a/sys/kern/kern_zcond.c b/sys/kern/kern_zcond.c new file mode 100644 --- /dev/null +++ b/sys/kern/kern_zcond.c @@ -0,0 +1,156 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Marko Vlaić + * + * This code was developed as a Google Summer of Code 2024. project + * under the guidance of Bojan Novković . + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct patch_point { + vm_offset_t patch_addr; + vm_offset_t lbl_true_addr; + struct zcond *zcond; + SLIST_ENTRY(patch_point) next; +} __attribute__((packed)); + +struct zcond_patch_arg { + int patching_cpu; + struct zcond *cond; + struct zcond_md_ctxt *md_ctxt; +}; + +static void +zcond_load_patch_points(linker_file_t lf) +{ + struct patch_point *begin, *end; + struct patch_point *ins_p; + struct zcond *owning_zcond; + + if (linker_file_lookup_set(lf, __XSTRING(ZCOND_LINKER_SET), &begin, + &end, NULL) == 0) { + for (ins_p = begin; ins_p < end; ins_p++) { + owning_zcond = ins_p->zcond; + owning_zcond->refcnt = 1; + + if (owning_zcond->patch_points.slh_first == NULL) { + SLIST_INIT(&owning_zcond->patch_points); + } + + SLIST_INSERT_HEAD(&owning_zcond->patch_points, ins_p, + next); + } + } +} + +static void +zcond_kld_load(void *arg __unused, struct linker_file *lf) +{ + zcond_load_patch_points(lf); +} + +static int +zcond_load_patch_points_cb(linker_file_t lf, void *arg __unused) +{ + zcond_load_patch_points(lf); + return (0); +} + +static void +zcond_init(const void *unused) +{ + EVENTHANDLER_REGISTER(kld_load, zcond_kld_load, NULL, + EVENTHANDLER_PRI_ANY); + linker_file_foreach(zcond_load_patch_points_cb, NULL); +} +SYSINIT(zcond, SI_SUB_ZCOND, SI_ORDER_SECOND, zcond_init, NULL); + +/* + * Patch all patch_points belonging to cond. + */ +static void +zcond_patch(struct zcond *cond, struct zcond_md_ctxt *ctxt) +{ + struct patch_point *p; + uint8_t *insn; + size_t insn_size; + + SLIST_FOREACH(p, &cond->patch_points, next) { + insn = zcond_get_patch_insn(p->patch_addr, p->lbl_true_addr, + &insn_size); + + zcond_before_patch(ctxt); + memcpy((void *)p->patch_addr, insn, insn_size); + zcond_after_patch(ctxt); + } +} + +static void +rendezvous_action(void *arg) +{ + struct zcond_patch_arg *data; + + data = (struct zcond_patch_arg *)arg; + + if (data->patching_cpu == curcpu) { + zcond_patch(data->cond, data->md_ctxt); + } +} + +void +__zcond_toggle(struct zcond *cond, bool enable) +{ + struct zcond_md_ctxt ctxt; + + if (enable && refcount_acquire(&cond->refcnt) > 1) { + return; + } else if (!enable) { + if (!refcount_release_if_not_last(&cond->refcnt) || + refcount_load(&cond->refcnt) != 1) { + return; + } + } + + struct zcond_patch_arg arg = { + .patching_cpu = curcpu, + .cond = cond, + .md_ctxt = &ctxt, + }; + + smp_rendezvous(NULL, rendezvous_action, NULL, &arg); +} diff --git a/sys/powerpc/include/sdt_machdep.h b/sys/powerpc/include/sdt_machdep.h deleted file mode 100644 --- a/sys/powerpc/include/sdt_machdep.h +++ /dev/null @@ -1,12 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2024 Mark Johnston - */ - -#ifndef _SYS_SDT_MACHDEP_H_ -#define _SYS_SDT_MACHDEP_H_ - -#define _SDT_ASM_PATCH_INSTR "nop" - -#endif diff --git a/sys/powerpc/powerpc/sdt_machdep.c b/sys/powerpc/powerpc/sdt_machdep.c deleted file mode 100644 --- a/sys/powerpc/powerpc/sdt_machdep.c +++ /dev/null @@ -1,59 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2024 Mark Johnston - */ - -#include -#include - -#include - -/* - * Return true if we can overwrite a nop at "patchpoint" with a jump to the - * target address. - */ -bool -sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target) -{ - int64_t offset; - - if (patchpoint == target || - (patchpoint & 3) != 0 || (target & 3) != 0) - return (false); - offset = target - patchpoint; - if (offset < -(1 << 26) || offset > (1 << 26)) - return (false); - return (true); -} - -/* - * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to - * the target address. - */ -void -sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target) -{ - uint32_t instr; - - KASSERT(sdt_tracepoint_valid(patchpoint, target), - ("%s: invalid tracepoint %#lx -> %#lx", - __func__, patchpoint, target)); - - instr = ((target - patchpoint) & 0x7fffffful) | 0x48000000; - memcpy((void *)patchpoint, &instr, sizeof(instr)); - __syncicache((void *)patchpoint, sizeof(instr)); -} - -/* - * Overwrite the patchpoint with a nop instruction. - */ -void -sdt_tracepoint_restore(uintptr_t patchpoint) -{ - uint32_t instr; - - instr = 0x60000000; - memcpy((void *)patchpoint, &instr, sizeof(instr)); - __syncicache((void *)patchpoint, sizeof(instr)); -} diff --git a/sys/riscv/include/sdt_machdep.h b/sys/riscv/include/sdt_machdep.h deleted file mode 100644 --- a/sys/riscv/include/sdt_machdep.h +++ /dev/null @@ -1,12 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2024 Mark Johnston - */ - -#ifndef _SYS_SDT_MACHDEP_H_ -#define _SYS_SDT_MACHDEP_H_ - -#define _SDT_ASM_PATCH_INSTR ".option push; .option norvc; nop; .option pop" - -#endif /* _SYS_SDT_MACHDEP_H_ */ diff --git a/sys/riscv/riscv/sdt_machdep.c b/sys/riscv/riscv/sdt_machdep.c deleted file mode 100644 --- a/sys/riscv/riscv/sdt_machdep.c +++ /dev/null @@ -1,68 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2024 Mark Johnston - */ - -#include -#include - -#include - -/* - * Return true if we can overwrite a nop at "patchpoint" with a jump to the - * target address. - */ -bool -sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target) -{ - int64_t offset; - - if (patchpoint == target || - (patchpoint & (INSN_C_SIZE - 1)) != 0 || - (target & (INSN_C_SIZE - 1)) != 0) - return (false); - offset = target - patchpoint; - if (offset < -(1 << 19) || offset > (1 << 19)) - return (false); - return (true); -} - -/* - * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to - * the target address. - */ -void -sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target) -{ - int32_t imm; - uint32_t instr; - - KASSERT(sdt_tracepoint_valid(patchpoint, target), - ("%s: invalid tracepoint %#lx -> %#lx", - __func__, patchpoint, target)); - - imm = target - patchpoint; - imm = (imm & 0x100000) | - ((imm & 0x7fe) << 8) | - ((imm & 0x800) >> 2) | - ((imm & 0xff000) >> 12); - instr = (imm << 12) | MATCH_JAL; - - memcpy((void *)patchpoint, &instr, sizeof(instr)); - fence_i(); -} - -/* - * Overwrite the patchpoint with a nop instruction. - */ -void -sdt_tracepoint_restore(uintptr_t patchpoint) -{ - uint32_t instr; - - instr = 0x13; /* uncompressed nop */ - - memcpy((void *)patchpoint, &instr, sizeof(instr)); - fence_i(); -} diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h --- a/sys/sys/kernel.h +++ b/sys/sys/kernel.h @@ -107,6 +107,7 @@ SI_SUB_RACCT = 0x2110000, /* resource accounting */ SI_SUB_KDTRACE = 0x2140000, /* Kernel dtrace hooks */ SI_SUB_RANDOM = 0x2160000, /* random number generator */ + SI_SUB_ZCOND = 0x2170000, SI_SUB_MAC = 0x2180000, /* TrustedBSD MAC subsystem */ SI_SUB_MAC_POLICY = 0x21C0000, /* TrustedBSD MAC policies */ SI_SUB_MAC_LATE = 0x21D0000, /* TrustedBSD MAC subsystem */ diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h --- a/sys/sys/sdt.h +++ b/sys/sys/sdt.h @@ -77,8 +77,9 @@ #else /* _KERNEL */ +#include #include -#include +#include extern volatile bool sdt_probes_enabled; @@ -177,6 +178,7 @@ .mod = #_mod, \ .func = #_func, \ .name = #_name, \ + .enabled = ZCOND_INIT() \ }, \ }; \ DATA_SET(sdt_probes_set, _SDT_PROBE_NAME(_prov, _mod, _func, _name)) @@ -186,58 +188,17 @@ #define SDT_PROBES_ENABLED() __predict_false(sdt_probes_enabled) -#ifdef _ILP32 -#define _SDT_ASM_WORD ".long" -#else -#define _SDT_ASM_WORD ".quad" -#endif - -#ifndef _SDT_ASM_PROBE_CONSTRAINT -#define _SDT_ASM_PROBE_CONSTRAINT "i" -#endif -#ifndef _SDT_ASM_PROBE_OPERAND -#define _SDT_ASM_PROBE_OPERAND "c" -#endif +#define _SDT_PROBE(prov, mod, func, name, f, ...) do { \ + if(SDT_PROBES_ENABLED()) { \ + if (zcond_true(_SDT_PROBE_NAME(prov, mod, func, name)->enabled)) \ + f(_SDT_PROBE_NAME(prov, mod, func, name)->id, __VA_ARGS__); \ + } \ + } while(0) -/* - * The asm below generates records corresponding to the structure's layout, so - * the two must be kept in sync. - */ -struct sdt_tracepoint { - struct sdt_probe *probe; - uintptr_t patchpoint; - uintptr_t target; - STAILQ_ENTRY(sdt_tracepoint) tracepoint_entry; -}; -#define __SDT_PROBE(prov, mod, func, name, uniq, f, ...) do { \ - __WEAK(__CONCAT(__start_set_, _SDT_TRACEPOINT_SET)); \ - __WEAK(__CONCAT(__stop_set_, _SDT_TRACEPOINT_SET)); \ - asm goto( \ - "0:\n" \ - _SDT_ASM_PATCH_INSTR "\n" \ - ".pushsection " _SDT_TRACEPOINT_SECTION ", \"aw\"\n" \ - _SDT_ASM_WORD " %" _SDT_ASM_PROBE_OPERAND "0\n" \ - _SDT_ASM_WORD " 0b\n" \ - _SDT_ASM_WORD " %l1\n" \ - _SDT_ASM_WORD " 0\n" \ - ".popsection\n" \ - : \ - : _SDT_ASM_PROBE_CONSTRAINT (_SDT_PROBE_NAME(prov, mod, \ - func, name)) \ - : \ - : __sdt_probe##uniq); \ - if (0) { \ -__sdt_probe##uniq:; \ - f(_SDT_PROBE_NAME(prov, mod, func, name)->id, __VA_ARGS__); \ - } \ -} while (0) -#define _SDT_PROBE(prov, mod, func, name, uniq, f, ...) \ - __SDT_PROBE(prov, mod, func, name, uniq, f, __VA_ARGS__) #define SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) \ - _SDT_PROBE(prov, mod, func, name, __COUNTER__, sdt_probe, \ - (uintptr_t)arg0, (uintptr_t)arg1, (uintptr_t)arg2, \ - (uintptr_t)arg3, (uintptr_t)arg4) + _SDT_PROBE(prov, mod, func, name, sdt_probe, \ + (uintptr_t)arg0, (uintptr_t)arg1, (uintptr_t)arg2, (uintptr_t)arg3, (uintptr_t)arg4) #define SDT_PROBE_ARGTYPE(_prov, _mod, _func, _name, _num, _type, _xtype) \ static struct sdt_argtype \ @@ -355,9 +316,7 @@ #define SDT_PROBE5(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) \ SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) #define SDT_PROBE6(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4, arg5) \ - _SDT_PROBE(prov, mod, func, name, __COUNTER__, sdt_probe6, \ - (uintptr_t)arg0, (uintptr_t)arg1, (uintptr_t)arg2, \ - (uintptr_t)arg3, (uintptr_t)arg4, (uintptr_t)arg5) + _SDT_PROBE(prov, mod, func, name, sdt_probe6, (uintptr_t)arg0, (uintptr_t)arg1, (uintptr_t)arg2, (uintptr_t)arg3, (uintptr_t)arg4, (uintptr_t)arg5) #ifndef KDTRACE_NO_MIB_SDT #define MIB_SDT_PROBE1(...) SDT_PROBE1(mib, __VA_ARGS__) @@ -419,12 +378,15 @@ * way to avoid having to rely on CDDL code. */ typedef void (*sdt_probe_func_t)(uint32_t, uintptr_t arg0, uintptr_t arg1, - uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5); + uintptr_t arg2, uintptr_t arg3, uintptr_t arg4); +typedef void (*sdt_probe6_func_t)(uint32_t, uintptr_t arg0, uintptr_t arg1, + uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5); /* * The 'sdt' provider will set it to dtrace_probe when it loads. */ extern sdt_probe_func_t sdt_probe_func; +extern sdt_probe6_func_t sdt_probe6_func; struct sdt_probe; struct sdt_provider; @@ -452,6 +414,7 @@ id_t id; /* DTrace probe ID. */ int n_args; /* Number of arguments. */ struct linker_file *sdtp_lf; /* Module in which we're defined. */ + DECLARE_ZCOND_FALSE(enabled); }; struct sdt_provider { @@ -463,7 +426,7 @@ }; void sdt_probe_stub(uint32_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, - uintptr_t, uintptr_t); + uintptr_t); SDT_PROVIDER_DECLARE(sdt); diff --git a/sys/sys/zcond.h b/sys/sys/zcond.h new file mode 100644 --- /dev/null +++ b/sys/sys/zcond.h @@ -0,0 +1,249 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Marko Vlaić + * + * This code was developed as a Google Summer of Code 2024. project + * under the guidance of Bojan Novković . + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifdef _KERNEL +#ifndef _SYS_ZCOND_H +#define _SYS_ZCOND_H + +#include +#include +#include +#include + +#include + +/* + * The zcond interface provides a low-cost mechanism for conditional execution. + * It is applicable in situations where branch selection is performed by + * inspecting the state of a single boolean flag i.e blocks of the following + * form: if(flag) { + * // do something + * } + * + * This kind of block is compiled into some sequence of load, test, jump + * assembly instructions. The low cost provided by zcond is achieved by "baking + * in" a single branch direction at compile time. This means outputting either + * an unconditional jump or a nop, while the memory access is avoided. + * + * When the time comes to switch the branch direction, the current instruction + * (jump or nop) is patched at runtime to a corresponding instruction (nop or + * jump). Keep in mind that this is an expensive operation, since all cpus + * except the one performing the patch need to be halted. + * + * + * To use a zcond, first define it with: ZCOND_DEFINE_TRUE(name) or + * ZCOND_DEFINE_FALSE(name) Alternatively, declare it with + * ZCOND_DECLARE_TRUE(name) or ZCOND_DECLARE_FALSE(name). Then initialize it + * with ZCOND_INIT(true) or ZCOND_INIT(false). + * + * Use zcond_false(cond) or zcond_true(cond) to inspect the state of a zcond. + * + * To flip the state of a zcond, use zcond_enable(cond) and zcond_disable(cond). + * + * This header includes the interface intended to be used by consumers, as well + * as some MI code. MD support can be found in sys//include/zcond.h and + * sys///zcond_machdep.c + */ + +/* + * Describes a single inspection of the zcond state (performed with an if + * statement). Holds all the data neccessary to perform a safe instruction + * patch. + */ + +/* + * A single optimized boolean. + */ +struct zcond { + int refcnt; + SLIST_HEAD(, patch_point) patch_points; +}; + +/* + * Wrapper types are needed for compile time decision making. + */ +struct zcond_true { + struct zcond cond; +}; + +struct zcond_false { + struct zcond cond; +}; + +#define ZCOND_ELF_SECTION "set_zcond_patch_points_set" +#define ZCOND_LINKER_SET zcond_patch_points_set + +/* + * __zcond_table is an ELF section which keeps + * all the data related to the zcond mechanism. + * A single entry describes a single patch_point. + */ +#define ZCOND_TABLE_ENTRY \ + ".pushsection " ZCOND_ELF_SECTION ", \"aw\" \n\t" \ + ".quad 1b \n\t" \ + ".quad %l[l_true] \n\t" \ + ".quad %c0 \n\t" \ + ".quad 0 \n\t" \ + ".popsection \n\t" + +#define ZCOND_SET_START_STOP \ + do { \ + __WEAK(__CONCAT(__start_set_, ZCOND_LINKER_SET)); \ + __WEAK(__CONCAT(__stop_set_, ZCOND_LINKER_SET)); \ + } while (0); + +/* + * Emits a __zcond_table entry, describing one patch_point. + * Bakes in a nop instruction instruction, so the return value is initially + * false. + */ +static __always_inline bool +zcond_nop(struct zcond *const zcond_p) +{ + ZCOND_SET_START_STOP + asm goto("1: " ZCOND_NOP_ASM ZCOND_TABLE_ENTRY + : + : "i"(zcond_p) + : + : l_true); + + return (false); +l_true: + return (true); +} + +/* + * Emits a __zcond_table entry, describing one patch_point. + * Bakes in a jmp instruction instruction, so the return value is initially + * true. + */ +static __always_inline bool +zcond_jmp(struct zcond *const zcond_p) +{ + ZCOND_SET_START_STOP + asm goto("1:" ZCOND_JMP_ASM " %[l_true] \n\t" ZCOND_TABLE_ENTRY + : + : "i"(zcond_p) + : + : l_true); + return (false); +l_true: + return (true); +} + +/* + * These macros declare and initialize a new zcond. + */ + +#define ZCOND_INIT() \ + { \ + { \ + .patch_points = SLIST_HEAD_INITIALIZER() \ + } \ + } + +#define DEFINE_ZCOND_TRUE(name) struct zcond_true name = ZCOND_INIT() + +#define DEFINE_ZCOND_FALSE(name) struct zcond_false name = ZCOND_INIT() + +#define DECLARE_ZCOND_TRUE(name) struct zcond_true name; + +#define DECLARE_ZCOND_FALSE(name) struct zcond_false name; + +/* + * These macros inspect the state of a zcond (is it true or false) + * thus instatiating a patch_point. + */ +#define zcond_true(cond_wrapped) \ + ({ \ + bool branch; \ + if (__builtin_types_compatible_p(typeof(cond_wrapped), \ + struct zcond_true)) { \ + branch = zcond_jmp(&(cond_wrapped.cond)); \ + } else if (__builtin_types_compatible_p(typeof(cond_wrapped), \ + struct zcond_false)) { \ + branch = zcond_nop(&(cond_wrapped.cond)); \ + } \ + \ + branch; \ + }) + +#define zcond_false(cond_wrapped) \ + ({ \ + bool branch; \ + if (__builtin_types_compatible_p(typeof(cond_wrapped), \ + struct zcond_true)) { \ + branch = zcond_nop(&(cond_wrapped.cond)); \ + } else if (__builtin_types_compatible_p(typeof(cond_wrapped), \ + struct zcond_false)) { \ + branch = zcond_jmp(&(cond_wrapped.cond)); \ + } \ + \ + branch; \ + }) + +/* + * These macros change the state of a zcond. + */ +#define zcond_enable(cond_wrapped) __zcond_toggle(&cond_wrapped.cond, true) +#define zcond_disable(cond_wrapped) __zcond_toggle(&cond_wrapped.cond, false) + +/* + * Forward declaration of a struct, defined separately for each architecture in + * + */ +struct zcond_md_ctxt; + +/* + * Change the state of a zcond by safely patching all of its + * inspection points with appropriate instructions. + */ +void __zcond_toggle(struct zcond *cond, bool enable); + +/* + * Called before a single patch_point is patched. + */ +void zcond_before_patch(struct zcond_md_ctxt *); + +/* + * Called after a single patch_point was patched. + */ +void zcond_after_patch(struct zcond_md_ctxt *); + +/* + * Calculates the bytes of instruction with which the ins_p inspection point is + * to be patched with. insn[] is populated with the instruction bytes and size + * is set to the number of instruction bytes. + */ +uint8_t *zcond_get_patch_insn(vm_offset_t patch_addr, vm_offset_t lbl_true_addr, + size_t *size); + +#endif +#endif diff --git a/sys/x86/x86/sdt_machdep.c b/sys/x86/x86/sdt_machdep.c deleted file mode 100644 --- a/sys/x86/x86/sdt_machdep.c +++ /dev/null @@ -1,84 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2024 Mark Johnston - */ - -#include -#include - -#include -#include - -#include -#include - -#define SDT_PATCH_SIZE 5 - -/* - * Return true if we can overwrite a nop at "patchpoint" with a jump to the - * target address. - */ -bool -sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target) -{ -#ifdef __amd64__ - if (patchpoint < KERNSTART || target < KERNSTART) - return (false); -#endif - if (patchpoint == target || - patchpoint + SDT_PATCH_SIZE < patchpoint) - return (false); -#ifdef __amd64__ - int64_t offset = target - (patchpoint + SDT_PATCH_SIZE); - if (offset < -(1l << 31) || offset > (1l << 31)) - return (false); -#endif - return (true); -} - -/* - * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to - * the target address. - */ -void -sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target) -{ - uint8_t instr[SDT_PATCH_SIZE]; - int32_t disp; - bool old_wp; - - KASSERT(sdt_tracepoint_valid(patchpoint, target), - ("%s: invalid tracepoint %#jx -> %#jx", - __func__, (uintmax_t)patchpoint, (uintmax_t)target)); - - instr[0] = 0xe9; - disp = target - (patchpoint + SDT_PATCH_SIZE); - memcpy(&instr[1], &disp, sizeof(disp)); - - old_wp = disable_wp(); - memcpy((void *)patchpoint, instr, sizeof(instr)); - restore_wp(old_wp); -} - -/* - * Overwrite the patchpoint with a nop instruction. - */ -void -sdt_tracepoint_restore(uintptr_t patchpoint) -{ - uint8_t instr[SDT_PATCH_SIZE]; - bool old_wp; - -#ifdef __amd64__ - KASSERT(patchpoint >= KERNSTART, - ("%s: invalid patchpoint %#lx", __func__, patchpoint)); -#endif - - for (int i = 0; i < SDT_PATCH_SIZE; i++) - instr[i] = 0x90; - - old_wp = disable_wp(); - memcpy((void *)patchpoint, instr, sizeof(instr)); - restore_wp(old_wp); -}