Page MenuHomeFreeBSD
Files F162214245

D46379.id179919.diff
No OneTemporary
Actions

D46379.id179919.diff
View Options

diff --git a/sys/amd64/amd64/zcond_machdep.c b/sys/amd64/amd64/zcond_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/amd64/amd64/zcond_machdep.c
@@ -0,0 +1,169 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Marko Vlaić <mvlaic@freebsd.org>
+ *
+ * This code was developed as a Google Summer of Code 2024. project
+ * under the guidance of Bojan Novković <bnovkov@freebsdorg>.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+#include <sys/pcpu.h>
+#include <sys/kassert.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/cpufunc.h>
+#include <machine/zcond.h>
+#include <machine/md_var.h>
+#include <machine/vmparam.h>
+
+#define INSN_SHORT_SIZE 2
+#define INSN_LONG_SIZE 5
+#define INSN_MAX_SIZE 5
+
+#define JMP_SHORT_OPCODE 0xeb
+#define JMP_LONG_OPCODE 0xe9
+
+/* from Intel® 64 and IA-32 Architectures Software Developer’s Manual, Volume 2B
+ * 4-165 */
+static const uint8_t nop_short_bytes[INSN_SHORT_SIZE] = { 0x66, 0x90 };
+static const uint8_t nop_long_bytes[INSN_LONG_SIZE] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 };
+
+static uint8_t insn[INSN_MAX_SIZE];
+
+static size_t
+insn_size(uintptr_t addr)
+{
+
+ uint8_t *paddr;
+
+ paddr = (uint8_t *)addr;
+ if (*paddr == nop_short_bytes[0]) {
+ /* two byte nop */
+ return (INSN_SHORT_SIZE);
+ } else if (*paddr == nop_long_bytes[0]) {
+ return (INSN_LONG_SIZE);
+ } else if (*paddr == JMP_SHORT_OPCODE) {
+ /* two byte jump */
+ return (INSN_SHORT_SIZE);
+ } else if (*paddr == JMP_LONG_OPCODE) {
+ /* five byte jump */
+ return (INSN_LONG_SIZE);
+ }
+
+ panic("%s: unexpected opcode: %02hhx", __func__, *paddr);
+}
+
+static const uint8_t *
+insn_nop(size_t size)
+{
+ if (size == INSN_SHORT_SIZE) {
+ return &nop_short_bytes[0];
+ }
+ return &nop_long_bytes[0];
+}
+
+static const uint8_t *
+insn_jmp(size_t size, uintptr_t patch_addr, uintptr_t target)
+{
+ int i;
+ uintptr_t offset;
+
+ offset = target - patch_addr - size;
+
+ if (size == INSN_SHORT_SIZE) {
+ insn[0] = JMP_SHORT_OPCODE;
+ insn[1] = offset;
+ } else {
+ insn[0] = JMP_LONG_OPCODE;
+ for (i = 0; i < INSN_LONG_SIZE - 1; i++) {
+ insn[i + 1] = (offset >> (i * 8)) & 0xFF;
+ }
+ }
+
+ return &insn[0];
+}
+
+static const uint8_t *
+get_patch_insn(uintptr_t patch_addr, uintptr_t target,
+ size_t *size)
+{
+ const uint8_t *pa;
+
+ *size = insn_size(patch_addr);
+ pa = (uint8_t *)patch_addr;
+ if (*pa == nop_short_bytes[0]) {
+ /* two byte nop */
+ return insn_jmp(*size, patch_addr, target);
+ } else if (*pa == nop_long_bytes[0]) {
+ return insn_jmp(*size, patch_addr, target);
+ } else if (*pa == JMP_SHORT_OPCODE) {
+ /* two byte jump */
+ return insn_nop(*size);
+ } else if (*pa == JMP_LONG_OPCODE) {
+ /* five byte jump */
+ return insn_nop(*size);
+ } else {
+ panic("%s: unexpected opcode: %02hhx", __func__, *pa);
+ }
+}
+
+
+static bool
+patchpoint_valid(uintptr_t patch_addr, uintptr_t target)
+{
+ if(patch_addr < KERNSTART || target < KERNSTART)
+ return (false);
+
+ size_t size = insn_size(patch_addr);
+ if(patch_addr == target || patch_addr + size < patch_addr)
+ return (false);
+
+ intptr_t offset = target - patch_addr - size;
+ if(offset < -(1l << 31) || offset > (1l << 31))
+ return (false);
+
+ return (true);
+}
+
+void
+zcond_patchpoint_patch(uintptr_t patch_addr, uintptr_t target)
+{
+
+ KASSERT(patchpoint_valid(patch_addr, target),
+ ("%s: invalid zcond patchpoint %#jx -> %#jx",
+ __func__, (uintmax_t)patch_addr, target));
+
+ size_t size;
+ const uint8_t *insn;
+ bool wp;
+
+ insn = get_patch_insn(patch_addr, target, &size);
+ wp = disable_wp();
+ memcpy((void *)patch_addr, insn, size);
+ restore_wp(wp);
+}
diff --git a/sys/amd64/include/zcond.h b/sys/amd64/include/zcond.h
new file mode 100644
--- /dev/null
+++ b/sys/amd64/include/zcond.h
@@ -0,0 +1,50 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Marko Vlaić <mvlaic@freebsd.org>
+ *
+ * This code was developed as a Google Summer of Code 2024. project
+ * under the guidance of Bojan Novković <bnovkov@freebsdorg>.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifdef _KERNEL
+#ifndef _MACHINE_ZCOND_H
+#define _MACHINE_ZCOND_H
+
+#include <sys/types.h>
+
+#define ZCOND_NOP_ASM \
+ ".byte 0x0f \n\t" \
+ ".byte 0x1f \n\t" \
+ ".byte 0x44 \n\t" \
+ ".byte 0x00 \n\t" \
+ ".byte 0x00 \n\t"
+
+#define ZCOND_JMP_ASM(target) "jmp " target
+
+void
+zcond_patchpoint_patch(uintptr_t patch_addr, uintptr_t lbl_true_addr);
+
+#endif /*_MACHINE_ZCOND_H*/
+#endif /* _KERNEL */
diff --git a/sys/arm/arm/sdt_machdep.c b/sys/arm/arm/sdt_machdep.c
deleted file mode 100644
--- a/sys/arm/arm/sdt_machdep.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#include <sys/systm.h>
-#include <sys/sdt.h>
-
-#include <machine/cpu.h>
-
-/*
- * Return true if we can overwrite a nop at "patchpoint" with a jump to the
- * target address.
- */
-bool
-sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
-{
- int32_t offset;
-
- if (patchpoint == target ||
- (patchpoint & (INSN_SIZE - 1)) != 0 ||
- (target & (INSN_SIZE - 1)) != 0 ||
- patchpoint + 2 * INSN_SIZE < patchpoint)
- return (false);
- offset = target - (patchpoint + 2 * INSN_SIZE);
- if (offset < -(1 << 24) || offset > (1 << 24))
- return (false);
- return (true);
-}
-
-/*
- * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to
- * the target address.
- */
-void
-sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
-{
- uint32_t instr;
-
- KASSERT(sdt_tracepoint_valid(patchpoint, target),
- ("%s: invalid tracepoint %#x -> %#x",
- __func__, patchpoint, target));
-
- instr =
- (((target - (patchpoint + 2 * INSN_SIZE)) >> 2) & ((1 << 24) - 1)) |
- 0xea000000;
- memcpy((void *)patchpoint, &instr, sizeof(instr));
- icache_sync(patchpoint, sizeof(instr));
-}
-
-/*
- * Overwrite the patchpoint with a nop instruction.
- */
-void
-sdt_tracepoint_restore(uintptr_t patchpoint)
-{
- uint32_t instr;
-
- instr = 0xe320f000u;
- memcpy((void *)patchpoint, &instr, sizeof(instr));
- icache_sync(patchpoint, sizeof(instr));
-}
diff --git a/sys/arm/arm/zcond_machdep.c b/sys/arm/arm/zcond_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/arm/arm/zcond_machdep.c
@@ -0,0 +1,47 @@
+#include <sys/systm.h>
+#include <machine/cpu.h>
+#include <machine/zcond.h>
+
+#define INSN_SIZE 4
+
+static const uint32_t nop_insn = 0xe320f000u;
+
+static bool
+patch_addr_valid(uintptr_t patch_addr, uintptr_t target)
+{
+ int32_t offset;
+
+ if (patch_addr == target ||
+ (patch_addr & (INSN_SIZE - 1)) != 0 ||
+ (target & (INSN_SIZE - 1)) != 0 ||
+ patch_addr + 2 * INSN_SIZE < patch_addr)
+ return (false);
+
+ offset = target - (patch_addr + 2 * INSN_SIZE);
+ if (offset < -(1 << 24) || offset > (1 << 24))
+ return (false);
+ return (true);
+}
+
+void
+zcond_patchpoint_patch(uintptr_t patch_addr, uintptr_t target)
+{
+ uint32_t instr;
+
+ KASSERT(patch_addr_valid(patch_addr, target),
+ ("%s: invalid tracepoint %#x -> %#x",
+ __func__, patch_addr, target));
+
+ if(*((uint32_t*)patch_addr) == nop_insn) {
+ // Replace nop with jump
+ instr =
+ (((target - (patch_addr + 2 * INSN_SIZE)) >> 2) & ((1 << 24) - 1)) |
+ 0xea000000;
+ } else {
+ // Replace jump with nop
+ instr = nop_insn;
+ }
+
+ memcpy((void *)patch_addr, &instr, sizeof(instr));
+ icache_sync(patch_addr, INSN_SIZE);
+}
diff --git a/sys/arm/include/sdt_machdep.h b/sys/arm/include/sdt_machdep.h
deleted file mode 100644
--- a/sys/arm/include/sdt_machdep.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#ifndef _SYS_SDT_MACHDEP_H_
-#define _SYS_SDT_MACHDEP_H_
-
-#define _SDT_ASM_PATCH_INSTR "nop"
-
-#endif /* _SYS_SDT_MACHDEP_H_ */
diff --git a/sys/arm/include/zcond.h b/sys/arm/include/zcond.h
new file mode 100644
--- /dev/null
+++ b/sys/arm/include/zcond.h
@@ -0,0 +1,14 @@
+#ifdef _KERNEL
+#ifndef _MACHINE_ZCOND_H
+#define _MACHINE_ZCOND_H
+
+#include <sys/types.h>
+
+#define ZCOND_NOP_ASM "nop"
+#define ZCOND_JMP_ASM(target) "b " target
+
+void
+zcond_patchpoint_patch(uintptr_t patch_addr, uintptr_t lbl_true_addr);
+
+#endif /* _MACHINE_ZCOND_H */
+#endif /* _KERNEL */
diff --git a/sys/arm64/arm64/sdt_machdep.c b/sys/arm64/arm64/sdt_machdep.c
deleted file mode 100644
--- a/sys/arm64/arm64/sdt_machdep.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#include <sys/systm.h>
-#include <sys/sdt.h>
-
-#include <vm/vm.h>
-#include <vm/pmap.h>
-
-#include <machine/cpufunc.h>
-#include <machine/md_var.h>
-#include <machine/vmparam.h>
-
-/*
- * Return true if we can overwrite a nop at "patchpoint" with a jump to the
- * target address.
- */
-bool
-sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
-{
- void *addr;
- int64_t offset;
-
- if (!arm64_get_writable_addr((void *)patchpoint, &addr))
- return (false);
-
- if (patchpoint == target ||
- (patchpoint & (INSN_SIZE - 1)) != 0 ||
- (target & (INSN_SIZE - 1)) != 0)
- return (false);
- offset = target - patchpoint;
- if (offset < -(1 << 26) || offset > (1 << 26))
- return (false);
- return (true);
-}
-
-/*
- * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to the
- * target address.
- */
-void
-sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
-{
- void *addr;
- uint32_t instr;
-
- KASSERT(sdt_tracepoint_valid(patchpoint, target),
- ("%s: invalid tracepoint %#lx -> %#lx",
- __func__, patchpoint, target));
-
- if (!arm64_get_writable_addr((void *)patchpoint, &addr))
- panic("%s: Unable to write new instruction", __func__);
-
- instr = (((target - patchpoint) >> 2) & 0x3fffffful) | 0x14000000;
- memcpy(addr, &instr, sizeof(instr));
- cpu_icache_sync_range((void *)patchpoint, INSN_SIZE);
-}
-
-/*
- * Overwrite the patchpoint with a nop instruction.
- */
-void
-sdt_tracepoint_restore(uintptr_t patchpoint)
-{
- void *addr;
- uint32_t instr;
-
- if (!arm64_get_writable_addr((void *)patchpoint, &addr))
- panic("%s: Unable to write new instruction", __func__);
-
- instr = 0xd503201f;
- memcpy(addr, &instr, sizeof(instr));
- cpu_icache_sync_range((void *)patchpoint, INSN_SIZE);
-}
diff --git a/sys/arm64/arm64/zcond_machdep.c b/sys/arm64/arm64/zcond_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/arm64/zcond_machdep.c
@@ -0,0 +1,58 @@
+#include <sys/systm.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/cpufunc.h>
+#include <machine/md_var.h>
+#include <machine/vmparam.h>
+#include <machine/zcond.h>
+
+#define INSN_SIZE 4
+
+static const uint32_t nop_insn = 0xd503201f;
+
+static bool
+patch_addr_valid(uintptr_t patch_addr, uintptr_t lbl_true_addr)
+{
+ void *addr;
+ int64_t offset;
+
+ if (!arm64_get_writable_addr((void *)patch_addr, &addr))
+ return (false);
+
+ if (patch_addr == lbl_true_addr ||
+ (patch_addr & (INSN_SIZE - 1)) != 0 ||
+ (lbl_true_addr & (INSN_SIZE - 1)) != 0)
+ return (false);
+ offset = lbl_true_addr - patch_addr;
+ if (offset < -(1 << 26) || offset > (1 << 26))
+ return (false);
+ return (true);
+}
+
+void
+zcond_patchpoint_patch(uintptr_t patch_addr, uintptr_t lbl_true_addr)
+{
+ void *addr;
+ uint32_t instr;
+
+ KASSERT(patch_addr_valid(patch_addr, lbl_true_addr),
+ ("%s: invalid tracepoint %#lx -> %#lx",
+ __func__, patch_addr, lbl_true_addr));
+
+ if(*((uint32_t*)patch_addr) == nop_insn) {
+ // Replace nop with jump
+ instr = (((lbl_true_addr - patch_addr) >> 2) & 0x3fffffful) | 0x14000000;
+ } else {
+ // Replace jump with nop
+ instr = nop_insn;
+ }
+
+
+ if (!arm64_get_writable_addr((void *)patch_addr, &addr))
+ panic("%s: Unable to write new instruction", __func__);
+
+ memcpy(addr, &instr, sizeof(instr));
+ cpu_icache_sync_range((void *)patch_addr, INSN_SIZE);
+}
diff --git a/sys/arm64/include/sdt_machdep.h b/sys/arm64/include/sdt_machdep.h
deleted file mode 100644
--- a/sys/arm64/include/sdt_machdep.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#ifndef _SYS_SDT_MACHDEP_H_
-#define _SYS_SDT_MACHDEP_H_
-
-#define _SDT_ASM_PATCH_INSTR "nop"
-
-#endif /* _SYS_SDT_MACHDEP_H_ */
diff --git a/sys/arm64/include/zcond.h b/sys/arm64/include/zcond.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/include/zcond.h
@@ -0,0 +1,14 @@
+#ifdef _KERNEL
+#ifndef _MACHINE_ZCOND_H
+#define _MACHINE_ZCOND_H
+
+#include <sys/types.h>
+
+#define ZCOND_NOP_ASM "nop"
+#define ZCOND_JMP_ASM(target) "b " target
+
+void
+zcond_patchpoint_patch(uintptr_t patch_addr, uintptr_t lbl_true_addr);
+
+#endif /* _MACHINE_ZCOND_H */
+#endif /* _KERNEL */
diff --git a/sys/cddl/dev/sdt/sdt.c b/sys/cddl/dev/sdt/sdt.c
--- a/sys/cddl/dev/sdt/sdt.c
+++ b/sys/cddl/dev/sdt/sdt.c
@@ -213,7 +213,6 @@
{
probe->sdtp_lf = lf;
TAILQ_INIT(&probe->argtype_list);
- STAILQ_INIT(&probe->tracepoint_list);
}
/*
@@ -226,56 +225,6 @@
{
}
-struct sdt_enable_cb_arg {
- struct sdt_probe *probe;
- int cpu;
- int arrived;
- int done;
- bool enable;
-};
-
-static void
-sdt_probe_update_cb(void *_arg)
-{
- struct sdt_enable_cb_arg *arg;
- struct sdt_tracepoint *tp;
-
- arg = _arg;
- if (arg->cpu != curcpu) {
- atomic_add_rel_int(&arg->arrived, 1);
- while (atomic_load_acq_int(&arg->done) == 0)
- cpu_spinwait();
- return;
- } else {
- while (atomic_load_acq_int(&arg->arrived) != mp_ncpus - 1)
- cpu_spinwait();
- }
-
- STAILQ_FOREACH(tp, &arg->probe->tracepoint_list, tracepoint_entry) {
- if (arg->enable)
- sdt_tracepoint_patch(tp->patchpoint, tp->target);
- else
- sdt_tracepoint_restore(tp->patchpoint);
- }
-
- atomic_store_rel_int(&arg->done, 1);
-}
-
-static void
-sdt_probe_update(struct sdt_probe *probe, bool enable)
-{
- struct sdt_enable_cb_arg cbarg;
-
- sched_pin();
- cbarg.probe = probe;
- cbarg.cpu = curcpu;
- atomic_store_rel_int(&cbarg.arrived, 0);
- atomic_store_rel_int(&cbarg.done, 0);
- cbarg.enable = enable;
- smp_rendezvous(NULL, sdt_probe_update_cb, NULL, &cbarg);
- sched_unpin();
-}
-
static void
sdt_enable(void *arg __unused, dtrace_id_t id, void *parg)
{
@@ -294,7 +243,7 @@
if (sdt_probes_enabled_count == 1)
sdt_probes_enabled = true;
- sdt_probe_update(probe, true);
+ zcond_enable(probe->enabled);
}
static void
@@ -305,7 +254,7 @@
probe = parg;
KASSERT(probe->sdtp_lf->nenabled > 0, ("no probes enabled"));
- sdt_probe_update(probe, false);
+ zcond_disable(probe->enabled);
sdt_probes_enabled_count--;
if (sdt_probes_enabled_count == 0)
@@ -393,7 +342,6 @@
{
struct sdt_probe **p_begin, **p_end;
struct sdt_argtype **a_begin, **a_end;
- struct sdt_tracepoint *tp_begin, *tp_end;
if (linker_file_lookup_set(lf, "sdt_probes_set", &p_begin, &p_end,
NULL) == 0) {
@@ -410,23 +358,6 @@
*argtype, argtype_entry);
}
}
-
- if (linker_file_lookup_set(lf, __XSTRING(_SDT_TRACEPOINT_SET),
- &tp_begin, &tp_end, NULL) == 0) {
- for (struct sdt_tracepoint *tp = tp_begin; tp < tp_end; tp++) {
- if (!sdt_tracepoint_valid(tp->patchpoint, tp->target)) {
- printf(
- "invalid tracepoint %#jx->%#jx for %s:%s:%s:%s\n",
- (uintmax_t)tp->patchpoint,
- (uintmax_t)tp->target,
- tp->probe->prov->name, tp->probe->mod,
- tp->probe->func, tp->probe->name);
- continue;
- }
- STAILQ_INSERT_TAIL(&tp->probe->tracepoint_list, tp,
- tracepoint_entry);
- }
- }
}
/*
@@ -483,40 +414,6 @@
{
struct sdt_probe **p_begin, **p_end;
struct sdt_argtype **a_begin, **a_end;
- struct sdt_tracepoint *tp_begin, *tp_end;
-
- if (linker_file_lookup_set(lf, __XSTRING(_SDT_TRACEPOINT_SET),
- &tp_begin, &tp_end, NULL) == 0) {
- for (struct sdt_tracepoint *tp = tp_begin; tp < tp_end; tp++) {
- struct sdt_tracepoint *tp2;
-
- if (!sdt_tracepoint_valid(tp->patchpoint, tp->target))
- continue;
-
- /* Only remove the entry if it is in the list. */
- tp2 = STAILQ_FIRST(&tp->probe->tracepoint_list);
- if (tp2 == tp) {
- STAILQ_REMOVE_HEAD(&tp->probe->tracepoint_list,
- tracepoint_entry);
- } else if (tp2 != NULL) {
- struct sdt_tracepoint *tp3;
-
- for (;;) {
- tp3 = STAILQ_NEXT(tp2,
- tracepoint_entry);
- if (tp3 == NULL)
- break;
- if (tp3 == tp) {
- STAILQ_REMOVE_AFTER(
- &tp->probe->tracepoint_list,
- tp2, tracepoint_entry);
- break;
- }
- tp2 = tp3;
- }
- }
- }
- }
if (linker_file_lookup_set(lf, "sdt_argtypes_set", &a_begin, &a_end,
NULL) == 0) {
@@ -545,9 +442,6 @@
if ((*probe)->sdtp_lf == lf) {
if (!TAILQ_EMPTY(&(*probe)->argtype_list))
return (false);
- if (!STAILQ_EMPTY(&(*probe)->tracepoint_list))
- return (false);
-
/*
* Don't destroy the probe as there
* might be multiple instances of the
diff --git a/sys/conf/files b/sys/conf/files
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -3937,6 +3937,7 @@
kern/kern_uuid.c standard
kern/kern_vnodedumper.c standard
kern/kern_xxx.c standard
+kern/kern_zcond.c optional zcond_patch | kdtrace_hooks
kern/link_elf.c standard
kern/linker_if.m standard
kern/p1003_1b.c standard
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -95,6 +95,7 @@
amd64/amd64/trap.c standard
amd64/amd64/uio_machdep.c standard
amd64/amd64/vm_machdep.c standard
+amd64/amd64/zcond_machdep.c optional zcond_patch | kdtrace_hooks
amd64/pci/pci_cfgreg.c optional pci
cddl/dev/dtrace/amd64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}"
cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}"
diff --git a/sys/conf/files.arm b/sys/conf/files.arm
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -57,7 +57,6 @@
arm/arm/pmu_fdt.c optional fdt pmu | fdt hwpmc
arm/arm/ptrace_machdep.c standard
arm/arm/sc_machdep.c optional sc
-arm/arm/sdt_machdep.c optional kdtrace_hooks
arm/arm/setcpsr.S standard
arm/arm/setstack.S standard
arm/arm/stack_machdep.c optional ddb | stack
@@ -74,6 +73,7 @@
arm/arm/unwind.c optional ddb | kdtrace_hooks | stack
arm/arm/vm_machdep.c standard
arm/arm/vfp.c optional vfp
+arm/arm/zcond_machdep.c optional zcond_patch | kdtrace_hooks
cddl/dev/dtrace/arm/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}"
cddl/dev/dtrace/arm/dtrace_isa.c optional dtrace compile-with "${DTRACE_C}"
cddl/dev/dtrace/arm/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}"
diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -80,7 +80,6 @@
arm64/arm64/pmap.c standard
arm64/arm64/ptrace_machdep.c standard
arm64/arm64/rsi.c standard
-arm64/arm64/sdt_machdep.c optional kdtrace_hooks
arm64/arm64/sigtramp.S standard
arm64/arm64/spec_workaround.c standard
arm64/arm64/stack_machdep.c optional ddb | stack
@@ -97,6 +96,7 @@
compile-with "${NOSAN_C}"
arm64/arm64/vfp.c standard
arm64/arm64/vm_machdep.c standard
+arm64/arm64/zcond_machdep.c optional zcond_patch | kdtrace_hooks
arm64/coresight/coresight.c standard
arm64/coresight/coresight_acpi.c optional acpi
diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -413,7 +413,6 @@
powerpc/powerpc/platform_if.m standard
powerpc/powerpc/ptrace_machdep.c standard
powerpc/powerpc/sc_machdep.c optional sc
-powerpc/powerpc/sdt_machdep.c optional kdtrace_hooks
powerpc/powerpc/setjmp.S standard
powerpc/powerpc/sigcode32.S optional powerpc | compat_freebsd32
powerpc/powerpc/sigcode64.S optional powerpc64 | powerpc64le
@@ -427,6 +426,7 @@
powerpc/powerpc/uio_machdep.c standard
powerpc/powerpc/uma_machdep.c standard
powerpc/powerpc/vm_machdep.c standard
+powerpc/powerpc/zcond_machdep.c optional zcond_patch | kdtrace_hooks
powerpc/ps3/ehci_ps3.c optional ps3 ehci
powerpc/ps3/ohci_ps3.c optional ps3 ohci
powerpc/ps3/if_glc.c optional ps3 glc
diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv
--- a/sys/conf/files.riscv
+++ b/sys/conf/files.riscv
@@ -88,7 +88,6 @@
riscv/riscv/sigtramp.S standard
riscv/riscv/sbi.c standard
riscv/riscv/sbi_ipi.c optional smp
-riscv/riscv/sdt_machdep.c optional kdtrace_hooks
riscv/riscv/stack_machdep.c optional ddb | stack
riscv/riscv/support.S standard
riscv/riscv/swtch.S standard
@@ -98,6 +97,7 @@
riscv/riscv/uio_machdep.c standard
riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack
riscv/riscv/vm_machdep.c standard
+riscv/riscv/zcond_machdep.c optional zcond_patch | kdtrace_hooks
riscv/vmm/vmm.c optional vmm
riscv/vmm/vmm_aplic.c optional vmm
riscv/vmm/vmm_dev_machdep.c optional vmm
diff --git a/sys/conf/files.x86 b/sys/conf/files.x86
--- a/sys/conf/files.x86
+++ b/sys/conf/files.x86
@@ -387,7 +387,6 @@
x86/x86/mp_x86.c optional smp
x86/x86/nexus.c standard
x86/x86/pvclock.c optional kvm_clock | xenhvm
-x86/x86/sdt_machdep.c optional kdtrace_hooks
x86/x86/stack_machdep.c optional ddb | stack
x86/x86/tsc.c standard
x86/x86/ucode.c standard
diff --git a/sys/conf/options b/sys/conf/options
--- a/sys/conf/options
+++ b/sys/conf/options
@@ -240,6 +240,7 @@
UMTX_CHAINS opt_global.h
VERBOSE_SYSINIT
ZSTDIO opt_zstdio.h
+ZCOND_PATCH opt_global.h
# Sanitizers
COVERAGE opt_global.h
diff --git a/sys/kern/kern_zcond.c b/sys/kern/kern_zcond.c
new file mode 100644
--- /dev/null
+++ b/sys/kern/kern_zcond.c
@@ -0,0 +1,225 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Marko Vlaić <mvlaic@freebsd.org>
+ *
+ * This code was developed as a Google Summer of Code 2024. project
+ * under the guidance of Bojan Novković <bnovkov@freebsdorg>.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifdef ZCOND_PATCH
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/eventhandler.h>
+#include <sys/kernel.h>
+#include <sys/linker.h>
+#include <sys/linker_set.h>
+#include <sys/refcount.h>
+#include <sys/sbuf.h>
+#include <sys/smp.h>
+#include <sys/sysctl.h>
+#include <sys/zcond.h>
+
+/*
+ * Describes a single inspection of the zcond state (performed with an if
+ * statement). Holds all the data neccessary to perform an instruction patch.
+ */
+struct patch_point {
+ uintptr_t patch_addr;
+ uintptr_t target;
+ struct zcond *zcond;
+ SLIST_ENTRY(patch_point) next;
+} __attribute__((packed));
+
+struct zcond_patch_arg {
+ int patching_cpu;
+ struct zcond *cond;
+};
+
+static void
+zcond_load_patch_points(linker_file_t lf)
+{
+ struct patch_point *begin, *end;
+ struct patch_point *pp;
+ struct zcond *owning_zcond;
+
+ if (linker_file_lookup_set(lf, __XSTRING(ZCOND_LINKER_SET), &begin,
+ &end, NULL) == 0) {
+ for (pp = begin; pp < end; pp++) {
+ owning_zcond = pp->zcond;
+ if (owning_zcond->patch_points.slh_first == NULL) {
+ SLIST_INIT(&owning_zcond->patch_points);
+ }
+
+ SLIST_INSERT_HEAD(&owning_zcond->patch_points, pp,
+ next);
+ }
+ }
+}
+
+static void
+zcond_unload_patch_points(linker_file_t lf)
+{
+ struct patch_point *begin, *end;
+ struct patch_point *pp;
+
+ if (linker_file_lookup_set(lf, __XSTRING(ZCOND_LINKER_SET), &begin,
+ &end, NULL) != 0) {
+ return;
+ }
+
+ for(pp = begin; pp < end; pp++) {
+ struct patch_point *pp2 = SLIST_FIRST(&pp->zcond->patch_points);
+ if(pp2 == pp) {
+ SLIST_REMOVE_HEAD(&pp->zcond->patch_points, next);
+ } else if(pp2 != NULL) {
+ struct patch_point *pp3;
+
+ for(;;) {
+ pp3 = SLIST_NEXT(pp2, next);
+ if(pp3 == NULL) {
+ break;
+ }
+ if(pp3 == pp) {
+ SLIST_REMOVE_AFTER(pp2, next);
+ break;
+ }
+ pp2 = pp3;
+ }
+ }
+ }
+}
+
+static void
+zcond_kld_load(void *arg __unused, struct linker_file *lf)
+{
+ zcond_load_patch_points(lf);
+}
+
+static void
+zcond_kld_unload(void *arg __unused, struct linker_file *lf, int *error)
+{
+ if(*error != 0) {
+ return;
+ }
+
+ zcond_unload_patch_points(lf);
+}
+
+static int
+zcond_load_patch_points_cb(linker_file_t lf, void *arg __unused)
+{
+ zcond_load_patch_points(lf);
+ return (0);
+}
+
+/*
+ * Collect patch_points from the __zcond_table ELF section into a list.
+ * Prepare a CPU local copy of the kernel_pmap, used to safely patch
+ * an instruction.
+ */
+static void
+zcond_init(const void *unused)
+{
+ EVENTHANDLER_REGISTER(kld_load, zcond_kld_load, NULL,
+ EVENTHANDLER_PRI_ANY);
+ EVENTHANDLER_REGISTER(kld_unload_try, zcond_kld_unload, NULL, EVENTHANDLER_PRI_ANY);
+ linker_file_foreach(zcond_load_patch_points_cb, NULL);
+}
+
+SYSINIT(zcond, SI_SUB_ZCOND, SI_ORDER_SECOND, zcond_init, NULL);
+
+
+/*
+ * Patch all patch_points belonging to cond.
+ */
+static void
+zcond_patch(struct zcond *cond)
+{
+ struct patch_point *p;
+ SLIST_FOREACH(p, &cond->patch_points, next) {
+ zcond_patchpoint_patch(p->patch_addr, p->target);
+ }
+}
+
+static void
+rendezvous_action(void *arg)
+{
+ struct zcond_patch_arg *data;
+
+ data = (struct zcond_patch_arg *)arg;
+
+ if (data->patching_cpu == curcpu) {
+ zcond_patch(data->cond);
+ }
+}
+
+void
+__zcond_toggle(struct zcond *cond, bool enable)
+{
+ if(enable && refcount_acquire(&cond->refcnt) > 0) {
+ // cond is already enabled
+ return;
+ } else if(!enable) {
+ if(refcount_load(&cond->refcnt) == 0) {
+ // cond is already disabled
+ return;
+ }
+
+ if(refcount_release_if_not_last(&cond->refcnt)) {
+ // cond stays disabled after release
+ return;
+ }
+
+ // release the last reference to cond
+ refcount_release(&cond->refcnt);
+ }
+
+ struct zcond_patch_arg arg = {
+ .patching_cpu = curcpu,
+ .cond = cond,
+ };
+
+ smp_rendezvous(NULL, rendezvous_action, NULL, &arg);
+}
+
+#else
+
+#include <sys/zcond.h>
+#include <sys/refcount.h>
+
+void
+__zcond_toggle(struct zcond *cond, bool enable)
+{
+ if(enable) {
+ refcount_acquire(&cond->refcnt);
+ } else {
+ refcount_release(&cond->refcnt);
+ }
+}
+
+#endif // ZCOND_PATCH
diff --git a/sys/powerpc/include/sdt_machdep.h b/sys/powerpc/include/sdt_machdep.h
deleted file mode 100644
--- a/sys/powerpc/include/sdt_machdep.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#ifndef _SYS_SDT_MACHDEP_H_
-#define _SYS_SDT_MACHDEP_H_
-
-#define _SDT_ASM_PATCH_INSTR "nop"
-
-#endif
diff --git a/sys/powerpc/include/zcond.h b/sys/powerpc/include/zcond.h
new file mode 100644
--- /dev/null
+++ b/sys/powerpc/include/zcond.h
@@ -0,0 +1,14 @@
+#ifdef _KERNEL
+#ifndef _MACHINE_ZCOND_H
+#define _MACHINE_ZCOND_H
+
+#include <sys/types.h>
+
+#define ZCOND_NOP_ASM "nop"
+#define ZCOND_JMP_ASM(target) "b " target
+
+void
+zcond_patchpoint_patch(uintptr_t patch_addr, uintptr_t lbl_true_addr);
+
+#endif /* _MACHINE_ZCOND_H */
+#endif /* _KERNEL */
diff --git a/sys/powerpc/powerpc/sdt_machdep.c b/sys/powerpc/powerpc/sdt_machdep.c
deleted file mode 100644
--- a/sys/powerpc/powerpc/sdt_machdep.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#include <sys/systm.h>
-#include <sys/sdt.h>
-
-#include <machine/md_var.h>
-
-/*
- * Return true if we can overwrite a nop at "patchpoint" with a jump to the
- * target address.
- */
-bool
-sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
-{
- int64_t offset;
-
- if (patchpoint == target ||
- (patchpoint & 3) != 0 || (target & 3) != 0)
- return (false);
- offset = target - patchpoint;
- if (offset < -(1 << 26) || offset > (1 << 26))
- return (false);
- return (true);
-}
-
-/*
- * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to
- * the target address.
- */
-void
-sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
-{
- uint32_t instr;
-
- KASSERT(sdt_tracepoint_valid(patchpoint, target),
- ("%s: invalid tracepoint %#jx -> %#jx",
- __func__, (uintmax_t)patchpoint, (uintmax_t)target));
-
- instr = ((target - patchpoint) & 0x7fffffful) | 0x48000000;
- memcpy((void *)patchpoint, &instr, sizeof(instr));
- __syncicache((void *)patchpoint, sizeof(instr));
-}
-
-/*
- * Overwrite the patchpoint with a nop instruction.
- */
-void
-sdt_tracepoint_restore(uintptr_t patchpoint)
-{
- uint32_t instr;
-
- instr = 0x60000000;
- memcpy((void *)patchpoint, &instr, sizeof(instr));
- __syncicache((void *)patchpoint, sizeof(instr));
-}
diff --git a/sys/powerpc/powerpc/zcond_machdep.c b/sys/powerpc/powerpc/zcond_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/powerpc/powerpc/zcond_machdep.c
@@ -0,0 +1,44 @@
+#include <sys/systm.h>
+
+#include <machine/md_var.h>
+#include <machine/zcond.h>
+
+#define INSN_SIZE 4
+
+static const uint32_t nop_insn = 0x60000000;
+
+static bool
+patch_addr_valid(uintptr_t patch_addr, uintptr_t lbl_true_addr)
+{
+
+ int64_t offset;
+
+ if (patch_addr == lbl_true_addr ||
+ (patch_addr & 3) != 0 || (lbl_true_addr & 3) != 0)
+ return (false);
+ offset = lbl_true_addr - patch_addr;
+ if (offset < -(1 << 26) || offset > (1 << 26))
+ return (false);
+ return (true);
+}
+
+void
+zcond_patchpoint_patch(uintptr_t patch_addr, uintptr_t lbl_true_addr)
+{
+ uint32_t instr;
+
+ KASSERT(patch_addr_valid(patch_addr, lbl_true_addr),
+ ("%s: invalid tracepoint %#lx -> %#lx",
+ __func__, (uintmax_t)patch_addr, (uintmax_t)lbl_true_addr));
+
+ if(*((uint32_t*)patch_addr) == nop_insn) {
+ // Replace nop with jump
+ instr = ((patch_addr - lbl_true_addr) & 0x7fffffful) | 0x48000000;
+ } else {
+ // Replace jump with nop
+ instr = nop_insn;
+ }
+
+ memcpy((void *)patch_addr, &instr, sizeof(instr));
+ __syncicache((void *)patch_addr, sizeof(instr));
+}
diff --git a/sys/riscv/include/sdt_machdep.h b/sys/riscv/include/sdt_machdep.h
deleted file mode 100644
--- a/sys/riscv/include/sdt_machdep.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#ifndef _SYS_SDT_MACHDEP_H_
-#define _SYS_SDT_MACHDEP_H_
-
-#define _SDT_ASM_PATCH_INSTR ".option push; .option norvc; nop; .option pop"
-
-#endif /* _SYS_SDT_MACHDEP_H_ */
diff --git a/sys/riscv/include/zcond.h b/sys/riscv/include/zcond.h
new file mode 100644
--- /dev/null
+++ b/sys/riscv/include/zcond.h
@@ -0,0 +1,12 @@
+#ifndef ZCOND_MACHDEP_H
+#define ZCOND_MACHDEP_H
+
+#include <sys/types.h>
+
+#define ZCOND_NOP_ASM ".option push; .option norvc; nop; .option pop"
+#define ZCOND_JMP_ASM(target) ".option push; .option norvc; j " target "; .option pop"
+
+void
+zcond_patchpoint_patch(uintptr_t patch_addr, uintptr_t lbl_true_addr);
+
+#endif // ZCOND_MACHDEP_H
diff --git a/sys/riscv/riscv/sdt_machdep.c b/sys/riscv/riscv/sdt_machdep.c
deleted file mode 100644
--- a/sys/riscv/riscv/sdt_machdep.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#include <sys/systm.h>
-#include <sys/sdt.h>
-
-#include <machine/encoding.h>
-
-/*
- * Return true if we can overwrite a nop at "patchpoint" with a jump to the
- * target address.
- */
-bool
-sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
-{
- int64_t offset;
-
- if (patchpoint == target ||
- (patchpoint & (INSN_C_SIZE - 1)) != 0 ||
- (target & (INSN_C_SIZE - 1)) != 0)
- return (false);
- offset = target - patchpoint;
- if (offset < -(1 << 19) || offset > (1 << 19))
- return (false);
- return (true);
-}
-
-/*
- * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to
- * the target address.
- */
-void
-sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
-{
- int32_t imm;
- uint32_t instr;
-
- KASSERT(sdt_tracepoint_valid(patchpoint, target),
- ("%s: invalid tracepoint %#lx -> %#lx",
- __func__, patchpoint, target));
-
- imm = target - patchpoint;
- imm = (imm & 0x100000) |
- ((imm & 0x7fe) << 8) |
- ((imm & 0x800) >> 2) |
- ((imm & 0xff000) >> 12);
- instr = (imm << 12) | MATCH_JAL;
-
- memcpy((void *)patchpoint, &instr, sizeof(instr));
- fence_i();
-}
-
-/*
- * Overwrite the patchpoint with a nop instruction.
- */
-void
-sdt_tracepoint_restore(uintptr_t patchpoint)
-{
- uint32_t instr;
-
- instr = 0x13; /* uncompressed nop */
-
- memcpy((void *)patchpoint, &instr, sizeof(instr));
- fence_i();
-}
diff --git a/sys/riscv/riscv/zcond_machdep.c b/sys/riscv/riscv/zcond_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/riscv/riscv/zcond_machdep.c
@@ -0,0 +1,51 @@
+#include <sys/systm.h>
+#include <sys/sdt.h>
+
+#include <machine/encoding.h>
+#include <machine/zcond.h>
+
+#define INSN_SIZE 4
+
+static const uint32_t nop_insn = 0x13;
+
+static bool
+patch_addr_valid(uintptr_t patch_addr, uintptr_t lbl_true_addr)
+{
+ int64_t offset;
+
+ if (patch_addr == lbl_true_addr ||
+ (patch_addr & (INSN_C_SIZE - 1)) != 0 ||
+ (lbl_true_addr & (INSN_C_SIZE - 1)) != 0)
+ return (false);
+ offset = lbl_true_addr - patch_addr;
+ if (offset < -(1 << 19) || offset > (1 << 19))
+ return (false);
+ return (true);
+}
+
+void
+zcond_patchpoint_patch(uintptr_t patch_addr, uintptr_t lbl_true_addr)
+{
+ uint32_t instr;
+ int32_t imm;
+
+ KASSERT(patch_addr_valid(patch_addr, lbl_true_addr),
+ ("%s: invalid tracepoint %#lx -> %#lx",
+ __func__, patch_addr, lbl_true_addr));
+
+ if(*((uint32_t*)patch_addr) == nop_insn) {
+ // Replace nop with jump
+ imm = lbl_true_addr - patch_addr;
+ imm = (imm & 0x100000) |
+ ((imm & 0x7fe) << 8) |
+ ((imm & 0x800) >> 2) |
+ ((imm & 0xff000) >> 12);
+ instr = (imm << 12) | MATCH_JAL;
+ } else {
+ // Replace jump with nop
+ instr = nop_insn;
+ }
+
+ memcpy((void *)patch_addr, &instr, sizeof(instr));
+ fence_i();
+}
diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h
--- a/sys/sys/kernel.h
+++ b/sys/sys/kernel.h
@@ -115,25 +115,26 @@
SI_SUB_RACCT = 0x2110000, /* resource accounting */
SI_SUB_KDTRACE = 0x2140000, /* Kernel dtrace hooks */
SI_SUB_RANDOM = 0x2160000, /* random number generator */
+ SI_SUB_ZCOND = 0x2170000,
SI_SUB_MAC = 0x2180000, /* TrustedBSD MAC subsystem */
SI_SUB_MAC_POLICY = 0x21C0000, /* TrustedBSD MAC policies */
SI_SUB_MAC_LATE = 0x21D0000, /* TrustedBSD MAC subsystem */
SI_SUB_VNET = 0x21E0000, /* vnet 0 */
- SI_SUB_INTRINSIC = 0x2200000, /* proc 0 */
- SI_SUB_VM_CONF = 0x2300000, /* config VM, set limits */
+ SI_SUB_INTRINSIC = 0x2200000, /* proc 0*/
+ SI_SUB_VM_CONF = 0x2300000, /* config VM, set limits*/
SI_SUB_DDB_SERVICES = 0x2380000, /* capture, scripting, etc. */
- SI_SUB_RUN_QUEUE = 0x2400000, /* set up run queue */
+ SI_SUB_RUN_QUEUE = 0x2400000, /* set up run queue*/
SI_SUB_KTRACE = 0x2480000, /* ktrace */
SI_SUB_OPENSOLARIS = 0x2490000, /* OpenSolaris compatibility */
SI_SUB_AUDIT = 0x24C0000, /* audit */
- SI_SUB_CREATE_INIT = 0x2500000, /* create init process */
+ SI_SUB_CREATE_INIT = 0x2500000, /* create init process*/
SI_SUB_SCHED_IDLE = 0x2600000, /* required idle procs */
SI_SUB_MBUF = 0x2700000, /* mbuf subsystem */
SI_SUB_INTR = 0x2800000, /* interrupt threads */
SI_SUB_TASKQ = 0x2880000, /* task queues */
SI_SUB_EPOCH = 0x2888000, /* epoch subsystem */
#ifdef EARLY_AP_STARTUP
- SI_SUB_SMP = 0x2900000, /* start the APs */
+ SI_SUB_SMP = 0x2900000, /* start the APs*/
#endif
SI_SUB_SOFTINTR = 0x2A00000, /* start soft interrupt thread */
SI_SUB_DEVFS = 0x2F00000, /* devfs ready for devices */
@@ -144,13 +145,13 @@
SI_SUB_DTRACE_ANON = 0x308C000, /* DTrace anon enabling */
SI_SUB_DRIVERS = 0x3100000, /* Let Drivers initialize */
SI_SUB_CONFIGURE = 0x3800000, /* Configure devices */
- SI_SUB_VFS = 0x4000000, /* virtual filesystem */
- SI_SUB_CLOCKS = 0x4800000, /* real time and stat clocks */
- SI_SUB_SYSV_SHM = 0x6400000, /* System V shared memory */
- SI_SUB_SYSV_SEM = 0x6800000, /* System V semaphores */
- SI_SUB_SYSV_MSG = 0x6C00000, /* System V message queues */
+ SI_SUB_VFS = 0x4000000, /* virtual filesystem*/
+ SI_SUB_CLOCKS = 0x4800000, /* real time and stat clocks*/
+ SI_SUB_SYSV_SHM = 0x6400000, /* System V shared memory*/
+ SI_SUB_SYSV_SEM = 0x6800000, /* System V semaphores*/
+ SI_SUB_SYSV_MSG = 0x6C00000, /* System V message queues*/
SI_SUB_P1003_1B = 0x6E00000, /* P1003.1B realtime */
- SI_SUB_PSEUDO = 0x7000000, /* pseudo devices */
+ SI_SUB_PSEUDO = 0x7000000, /* pseudo devices*/
SI_SUB_EXEC = 0x7400000, /* execve() handlers */
SI_SUB_PROTO_BEGIN = 0x8000000, /* VNET initialization */
SI_SUB_PROTO_PFIL = 0x8100000, /* Initialize pfil before FWs */
@@ -161,23 +162,23 @@
SI_SUB_PROTO_FIREWALL = 0x8806000, /* Firewalls */
SI_SUB_PROTO_IFATTACHDOMAIN = 0x8808000,/* domain dependent data init */
SI_SUB_PROTO_END = 0x8ffffff, /* VNET helper functions */
- SI_SUB_KPROF = 0x9000000, /* kernel profiling */
- SI_SUB_KICK_SCHEDULER = 0xa000000, /* start the timeout events */
+ SI_SUB_KPROF = 0x9000000, /* kernel profiling*/
+ SI_SUB_KICK_SCHEDULER = 0xa000000, /* start the timeout events*/
SI_SUB_INT_CONFIG_HOOKS = 0xa800000, /* Interrupts enabled config */
SI_SUB_ROOT_CONF = 0xb000000, /* Find root devices */
- SI_SUB_INTRINSIC_POST = 0xd000000, /* proc 0 cleanup */
+ SI_SUB_INTRINSIC_POST = 0xd000000, /* proc 0 cleanup*/
SI_SUB_SYSCALLS = 0xd800000, /* register system calls */
SI_SUB_VNET_DONE = 0xdc00000, /* vnet registration complete */
- SI_SUB_KTHREAD_INIT = 0xe000000, /* init process */
- SI_SUB_KTHREAD_PAGE = 0xe400000, /* pageout daemon */
- SI_SUB_KTHREAD_VM = 0xe800000, /* vm daemon */
- SI_SUB_KTHREAD_BUF = 0xea00000, /* buffer daemon */
- SI_SUB_KTHREAD_UPDATE = 0xec00000, /* update daemon */
- SI_SUB_KTHREAD_IDLE = 0xee00000, /* idle procs */
+ SI_SUB_KTHREAD_INIT = 0xe000000, /* init process*/
+ SI_SUB_KTHREAD_PAGE = 0xe400000, /* pageout daemon*/
+ SI_SUB_KTHREAD_VM = 0xe800000, /* vm daemon*/
+ SI_SUB_KTHREAD_BUF = 0xea00000, /* buffer daemon*/
+ SI_SUB_KTHREAD_UPDATE = 0xec00000, /* update daemon*/
+ SI_SUB_KTHREAD_IDLE = 0xee00000, /* idle procs*/
#ifndef EARLY_AP_STARTUP
- SI_SUB_SMP = 0xf000000, /* start the APs */
-#endif
- SI_SUB_RACCTD = 0xf100000, /* start racctd */
+ SI_SUB_SMP = 0xf000000, /* start the APs*/
+#endif
+ SI_SUB_RACCTD = 0xf100000, /* start racctd*/
SI_SUB_LAST = 0xfffffff /* final initialization */
};
@@ -185,16 +186,16 @@
* Some enumerated orders; "ANY" sorts last.
*/
enum sysinit_elem_order {
- SI_ORDER_FIRST = 0x0000000, /* first */
- SI_ORDER_SECOND = 0x0000001, /* second */
- SI_ORDER_THIRD = 0x0000002, /* third */
- SI_ORDER_FOURTH = 0x0000003, /* fourth */
- SI_ORDER_FIFTH = 0x0000004, /* fifth */
- SI_ORDER_SIXTH = 0x0000005, /* sixth */
- SI_ORDER_SEVENTH = 0x0000006, /* seventh */
- SI_ORDER_EIGHTH = 0x0000007, /* eighth */
+ SI_ORDER_FIRST = 0x0000000, /* first*/
+ SI_ORDER_SECOND = 0x0000001, /* second*/
+ SI_ORDER_THIRD = 0x0000002, /* third*/
+ SI_ORDER_FOURTH = 0x0000003, /* fourth*/
+ SI_ORDER_FIFTH = 0x0000004, /* fifth*/
+ SI_ORDER_SIXTH = 0x0000005, /* sixth*/
+ SI_ORDER_SEVENTH = 0x0000006, /* seventh*/
+ SI_ORDER_EIGHTH = 0x0000007, /* eighth*/
SI_ORDER_MIDDLE = 0x1000000, /* somewhere in the middle */
- SI_ORDER_ANY = 0xfffffff /* last */
+ SI_ORDER_ANY = 0xfffffff /* last*/
};
/*
@@ -224,10 +225,10 @@
typedef void (*sysinit_cfunc_t)(const void *);
struct sysinit {
- enum sysinit_sub_id subsystem; /* subsystem identifier */
- enum sysinit_elem_order order; /* init order within subsystem */
+ enum sysinit_sub_id subsystem; /* subsystem identifier*/
+ enum sysinit_elem_order order; /* init order within subsystem*/
STAILQ_ENTRY(sysinit) next; /* singly-linked list */
- sysinit_cfunc_t func; /* function */
+ sysinit_cfunc_t func; /* function */
const void *udata; /* multiplexer/argument */
};
@@ -246,8 +247,8 @@
#ifdef TSLOG
struct sysinit_tslog {
sysinit_cfunc_t func;
- const void *data;
- const char *name;
+ const void * data;
+ const char * name;
};
void sysinit_tslog_shim(const void *);
@@ -264,7 +265,7 @@
sysinit_tslog_shim, \
&uniquifier ## _sys_init_tslog \
}; \
- DATA_WSET(sysinit_set, uniquifier ## _sys_init)
+ DATA_WSET(sysinit_set,uniquifier ## _sys_init)
#else
#define C_SYSINIT(uniquifier, subsystem, order, func, ident) \
static struct sysinit uniquifier ## _sys_init = { \
@@ -274,7 +275,7 @@
func, \
(ident) \
}; \
- DATA_WSET(sysinit_set, uniquifier ## _sys_init)
+ DATA_WSET(sysinit_set,uniquifier ## _sys_init)
#endif
#define SYSINIT(uniquifier, subsystem, order, func, ident) \
@@ -292,7 +293,7 @@
func, \
(ident) \
}; \
- DATA_WSET(sysuninit_set, uniquifier ## _sys_uninit)
+ DATA_WSET(sysuninit_set,uniquifier ## _sys_uninit)
#define SYSUNINIT(uniquifier, subsystem, order, func, ident) \
C_SYSUNINIT(uniquifier, subsystem, order, \
diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h
--- a/sys/sys/sdt.h
+++ b/sys/sys/sdt.h
@@ -78,7 +78,7 @@
#else /* _KERNEL */
#include <sys/linker_set.h>
-#include <machine/sdt_machdep.h>
+#include <sys/zcond.h>
extern volatile bool sdt_probes_enabled;
@@ -142,13 +142,6 @@
void sdt_probe6(uint32_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t,
uintptr_t, uintptr_t);
-#define _SDT_TRACEPOINT_SET sdt_tracepoint_set
-#define _SDT_TRACEPOINT_SECTION "set_sdt_tracepoint_set"
-
-bool sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target);
-void sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target);
-void sdt_tracepoint_restore(uintptr_t patchpoint);
-
#define __sdt_used
SET_DECLARE(sdt_providers_set, struct sdt_provider);
@@ -176,6 +169,7 @@
.mod = #_mod, \
.func = #_func, \
.name = #_name, \
+ .enabled = ZCOND_INIT(ZCOND_DISABLED), \
}; \
DATA_SET(sdt_probes_set, _SDT_PROBE_NAME(_prov, _mod, _func, _name))
@@ -184,34 +178,6 @@
#define SDT_PROBES_ENABLED() __predict_false(sdt_probes_enabled)
-#ifdef _ILP32
-#define _SDT_ASM_WORD ".long"
-#else
-#define _SDT_ASM_WORD ".quad"
-#endif
-
-#ifndef _SDT_ASM_PROBE_CONSTRAINT
-#define _SDT_ASM_PROBE_CONSTRAINT "i"
-#endif
-#ifndef _SDT_ASM_PROBE_OPERAND
-#if !defined(__clang__) && __GNUC_PREREQ__(15, 0)
-#define _SDT_ASM_PROBE_OPERAND "cc"
-#else
-#define _SDT_ASM_PROBE_OPERAND "c"
-#endif
-#endif
-
-/*
- * The asm below generates records corresponding to the structure's layout, so
- * the two must be kept in sync.
- */
-struct sdt_tracepoint {
- struct sdt_probe *probe;
- uintptr_t patchpoint;
- uintptr_t target;
- STAILQ_ENTRY(sdt_tracepoint) tracepoint_entry;
-};
-
/* XXX: GCC is not able to compile probes in kernel modules for aarch64. */
#if !defined(__clang__) && defined(KLD_MODULE) && defined(__aarch64__)
#undef __sdt_used
@@ -219,24 +185,7 @@
#define __SDT_PROBE(prov, mod, func, name, uniq, f, ...)
#else
#define __SDT_PROBE(prov, mod, func, name, uniq, f, ...) do { \
- __WEAK(__CONCAT(__start_set_, _SDT_TRACEPOINT_SET)); \
- __WEAK(__CONCAT(__stop_set_, _SDT_TRACEPOINT_SET)); \
- asm goto( \
- "0:\n" \
- _SDT_ASM_PATCH_INSTR "\n" \
- ".pushsection " _SDT_TRACEPOINT_SECTION ", \"aw\"\n" \
- _SDT_ASM_WORD " %" _SDT_ASM_PROBE_OPERAND "0\n" \
- _SDT_ASM_WORD " 0b\n" \
- _SDT_ASM_WORD " %l1\n" \
- _SDT_ASM_WORD " 0\n" \
- ".popsection\n" \
- : \
- : _SDT_ASM_PROBE_CONSTRAINT (&_SDT_PROBE_NAME(prov, mod, \
- func, name)) \
- : \
- : __sdt_probe##uniq); \
- if (0) { \
-__sdt_probe##uniq:; \
+ if (zcond_branch_likely(_SDT_PROBE_NAME(prov, mod, func, name).enabled)) { \
f(_SDT_PROBE_NAME(prov, mod, func, name).id, __VA_ARGS__); \
} \
} while (0)
@@ -454,7 +403,7 @@
TAILQ_ENTRY(sdt_probe)
probe_entry; /* SDT probe list entry. */
TAILQ_HEAD(, sdt_argtype) argtype_list;
- STAILQ_HEAD(, sdt_tracepoint) tracepoint_list;
+ DECLARE_ZCOND_FALSE(enabled);
const char *mod;
const char *func;
const char *name;
diff --git a/sys/sys/zcond.h b/sys/sys/zcond.h
new file mode 100644
--- /dev/null
+++ b/sys/sys/zcond.h
@@ -0,0 +1,283 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Marko Vlaić <mvlaic@freebsd.org>
+ *
+ * This code was developed as a Google Summer of Code 2024. project
+ * under the guidance of Bojan Novković <bnovkov@freebsdorg>.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifdef _KERNEL
+#ifndef _SYS_ZCOND_H
+#define _SYS_ZCOND_H
+
+#ifdef ZCOND_PATCH
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/queue.h>
+
+#include <machine/zcond.h>
+
+/*
+ * The zcond interface provides a low-cost mechanism for conditional execution.
+ * It is applicable in situations where branch selection is performed by
+ * inspecting the state of a single boolean flag i.e blocks of the following
+ * form: if(flag) {
+ * // do something
+ * }
+ *
+ * This kind of block is compiled into some sequence of load, test, jump
+ * assembly instructions. The low cost provided by zcond is achieved by "baking
+ * in" a single branch direction at compile time. This means outputting either
+ * an unconditional jump or a nop, while the memory access is avoided.
+ *
+ * When the time comes to switch the branch direction, the current instruction
+ * (jump or nop) is patched at runtime to a corresponding instruction (nop or
+ * jump). Keep in mind that this is an expensive operation, since all cpus
+ * except the one performing the patch need to be halted.
+ *
+ * Zconds expand boolean semantics with reference counting. A zcond is in a
+ * false (disabled) state when its reference count is 0 and in a true (enabled)
+ * state when the reference count is greater than 0.
+ *
+ * To use a zcond, first define it with: DEFINE_ZCOND_TRUE(name) or
+ * DEFINE_ZCOND_FALSE(name) Alternatively, declare it with
+ * DECLARE_ZCOND_TRUE(name) or DECLARE_ZCOND_FALSE(name). Then initialize it
+ * with ZCOND_INIT(ZCOND_ENABLED) or ZCOND_INIT(ZCOND_DISABLED).
+ *
+ * Use zcond_branch_likely(cond) or zcond_branch_unlikely(cond) to perform branch selection
+ * based on a zcond. Both functions execute a branch corresponding to the zcond state (true/false).
+ * The likely/unlikely suffix is just a hint indicating which branch is expected to be
+ * executed more frequently.
+ *
+ * To alter the state of a zcond, use zcond_enable(cond) and zcond_disable(cond).
+ * These increase/decrease a zcond's reference count by 1 and perform the
+ * instruction patch on the transition between false and true states.
+ *
+ * This header includes the interface intended to be used by consumers, as well
+ * as some MI code. MD support can be found in sys/<arch>/include/zcond.h and
+ * sys/<arch>/<arch>/zcond_machdep.c
+ */
+
+struct zcond {
+ int refcnt;
+ SLIST_HEAD(, patch_point) patch_points;
+};
+
+/*
+ * Wrapper types are needed for compile time decision making.
+ */
+struct zcond_true {
+ struct zcond cond;
+};
+
+struct zcond_false {
+ struct zcond cond;
+};
+
+#define ZCOND_ELF_SECTION "set_zcond_patch_points_set"
+#define ZCOND_LINKER_SET zcond_patch_points_set
+
+
+#ifdef __ILP32__
+#define _ZCOND_ASM_WORD ".long"
+#else
+#define _ZCOND_ASM_WORD ".quad"
+#endif
+
+/*
+ * __zcond_table is an ELF section which keeps
+ * all the data related to the zcond mechanism.
+ * A single entry describes a single patch_point.
+ */
+#define ZCOND_TABLE_ENTRY \
+ ".pushsection " ZCOND_ELF_SECTION ", \"aw\" \n\t" \
+ _ZCOND_ASM_WORD " 1b \n\t" \
+ _ZCOND_ASM_WORD " %l[l_true] \n\t" \
+ _ZCOND_ASM_WORD " %c0 \n\t" \
+ _ZCOND_ASM_WORD " 0 \n\t" \
+ ".popsection \n\t"
+
+#define ZCOND_SET_START_STOP \
+ do { \
+ __WEAK(__CONCAT(__start_set_, ZCOND_LINKER_SET)); \
+ __WEAK(__CONCAT(__stop_set_, ZCOND_LINKER_SET)); \
+ } while (0);
+
+/*
+ * Emits a __zcond_table entry, describing one patch_point.
+ * Bakes in a nop instruction instruction, so the return value is initially
+ * false.
+ */
+static __always_inline bool
+zcond_nop(struct zcond *const zcond_p)
+{
+ ZCOND_SET_START_STOP
+ asm goto("1: " ZCOND_NOP_ASM "\n\t" ZCOND_TABLE_ENTRY
+ :
+ : "i"(zcond_p)
+ :
+ : l_true);
+
+ return (false);
+l_true:
+ return (true);
+}
+
+/*
+ * Emits a __zcond_table entry, describing one patch_point.
+ * Bakes in a jmp instruction instruction, so the return value is initially
+ * true.
+ */
+static __always_inline bool
+zcond_jmp(struct zcond *const zcond_p)
+{
+ ZCOND_SET_START_STOP
+ asm goto("1:" ZCOND_JMP_ASM("%[l_true]") "\n\t" ZCOND_TABLE_ENTRY
+ :
+ : "i"(zcond_p)
+ :
+ : l_true);
+ return (false);
+l_true:
+ return (true);
+}
+
+/*
+ * These macros declare and initialize a new zcond.
+ */
+
+#define ZCOND_INIT(enabled) \
+ { \
+ { \
+ .patch_points = SLIST_HEAD_INITIALIZER(), \
+ .refcnt = (enabled) \
+ } \
+ }
+
+#define ZCOND_ENABLED 1
+#define ZCOND_DISABLED 0
+
+#define DEFINE_ZCOND_TRUE(name) struct zcond_true name = ZCOND_INIT(ZCOND_ENABLED)
+
+#define DEFINE_ZCOND_FALSE(name) struct zcond_false name = ZCOND_INIT(ZCOND_DISABLED)
+
+#define DECLARE_ZCOND_TRUE(name) struct zcond_true name;
+
+#define DECLARE_ZCOND_FALSE(name) struct zcond_false name;
+
+#define zcond_likely(x) (__builtin_expect((x), 1))
+#define zcond_unlikely(x) (__builtin_expect((x), 0))
+
+/*
+ * These macros inspect the state of a zcond (is it true or false)
+ * thus instatiating a patch_point.
+ */
+#define zcond_branch_likely(cond_wrapped) \
+ ({ \
+ bool branch; \
+ if (__builtin_types_compatible_p(typeof(cond_wrapped), \
+ struct zcond_true)) { \
+ branch = !zcond_nop(&(cond_wrapped.cond)); \
+ } else if (__builtin_types_compatible_p(typeof(cond_wrapped), \
+ struct zcond_false)) { \
+ branch = !zcond_jmp(&(cond_wrapped.cond)); \
+ } \
+ \
+ zcond_likely(branch); \
+ })
+
+#define zcond_branch_unlikely(cond_wrapped) \
+ ({ \
+ bool branch; \
+ if (__builtin_types_compatible_p(typeof(cond_wrapped), \
+ struct zcond_true)) { \
+ branch = zcond_jmp(&(cond_wrapped.cond)); \
+ } else if (__builtin_types_compatible_p(typeof(cond_wrapped), \
+ struct zcond_false)) { \
+ branch = zcond_nop(&(cond_wrapped.cond)); \
+ } \
+ \
+ zcond_unlikely(branch); \
+ })
+
+/*
+ * These macros change the state of a zcond.
+ */
+#define zcond_enable(cond_wrapped) __zcond_toggle(&cond_wrapped.cond, true)
+#define zcond_disable(cond_wrapped) __zcond_toggle(&cond_wrapped.cond, false)
+
+/*
+ * Change the state of a zcond by safely patching all of its
+ * inspection points with appropriate instructions.
+ */
+void __zcond_toggle(struct zcond *cond, bool enable);
+
+#else
+// Fallback implementation when instruction patching is disabled
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/param.h>
+
+struct zcond {
+ int refcnt;
+};
+
+#define ZCOND_INIT(enabled) \
+{ \
+ .refcnt = enabled \
+} \
+
+#define ZCOND_ENABLED 1
+#define ZCOND_DISABLED 0
+
+#define DEFINE_ZCOND_TRUE(name) struct zcond name = ZCOND_INIT(ZCOND_ENABLED)
+
+#define DEFINE_ZCOND_FALSE(name) struct zcond name = ZCOND_INIT(ZCOND_DISABLED)
+
+#define DECLARE_ZCOND_TRUE(name) struct zcond name;
+
+#define DECLARE_ZCOND_FALSE(name) struct zcond name;
+
+#define zcond_likely(x) (__builtin_expect((x), 1))
+#define zcond_unlikely(x) (__builtin_expect((x), 0))
+
+/*
+ * These macros inspect the state of a zcond (is it true or false)
+ * thus instatiating a patch_point.
+ */
+#define zcond_branch_likely(cond) (zcond_likely(cond.refcnt))
+#define zcond_branch_unlikely(cond) (zcond_unlikely(cond.refcnt))
+
+#define zcond_enable(cond) __zcond_toggle(&cond, true)
+#define zcond_disable(cond) __zcond_toggle(&cond, false)
+
+void __zcond_toggle(struct zcond *cond, bool enable);
+
+#endif // ZCOND_PATCH
+#endif // _SYS_ZCOND_H
+#endif // _KERNEL
diff --git a/sys/x86/x86/sdt_machdep.c b/sys/x86/x86/sdt_machdep.c
deleted file mode 100644
--- a/sys/x86/x86/sdt_machdep.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#include <sys/systm.h>
-#include <sys/sdt.h>
-
-#include <vm/vm.h>
-#include <vm/pmap.h>
-
-#include <machine/md_var.h>
-#include <machine/vmparam.h>
-
-#define SDT_PATCH_SIZE 5
-
-/*
- * Return true if we can overwrite a nop at "patchpoint" with a jump to the
- * target address.
- */
-bool
-sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
-{
-#ifdef __amd64__
- if (patchpoint < KERNSTART || target < KERNSTART)
- return (false);
-#endif
- if (patchpoint == target ||
- patchpoint + SDT_PATCH_SIZE < patchpoint)
- return (false);
-#ifdef __amd64__
- int64_t offset = target - (patchpoint + SDT_PATCH_SIZE);
- if (offset < -(1l << 31) || offset > (1l << 31))
- return (false);
-#endif
- return (true);
-}
-
-/*
- * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to
- * the target address.
- */
-void
-sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
-{
- uint8_t instr[SDT_PATCH_SIZE];
- int32_t disp;
- bool old_wp;
-
- KASSERT(sdt_tracepoint_valid(patchpoint, target),
- ("%s: invalid tracepoint %#jx -> %#jx",
- __func__, (uintmax_t)patchpoint, (uintmax_t)target));
-
- instr[0] = 0xe9;
- disp = target - (patchpoint + SDT_PATCH_SIZE);
- memcpy(&instr[1], &disp, sizeof(disp));
-
- old_wp = disable_wp();
- memcpy((void *)patchpoint, instr, sizeof(instr));
- restore_wp(old_wp);
-}
-
-/*
- * Overwrite the patchpoint with a nop instruction.
- */
-void
-sdt_tracepoint_restore(uintptr_t patchpoint)
-{
- uint8_t instr[SDT_PATCH_SIZE];
- bool old_wp;
-
-#ifdef __amd64__
- KASSERT(patchpoint >= KERNSTART,
- ("%s: invalid patchpoint %#lx", __func__, patchpoint));
-#endif
-
- for (int i = 0; i < SDT_PATCH_SIZE; i++)
- instr[i] = 0x90;
-
- old_wp = disable_wp();
- memcpy((void *)patchpoint, instr, sizeof(instr));
- restore_wp(old_wp);
-}

File Metadata

Mime Type
text/plain
Expires
Sat, Jul 11, 11:23 PM (14 h, 59 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
34978515
Default Alt Text
D46379.id179919.diff (58 KB)

Event Timeline