Page MenuHomeFreeBSD
Files F162612267

D58093.id181492.diff
No OneTemporary
Actions

D58093.id181492.diff
View Options

diff --git a/cddl/lib/libdtrace/Makefile b/cddl/lib/libdtrace/Makefile
--- a/cddl/lib/libdtrace/Makefile
+++ b/cddl/lib/libdtrace/Makefile
@@ -55,6 +55,7 @@
errno.d \
io.d \
ip.d \
+ priv.d \
psinfo.d \
sctp.d \
siftr.d \
diff --git a/cddl/lib/libdtrace/priv.d b/cddl/lib/libdtrace/priv.d
new file mode 100644
--- /dev/null
+++ b/cddl/lib/libdtrace/priv.d
@@ -0,0 +1,261 @@
+/*-
+ * Copyright (c) 2026 Devin Teske <dteske@FreeBSD.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+/*
+ * Map priv(9) privilege values to their symbolic names.
+ * NB: Mechanically generated from <sys/priv.h> -- keep in-sync.
+ */
+
+#pragma D binding "1.13" priv_string
+inline string priv_string[int priv] =
+ priv == 2 ? "PRIV_ACCT" :
+ priv == 3 ? "PRIV_MAXFILES" :
+ priv == 4 ? "PRIV_MAXPROC" :
+ priv == 5 ? "PRIV_KTRACE" :
+ priv == 6 ? "PRIV_SETDUMPER" :
+ priv == 8 ? "PRIV_REBOOT" :
+ priv == 9 ? "PRIV_SWAPON" :
+ priv == 10 ? "PRIV_SWAPOFF" :
+ priv == 11 ? "PRIV_MSGBUF" :
+ priv == 12 ? "PRIV_IO" :
+ priv == 13 ? "PRIV_KEYBOARD" :
+ priv == 14 ? "PRIV_DRIVER" :
+ priv == 15 ? "PRIV_ADJTIME" :
+ priv == 16 ? "PRIV_NTP_ADJTIME" :
+ priv == 17 ? "PRIV_CLOCK_SETTIME" :
+ priv == 18 ? "PRIV_SETTIMEOFDAY" :
+ priv == 40 ? "PRIV_AUDIT_CONTROL" :
+ priv == 41 ? "PRIV_AUDIT_FAILSTOP" :
+ priv == 42 ? "PRIV_AUDIT_GETAUDIT" :
+ priv == 43 ? "PRIV_AUDIT_SETAUDIT" :
+ priv == 44 ? "PRIV_AUDIT_SUBMIT" :
+ priv == 50 ? "PRIV_CRED_SETUID" :
+ priv == 51 ? "PRIV_CRED_SETEUID" :
+ priv == 52 ? "PRIV_CRED_SETGID" :
+ priv == 53 ? "PRIV_CRED_SETEGID" :
+ priv == 54 ? "PRIV_CRED_SETGROUPS" :
+ priv == 55 ? "PRIV_CRED_SETREUID" :
+ priv == 56 ? "PRIV_CRED_SETREGID" :
+ priv == 57 ? "PRIV_CRED_SETRESUID" :
+ priv == 58 ? "PRIV_CRED_SETRESGID" :
+ priv == 59 ? "PRIV_SEEOTHERGIDS" :
+ priv == 60 ? "PRIV_SEEOTHERUIDS" :
+ priv == 61 ? "PRIV_SEEJAILPROC" :
+ priv == 62 ? "PRIV_CRED_SETCRED" :
+ priv == 80 ? "PRIV_DEBUG_DIFFCRED" :
+ priv == 81 ? "PRIV_DEBUG_SUGID" :
+ priv == 82 ? "PRIV_DEBUG_UNPRIV" :
+ priv == 83 ? "PRIV_DEBUG_DENIED" :
+ priv == 84 ? "PRIV_DEBUG_DIFFJAIL" :
+ priv == 90 ? "PRIV_DTRACE_KERNEL" :
+ priv == 91 ? "PRIV_DTRACE_PROC" :
+ priv == 92 ? "PRIV_DTRACE_USER" :
+ priv == 100 ? "PRIV_FIRMWARE_LOAD" :
+ priv == 110 ? "PRIV_JAIL_ATTACH" :
+ priv == 111 ? "PRIV_JAIL_SET" :
+ priv == 112 ? "PRIV_JAIL_REMOVE" :
+ priv == 120 ? "PRIV_KENV_SET" :
+ priv == 121 ? "PRIV_KENV_UNSET" :
+ priv == 130 ? "PRIV_KLD_LOAD" :
+ priv == 131 ? "PRIV_KLD_UNLOAD" :
+ priv == 140 ? "PRIV_MAC_PARTITION" :
+ priv == 141 ? "PRIV_MAC_PRIVS" :
+ priv == 160 ? "PRIV_PROC_LIMIT" :
+ priv == 161 ? "PRIV_PROC_SETLOGIN" :
+ priv == 162 ? "PRIV_PROC_SETRLIMIT" :
+ priv == 163 ? "PRIV_PROC_SETLOGINCLASS" :
+ priv == 170 ? "PRIV_IPC_READ" :
+ priv == 171 ? "PRIV_IPC_WRITE" :
+ priv == 172 ? "PRIV_IPC_ADMIN" :
+ priv == 173 ? "PRIV_IPC_MSGSIZE" :
+ priv == 180 ? "PRIV_MQ_ADMIN" :
+ priv == 190 ? "PRIV_PMC_MANAGE" :
+ priv == 191 ? "PRIV_PMC_SYSTEM" :
+ priv == 200 ? "PRIV_SCHED_DIFFCRED" :
+ priv == 201 ? "PRIV_SCHED_SETPRIORITY" :
+ priv == 202 ? "PRIV_SCHED_RTPRIO" :
+ priv == 203 ? "PRIV_SCHED_SETPOLICY" :
+ priv == 204 ? "PRIV_SCHED_SET" :
+ priv == 205 ? "PRIV_SCHED_SETPARAM" :
+ priv == 206 ? "PRIV_SCHED_CPUSET" :
+ priv == 207 ? "PRIV_SCHED_CPUSET_INTR" :
+ priv == 208 ? "PRIV_SCHED_IDPRIO" :
+ priv == 209 ? "PRIV_SCHED_DIFFJAIL" :
+ priv == 220 ? "PRIV_SEM_WRITE" :
+ priv == 230 ? "PRIV_SIGNAL_DIFFCRED" :
+ priv == 231 ? "PRIV_SIGNAL_SUGID" :
+ priv == 232 ? "PRIV_SIGNAL_DIFFJAIL" :
+ priv == 240 ? "PRIV_SYSCTL_DEBUG" :
+ priv == 241 ? "PRIV_SYSCTL_WRITE" :
+ priv == 242 ? "PRIV_SYSCTL_WRITEJAIL" :
+ priv == 243 ? "PRIV_SYSCTL_MEMLOCK" :
+ priv == 250 ? "PRIV_TTY_CONSOLE" :
+ priv == 251 ? "PRIV_TTY_DRAINWAIT" :
+ priv == 252 ? "PRIV_TTY_DTRWAIT" :
+ priv == 253 ? "PRIV_TTY_EXCLUSIVE" :
+ priv == 255 ? "PRIV_TTY_STI" :
+ priv == 256 ? "PRIV_TTY_SETA" :
+ priv == 270 ? "PRIV_UFS_EXTATTRCTL" :
+ priv == 271 ? "PRIV_UFS_QUOTAOFF" :
+ priv == 272 ? "PRIV_UFS_QUOTAON" :
+ priv == 273 ? "PRIV_UFS_SETUSE" :
+ priv == 280 ? "PRIV_ZFS_POOL_CONFIG" :
+ priv == 281 ? "PRIV_ZFS_INJECT" :
+ priv == 282 ? "PRIV_ZFS_JAIL" :
+ priv == 290 ? "PRIV_NFS_DAEMON" :
+ priv == 291 ? "PRIV_NFS_LOCKD" :
+ priv == 310 ? "PRIV_VFS_READ" :
+ priv == 311 ? "PRIV_VFS_WRITE" :
+ priv == 312 ? "PRIV_VFS_ADMIN" :
+ priv == 313 ? "PRIV_VFS_EXEC" :
+ priv == 314 ? "PRIV_VFS_LOOKUP" :
+ priv == 315 ? "PRIV_VFS_BLOCKRESERVE" :
+ priv == 316 ? "PRIV_VFS_CHFLAGS_DEV" :
+ priv == 317 ? "PRIV_VFS_CHOWN" :
+ priv == 318 ? "PRIV_VFS_CHROOT" :
+ priv == 319 ? "PRIV_VFS_RETAINSUGID" :
+ priv == 320 ? "PRIV_VFS_EXCEEDQUOTA" :
+ priv == 321 ? "PRIV_VFS_EXTATTR_SYSTEM" :
+ priv == 322 ? "PRIV_VFS_FCHROOT" :
+ priv == 323 ? "PRIV_VFS_FHOPEN" :
+ priv == 324 ? "PRIV_VFS_FHSTAT" :
+ priv == 325 ? "PRIV_VFS_FHSTATFS" :
+ priv == 326 ? "PRIV_VFS_GENERATION" :
+ priv == 327 ? "PRIV_VFS_GETFH" :
+ priv == 328 ? "PRIV_VFS_GETQUOTA" :
+ priv == 329 ? "PRIV_VFS_LINK" :
+ priv == 330 ? "PRIV_VFS_MKNOD_BAD" :
+ priv == 331 ? "PRIV_VFS_MKNOD_DEV" :
+ priv == 332 ? "PRIV_VFS_MKNOD_WHT" :
+ priv == 333 ? "PRIV_VFS_MOUNT" :
+ priv == 334 ? "PRIV_VFS_MOUNT_OWNER" :
+ priv == 335 ? "PRIV_VFS_MOUNT_EXPORTED" :
+ priv == 336 ? "PRIV_VFS_MOUNT_PERM" :
+ priv == 337 ? "PRIV_VFS_MOUNT_SUIDDIR" :
+ priv == 338 ? "PRIV_VFS_MOUNT_NONUSER" :
+ priv == 339 ? "PRIV_VFS_SETGID" :
+ priv == 340 ? "PRIV_VFS_SETQUOTA" :
+ priv == 341 ? "PRIV_VFS_STICKYFILE" :
+ priv == 342 ? "PRIV_VFS_SYSFLAGS" :
+ priv == 343 ? "PRIV_VFS_UNMOUNT" :
+ priv == 344 ? "PRIV_VFS_STAT" :
+ priv == 345 ? "PRIV_VFS_READ_DIR" :
+ priv == 360 ? "PRIV_VM_MADV_PROTECT" :
+ priv == 361 ? "PRIV_VM_MLOCK" :
+ priv == 362 ? "PRIV_VM_MUNLOCK" :
+ priv == 363 ? "PRIV_VM_SWAP_NOQUOTA" :
+ priv == 364 ? "PRIV_VM_SWAP_NORLIMIT" :
+ priv == 370 ? "PRIV_DEVFS_RULE" :
+ priv == 371 ? "PRIV_DEVFS_SYMLINK" :
+ priv == 380 ? "PRIV_RANDOM_RESEED" :
+ priv == 390 ? "PRIV_NET_BRIDGE" :
+ priv == 391 ? "PRIV_NET_GRE" :
+ priv == 394 ? "PRIV_NET_BPF" :
+ priv == 395 ? "PRIV_NET_RAW" :
+ priv == 396 ? "PRIV_NET_ROUTE" :
+ priv == 397 ? "PRIV_NET_TAP" :
+ priv == 398 ? "PRIV_NET_SETIFMTU" :
+ priv == 399 ? "PRIV_NET_SETIFFLAGS" :
+ priv == 400 ? "PRIV_NET_SETIFCAP" :
+ priv == 401 ? "PRIV_NET_SETIFNAME" :
+ priv == 402 ? "PRIV_NET_SETIFMETRIC" :
+ priv == 403 ? "PRIV_NET_SETIFPHYS" :
+ priv == 404 ? "PRIV_NET_SETIFMAC" :
+ priv == 405 ? "PRIV_NET_ADDMULTI" :
+ priv == 406 ? "PRIV_NET_DELMULTI" :
+ priv == 407 ? "PRIV_NET_HWIOCTL" :
+ priv == 408 ? "PRIV_NET_SETLLADDR" :
+ priv == 409 ? "PRIV_NET_ADDIFGROUP" :
+ priv == 410 ? "PRIV_NET_DELIFGROUP" :
+ priv == 411 ? "PRIV_NET_IFCREATE" :
+ priv == 412 ? "PRIV_NET_IFDESTROY" :
+ priv == 413 ? "PRIV_NET_ADDIFADDR" :
+ priv == 414 ? "PRIV_NET_DELIFADDR" :
+ priv == 415 ? "PRIV_NET_LAGG" :
+ priv == 416 ? "PRIV_NET_GIF" :
+ priv == 417 ? "PRIV_NET_SETIFVNET" :
+ priv == 418 ? "PRIV_NET_SETIFDESCR" :
+ priv == 419 ? "PRIV_NET_SETIFFIB" :
+ priv == 420 ? "PRIV_NET_VXLAN" :
+ priv == 421 ? "PRIV_NET_SETLANPCP" :
+ priv == 422 ? "PRIV_NET_OVPN" :
+ priv == 423 ? "PRIV_NET_ME" :
+ priv == 424 ? "PRIV_NET_WG" :
+ priv == 425 ? "PRIV_NET_GENEVE" :
+ priv == 440 ? "PRIV_NET80211_VAP_GETKEY" :
+ priv == 441 ? "PRIV_NET80211_VAP_MANAGE" :
+ priv == 442 ? "PRIV_NET80211_VAP_SETMAC" :
+ priv == 443 ? "PRIV_NET80211_CREATE_VAP" :
+ priv == 460 ? "PRIV_NETATM_CFG" :
+ priv == 461 ? "PRIV_NETATM_ADD" :
+ priv == 462 ? "PRIV_NETATM_DEL" :
+ priv == 463 ? "PRIV_NETATM_SET" :
+ priv == 470 ? "PRIV_NETBLUETOOTH_RAW" :
+ priv == 480 ? "PRIV_NETGRAPH_CONTROL" :
+ priv == 481 ? "PRIV_NETGRAPH_TTY" :
+ priv == 490 ? "PRIV_NETINET_RESERVEDPORT" :
+ priv == 491 ? "PRIV_NETINET_IPFW" :
+ priv == 492 ? "PRIV_NETINET_DIVERT" :
+ priv == 493 ? "PRIV_NETINET_PF" :
+ priv == 494 ? "PRIV_NETINET_DUMMYNET" :
+ priv == 495 ? "PRIV_NETINET_CARP" :
+ priv == 496 ? "PRIV_NETINET_MROUTE" :
+ priv == 497 ? "PRIV_NETINET_RAW" :
+ priv == 498 ? "PRIV_NETINET_GETCRED" :
+ priv == 499 ? "PRIV_NETINET_ADDRCTRL6" :
+ priv == 500 ? "PRIV_NETINET_ND6" :
+ priv == 501 ? "PRIV_NETINET_SCOPE6" :
+ priv == 502 ? "PRIV_NETINET_ALIFETIME6" :
+ priv == 503 ? "PRIV_NETINET_IPSEC" :
+ priv == 504 ? "PRIV_NETINET_REUSEPORT" :
+ priv == 505 ? "PRIV_NETINET_SETHDROPTS" :
+ priv == 506 ? "PRIV_NETINET_BINDANY" :
+ priv == 507 ? "PRIV_NETINET_HASHKEY" :
+ priv == 508 ? "PRIV_NETINET_KTLSKEYS" :
+ priv == 530 ? "PRIV_NETNCP" :
+ priv == 540 ? "PRIV_NETSMB" :
+ priv == 550 ? "PRIV_VM86_INTCALL" :
+ priv == 560 ? "PRIV_PIPEBUF" :
+ priv == 600 ? "PRIV_MODULE0" :
+ priv == 601 ? "PRIV_MODULE1" :
+ priv == 602 ? "PRIV_MODULE2" :
+ priv == 603 ? "PRIV_MODULE3" :
+ priv == 604 ? "PRIV_MODULE4" :
+ priv == 605 ? "PRIV_MODULE5" :
+ priv == 606 ? "PRIV_MODULE6" :
+ priv == 607 ? "PRIV_MODULE7" :
+ priv == 608 ? "PRIV_MODULE8" :
+ priv == 609 ? "PRIV_MODULE9" :
+ priv == 610 ? "PRIV_MODULE10" :
+ priv == 611 ? "PRIV_MODULE11" :
+ priv == 612 ? "PRIV_MODULE12" :
+ priv == 613 ? "PRIV_MODULE13" :
+ priv == 614 ? "PRIV_MODULE14" :
+ priv == 615 ? "PRIV_MODULE15" :
+ priv == 620 ? "PRIV_DDB_CAPTURE" :
+ priv == 630 ? "PRIV_NNPFS_DEBUG" :
+ priv == 640 ? "PRIV_CPUCTL_WRMSR" :
+ priv == 641 ? "PRIV_CPUCTL_UPDATE" :
+ priv == 650 ? "PRIV_C4B_RESET_CTLR" :
+ priv == 651 ? "PRIV_C4B_TRACE" :
+ priv == 660 ? "PRIV_AFS_ADMIN" :
+ priv == 661 ? "PRIV_AFS_DAEMON" :
+ priv == 670 ? "PRIV_RCTL_GET_RACCT" :
+ priv == 671 ? "PRIV_RCTL_GET_RULES" :
+ priv == 672 ? "PRIV_RCTL_GET_LIMITS" :
+ priv == 673 ? "PRIV_RCTL_ADD_RULE" :
+ priv == 674 ? "PRIV_RCTL_REMOVE_RULE" :
+ priv == 680 ? "PRIV_KMEM_READ" :
+ priv == 681 ? "PRIV_KMEM_WRITE" :
+ priv == 682 ? "PRIV_PROC_MEM_WRITE" :
+ priv == 690 ? "PRIV_KDB_SET_BACKEND" :
+ priv == 700 ? "PRIV_VERIEXEC_DIRECT" :
+ priv == 701 ? "PRIV_VERIEXEC_NOVERIFY" :
+ priv == 702 ? "PRIV_VERIEXEC_CONTROL" :
+ priv == 710 ? "PRIV_VMM_PPTDEV" :
+ priv == 711 ? "PRIV_VMM_CREATE" :
+ priv == 712 ? "PRIV_VMM_DESTROY" :
+ strjoin("PRIV_UNKNOWN#", lltostr(priv));
diff --git a/cddl/usr.sbin/dwatch/dwatch.1 b/cddl/usr.sbin/dwatch/dwatch.1
--- a/cddl/usr.sbin/dwatch/dwatch.1
+++ b/cddl/usr.sbin/dwatch/dwatch.1
@@ -1,4 +1,4 @@
-.\" Copyright (c) 2014-2018 Devin Teske
+.\" Copyright (c) 2014-2026 Devin Teske
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
@@ -22,7 +22,7 @@
.\" ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
.\" POSSIBILITY OF SUCH DAMAGE.
.\"
-.Dd April 18, 2025
+.Dd July 4, 2026
.Dt DWATCH 1
.Os
.Sh NAME
@@ -382,17 +382,69 @@
.Xr chmod 2 ,
.Xr lchmod 2 ,
.Xr fchmodat 2
+.It coredump
+Print coredump-worthy signal deliveries with a verdict of whether a
+core will actually be written
+.It coredump-top
+Maintain a running catalog of coredump-worthy signals by process and
+signal
+.It dtmalloc
+Print kernel
+.Xr malloc 9
+and
+.Xr free 9
+activity by malloc type,
+meeting a size threshold
+.It dtmalloc-top
+Maintain a running catalog of net bytes and allocation balance by
+malloc type
+.Pq leak suspects rise
.It errno
Print non-zero errno results from system calls
+.It errno- Ns Ar NAME
+Print system calls returning the given errno,
+by symbolic name or number
+.Pq e.g., errno-ENOENT, errno-ENOTCAPABLE, errno-13
+.It hang
+Print threads that were asleep longer than a threshold,
+as they wake,
+naming the sleeper and the waker
+.It hang-top
+Maintain a running catalog of long sleeps by process
.It io
Print disk I/O details provided by
.Xr dtrace_io 4
+.It io-slow
+Print disk I/O requests exceeding a latency threshold
.It ip
Print IPv4 and IPv6 details provided by
.Xr dtrace_ip 4
+.It jail
+Print jail management activity from
+.Xr jail 2 ,
+.Xr jail_set 2 ,
+.Xr jail_get 2 ,
+.Xr jail_attach 2 ,
+.Xr jail_remove 2
.It kill
Print signal and pid from
.Xr kill 2
+.It lock
+Print kernel lock contention details provided by
+.Xr dtrace_lockstat 4
+.It lock-spin
+Print kernel lock spinning details provided by
+.Xr dtrace_lockstat 4
+.It mib
+Print network stack statistics counters as they increment
+.Pq requires a kernel built with options KDTRACE_MIB_SDT
+.It namei
+Print pathname resolution details provided by the
+.Fn vfs:namei:lookup
+probes
+.It namei-failure
+Print failed pathname resolutions
+.Pq also namei-enoent for ENOENT only
.It nanosleep
Print requested time from
.Xr nanosleep 2
@@ -400,12 +452,19 @@
Print path from
.Xr open 2 ,
.Xr openat 2
+.It priv
+Print privilege requests denied by
+.Xr priv_check 9
+.Pq also priv-ok for those granted
.It proc
Print process execution details provided by
.Xr dtrace_proc 4
.It proc-signal
Print process signal details provided by
.Xr dtrace_proc 4
+.It proc-signal-fatal
+Print only signals whose default disposition is to terminate the
+receiving process
.It rw
Print buffer contents from
.Xr read 2 ,
@@ -413,12 +472,24 @@
.It sched
Print CPU scheduling details provided by
.Xr dtrace_sched 4
+.It sched-latency
+Print threads that waited longer than a threshold between becoming
+runnable and running
+.It slow
+Print filesystem-related syscalls exceeding a latency threshold
+.It slow-syscall
+Print any syscall exceeding a latency threshold
.It tcp
Print TCP address/port details provided by
.Xr dtrace_tcp 4
.It tcp-io
Print TCP I/O details provided by
.Xr dtrace_tcp 4
+.It tcp-retransmit
+Print TCP retransmissions,
+retransmit timeouts,
+and connections dropped by retransmit exhaustion
+.Pq requires a kernel built with options KDTRACE_MIB_SDT
.It udp
Print UDP I/O details provided by
.Xr dtrace_udp 4
@@ -454,6 +525,26 @@
These environment variables affect the execution of
.Nm :
.Bl -tag -width "DWATCH_PROFILES_PATH"
+.It Ev DWATCH_HANG_MS
+Minimum sleep duration,
+in milliseconds,
+displayed by the hang profiles
+.Pq default 1000; 0 to show everything .
+.It Ev DWATCH_IO_MS
+Minimum disk I/O request latency,
+in milliseconds,
+displayed by the io-slow profile
+.Pq default 100; 0 to show everything .
+.It Ev DWATCH_LOCK_MS
+Minimum lock holdoff duration,
+in milliseconds,
+displayed by the lock profiles
+.Pq default 1; 0 to show everything .
+.It Ev DWATCH_MALLOC_MIN
+Minimum allocation size,
+in bytes,
+displayed by the dtmalloc profiles
+.Pq default 65536; 0 to show everything .
.It Ev DWATCH_PROFILES_PATH
If
.Ev DWATCH_PROFILES_PATH
@@ -464,6 +555,16 @@
.Ql Li /usr/libexec/dwatch:/usr/local/libexec/dwatch .
If set to NULL,
profiles are not loaded.
+.It Ev DWATCH_SCHED_MS
+Minimum run-queue wait,
+in milliseconds,
+displayed by the sched-latency profile
+.Pq default 10; 0 to show everything .
+.It Ev DWATCH_SLOW_MS
+Minimum syscall latency,
+in milliseconds,
+displayed by the slow profiles
+.Pq default 100 .
.El
.Sh EXIT STATUS
.Ex -std
diff --git a/cddl/usr.sbin/dwatch/libexec/Makefile b/cddl/usr.sbin/dwatch/libexec/Makefile
--- a/cddl/usr.sbin/dwatch/libexec/Makefile
+++ b/cddl/usr.sbin/dwatch/libexec/Makefile
@@ -1,16 +1,25 @@
PACKAGE= dwatch
FILESDIR= ${LIBEXECDIR}/dwatch
FILES= chmod \
+ coredump \
+ dtmalloc \
errno \
+ hang \
io \
ip \
+ jail \
kill \
+ lock \
+ mib \
+ namei \
nanosleep \
open \
+ priv \
proc \
rw \
sched \
sendrecv \
+ slow \
systop \
tcp \
udp \
@@ -23,11 +32,37 @@
LINKMODE= ${SHAREMODE}
LINKS= ${LIBEXECDIR}/dwatch/chmod ${LIBEXECDIR}/dwatch/fchmodat
LINKS+= ${LIBEXECDIR}/dwatch/chmod ${LIBEXECDIR}/dwatch/lchmod
+LINKS+= ${LIBEXECDIR}/dwatch/coredump ${LIBEXECDIR}/dwatch/coredump-top
+LINKS+= ${LIBEXECDIR}/dwatch/dtmalloc ${LIBEXECDIR}/dwatch/dtmalloc-top
+LINKS+= ${LIBEXECDIR}/dwatch/errno ${LIBEXECDIR}/dwatch/errno-EACCES
+LINKS+= ${LIBEXECDIR}/dwatch/errno ${LIBEXECDIR}/dwatch/errno-ECAPMODE
+LINKS+= ${LIBEXECDIR}/dwatch/errno ${LIBEXECDIR}/dwatch/errno-ENOENT
+LINKS+= ${LIBEXECDIR}/dwatch/errno ${LIBEXECDIR}/dwatch/errno-ENOTCAPABLE
+LINKS+= ${LIBEXECDIR}/dwatch/errno ${LIBEXECDIR}/dwatch/errno-EPERM
+LINKS+= ${LIBEXECDIR}/dwatch/hang ${LIBEXECDIR}/dwatch/hang-top
LINKS+= ${LIBEXECDIR}/dwatch/io ${LIBEXECDIR}/dwatch/io-done
+LINKS+= ${LIBEXECDIR}/dwatch/io ${LIBEXECDIR}/dwatch/io-slow
LINKS+= ${LIBEXECDIR}/dwatch/io ${LIBEXECDIR}/dwatch/io-start
LINKS+= ${LIBEXECDIR}/dwatch/ip ${LIBEXECDIR}/dwatch/ip-receive
LINKS+= ${LIBEXECDIR}/dwatch/ip ${LIBEXECDIR}/dwatch/ip-send
+LINKS+= ${LIBEXECDIR}/dwatch/jail ${LIBEXECDIR}/dwatch/jail-attach
+LINKS+= ${LIBEXECDIR}/dwatch/jail ${LIBEXECDIR}/dwatch/jail-get
+LINKS+= ${LIBEXECDIR}/dwatch/jail ${LIBEXECDIR}/dwatch/jail-remove
+LINKS+= ${LIBEXECDIR}/dwatch/jail ${LIBEXECDIR}/dwatch/jail-set
+LINKS+= ${LIBEXECDIR}/dwatch/lock ${LIBEXECDIR}/dwatch/lock-adaptive
+LINKS+= ${LIBEXECDIR}/dwatch/lock ${LIBEXECDIR}/dwatch/lock-block
+LINKS+= ${LIBEXECDIR}/dwatch/lock ${LIBEXECDIR}/dwatch/lock-lockmgr
+LINKS+= ${LIBEXECDIR}/dwatch/lock ${LIBEXECDIR}/dwatch/lock-rw
+LINKS+= ${LIBEXECDIR}/dwatch/lock ${LIBEXECDIR}/dwatch/lock-spin
+LINKS+= ${LIBEXECDIR}/dwatch/lock ${LIBEXECDIR}/dwatch/lock-sx
+LINKS+= ${LIBEXECDIR}/dwatch/lock ${LIBEXECDIR}/dwatch/lock-thread
+LINKS+= ${LIBEXECDIR}/dwatch/mib ${LIBEXECDIR}/dwatch/tcp-retransmit
+LINKS+= ${LIBEXECDIR}/dwatch/namei ${LIBEXECDIR}/dwatch/namei-enoent
+LINKS+= ${LIBEXECDIR}/dwatch/namei ${LIBEXECDIR}/dwatch/namei-entry
+LINKS+= ${LIBEXECDIR}/dwatch/namei ${LIBEXECDIR}/dwatch/namei-failure
LINKS+= ${LIBEXECDIR}/dwatch/open ${LIBEXECDIR}/dwatch/openat
+LINKS+= ${LIBEXECDIR}/dwatch/priv ${LIBEXECDIR}/dwatch/priv-err
+LINKS+= ${LIBEXECDIR}/dwatch/priv ${LIBEXECDIR}/dwatch/priv-ok
LINKS+= ${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-create
LINKS+= ${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-exec
LINKS+= ${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-exec-failure
@@ -36,6 +71,7 @@
LINKS+= ${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-signal
LINKS+= ${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-signal-clear
LINKS+= ${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-signal-discard
+LINKS+= ${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-signal-fatal
LINKS+= ${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-signal-send
LINKS+= ${LIBEXECDIR}/dwatch/proc ${LIBEXECDIR}/dwatch/proc-status
LINKS+= ${LIBEXECDIR}/dwatch/rw ${LIBEXECDIR}/dwatch/read
@@ -45,6 +81,7 @@
LINKS+= ${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-dequeue
LINKS+= ${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-enqueue
LINKS+= ${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-exec
+LINKS+= ${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-latency
LINKS+= ${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-lend-pri
LINKS+= ${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-load-change
LINKS+= ${LIBEXECDIR}/dwatch/sched ${LIBEXECDIR}/dwatch/sched-off-cpu
@@ -63,6 +100,11 @@
LINKS+= ${LIBEXECDIR}/dwatch/sendrecv ${LIBEXECDIR}/dwatch/send
LINKS+= ${LIBEXECDIR}/dwatch/sendrecv ${LIBEXECDIR}/dwatch/sendmsg
LINKS+= ${LIBEXECDIR}/dwatch/sendrecv ${LIBEXECDIR}/dwatch/sendto
+LINKS+= ${LIBEXECDIR}/dwatch/slow ${LIBEXECDIR}/dwatch/slow-fsync
+LINKS+= ${LIBEXECDIR}/dwatch/slow ${LIBEXECDIR}/dwatch/slow-open
+LINKS+= ${LIBEXECDIR}/dwatch/slow ${LIBEXECDIR}/dwatch/slow-read
+LINKS+= ${LIBEXECDIR}/dwatch/slow ${LIBEXECDIR}/dwatch/slow-syscall
+LINKS+= ${LIBEXECDIR}/dwatch/slow ${LIBEXECDIR}/dwatch/slow-write
LINKS+= ${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-accept
LINKS+= ${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-accept-established
LINKS+= ${LIBEXECDIR}/dwatch/tcp ${LIBEXECDIR}/dwatch/tcp-accept-refused
diff --git a/cddl/usr.sbin/dwatch/libexec/coredump b/cddl/usr.sbin/dwatch/libexec/coredump
new file mode 100644
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/coredump
@@ -0,0 +1,210 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for coredump-worthy signal delivery $
+# $Copyright: 2026 Devin Teske. All rights reserved. $
+#
+############################################################ DESCRIPTION
+#
+# Print signals whose default action produces a coredump, as they are
+# sent, together with a verdict of whether a core will actually be
+# written, rendered the same way and in the same order the kernel will
+# decide it: the target may be ignoring or catching the signal, dumps
+# may be disabled (kern.coredump), the target may be flagged sugid
+# (kern.sugid_coredump) or may have disabled tracing via procctl(2)
+# PROC_TRACE_CTL, or RLIMIT_CORE may be 0.
+# Coredump-worthy is defined by the SIGPROP_CORE entries of the
+# sigproptbl in kern_sig.c: SIGQUIT, SIGILL, SIGTRAP, SIGABRT, SIGEMT,
+# SIGFPE, SIGBUS, SIGSEGV, and SIGSYS.
+# The coredump-top profile maintains a running catalog, updated
+# every 3 seconds, of coredump-worthy signals by process and signal.
+# Combine with `-O cmd' to capture state as each event occurs.
+#
+############################################################ PRAGMAS
+
+case "$PROFILE" in
+coredump-top)
+ DTRACE_PRAGMA="
+ option quiet
+ option aggsortrev
+ " # END-QUOTE
+ ;;
+esac
+
+############################################################ PROBE
+
+case "$PROFILE" in
+coredump-top)
+ : ${PROBE:=profile:::tick-3s} ;;
+*)
+ : ${PROBE:=proc:::signal-send}
+esac
+
+############################################################ GLOBALS
+
+exec 9<<EOF
+/*
+ * Signals whose default action includes a coredump, per the
+ * SIGPROP_CORE entries of the sigproptbl in kern_sig.c of FreeBSD
+ */
+inline int coredump_sig[int sig] =
+ sig == SIGQUIT ? 1 :
+ sig == SIGILL ? 1 :
+ sig == SIGTRAP ? 1 :
+ sig == SIGABRT ? 1 : /* SIGIOT */
+ sig == SIGEMT ? 1 :
+ sig == SIGFPE ? 1 :
+ sig == SIGBUS ? 1 :
+ sig == SIGSEGV ? 1 :
+ sig == SIGSYS ? 1 :
+ 0;
+EOF
+GLOBALS=$( cat <&9 )
+
+############################################################ EVENT ACTION
+
+[ "$CUSTOM_TEST" ] || case "$PROFILE" in
+coredump-top) ;;
+*) EVENT_TEST="coredump_sig[this->sig]"
+esac
+
+############################################################ ACTIONS
+
+if [ "$PROFILE" = "coredump-top" ]; then
+exec 9<<EOF
+$GLOBALS
+
+BEGIN { printf("Cataloging coredump-worthy signals ...") } /* probe ID $ID */
+
+proc:::signal-send /coredump_sig[(int)arg2]/ /* probe ID $(( $ID + 1 )) */
+{
+ @cores[stringof(((struct proc *)args[1])->p_comm),
+ signal_string[(int)arg2]] = count();
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 2 ))
+else
+exec 9<<EOF
+$GLOBALS
+
+this int sig;
+this pid_t pid;
+this string verdict;
+this struct proc * target;
+this u_int core_catch;
+this u_int core_ign;
+this u_int core_notrace;
+this u_int core_sugid;
+this uint64_t core_limit;
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+ printf("<$ID>");
+}
+ this->target = (struct proc *)args[1];
+ this->pid = (pid_t)this->target->p_pid;
+ this->sig = (int)arg2;
+}
+
+$PROBE /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+ printf("<$(( $ID + 1 ))>");
+}
+ /*
+ * Render a verdict the way the kernel will: disposition first
+ * (struct sigacts), then the coredump() gauntlet in its order:
+ * kern.coredump, kern.sugid_coredump vs P_SUGID, procctl(2)
+ * PROC_TRACE_CTL (P2_NOTRACE), and finally RLIMIT_CORE
+ */
+ this->core_ign =
+ this->target->p_sigacts->ps_sigignore.__bits[
+ (this->sig - 1) >> 5] &
+ (1 << ((this->sig - 1) & 31));
+ this->core_catch =
+ this->target->p_sigacts->ps_sigcatch.__bits[
+ (this->sig - 1) >> 5] &
+ (1 << ((this->sig - 1) & 31));
+ this->core_sugid =
+ this->target->p_flag & 0x00000100; /* P_SUGID */
+ this->core_notrace =
+ this->target->p_flag2 & 0x00000002; /* P2_NOTRACE */
+ this->core_limit = this->target->p_limit->
+ pl_rlimit[4].rlim_cur; /* RLIMIT_CORE */
+
+ this->verdict =
+ this->core_ign ? "ignored" :
+ this->core_catch ? "caught" :
+ \`do_coredump == 0 ? "denied by kern.coredump" :
+ this->core_sugid && \`sugid_coredump == 0 ?
+ "denied by kern.sugid_coredump" :
+ this->core_notrace ? "denied by procctl trace ctl" :
+ this->core_limit == 0 ? "denied by RLIMIT_CORE" :
+ "will dump core";
+
+ $( pproc -P _core "(struct proc *)args[1]" )
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 2 ))
+fi
+
+############################################################ EVENT TAG
+
+# For the running catalog, override the default `UID.GID CMD[PID]: ' tag
+# with ANSI cursor-homing and screen-clearing codes plus column headers.
+
+if [ "$PROFILE" = "coredump-top" ]; then
+size=$( stty size 2> /dev/null )
+rows="${size%% *}"
+cols="${size#* }"
+
+exec 9<<EOF
+ printf("\033[H"); /* Position the cursor at top-left */
+ printf("\033[J"); /* Clear display from cursor to end */
+
+ /* Header line containing probe (left) and date (right) */
+ printf("%-*s%s%Y%s\n",
+ $(( ${cols:-80} - 20 )), "$PROBE",
+ console ? "\033[32m" : "",
+ walltimestamp,
+ console ? "\033[39m" : "");
+
+ /* Column headers */
+ printf("%s%8s %-20s %s%s\n",
+ console ? "\033[1m" : "",
+ "COUNT",
+ "EXECNAME",
+ "SIGNAL",
+ console ? "\033[22m" : "");
+EOF
+EVENT_TAG=$( cat <&9 )
+fi
+
+############################################################ EVENT DETAILS
+
+if [ "$PROFILE" = "coredump-top" ]; then
+exec 9<<EOF
+ /* NB: Cumulative; not truncated between updates */
+ printa("%@8u %-20s %s\n", @cores);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+elif [ ! "$CUSTOM_DETAILS" ]; then
+exec 9<<EOF
+ /*
+ * Print coredump-worthy signal details
+ */
+ printf("%s[%d] pid %d (%s) -- %s",
+ signal_string[this->sig],
+ this->sig,
+ this->pid,
+ this->verdict,
+ this->args_core);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+fi
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/dtmalloc b/cddl/usr.sbin/dwatch/libexec/dtmalloc
new file mode 100644
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/dtmalloc
@@ -0,0 +1,146 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for malloc(9) type activity $
+# $Copyright: 2026 Devin Teske. All rights reserved. $
+#
+############################################################ DESCRIPTION
+#
+# Print kernel malloc(9) and free(9) activity by malloc type, via the
+# dtmalloc provider (one malloc and one free probe per type; see
+# vmstat -m for the types). The default profile prints allocations and
+# frees meeting a size threshold (default 65536 bytes; tunable via
+# DWATCH_MALLOC_MIN in the environment, 0 to show everything),
+# answering "who is allocating huge kernel buffers?" Use
+# dtmalloc-NAME to watch a single type (e.g., dtmalloc-devbuf).
+# The dtmalloc-top profile maintains a running catalog, updated every
+# 3 seconds, of net bytes and outstanding allocation balance by type:
+# a type whose net bytes climb without bound while the system is in
+# steady state is a leak suspect. NB: the catalog reflects activity
+# since the watch began, not preexisting allocations, and caches
+# legitimately hold what they allocate.
+#
+############################################################ PRAGMAS
+
+case "$PROFILE" in
+dtmalloc-top)
+ DTRACE_PRAGMA="
+ option quiet
+ option aggsortrev
+ " # END-QUOTE
+ ;;
+esac
+
+############################################################ PROBE
+
+case "$PROFILE" in
+dtmalloc)
+ : ${PROBE:=dtmalloc:::malloc, dtmalloc:::free} ;;
+dtmalloc-top)
+ : ${PROBE:=profile:::tick-3s} ;;
+*)
+ : ${PROBE:=$( echo \
+ dtmalloc::${PROFILE#dtmalloc-}:malloc, \
+ dtmalloc::${PROFILE#dtmalloc-}:free )} ;;
+esac
+
+############################################################ EVENT ACTION
+
+: ${DWATCH_MALLOC_MIN:=65536}
+
+case "$DWATCH_MALLOC_MIN" in
+""|*[!0-9]*) die "DWATCH_MALLOC_MIN must be a number" ;; # NOTREACHED
+esac
+
+[ "$CUSTOM_TEST" ] || case "$PROFILE" in
+dtmalloc-top) ;;
+*) EVENT_TEST="(uint64_t)arg3 >= $DWATCH_MALLOC_MIN"
+esac
+
+############################################################ ACTIONS
+
+if [ "$PROFILE" = "dtmalloc-top" ]; then
+exec 9<<EOF
+this int64_t dtmalloc_delta;
+
+BEGIN { printf("Cataloging malloc(9) activity ...") } /* probe ID $ID */
+
+dtmalloc:::malloc /* probe ID $(( $ID + 1 )) */
+{
+ this->dtmalloc_delta = (int64_t)arg3;
+}
+
+dtmalloc:::free /* probe ID $(( $ID + 2 )) */
+{
+ this->dtmalloc_delta = -(int64_t)arg3;
+}
+
+dtmalloc:::malloc, dtmalloc:::free /* probe ID $(( $ID + 3 )) */
+{
+ @dtmalloc_bytes[probefunc] = sum(this->dtmalloc_delta);
+ @dtmalloc_allocs[probefunc] =
+ sum(this->dtmalloc_delta > 0 ? 1 : -1);
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 4 ))
+fi
+
+############################################################ EVENT TAG
+
+# For the running catalog, override the default `UID.GID CMD[PID]: ' tag
+# with ANSI cursor-homing and screen-clearing codes plus column headers.
+
+if [ "$PROFILE" = "dtmalloc-top" ]; then
+size=$( stty size 2> /dev/null )
+rows="${size%% *}"
+cols="${size#* }"
+
+exec 9<<EOF
+ printf("\033[H"); /* Position the cursor at top-left */
+ printf("\033[J"); /* Clear display from cursor to end */
+
+ /* Header line containing probe (left) and date (right) */
+ printf("%-*s%s%Y%s\n",
+ $(( ${cols:-80} - 20 )), "$PROBE",
+ console ? "\033[32m" : "",
+ walltimestamp,
+ console ? "\033[39m" : "");
+
+ /* Column headers */
+ printf("%s%14s %10s %s%s\n",
+ console ? "\033[1m" : "",
+ "NET(B)",
+ "BALANCE",
+ "TYPE",
+ console ? "\033[22m" : "");
+EOF
+EVENT_TAG=$( cat <&9 )
+fi
+
+############################################################ EVENT DETAILS
+
+if [ "$PROFILE" = "dtmalloc-top" ]; then
+exec 9<<EOF
+ /* NB: Cumulative; not truncated between updates */
+ printa("%@14d %@10d %s\n", @dtmalloc_bytes, @dtmalloc_allocs);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+elif [ ! "$CUSTOM_DETAILS" ]; then
+exec 9<<EOF
+ /*
+ * Print malloc(9) activity details
+ */
+ printf("%s(9) type %s %d byte%s",
+ probename,
+ probefunc,
+ (long)arg3,
+ (long)arg3 == 1 ? "" : "s");
+EOF
+EVENT_DETAILS=$( cat <&9 )
+fi
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/errno b/cddl/usr.sbin/dwatch/libexec/errno
--- a/cddl/usr.sbin/dwatch/libexec/errno
+++ b/cddl/usr.sbin/dwatch/libexec/errno
@@ -3,7 +3,7 @@
############################################################ IDENT(1)
#
# $Title: dwatch(8) module for syscall errno logging $
-# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $Copyright: 2014-2026 Devin Teske. All rights reserved. $
#
############################################################ DESCRIPTION
#
@@ -11,13 +11,24 @@
# To override the default test condition, use (for example) `-t errno==2' to
# test for specific value or simply `-t 1' to unconditionally show all values.
#
+# When invoked as errno-NAME, only syscalls returning that errno are shown,
+# where NAME is either a symbolic name from errno.d of libdtrace(1) or a
+# number; e.g., errno-ENOENT for failed pathname lookups, errno-EACCES or
+# errno-EPERM for permission problems, and errno-ENOTCAPABLE or
+# errno-ECAPMODE for capsicum(4) capability-mode violations. Links are
+# installed for the aforementioned; others need only a new link to this
+# module (e.g., `ln -s errno errno-EDEADLK').
+#
############################################################ PROBE
: ${PROBE:=syscall:::return}
############################################################ EVENT ACTION
-[ "$CUSTOM_TEST" ] || EVENT_TEST="errno > 0"
+[ "$CUSTOM_TEST" ] || case "$PROFILE" in
+errno) EVENT_TEST="errno > 0" ;;
+ *) EVENT_TEST="errno == ${PROFILE#errno-}"
+esac
############################################################ EVENT DETAILS
diff --git a/cddl/usr.sbin/dwatch/libexec/hang b/cddl/usr.sbin/dwatch/libexec/hang
new file mode 100644
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/hang
@@ -0,0 +1,174 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for long sleeps and hung threads $
+# $Copyright: 2026 Devin Teske. All rights reserved. $
+#
+############################################################ DESCRIPTION
+#
+# Print threads that were asleep for a long time, as they wake, naming
+# the sleeper, how long it slept, and (in the standard event tag) the
+# process that woke it. Sleeps shorter than a threshold are suppressed
+# (default 1000 ms; tunable via DWATCH_HANG_MS in the environment, 0 to
+# show everything). Answers "what was my process stuck on?" -- the
+# blocking that the slow profile cannot see, because a syscall that
+# never returns never reports its latency. NB: the report fires at
+# wakeup with the full duration; a thread still asleep has not yet
+# been reported.
+# The hang-top profile maintains a running catalog, updated every
+# 3 seconds, of long sleeps by process. Combine with `-O cmd' to
+# capture state as each event occurs.
+#
+############################################################ PRAGMAS
+
+case "$PROFILE" in
+hang-top)
+ DTRACE_PRAGMA="
+ option quiet
+ option aggsortrev
+ " # END-QUOTE
+ ;;
+esac
+
+############################################################ PROBE
+
+case "$PROFILE" in
+hang-top)
+ : ${PROBE:=profile:::tick-3s} ;;
+*)
+ : ${PROBE:=sched:::wakeup}
+esac
+
+############################################################ EVENT ACTION
+
+: ${DWATCH_HANG_MS:=1000}
+
+case "$DWATCH_HANG_MS" in
+""|*[!0-9]*) die "DWATCH_HANG_MS must be a number" ;; # NOTREACHED
+esac
+
+[ "$CUSTOM_TEST" ] || case "$PROFILE" in
+hang-top) ;;
+*) EVENT_TEST="this->hang_ns >= (int64_t)$DWATCH_HANG_MS * 1000000"
+esac
+
+############################################################ ACTIONS
+
+if [ "$PROFILE" = "hang-top" ]; then
+exec 9<<EOF
+this int64_t hang_ns;
+int64_t hang_ts[int];
+
+BEGIN { printf("Cataloging long sleeps ...") } /* probe ID $ID */
+
+sched:::sleep /* probe ID $(( $ID + 1 )) */
+{
+ hang_ts[curthread->td_tid] = timestamp;
+}
+
+sched:::wakeup /* probe ID $(( $ID + 2 )) */
+{
+ /* NB: -1 if we did not see the sleep (enabled mid-sleep) */
+ this->hang_ns =
+ hang_ts[((struct thread *)args[0])->td_tid] ? timestamp -
+ hang_ts[((struct thread *)args[0])->td_tid] : -1;
+ hang_ts[((struct thread *)args[0])->td_tid] = 0;
+}
+
+sched:::wakeup /this->hang_ns >=
+ (int64_t)$DWATCH_HANG_MS * 1000000/ /* probe ID $(( $ID + 3 )) */
+{
+ @hang_cnt[stringof(((struct proc *)args[1])->p_comm)] = count();
+ @hang_max[stringof(((struct proc *)args[1])->p_comm)] =
+ max(this->hang_ns / 1000000);
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 4 ))
+else
+exec 9<<EOF
+this int64_t hang_ns;
+int64_t hang_ts[int];
+
+sched:::sleep /* probe ID $ID */
+{${TRACE:+
+ printf("<$ID>");
+}
+ hang_ts[curthread->td_tid] = timestamp;
+}
+
+$PROBE /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+ printf("<$(( $ID + 1 ))>");
+}
+ /* NB: -1 if we did not see the sleep (enabled mid-sleep) */
+ this->hang_ns =
+ hang_ts[((struct thread *)args[0])->td_tid] ? timestamp -
+ hang_ts[((struct thread *)args[0])->td_tid] : -1;
+ hang_ts[((struct thread *)args[0])->td_tid] = 0;
+
+ $( pproc -P _hang "(struct proc *)args[1]" )
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 2 ))
+fi
+
+############################################################ EVENT TAG
+
+# For the running catalog, override the default `UID.GID CMD[PID]: ' tag
+# with ANSI cursor-homing and screen-clearing codes plus column headers.
+
+if [ "$PROFILE" = "hang-top" ]; then
+size=$( stty size 2> /dev/null )
+rows="${size%% *}"
+cols="${size#* }"
+
+exec 9<<EOF
+ printf("\033[H"); /* Position the cursor at top-left */
+ printf("\033[J"); /* Clear display from cursor to end */
+
+ /* Header line containing probe (left) and date (right) */
+ printf("%-*s%s%Y%s\n",
+ $(( ${cols:-80} - 20 )), "$PROBE",
+ console ? "\033[32m" : "",
+ walltimestamp,
+ console ? "\033[39m" : "");
+
+ /* Column headers */
+ printf("%s%8s %10s %s%s\n",
+ console ? "\033[1m" : "",
+ "COUNT",
+ "MAX(ms)",
+ "EXECNAME",
+ console ? "\033[22m" : "");
+EOF
+EVENT_TAG=$( cat <&9 )
+fi
+
+############################################################ EVENT DETAILS
+
+if [ "$PROFILE" = "hang-top" ]; then
+exec 9<<EOF
+ /* NB: Cumulative; not truncated between updates */
+ printa("%@8u %@10d %s\n", @hang_cnt, @hang_max);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+elif [ ! "$CUSTOM_DETAILS" ]; then
+exec 9<<EOF
+ /*
+ * Print long sleep details (tag shows the waker)
+ */
+ printf("pid %d slept %d.%03d ms -- %s",
+ this->pid_hang,
+ this->hang_ns / 1000000,
+ (this->hang_ns % 1000000) / 1000,
+ this->args_hang);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+fi
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/io b/cddl/usr.sbin/dwatch/libexec/io
--- a/cddl/usr.sbin/dwatch/libexec/io
+++ b/cddl/usr.sbin/dwatch/libexec/io
@@ -3,22 +3,42 @@
############################################################ IDENT(1)
#
# $Title: dwatch(8) module for dtrace_io(4) $
-# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $Copyright: 2014-2026 Devin Teske. All rights reserved. $
#
############################################################ DESCRIPTION
#
-# Display activity related to disk I/O
+# Display activity related to disk I/O. The io-slow profile instead
+# measures per-request latency -- the time between io:::start and
+# io:::done for the same struct bio -- printing any request that meets
+# a threshold (default 100 ms; tunable via DWATCH_IO_MS in the
+# environment, 0 to show everything). Answers "is the application slow
+# because the disk is slow?" and, watched per-device, hunts I/O
+# starvation; on ZFS, watched against zvols and the pool's leaf vdevs,
+# it brackets where in the stack the time is going.
#
############################################################ PROBE
case "$PROFILE" in
io) : ${PROBE:=io:::start, io:::done} ;;
+io-slow) : ${PROBE:=io:::done} ;;
*) : ${PROBE:=io:::${PROFILE#io-}}
esac
############################################################ EVENT ACTION
-[ "$CUSTOM_TEST" ] || EVENT_TEST='this->devinfo.dev_name != ""'
+if [ "$PROFILE" = "io-slow" ]; then
+ : ${DWATCH_IO_MS:=100}
+
+ case "$DWATCH_IO_MS" in
+ ""|*[!0-9]*) die "DWATCH_IO_MS must be a number" ;; # NOTREACHED
+ esac
+
+ [ "$CUSTOM_TEST" ] ||
+ EVENT_TEST="this->devinfo.dev_name != \"\" &&
+ this->io_ns >= (int64_t)$DWATCH_IO_MS * 1000000"
+else
+ [ "$CUSTOM_TEST" ] || EVENT_TEST='this->devinfo.dev_name != ""'
+fi
############################################################ ACTIONS
@@ -82,9 +102,53 @@
ACTIONS=$( cat <&9 )
ID=$(( $ID + 1 ))
+if [ "$PROFILE" = "io-slow" ]; then
+exec 9<<EOF
+$ACTIONS
+
+this int64_t io_ns;
+int64_t io_ts[uintptr_t];
+
+io:::start /(struct bio *)args[0] != NULL/ /* probe ID $ID */
+{${TRACE:+
+ printf("<$ID>");
+}
+ io_ts[(uintptr_t)args[0]] = timestamp;
+}
+
+$PROBE /(struct bio *)args[0] != NULL/ /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+ printf("<$(( $ID + 1 ))>");
+}
+ /* NB: -1 if we did not see the start (enabled mid-request) */
+ this->io_ns = io_ts[(uintptr_t)args[0]] ?
+ timestamp - io_ts[(uintptr_t)args[0]] : -1;
+ io_ts[(uintptr_t)args[0]] = 0;
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 2 ))
+fi
+
############################################################ EVENT DETAILS
-if [ ! "$CUSTOM_DETAILS" ]; then
+if [ "$PROFILE" = "io-slow" ] && [ ! "$CUSTOM_DETAILS" ]; then
+exec 9<<EOF
+ /*
+ * Print disk I/O latency details
+ */
+ printf("%s %s %s %s %d byte%s %d.%03d ms",
+ this->device_type,
+ this->device_entry,
+ this->bio_cmd,
+ this->bio_flags,
+ this->bio_length,
+ this->bio_length == 1 ? "" : "s",
+ this->io_ns / 1000000,
+ (this->io_ns % 1000000) / 1000);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+elif [ ! "$CUSTOM_DETAILS" ]; then
exec 9<<EOF
/*
* Print disk I/O details
diff --git a/cddl/usr.sbin/dwatch/libexec/jail b/cddl/usr.sbin/dwatch/libexec/jail
new file mode 100644
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/jail
@@ -0,0 +1,92 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for jail(2) management syscalls $
+# $Copyright: 2026 Devin Teske. All rights reserved. $
+#
+############################################################ DESCRIPTION
+#
+# Print jail management activity -- jail(2), jail_set(2), jail_get(2),
+# jail_attach(2), and jail_remove(2) -- naming the syscall, the jail id
+# operated on, and any errno returned. Answers "who is creating,
+# entering, or destroying jails?" Complements the dwatch `-j jail'
+# filter, which scopes any profile to processes inside one jail; this
+# profile watches the management plane itself, from any jail or none.
+# Use jail-attach, jail-get, jail-remove, or jail-set to watch a
+# single operation by name.
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+jail)
+ : ${PROBE:=$( echo \
+ syscall::jail:return, \
+ syscall::jail_set:return, \
+ syscall::jail_get:return, \
+ syscall::jail_attach:return, \
+ syscall::jail_remove:return )} ;;
+*)
+ : ${PROBE:=syscall::jail_${PROFILE#jail-}:return}
+esac
+
+#
+# Derive the matching entry probes from the return probes being watched
+#
+ENTRY_PROBE=$( echo "$PROBE" | awk 'gsub(/:return/, ":entry") || 1' )
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+self int jail_arg;
+this int jail_jid;
+
+$ENTRY_PROBE /* probe ID $ID */
+{${TRACE:+
+ printf("<$ID>");}
+ self->jail_arg = -1;
+}
+
+syscall::jail_attach:entry,
+syscall::jail_remove:entry /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+ printf("<$(( $ID + 1 ))>");}
+ /* arg0 is the jail id being attached-to or removed */
+ self->jail_arg = (int)arg0;
+}
+
+$PROBE /* probe ID $(( $ID + 2 )) */
+{${TRACE:+
+ printf("<$(( $ID + 2 ))>");
+}
+ /*
+ * jail(2), jail_set(2), and jail_get(2) return the jail id;
+ * jail_attach(2) and jail_remove(2) were given it at entry
+ */
+ this->jail_jid = self->jail_arg >= 0 ?
+ self->jail_arg : (int)arg0;
+ self->jail_arg = 0;
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 3 ))
+
+############################################################ EVENT DETAILS
+
+if [ ! "$CUSTOM_DETAILS" ]; then
+exec 9<<EOF
+ /*
+ * Print jail management details
+ */
+ printf("%s(2) jid %d%s%s",
+ probefunc,
+ this->jail_jid,
+ errno > 0 ? " -- " : "",
+ errno > 0 ? strerror[errno] : "");
+EOF
+EVENT_DETAILS=$( cat <&9 )
+fi
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/lock b/cddl/usr.sbin/dwatch/libexec/lock
new file mode 100644
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/lock
@@ -0,0 +1,136 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for dtrace_lockstat(4) contention $
+# $Copyright: 2026 Devin Teske. All rights reserved. $
+#
+############################################################ DESCRIPTION
+#
+# Print kernel lock contention events reported by dtrace_lockstat(4),
+# naming the lock, the thread held-off by it, and for how long. Answers
+# "the system is slow -- where is it fighting over locks?" The default
+# profile watches block events (thread went off-CPU waiting); lock-spin
+# watches spin events instead. Contention shorter than a threshold is
+# suppressed (default 1 ms; tunable via DWATCH_LOCK_MS in the environment,
+# 0 to show everything).
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+lock|lock-block)
+ : ${PROBE:=$( echo \
+ lockstat:::adaptive-block, \
+ lockstat:::lockmgr-block, \
+ lockstat:::rw-block, \
+ lockstat:::sx-block )} ;;
+lock-spin)
+ : ${PROBE:=$( echo \
+ lockstat:::adaptive-spin, \
+ lockstat:::rw-spin, \
+ lockstat:::spin-spin, \
+ lockstat:::sx-spin, \
+ lockstat:::thread-spin )} ;;
+lock-adaptive)
+ : ${PROBE:=lockstat:::adaptive-block, lockstat:::adaptive-spin} ;;
+lock-lockmgr)
+ : ${PROBE:=lockstat:::lockmgr-block} ;;
+lock-rw)
+ : ${PROBE:=lockstat:::rw-block, lockstat:::rw-spin} ;;
+lock-sx)
+ : ${PROBE:=lockstat:::sx-block, lockstat:::sx-spin} ;;
+lock-thread)
+ : ${PROBE:=lockstat:::thread-spin} ;;
+*)
+ : ${PROBE:=lockstat:::${PROFILE#lock-}}
+esac
+
+############################################################ EVENT ACTION
+
+: ${DWATCH_LOCK_MS:=1}
+
+case "$DWATCH_LOCK_MS" in
+""|*[!0-9]*) die "DWATCH_LOCK_MS must be a number" ;; # NOTREACHED
+esac
+
+[ "$CUSTOM_TEST" ] ||
+ EVENT_TEST="(int64_t)arg1 >= (int64_t)$DWATCH_LOCK_MS * 1000000"
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this string lock_class;
+this string lock_how;
+this string lock_name;
+this string lock_verb;
+
+/*
+ * Lock classes from sys/lock.h as-witnessed by dtrace_lockstat(4)
+ */
+inline string lockstat_class[string name] =
+ name == "adaptive-block" ? "mtx" :
+ name == "adaptive-spin" ? "mtx" :
+ name == "lockmgr-block" ? "lockmgr" :
+ name == "rw-block" ? "rw" :
+ name == "rw-spin" ? "rw" :
+ name == "spin-spin" ? "spin mtx" :
+ name == "sx-block" ? "sx" :
+ name == "sx-spin" ? "sx" :
+ name == "thread-spin" ? "thread" :
+ name;
+
+inline string lockstat_verb[string name] =
+ name == "adaptive-block" ? "blocked" :
+ name == "lockmgr-block" ? "blocked" :
+ name == "rw-block" ? "blocked" :
+ name == "sx-block" ? "blocked" :
+ "spun";
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+ printf("<$ID>");
+}
+ /*
+ * struct lock_object * (first member of every kernel lock)
+ */
+ this->lock_name =
+ stringof(((struct lock_object *)arg0)->lo_name);
+ this->lock_class = lockstat_class[probename];
+ this->lock_verb = lockstat_verb[probename];
+ this->lock_how = "";
+}
+
+lockstat:::lockmgr-block,
+lockstat:::rw-block,
+lockstat:::sx-block /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+ printf("<$(( $ID + 1 ))>");
+}
+ /* arg2 is 0 when acquiring as writer, 1 as reader */
+ this->lock_how = arg2 == 0 ? " as writer" : " as reader";
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 2 ))
+
+############################################################ EVENT DETAILS
+
+if [ ! "$CUSTOM_DETAILS" ]; then
+exec 9<<EOF
+ /*
+ * Print lock contention details
+ */
+ printf("%s %d.%03d ms on %s \"%s\"%s",
+ this->lock_verb,
+ (int64_t)arg1 / 1000000,
+ ((int64_t)arg1 % 1000000) / 1000,
+ this->lock_class,
+ this->lock_name,
+ this->lock_how);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+fi
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/mib b/cddl/usr.sbin/dwatch/libexec/mib
new file mode 100644
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/mib
@@ -0,0 +1,78 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for network stack counter probes $
+# $Copyright: 2026 Devin Teske. All rights reserved. $
+#
+############################################################ DESCRIPTION
+#
+# Print network stack statistics counters as they increment, via the
+# mib SDT probes (one probe per counter of the IP, ICMP, UDP, and TCP
+# MIBs). The tcp-retransmit profile watches the counters that signal
+# congestion or loss on the send path -- data packet retransmissions,
+# retransmit timer expirations, and connections dropped by retransmit
+# exhaustion -- answering "is this network slow because TCP is
+# resending?" as events with process context, not just netstat(1)
+# deltas. Use mib-NAME to watch a single counter by name (e.g.,
+# mib-tcps_sndrexmitpack). NB: the probes exist only in kernels built
+# with options KDTRACE_MIB_SDT (enabled by default in -CURRENT via
+# std.debug); elsewhere dtrace(1) will refuse the script. Counters
+# bumped outside syscall context (e.g., timers) are tagged with the
+# interrupted thread, commonly [clock] or [intr].
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+mib)
+ : ${PROBE:=mib:::} ;;
+tcp-retransmit)
+ : ${PROBE:=$( echo \
+ mib:tcp:count:tcps_sndrexmitpack, \
+ mib:tcp:count:tcps_sndrexmitbad, \
+ mib:tcp:count:tcps_rexmttimeo, \
+ mib:tcp:count:tcps_timeoutdrop )} ;;
+*)
+ : ${PROBE:=mib:::${PROFILE#mib-}}
+esac
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this string mib_desc;
+
+/*
+ * Counters that signal send-path congestion or loss, described
+ */
+inline string mib_tcp_desc[string name] =
+ name == "tcps_sndrexmitpack" ? "data packet retransmitted" :
+ name == "tcps_sndrexmitbad" ? "unnecessary retransmission" :
+ name == "tcps_rexmttimeo" ? "retransmit timeout" :
+ name == "tcps_timeoutdrop" ?
+ "connection dropped by retransmit timeout" :
+ name;
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+ printf("<$ID>");}
+ this->mib_desc = mib_tcp_desc[probename];
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 1 ))
+
+############################################################ EVENT DETAILS
+
+if [ ! "$CUSTOM_DETAILS" ]; then
+exec 9<<EOF
+ /*
+ * Print counter increment details
+ */
+ printf("%s (+%d)", this->mib_desc, (int)arg0);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+fi
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/namei b/cddl/usr.sbin/dwatch/libexec/namei
new file mode 100644
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/namei
@@ -0,0 +1,84 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for namei(9) pathname resolution $
+# $Copyright: 2026 Devin Teske. All rights reserved. $
+#
+############################################################ DESCRIPTION
+#
+# Print pathnames given to namei(9) for resolution, exactly as the process
+# requested them, together with the lookup result. Unlike the vop_lookup
+# profile, which reconstructs paths from the name cache one component at a
+# time, this rides the vfs:namei:lookup probes and so sees the whole path
+# in one piece. Use namei-failure to show only failed lookups or
+# namei-enoent to hunt file-not-found storms (the single most common
+# use of truss(1), without stopping the victim).
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+namei|namei-enoent|namei-failure)
+ : ${PROBE:=vfs:namei:lookup:return} ;;
+namei-entry)
+ : ${PROBE:=vfs:namei:lookup:entry} ;;
+*)
+ : ${PROBE:=vfs:namei:lookup:${PROFILE#namei-}}
+esac
+
+############################################################ EVENT ACTION
+
+[ "$CUSTOM_TEST" ] || case "$PROFILE" in
+namei-enoent) EVENT_TEST="this->namei_error == ENOENT" ;;
+namei-failure) EVENT_TEST="this->namei_error != 0" ;;
+esac
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+self string namei_path;
+this int namei_error;
+this string namei_pathstr;
+
+vfs:namei:lookup:entry /* probe ID $ID */
+{${TRACE:+
+ printf("<$ID>");
+}
+ /*
+ * char * (kernel copy of the path being resolved)
+ */
+ this->namei_pathstr = self->namei_path = stringof(arg1);
+ this->namei_error = 0;
+}
+
+vfs:namei:lookup:return /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+ printf("<$(( $ID + 1 ))>");
+}
+ /* NB: path was recorded at entry by the clause above */
+ this->namei_pathstr = self->namei_path;
+ this->namei_error = (int)arg0;
+ self->namei_path = 0;
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 2 ))
+
+############################################################ EVENT DETAILS
+
+if [ ! "$CUSTOM_DETAILS" ]; then
+exec 9<<EOF
+ /*
+ * Print pathname resolution details
+ */
+ printf("%s%s", this->namei_pathstr,
+ this->namei_error == 0 ? "" : strjoin(": ",
+ strjoin(strerror[this->namei_error],
+ strjoin(" (", strjoin(lltostr(this->namei_error), ")")))));
+EOF
+EVENT_DETAILS=$( cat <&9 )
+fi
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/priv b/cddl/usr.sbin/dwatch/libexec/priv
new file mode 100644
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/priv
@@ -0,0 +1,59 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for priv_check(9) privilege requests $
+# $Copyright: 2026 Devin Teske. All rights reserved. $
+#
+############################################################ DESCRIPTION
+#
+# Print privilege requests denied by priv_check(9), decoded to their
+# symbolic priv(9) names by the priv_string table in priv.d of
+# libdtrace(1). Answers "why is this process getting EPERM?" -- kernel
+# privilege denials (jails, unprivileged users, MAC policies) name the
+# exact privilege refused, something no amount of truss(1) will reveal.
+# Use priv-ok to instead watch privileges being granted.
+#
+# NB: Requires priv.d, a drop-in file for older releases like this module.
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+priv)
+ : ${PROBE:=priv:kernel:priv_check:priv-err} ;;
+*)
+ : ${PROBE:=priv:kernel:priv_check:${PROFILE#priv-}}
+esac
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+this int priv;
+
+$PROBE /* probe ID $ID */
+{${TRACE:+
+ printf("<$ID>");}
+ this->priv = (int)arg0;
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 1 ))
+
+############################################################ EVENT DETAILS
+
+if [ ! "$CUSTOM_DETAILS" ]; then
+exec 9<<EOF
+ /*
+ * Print privilege request details
+ */
+ printf("%s %s (%d)",
+ probename == "priv-err" ? "denied" : "granted",
+ priv_string[this->priv],
+ this->priv);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+fi
+
+################################################################################
+# END
+################################################################################
diff --git a/cddl/usr.sbin/dwatch/libexec/proc b/cddl/usr.sbin/dwatch/libexec/proc
--- a/cddl/usr.sbin/dwatch/libexec/proc
+++ b/cddl/usr.sbin/dwatch/libexec/proc
@@ -3,7 +3,7 @@
############################################################ IDENT(1)
#
# $Title: dwatch(8) module for dtrace_proc(4) activity $
-# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $Copyright: 2014-2026 Devin Teske. All rights reserved. $
#
############################################################ DESCRIPTION
#
@@ -29,6 +29,9 @@
proc:::signal-discard, \
proc:::signal-send )}
;;
+proc-signal-fatal)
+ : ${PROBE:=proc:::signal-send}
+ ;;
proc-status)
: ${PROBE:=$( echo \
proc:::create, \
@@ -41,6 +44,24 @@
: ${PROBE:=proc:::${PROFILE#proc-}}
esac
+############################################################ EVENT ACTION
+
+# Answer "what killed my process?" by showing only those signals whose
+# default disposition is to terminate the process (with or without core),
+# most-notably including kernel-generated SIGSEGV/SIGBUS/SIGILL/SIGFPE
+# that no kill(2) watcher will ever see.
+
+[ "$CUSTOM_TEST" ] || case "$PROFILE" in
+proc-signal-fatal)
+ EVENT_TEST="this->sig == SIGABRT || this->sig == SIGBUS ||
+ this->sig == SIGEMT || this->sig == SIGFPE ||
+ this->sig == SIGILL || this->sig == SIGKILL ||
+ this->sig == SIGQUIT || this->sig == SIGSEGV ||
+ this->sig == SIGSYS || this->sig == SIGTRAP ||
+ this->sig == SIGXCPU || this->sig == SIGXFSZ"
+ ;;
+esac
+
############################################################ ACTIONS
exec 9<<EOF
diff --git a/cddl/usr.sbin/dwatch/libexec/sched b/cddl/usr.sbin/dwatch/libexec/sched
--- a/cddl/usr.sbin/dwatch/libexec/sched
+++ b/cddl/usr.sbin/dwatch/libexec/sched
@@ -3,11 +3,17 @@
############################################################ IDENT(1)
#
# $Title: dwatch(8) module for dtrace_sched(4) $
-# $Copyright: 2014-2018 Devin Teske. All rights reserved. $
+# $Copyright: 2014-2026 Devin Teske. All rights reserved. $
#
############################################################ DESCRIPTION
#
-# Display CPU scheduling activity
+# Display CPU scheduling activity. The sched-latency profile instead
+# measures run-queue latency -- the time between a thread being made
+# runnable (enqueued) and actually running (on-cpu) -- printing any
+# wait that meets a threshold (default 10 ms; tunable via
+# DWATCH_SCHED_MS in the environment, 0 to show everything). Answers
+# "the system has idle CPU but feels sluggish -- who is waiting to
+# run, and for how long?"
#
############################################################ PROBE
@@ -18,6 +24,8 @@
: ${PROBE:=sched:::off-cpu, sched:::on-cpu, sched:::remain-cpu} ;;
sched-exec)
: ${PROBE:=sched:::sleep, sched:::wakeup} ;;
+sched-latency)
+ : ${PROBE:=sched:::on-cpu} ;;
sched-pri)
: ${PROBE:=sched:::change-pri, sched:::lend-pri} ;;
sched-queue)
@@ -26,8 +34,48 @@
: ${PROBE:=sched:::${PROFILE#sched-}}
esac
+############################################################ EVENT ACTION
+
+case "$PROFILE" in
+sched-latency)
+ : ${DWATCH_SCHED_MS:=10}
+
+ case "$DWATCH_SCHED_MS" in
+ ""|*[!0-9]*) die "DWATCH_SCHED_MS must be a number" ;; # NOTREACHED
+ esac
+
+ [ "$CUSTOM_TEST" ] ||
+ EVENT_TEST="this->sched_ns >= (int64_t)$DWATCH_SCHED_MS * 1000000"
+ ;;
+esac
+
############################################################ ACTIONS
+if [ "$PROFILE" = "sched-latency" ]; then
+exec 9<<EOF
+this int64_t sched_ns;
+int64_t sched_ts[int];
+
+sched:::enqueue /* probe ID $ID */
+{${TRACE:+
+ printf("<$ID>");
+}
+ sched_ts[((struct thread *)args[0])->td_tid] = timestamp;
+}
+
+$PROBE /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+ printf("<$(( $ID + 1 ))>");
+}
+ /* NB: -1 if we did not see the enqueue (enabled mid-wait) */
+ this->sched_ns = sched_ts[curthread->td_tid] ?
+ timestamp - sched_ts[curthread->td_tid] : -1;
+ sched_ts[curthread->td_tid] = 0;
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 2 ))
+else
exec 9<<EOF
this pid_t pid;
this string args;
@@ -90,10 +138,21 @@
EOF
ACTIONS=$( cat <&9 )
ID=$(( $ID + 6 ))
+fi
############################################################ EVENT DETAILS
-if [ ! "$CUSTOM_DETAILS" ]; then
+if [ "$PROFILE" = "sched-latency" ] && [ ! "$CUSTOM_DETAILS" ]; then
+exec 9<<EOF
+ /*
+ * Print run-queue latency details
+ */
+ printf("waited %d.%03d ms to run",
+ this->sched_ns / 1000000,
+ (this->sched_ns % 1000000) / 1000);
+EOF
+EVENT_DETAILS=$( cat <&9 )
+elif [ ! "$CUSTOM_DETAILS" ]; then
exec 9<<EOF
/*
* Print scheduling details
diff --git a/cddl/usr.sbin/dwatch/libexec/slow b/cddl/usr.sbin/dwatch/libexec/slow
new file mode 100644
--- /dev/null
+++ b/cddl/usr.sbin/dwatch/libexec/slow
@@ -0,0 +1,122 @@
+# -*- tab-width: 4 -*- ;; Emacs
+# vi: set filetype=sh tabstop=8 shiftwidth=8 noexpandtab :: Vi/ViM
+############################################################ IDENT(1)
+#
+# $Title: dwatch(8) module for slow syscall detection $
+# $Copyright: 2026 Devin Teske. All rights reserved. $
+#
+############################################################ DESCRIPTION
+#
+# Print syscalls whose entry-to-return latency meets or exceeds a threshold
+# (default 100 ms; tunable via DWATCH_SLOW_MS in the environment). Answers
+# "why is my application stalling?" by naming the slow operation, the time
+# it took, and any errno it returned. The default profile watches a curated
+# set of filesystem-related syscalls that are expected to be fast. Use
+# slow-syscall to watch every syscall (NB: intentionally-blocking syscalls
+# such as select(2), poll(2), kevent(2), and wait4(2) will dominate), or
+# slow-NAME (e.g., slow-connect) to watch a single syscall by name.
+#
+############################################################ PROBE
+
+case "$PROFILE" in
+slow)
+ : ${PROBE:=$( echo \
+ syscall::open:return, \
+ syscall::openat:return, \
+ syscall::close:return, \
+ syscall::read:return, \
+ syscall::readv:return, \
+ syscall::pread:return, \
+ syscall::preadv:return, \
+ syscall::write:return, \
+ syscall::writev:return, \
+ syscall::pwrite:return, \
+ syscall::pwritev:return, \
+ syscall::fsync:return, \
+ syscall::fdatasync:return, \
+ syscall::rename:return, \
+ syscall::renameat:return, \
+ syscall::unlink:return, \
+ syscall::unlinkat:return )} ;;
+slow-open)
+ : ${PROBE:=syscall::open:return, syscall::openat:return} ;;
+slow-read)
+ : ${PROBE:=$( echo \
+ syscall::read:return, \
+ syscall::readv:return, \
+ syscall::pread:return, \
+ syscall::preadv:return )} ;;
+slow-write)
+ : ${PROBE:=$( echo \
+ syscall::write:return, \
+ syscall::writev:return, \
+ syscall::pwrite:return, \
+ syscall::pwritev:return )} ;;
+slow-fsync)
+ : ${PROBE:=syscall::fsync:return, syscall::fdatasync:return} ;;
+slow-syscall)
+ : ${PROBE:=syscall:::return} ;;
+*)
+ : ${PROBE:=syscall::${PROFILE#slow-}:return}
+esac
+
+#
+# Derive the matching entry probes from the return probes being watched
+#
+ENTRY_PROBE=$( echo "$PROBE" | awk 'gsub(/:return/, ":entry") || 1' )
+
+############################################################ EVENT ACTION
+
+: ${DWATCH_SLOW_MS:=100}
+
+case "$DWATCH_SLOW_MS" in
+""|*[!0-9]*) die "DWATCH_SLOW_MS must be a number" ;; # NOTREACHED
+esac
+
+[ "$CUSTOM_TEST" ] ||
+ EVENT_TEST="this->slow_ns >= (int64_t)$DWATCH_SLOW_MS * 1000000"
+
+############################################################ ACTIONS
+
+exec 9<<EOF
+self int64_t slow_ts;
+this int64_t slow_ns;
+
+$ENTRY_PROBE /* probe ID $ID */
+{${TRACE:+
+ printf("<$ID>");}
+ self->slow_ts = timestamp;
+}
+
+$PROBE /* probe ID $(( $ID + 1 )) */
+{${TRACE:+
+ printf("<$(( $ID + 1 ))>");
+}
+ /* NB: -1 if we did not see the entry (enabled mid-syscall) */
+ this->slow_ns = self->slow_ts ? timestamp - self->slow_ts : -1;
+ self->slow_ts = 0;
+}
+EOF
+ACTIONS=$( cat <&9 )
+ID=$(( $ID + 2 ))
+
+############################################################ EVENT DETAILS
+
+if [ ! "$CUSTOM_DETAILS" ]; then
+exec 9<<EOF
+ /*
+ * Print syscall latency details
+ */
+ printf("%s(2) %d.%03d ms%s%s",
+ probefunc,
+ this->slow_ns / 1000000,
+ (this->slow_ns % 1000000) / 1000,
+ errno > 0 ? " -- " : "",
+ errno > 0 ? strerror[errno] : "");
+EOF
+EVENT_DETAILS=$( cat <&9 )
+fi
+
+################################################################################
+# END
+################################################################################

File Metadata

Mime Type
text/plain
Expires
Thu, Jul 16, 1:32 AM (10 h, 53 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
35098462
Default Alt Text
D58093.id181492.diff (62 KB)

Event Timeline