diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc --- a/ObsoleteFiles.inc +++ b/ObsoleteFiles.inc @@ -51,6 +51,10 @@ # xargs -n1 | sort | uniq -d; # done +# 20251003: kgdb python scripts moved +OLD_FILES+=usr/libexec/kgdb/acttrace.py +OLD_DIRS+=usr/libexec/kgdb + # 20251001: test helper sendto-IP_MULTICAST_IF renamed OLD_FILES+=usr/tests/sys/netinet/sendto-IP_MULTICAST_IF diff --git a/etc/mtree/BSD.usr.dist b/etc/mtree/BSD.usr.dist --- a/etc/mtree/BSD.usr.dist +++ b/etc/mtree/BSD.usr.dist @@ -181,8 +181,6 @@ .. hyperv .. - kgdb - .. lpr ru .. diff --git a/libexec/Makefile b/libexec/Makefile --- a/libexec/Makefile +++ b/libexec/Makefile @@ -10,7 +10,6 @@ flua \ getty \ ${_hyperv} \ - kgdb \ ${_mail.local} \ ${_makewhatis.local} \ ${_mknetid} \ diff --git a/libexec/kgdb/Makefile b/libexec/kgdb/Makefile deleted file mode 100644 --- a/libexec/kgdb/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -FILESDIR?= /usr/libexec/kgdb - -FILES= acttrace.py - -.include diff --git a/sys/conf/kern.post.mk b/sys/conf/kern.post.mk --- a/sys/conf/kern.post.mk +++ b/sys/conf/kern.post.mk @@ -398,6 +398,14 @@ .endif .endfor +# Install GDB plugins that are useful for kernel debugging. See the +# README in sys/tools/gdb for more information. +GDB_FILES= acttrace.py \ + freebsd.py \ + pcpu.py \ + selftest.py \ + vnet.py + ${_ILINKS}: @case ${.TARGET} in \ machine) \ @@ -447,6 +455,13 @@ .if defined(DEBUG) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no" mkdir -p ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ${INSTALL} -p -m ${KMODMODE} -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/ + ${INSTALL} -m ${KMODMODE} -o ${KMODOWN} -g ${KMODGRP} \ + $S/tools/kernel-gdb.py ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/${KERNEL_KO}-gdb.py + mkdir -p ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/gdb +.for file in ${GDB_FILES} + ${INSTALL} -m ${KMODMODE} -o ${KMODOWN} -g ${KMODGRP} \ + $S/tools/gdb/${file} ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/gdb/${file} +.endfor .endif .if defined(KERNEL_EXTRA_INSTALL) ${INSTALL} -p -m ${KMODMODE} -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_EXTRA_INSTALL} ${DESTDIR}${KODIR}/ diff --git a/sys/tools/gdb/README.txt b/sys/tools/gdb/README.txt new file mode 100644 --- /dev/null +++ b/sys/tools/gdb/README.txt @@ -0,0 +1,21 @@ +This directory contains Python scripts that can be loaded by GDB to help debug +FreeBSD kernel crashes. + +Add new commands and functions in their own files. Functions with general +utility should be added to freebsd.py. sys/tools/kernel-gdb.py is installed +into the kernel debug directory (typically /usr/lib/debug/boot/kernel). It will +be automatically loaded by kgdb when opening a vmcore, so if you add new GDB +commands or functions, that script should be updated to import them, and you +should document them here. + +To provide some rudimentary testing, selftest.py tries to exercise all of the +commands and functions defined here. To use it, run selftest.sh to panic the +system. Then, create a kernel dump or attach to the panicked kernel, and invoke +the script with "python import selftest" in (k)gdb. + +Commands: +acttrace Display a backtrace for all on-CPU threads + +Functions: +$PCPU([, ]) Display the value of a PCPU/DPCPU field +$V([, ]) Display the value of a VNET variable diff --git a/libexec/kgdb/acttrace.py b/sys/tools/gdb/acttrace.py rename from libexec/kgdb/acttrace.py rename to sys/tools/gdb/acttrace.py --- a/libexec/kgdb/acttrace.py +++ b/sys/tools/gdb/acttrace.py @@ -1,38 +1,23 @@ -#- +# # Copyright (c) 2022 The FreeBSD Foundation # # This software was developed by Mark Johnston under sponsorship from the # FreeBSD Foundation. # +# SPDX-License-Identifier: BSD-2-Clause +# import gdb - - -def symval(name): - return gdb.lookup_global_symbol(name).value() - - -def tid_to_gdb_thread(tid): - for thread in gdb.inferiors()[0].threads(): - if thread.ptid[2] == tid: - return thread - else: - return None - - -def all_pcpus(): - mp_maxid = symval("mp_maxid") - cpuid_to_pcpu = symval("cpuid_to_pcpu") - - cpu = 0 - while cpu <= mp_maxid: - pcpu = cpuid_to_pcpu[cpu] - if pcpu: - yield pcpu - cpu = cpu + 1 - +from freebsd import * +from pcpu import * class acttrace(gdb.Command): + """ + Register an acttrace command with gdb. + + When run, acttrace prints the stack trace of all threads that were on-CPU + at the time of the panic. + """ def __init__(self): super(acttrace, self).__init__("acttrace", gdb.COMMAND_USER) @@ -40,13 +25,13 @@ # Save the current thread so that we can switch back after. curthread = gdb.selected_thread() - for pcpu in all_pcpus(): + for pcpu in pcpu_foreach(): td = pcpu['pc_curthread'] tid = td['td_tid'] gdb_thread = tid_to_gdb_thread(tid) if gdb_thread is None: - print("failed to find GDB thread with TID {}".format(tid)) + raise gdb.error(f"failed to find GDB thread with TID {tid}") else: gdb_thread.switch() diff --git a/sys/tools/gdb/freebsd.py b/sys/tools/gdb/freebsd.py new file mode 100644 --- /dev/null +++ b/sys/tools/gdb/freebsd.py @@ -0,0 +1,75 @@ +# +# Copyright (c) 2025 Mark Johnston +# +# SPDX-License-Identifier: BSD-2-Clause +# + +import gdb + +def symval(name): + sym = gdb.lookup_global_symbol(name) + if sym is None: + sym = gdb.lookup_static_symbol(name) + if sym is None: + raise gdb.GdbError(f"Symbol '{name}' not found") + return sym.value() + + +def _queue_foreach(head, field, headf, nextf): + elm = head[headf] + while elm != 0: + yield elm + elm = elm[field][nextf] + + +def list_foreach(head, field): + """sys/queue.h-style iterator.""" + return _queue_foreach(head, field, "lh_first", "le_next") + + +def tailq_foreach(head, field): + """sys/queue.h-style iterator.""" + return _queue_foreach(head, field, "tqh_first", "tqe_next") + + +def linker_file_foreach(): + """Iterate over loaded linker files.""" + return tailq_foreach(symval("linker_files"), "link") + + +def pcpu_foreach(): + mp_maxid = symval("mp_maxid") + cpuid_to_pcpu = symval("cpuid_to_pcpu") + + cpu = 0 + while cpu <= mp_maxid: + pcpu = cpuid_to_pcpu[cpu] + if pcpu: + yield pcpu + cpu = cpu + 1 + + +def tid_to_gdb_thread(tid): + """Convert a FreeBSD kernel thread ID to a gdb inferior thread.""" + for thread in gdb.inferiors()[0].threads(): + if thread.ptid[2] == tid: + return thread + else: + return None + + +def tdfind(tid, pid=-1): + """Convert a FreeBSD kernel thread ID to a struct thread pointer.""" + td = tdfind.cached_threads.get(int(tid)) + if td: + return td + + for p in list_foreach(symval("allproc"), "p_list"): + if pid != -1 and pid != p['p_pid']: + continue + for td in tailq_foreach(p['p_threads'], "td_plist"): + ntid = td['td_tid'] + tdfind.cached_threads[int(ntid)] = td + if ntid == tid: + return td +tdfind.cached_threads = dict() diff --git a/sys/tools/gdb/pcpu.py b/sys/tools/gdb/pcpu.py new file mode 100644 --- /dev/null +++ b/sys/tools/gdb/pcpu.py @@ -0,0 +1,77 @@ +# +# Copyright (c) 2025 Mark Johnston +# +# SPDX-License-Identifier: BSD-2-Clause +# + +import gdb +from freebsd import * + +class pcpu(gdb.Function): + """ + Register a function to lookup PCPU and DPCPU variables by name. + + To look up the value of the PCPU field foo on CPU n, use + $PCPU("foo", n). This works for DPCPU fields too. If the CPU ID is + omitted, and the currently selected thread is on-CPU, that CPU is + used, otherwise an error is raised. + """ + def __init__(self): + super(pcpu, self).__init__("PCPU") + + def invoke(self, field, cpuid=-1): + if cpuid == -1: + cpuid = tdfind(gdb.selected_thread().ptid[2])['td_oncpu'] + if cpuid == -1: + raise gdb.error("Currently selected thread is off-CPU") + if cpuid < 0 or cpuid > symval("mp_maxid"): + raise gdb.error(f"Currently selected on invalid CPU {cpuid}") + pcpu = symval("cpuid_to_pcpu")[cpuid] + + # Are we dealing with a PCPU or DPCPU field? + field = field.string() + for f in gdb.lookup_type("struct pcpu").fields(): + if f.name == "pc_" + field: + return pcpu["pc_" + field] + + def uintptr_t(val): + return val.cast(gdb.lookup_type("uintptr_t")) + + # We're dealing with a DPCPU field. This is handled similarly + # to VNET symbols, see vnet.py for comments. + pcpu_base = pcpu['pc_dynamic'] + pcpu_entry = symval("pcpu_entry_" + field) + pcpu_entry_addr = uintptr_t(pcpu_entry.address) + + for lf in linker_file_foreach(): + block = gdb.block_for_pc(lf['ops']['cls']['methods'][0]['func']) + elf_file_t = gdb.lookup_type("elf_file_t", block).target() + ef = lf.cast(elf_file_t) + + file_type = lf['ops']['cls']['name'].string() + if file_type == "elf64": + start = uintptr_t(ef['pcpu_start']) + if start == 0: + continue + end = uintptr_t(ef['pcpu_stop']) + base = uintptr_t(ef['pcpu_base']) + elif file_type == "elf64_obj": + for i in range(ef['nprogtab']): + pe = ef['progtab'][i] + if pe['name'].string() == "set_pcpu": + start = uintptr_t(pe['origaddr']) + end = start + uintptr_t(pe['size']) + base = uintptr_t(pe['addr']) + break + else: + continue + else: + path = lf['pathname'].string() + raise gdb.error(f"{path} has unexpected linker file type {file_type}") + + if pcpu_entry_addr >= start and pcpu_entry_addr < end: + obj = gdb.Value(pcpu_base + pcpu_entry_addr - start + base) + return obj.cast(pcpu_entry.type.pointer()).dereference() + +# Register with gdb. +pcpu() diff --git a/sys/tools/gdb/selftest.py b/sys/tools/gdb/selftest.py new file mode 100644 --- /dev/null +++ b/sys/tools/gdb/selftest.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2025 Mark Johnston +# +# SPDX-License-Identifier: BSD-2-Clause +# + +import gdb + +cmds = ["acttrace", + "p $V(\"tcbinfo\")", + "p $V(\"tcbinfo\", vnet0)", + "p $V(\"pf_status\")", + "p $V(\"pf_status\", \"gdbselftest\")", + "p $PCPU(\"curthread\")", + "p $PCPU(\"curthread\", 0)", + "p/x $PCPU(\"hardclocktime\", 1)", + "p $PCPU(\"pqbatch\")[0][0]", + "p $PCPU(\"ss\", 1)", + ] + +for cmd in cmds: + try: + print(f"Running command: '{cmd}'") + gdb.execute(cmd) + except gdb.error as e: + print(f"Command '{cmd}' failed: {e}") + break + +# We didn't hit any unexpected errors. This isn't as good as actually +# verifying the output, but it's better than nothing. +print("Everything seems OK") diff --git a/sys/tools/gdb/selftest.sh b/sys/tools/gdb/selftest.sh new file mode 100644 --- /dev/null +++ b/sys/tools/gdb/selftest.sh @@ -0,0 +1,23 @@ +# +# Copyright (c) 2025 Mark Johnston +# +# SPDX-License-Identifier: BSD-2-Clause +# + +set -e + +n=$(sysctl -n hw.ncpu) +if [ $n -lt 2 ]; then + echo "This test requires at least 2 CPUs" + exit 1 +fi + +# Set up some things expected by selftest.py. +kldload -n pf siftr +pfctl -e || true +jail -c name=gdbselftest vnet persist + +echo "I'm about to panic your system, ctrl-C now if that's not what you want." +sleep 10 +sysctl debug.debugger_on_panic=0 +sysctl debug.kdb.panic=1 diff --git a/sys/tools/gdb/vnet.py b/sys/tools/gdb/vnet.py new file mode 100644 --- /dev/null +++ b/sys/tools/gdb/vnet.py @@ -0,0 +1,100 @@ +# +# Copyright (c) 2025 Mark Johnston +# +# SPDX-License-Identifier: BSD-2-Clause +# + +import gdb +import traceback +from freebsd import * + +class vnet(gdb.Function): + """ + Register a function to look up VNET variables by name. + + To look at the value of a VNET variable V_foo, print $V("foo"). The + currently selected thread's VNET is used by default, but can be optionally + specified as a second parameter, e.g., $V("foo", ), where is a + pointer to a struct vnet (e.g., vnet0 or allprison.tqh_first->pr_vnet) or a + string naming a jail. + """ + def __init__(self): + super(vnet, self).__init__("V") + + def invoke(self, sym, vnet=None): + sym = sym.string() + if sym.startswith("V_"): + sym = sym[len("V_"):] + if gdb.lookup_symbol("sysctl___kern_features_vimage")[0] is None: + return symval(sym) + + # Look up the VNET's base address. + if vnet is None: + vnet = tdfind(gdb.selected_thread().ptid[2])['td_vnet'] + if not vnet: + # If curthread->td_vnet == NULL, vnet0 is the current vnet. + vnet = symval("vnet0") + elif vnet.type.is_string_like: + vnet = vnet.string() + for prison in tailq_foreach(symval("allprison"), "pr_list"): + if prison['pr_name'].string() == vnet: + vnet = prison['pr_vnet'] + break + else: + raise gdb.error(f"No prison named {vnet}") + + def uintptr_t(val): + return val.cast(gdb.lookup_type("uintptr_t")) + + # Now the tricky part: compute the address of the symbol relative + # to the selected VNET. In the compiled kernel this is done at + # load time by applying a magic transformation to relocations + # against symbols in the vnet linker set. Here we have to apply + # the transformation manually. + vnet_data_base = vnet['vnet_data_base'] + vnet_entry = symval("vnet_entry_" + sym) + vnet_entry_addr = uintptr_t(vnet_entry.address) + + # First, which kernel module does the symbol belong to? + for lf in linker_file_foreach(): + # Find the bounds of this linker file's VNET linker set. The + # struct containing the bounds depends on the type of the linker + # file, and unfortunately both are called elf_file_t. So we use a + # PC value from the compilation unit (either link_elf.c or + # link_elf_obj.c) to disambiguate. + block = gdb.block_for_pc(lf['ops']['cls']['methods'][0]['func']) + elf_file_t = gdb.lookup_type("elf_file_t", block).target() + ef = lf.cast(elf_file_t) + + file_type = lf['ops']['cls']['name'].string() + if file_type == "elf64": + start = uintptr_t(ef['vnet_start']) + if start == 0: + # This linker file doesn't have a VNET linker set. + continue + end = uintptr_t(ef['vnet_stop']) + base = uintptr_t(ef['vnet_base']) + elif file_type == "elf64_obj": + for i in range(ef['nprogtab']): + pe = ef['progtab'][i] + if pe['name'].string() == "set_vnet": + start = uintptr_t(pe['origaddr']) + end = start + uintptr_t(pe['size']) + base = uintptr_t(pe['addr']) + break + else: + # This linker file doesn't have a VNET linker set. + continue + else: + path = lf['pathname'].string() + raise gdb.error(f"{path} has unexpected linker file type {file_type}") + + if vnet_entry_addr >= start and vnet_entry_addr < end: + # The symbol belongs to this linker file, so compute the final + # address. + obj = gdb.Value(vnet_data_base + vnet_entry_addr - start + base) + return obj.cast(vnet_entry.type.pointer()).dereference() + + +# Register with gdb. +vnet() diff --git a/sys/tools/kernel-gdb.py b/sys/tools/kernel-gdb.py new file mode 100644 --- /dev/null +++ b/sys/tools/kernel-gdb.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2025 Mark Johnston +# +# SPDX-License-Identifier: BSD-2-Clause +# + +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "gdb")) + +# Import FreeBSD kernel debugging commands and modules below. +import acttrace +import pcpu +import vnet diff --git a/usr.sbin/crashinfo/crashinfo.sh b/usr.sbin/crashinfo/crashinfo.sh --- a/usr.sbin/crashinfo/crashinfo.sh +++ b/usr.sbin/crashinfo/crashinfo.sh @@ -217,10 +217,7 @@ file=`mktemp /tmp/crashinfo.XXXXXX` if [ $? -eq 0 ]; then - scriptdir=/usr/libexec/kgdb - echo "bt -full" >> $file - echo "source ${scriptdir}/acttrace.py" >> $file echo "acttrace" >> $file echo "quit" >> $file ${GDB%gdb}kgdb -q $KERNEL $VMCORE < $file