diff --git a/sys/arm64/arm64/debug_monitor.c b/sys/arm64/arm64/debug_monitor.c index 3a5a40925c92..91594d5c3a78 100644 --- a/sys/arm64/arm64/debug_monitor.c +++ b/sys/arm64/arm64/debug_monitor.c @@ -1,600 +1,603 @@ /*- * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_ddb.h" +#include "opt_gdb.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #include #endif enum dbg_t { DBG_TYPE_BREAKPOINT = 0, DBG_TYPE_WATCHPOINT = 1, }; static int dbg_watchpoint_num; static int dbg_breakpoint_num; static struct debug_monitor_state kernel_monitor = { .dbg_flags = DBGMON_KERNEL }; /* Called from the exception handlers */ void dbg_monitor_enter(struct thread *); void dbg_monitor_exit(struct thread *, struct trapframe *); /* Watchpoints/breakpoints control register bitfields */ #define DBG_WATCH_CTRL_LEN_1 (0x1 << 5) #define DBG_WATCH_CTRL_LEN_2 (0x3 << 5) #define DBG_WATCH_CTRL_LEN_4 (0xf << 5) #define DBG_WATCH_CTRL_LEN_8 (0xff << 5) #define DBG_WATCH_CTRL_LEN_MASK(x) ((x) & (0xff << 5)) #define DBG_WATCH_CTRL_EXEC (0x0 << 3) #define DBG_WATCH_CTRL_LOAD (0x1 << 3) #define DBG_WATCH_CTRL_STORE (0x2 << 3) #define DBG_WATCH_CTRL_ACCESS_MASK(x) ((x) & (0x3 << 3)) /* Common for breakpoint and watchpoint */ #define DBG_WB_CTRL_EL1 (0x1 << 1) #define DBG_WB_CTRL_EL0 (0x2 << 1) #define DBG_WB_CTRL_ELX_MASK(x) ((x) & (0x3 << 1)) #define DBG_WB_CTRL_E (0x1 << 0) #define DBG_REG_BASE_BVR 0 #define DBG_REG_BASE_BCR (DBG_REG_BASE_BVR + 16) #define DBG_REG_BASE_WVR (DBG_REG_BASE_BCR + 16) #define DBG_REG_BASE_WCR (DBG_REG_BASE_WVR + 16) /* Watchpoint/breakpoint helpers */ #define DBG_WB_WVR "wvr" #define DBG_WB_WCR "wcr" #define DBG_WB_BVR "bvr" #define DBG_WB_BCR "bcr" #define DBG_WB_READ(reg, num, val) do { \ __asm __volatile("mrs %0, dbg" reg #num "_el1" : "=r" (val)); \ } while (0) #define DBG_WB_WRITE(reg, num, val) do { \ __asm __volatile("msr dbg" reg #num "_el1, %0" :: "r" (val)); \ } while (0) #define READ_WB_REG_CASE(reg, num, offset, val) \ case (num + offset): \ DBG_WB_READ(reg, num, val); \ break #define WRITE_WB_REG_CASE(reg, num, offset, val) \ case (num + offset): \ DBG_WB_WRITE(reg, num, val); \ break #define SWITCH_CASES_READ_WB_REG(reg, offset, val) \ READ_WB_REG_CASE(reg, 0, offset, val); \ READ_WB_REG_CASE(reg, 1, offset, val); \ READ_WB_REG_CASE(reg, 2, offset, val); \ READ_WB_REG_CASE(reg, 3, offset, val); \ READ_WB_REG_CASE(reg, 4, offset, val); \ READ_WB_REG_CASE(reg, 5, offset, val); \ READ_WB_REG_CASE(reg, 6, offset, val); \ READ_WB_REG_CASE(reg, 7, offset, val); \ READ_WB_REG_CASE(reg, 8, offset, val); \ READ_WB_REG_CASE(reg, 9, offset, val); \ READ_WB_REG_CASE(reg, 10, offset, val); \ READ_WB_REG_CASE(reg, 11, offset, val); \ READ_WB_REG_CASE(reg, 12, offset, val); \ READ_WB_REG_CASE(reg, 13, offset, val); \ READ_WB_REG_CASE(reg, 14, offset, val); \ READ_WB_REG_CASE(reg, 15, offset, val) #define SWITCH_CASES_WRITE_WB_REG(reg, offset, val) \ WRITE_WB_REG_CASE(reg, 0, offset, val); \ WRITE_WB_REG_CASE(reg, 1, offset, val); \ WRITE_WB_REG_CASE(reg, 2, offset, val); \ WRITE_WB_REG_CASE(reg, 3, offset, val); \ WRITE_WB_REG_CASE(reg, 4, offset, val); \ WRITE_WB_REG_CASE(reg, 5, offset, val); \ WRITE_WB_REG_CASE(reg, 6, offset, val); \ WRITE_WB_REG_CASE(reg, 7, offset, val); \ WRITE_WB_REG_CASE(reg, 8, offset, val); \ WRITE_WB_REG_CASE(reg, 9, offset, val); \ WRITE_WB_REG_CASE(reg, 10, offset, val); \ WRITE_WB_REG_CASE(reg, 11, offset, val); \ WRITE_WB_REG_CASE(reg, 12, offset, val); \ WRITE_WB_REG_CASE(reg, 13, offset, val); \ WRITE_WB_REG_CASE(reg, 14, offset, val); \ WRITE_WB_REG_CASE(reg, 15, offset, val) #ifdef DDB static uint64_t dbg_wb_read_reg(int reg, int n) { uint64_t val = 0; switch (reg + n) { SWITCH_CASES_READ_WB_REG(DBG_WB_WVR, DBG_REG_BASE_WVR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_WCR, DBG_REG_BASE_WCR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_BVR, DBG_REG_BASE_BVR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_BCR, DBG_REG_BASE_BCR, val); default: printf("trying to read from wrong debug register %d\n", n); } return val; } #endif /* DDB */ static void dbg_wb_write_reg(int reg, int n, uint64_t val) { switch (reg + n) { SWITCH_CASES_WRITE_WB_REG(DBG_WB_WVR, DBG_REG_BASE_WVR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_WCR, DBG_REG_BASE_WCR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_BVR, DBG_REG_BASE_BVR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_BCR, DBG_REG_BASE_BCR, val); default: printf("trying to write to wrong debug register %d\n", n); return; } isb(); } -#ifdef DDB +#if defined(DDB) || defined(GDB) void kdb_cpu_set_singlestep(void) { KASSERT((READ_SPECIALREG(daif) & PSR_D) == PSR_D, ("%s: debug exceptions are not masked", __func__)); kdb_frame->tf_spsr |= DBG_SPSR_SS; WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) | DBG_MDSCR_SS | DBG_MDSCR_KDE); /* * Disable breakpoints and watchpoints, e.g. stepping * over watched instruction will trigger break exception instead of * single-step exception and locks CPU on that instruction for ever. */ if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) { WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) & ~DBG_MDSCR_MDE); } } void kdb_cpu_clear_singlestep(void) { KASSERT((READ_SPECIALREG(daif) & PSR_D) == PSR_D, ("%s: debug exceptions are not masked", __func__)); WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) & ~(DBG_MDSCR_SS | DBG_MDSCR_KDE)); /* Restore breakpoints and watchpoints */ if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) { WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) | DBG_MDSCR_MDE); if ((kernel_monitor.dbg_flags & DBGMON_KERNEL) != 0) { WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) | DBG_MDSCR_KDE); } } } int kdb_cpu_set_watchpoint(vm_offset_t addr, vm_size_t size, int access) { enum dbg_access_t dbg_access; switch (access) { case KDB_DBG_ACCESS_R: dbg_access = HW_BREAKPOINT_R; break; case KDB_DBG_ACCESS_W: dbg_access = HW_BREAKPOINT_W; break; case KDB_DBG_ACCESS_RW: dbg_access = HW_BREAKPOINT_RW; break; default: return (EINVAL); } return (dbg_setup_watchpoint(NULL, addr, size, dbg_access)); } int kdb_cpu_clr_watchpoint(vm_offset_t addr, vm_size_t size) { return (dbg_remove_watchpoint(NULL, addr, size)); } +#endif /* DDB || GDB */ +#ifdef DDB static const char * dbg_watchtype_str(uint32_t type) { switch (type) { case DBG_WATCH_CTRL_EXEC: return ("execute"); case DBG_WATCH_CTRL_STORE: return ("write"); case DBG_WATCH_CTRL_LOAD: return ("read"); case DBG_WATCH_CTRL_LOAD | DBG_WATCH_CTRL_STORE: return ("read/write"); default: return ("invalid"); } } static int dbg_watchtype_len(uint32_t len) { switch (len) { case DBG_WATCH_CTRL_LEN_1: return (1); case DBG_WATCH_CTRL_LEN_2: return (2); case DBG_WATCH_CTRL_LEN_4: return (4); case DBG_WATCH_CTRL_LEN_8: return (8); default: return (0); } } void dbg_show_watchpoint(void) { uint32_t wcr, len, type; uint64_t addr; int i; db_printf("\nhardware watchpoints:\n"); db_printf(" watch status type len address symbol\n"); db_printf(" ----- -------- ---------- --- ------------------ ------------------\n"); for (i = 0; i < dbg_watchpoint_num; i++) { wcr = dbg_wb_read_reg(DBG_REG_BASE_WCR, i); if ((wcr & DBG_WB_CTRL_E) != 0) { type = DBG_WATCH_CTRL_ACCESS_MASK(wcr); len = DBG_WATCH_CTRL_LEN_MASK(wcr); addr = dbg_wb_read_reg(DBG_REG_BASE_WVR, i); db_printf(" %-5d %-8s %10s %3d 0x%16lx ", i, "enabled", dbg_watchtype_str(type), dbg_watchtype_len(len), addr); db_printsym((db_addr_t)addr, DB_STGY_ANY); db_printf("\n"); } else { db_printf(" %-5d disabled\n", i); } } } #endif /* DDB */ static int dbg_find_free_slot(struct debug_monitor_state *monitor, enum dbg_t type) { uint64_t *reg; u_int max, i; switch(type) { case DBG_TYPE_BREAKPOINT: max = dbg_breakpoint_num; reg = monitor->dbg_bcr; break; case DBG_TYPE_WATCHPOINT: max = dbg_watchpoint_num; reg = monitor->dbg_wcr; break; default: printf("Unsupported debug type\n"); return (i); } for (i = 0; i < max; i++) { if ((reg[i] & DBG_WB_CTRL_E) == 0) return (i); } return (-1); } static int dbg_find_slot(struct debug_monitor_state *monitor, enum dbg_t type, vm_offset_t addr) { uint64_t *reg_addr, *reg_ctrl; u_int max, i; switch(type) { case DBG_TYPE_BREAKPOINT: max = dbg_breakpoint_num; reg_addr = monitor->dbg_bvr; reg_ctrl = monitor->dbg_bcr; break; case DBG_TYPE_WATCHPOINT: max = dbg_watchpoint_num; reg_addr = monitor->dbg_wvr; reg_ctrl = monitor->dbg_wcr; break; default: printf("Unsupported debug type\n"); return (i); } for (i = 0; i < max; i++) { if (reg_addr[i] == addr && (reg_ctrl[i] & DBG_WB_CTRL_E) != 0) return (i); } return (-1); } int dbg_setup_watchpoint(struct debug_monitor_state *monitor, vm_offset_t addr, vm_size_t size, enum dbg_access_t access) { uint64_t wcr_size, wcr_priv, wcr_access; u_int i; if (monitor == NULL) monitor = &kernel_monitor; i = dbg_find_free_slot(monitor, DBG_TYPE_WATCHPOINT); if (i == -1) { printf("Can not find slot for watchpoint, max %d" " watchpoints supported\n", dbg_watchpoint_num); return (EBUSY); } switch(size) { case 1: wcr_size = DBG_WATCH_CTRL_LEN_1; break; case 2: wcr_size = DBG_WATCH_CTRL_LEN_2; break; case 4: wcr_size = DBG_WATCH_CTRL_LEN_4; break; case 8: wcr_size = DBG_WATCH_CTRL_LEN_8; break; default: printf("Unsupported address size for watchpoint: %zu\n", size); return (EINVAL); } if ((monitor->dbg_flags & DBGMON_KERNEL) == 0) wcr_priv = DBG_WB_CTRL_EL0; else wcr_priv = DBG_WB_CTRL_EL1; switch(access) { case HW_BREAKPOINT_X: wcr_access = DBG_WATCH_CTRL_EXEC; break; case HW_BREAKPOINT_R: wcr_access = DBG_WATCH_CTRL_LOAD; break; case HW_BREAKPOINT_W: wcr_access = DBG_WATCH_CTRL_STORE; break; case HW_BREAKPOINT_RW: wcr_access = DBG_WATCH_CTRL_LOAD | DBG_WATCH_CTRL_STORE; break; default: printf("Unsupported access type for watchpoint: %d\n", access); return (EINVAL); } monitor->dbg_wvr[i] = addr; monitor->dbg_wcr[i] = wcr_size | wcr_access | wcr_priv | DBG_WB_CTRL_E; monitor->dbg_enable_count++; monitor->dbg_flags |= DBGMON_ENABLED; dbg_register_sync(monitor); return (0); } int dbg_remove_watchpoint(struct debug_monitor_state *monitor, vm_offset_t addr, vm_size_t size) { u_int i; if (monitor == NULL) monitor = &kernel_monitor; i = dbg_find_slot(monitor, DBG_TYPE_WATCHPOINT, addr); if (i == -1) { printf("Can not find watchpoint for address 0%lx\n", addr); return (EINVAL); } monitor->dbg_wvr[i] = 0; monitor->dbg_wcr[i] = 0; monitor->dbg_enable_count--; if (monitor->dbg_enable_count == 0) monitor->dbg_flags &= ~DBGMON_ENABLED; dbg_register_sync(monitor); return (0); } void dbg_register_sync(struct debug_monitor_state *monitor) { uint64_t mdscr; int i; if (monitor == NULL) monitor = &kernel_monitor; mdscr = READ_SPECIALREG(mdscr_el1); if ((monitor->dbg_flags & DBGMON_ENABLED) == 0) { mdscr &= ~(DBG_MDSCR_MDE | DBG_MDSCR_KDE); } else { for (i = 0; i < dbg_breakpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_BCR, i, monitor->dbg_bcr[i]); dbg_wb_write_reg(DBG_REG_BASE_BVR, i, monitor->dbg_bvr[i]); } for (i = 0; i < dbg_watchpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, monitor->dbg_wcr[i]); dbg_wb_write_reg(DBG_REG_BASE_WVR, i, monitor->dbg_wvr[i]); } mdscr |= DBG_MDSCR_MDE; if ((monitor->dbg_flags & DBGMON_KERNEL) == DBGMON_KERNEL) mdscr |= DBG_MDSCR_KDE; } WRITE_SPECIALREG(mdscr_el1, mdscr); isb(); } void dbg_monitor_init(void) { u_int i; /* Find out many breakpoints and watchpoints we can use */ dbg_watchpoint_num = ((READ_SPECIALREG(id_aa64dfr0_el1) >> 20) & 0xf) + 1; dbg_breakpoint_num = ((READ_SPECIALREG(id_aa64dfr0_el1) >> 12) & 0xf) + 1; if (bootverbose && PCPU_GET(cpuid) == 0) { printf("%d watchpoints and %d breakpoints supported\n", dbg_watchpoint_num, dbg_breakpoint_num); } /* * We have limited number of {watch,break}points, each consists of * two registers: * - wcr/bcr regsiter configurates corresponding {watch,break}point * behaviour * - wvr/bvr register keeps address we are hunting for * * Reset all breakpoints and watchpoints. */ for (i = 0; i < dbg_watchpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_WVR, i, 0); } for (i = 0; i < dbg_breakpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_BCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_BVR, i, 0); } dbg_enable(); } void dbg_monitor_enter(struct thread *thread) { int i; if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) { /* Install the kernel version of the registers */ dbg_register_sync(&kernel_monitor); } else if ((thread->td_pcb->pcb_dbg_regs.dbg_flags & DBGMON_ENABLED) != 0) { /* Disable the user breakpoints until we return to userspace */ for (i = 0; i < dbg_watchpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_WVR, i, 0); } for (i = 0; i < dbg_breakpoint_num; ++i) { dbg_wb_write_reg(DBG_REG_BASE_BCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_BVR, i, 0); } WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) & ~(DBG_MDSCR_MDE | DBG_MDSCR_KDE)); isb(); } } void dbg_monitor_exit(struct thread *thread, struct trapframe *frame) { int i; /* * PSR_D is an aarch64-only flag. On aarch32, it switches * the processor to big-endian, so avoid setting it for * 32bits binaries. */ if (!(SV_PROC_FLAG(thread->td_proc, SV_ILP32))) frame->tf_spsr |= PSR_D; if ((thread->td_pcb->pcb_dbg_regs.dbg_flags & DBGMON_ENABLED) != 0) { /* Install the thread's version of the registers */ dbg_register_sync(&thread->td_pcb->pcb_dbg_regs); frame->tf_spsr &= ~PSR_D; } else if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) { /* Disable the kernel breakpoints until we re-enter */ for (i = 0; i < dbg_watchpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_WVR, i, 0); } for (i = 0; i < dbg_breakpoint_num; ++i) { dbg_wb_write_reg(DBG_REG_BASE_BCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_BVR, i, 0); } WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) & ~(DBG_MDSCR_MDE | DBG_MDSCR_KDE)); isb(); } } diff --git a/sys/conf/files.arm b/sys/conf/files.arm index 69986585bdf6..a3b0d166f020 100644 --- a/sys/conf/files.arm +++ b/sys/conf/files.arm @@ -1,156 +1,155 @@ # $FreeBSD$ arm/arm/autoconf.c standard arm/arm/bcopy_page.S standard arm/arm/bcopyinout.S standard arm/arm/blockio.S standard arm/arm/bus_space_asm_generic.S standard arm/arm/bus_space_base.c optional fdt arm/arm/bus_space_generic.c standard arm/arm/busdma_machdep.c standard arm/arm/copystr.S standard arm/arm/cpufunc.c standard arm/arm/cpufunc_asm.S standard arm/arm/cpufunc_asm_arm11x6.S optional cpu_arm1176 arm/arm/cpufunc_asm_armv7.S optional cpu_cortexa | cpu_krait | cpu_mv_pj4b arm/arm/cpufunc_asm_pj4b.S optional cpu_mv_pj4b arm/arm/cpuinfo.c standard arm/arm/cpu_asm-v6.S standard arm/arm/db_disasm.c optional ddb arm/arm/db_interface.c optional ddb arm/arm/db_trace.c optional ddb -arm/arm/debug_monitor.c optional ddb armv6 -arm/arm/debug_monitor.c optional ddb armv7 +arm/arm/debug_monitor.c optional ddb | gdb arm/arm/disassem.c optional ddb arm/arm/dump_machdep.c standard arm/arm/elf_machdep.c standard arm/arm/elf_note.S standard arm/arm/exception.S standard arm/arm/fiq.c standard arm/arm/fiq_subr.S standard arm/arm/fusu.S standard arm/arm/gdb_machdep.c optional gdb arm/arm/generic_timer.c optional generic_timer arm/arm/gic.c optional gic arm/arm/gic_fdt.c optional gic fdt arm/arm/identcpu-v6.c standard arm/arm/in_cksum.c optional inet | inet6 arm/arm/in_cksum_arm.S optional inet | inet6 kern/subr_intr.c standard arm/arm/locore.S standard no-obj arm/arm/hypervisor-stub.S standard arm/arm/machdep.c standard arm/arm/machdep_boot.c standard arm/arm/machdep_kdb.c standard arm/arm/machdep_intr.c standard arm/arm/machdep_ptrace.c standard arm/arm/mem.c optional mem arm/arm/minidump_machdep.c standard arm/arm/mp_machdep.c optional smp arm/arm/mpcore_timer.c optional mpcore_timer arm/arm/nexus.c standard arm/arm/ofw_machdep.c optional fdt arm/arm/pl190.c optional pl190 arm/arm/pl310.c optional pl310 arm/arm/platform.c optional platform arm/arm/platform_if.m optional platform arm/arm/platform_pl310_if.m optional platform pl310 arm/arm/pmap-v6.c standard arm/arm/pmu.c optional pmu | hwpmc arm/arm/pmu_fdt.c optional fdt pmu | fdt hwpmc arm/arm/ptrace_machdep.c standard arm/arm/sc_machdep.c optional sc arm/arm/setcpsr.S standard arm/arm/setstack.s standard arm/arm/stack_machdep.c optional ddb | stack arm/arm/stdatomic.c standard \ compile-with "${NORMAL_C:N-Wmissing-prototypes}" arm/arm/support.S standard arm/arm/swtch.S standard arm/arm/swtch-v6.S standard arm/arm/sys_machdep.c standard arm/arm/syscall.c standard arm/arm/trap-v6.c standard arm/arm/uio_machdep.c standard arm/arm/undefined.c standard arm/arm/unwind.c optional ddb | kdtrace_hooks | stack arm/arm/vm_machdep.c standard arm/arm/vfp.c standard arm/cloudabi32/cloudabi32_sysvec.c optional compat_cloudabi32 cddl/compat/opensolaris/kern/opensolaris_atomic.c optional !armv7 !armv6 zfs | !armv7 !armv6 dtrace compile-with "${CDDL_C}" cddl/dev/dtrace/arm/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/arm/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/arm/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" crypto/des/des_enc.c optional netsmb dev/cpufreq/cpufreq_dt.c optional cpufreq fdt dev/dwc/if_dwc.c optional dwc dev/dwc/if_dwc_if.m optional dwc dev/fb/fb.c optional sc dev/fdt/fdt_arm_platform.c optional platform fdt dev/hdmi/hdmi_if.m optional hdmi dev/hwpmc/hwpmc_arm.c optional hwpmc dev/hwpmc/hwpmc_armv7.c optional hwpmc armv6 dev/hwpmc/hwpmc_armv7.c optional hwpmc armv7 dev/iicbus/twsi/twsi.c optional twsi dev/ofw/ofwpci.c optional fdt pci dev/pci/pci_host_generic.c optional pci_host_generic pci dev/pci/pci_host_generic_fdt.c optional pci_host_generic pci fdt dev/psci/psci.c optional psci dev/psci/smccc_arm.S optional psci dev/syscons/scgfbrndr.c optional sc dev/uart/uart_cpu_fdt.c optional uart fdt kern/msi_if.m optional intrng kern/pic_if.m optional intrng kern/subr_busdma_bufalloc.c standard kern/subr_devmap.c standard kern/subr_physmem.c standard kern/subr_sfbuf.c standard libkern/arm/aeabi_unwind.c standard libkern/arm/divsi3.S standard libkern/arm/ffs.S optional !armv7 !armv6 libkern/arm/ldivmod.S standard libkern/arm/ldivmod_helper.c standard libkern/arm/memclr.S standard libkern/arm/memcpy.S standard libkern/arm/memset.S standard libkern/arm/muldi3.c standard libkern/ashldi3.c standard libkern/ashrdi3.c standard libkern/divdi3.c standard libkern/ffsl.c optional !armv7 !armv6 libkern/ffsll.c optional !armv7 !armv6 libkern/fls.c optional !armv7 !armv6 libkern/flsl.c optional !armv7 !armv6 libkern/flsll.c optional !armv7 !armv6 libkern/lshrdi3.c standard libkern/memcmp.c standard libkern/moddi3.c standard libkern/qdivrem.c standard libkern/strlen.c standard libkern/ucmpdi2.c standard libkern/udivdi3.c standard libkern/umoddi3.c standard # CloudABI support cloudabi32_vdso.o optional compat_cloudabi32 \ dependency "$S/contrib/cloudabi/cloudabi_vdso_armv6.S" \ compile-with "${CC} -x assembler-with-cpp -shared -nostdinc -nostdlib -Wl,-T$S/compat/cloudabi/cloudabi_vdso.lds $S/contrib/cloudabi/cloudabi_vdso_armv6.S -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "cloudabi32_vdso.o" # cloudabi32_vdso_blob.o optional compat_cloudabi32 \ dependency "cloudabi32_vdso.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf32-littlearm --binary-architecture arm cloudabi32_vdso.o ${.TARGET}" \ no-implicit-rule \ clean "cloudabi32_vdso_blob.o" # # Annapurna support arm/annapurna/alpine/alpine_ccu.c optional al_ccu fdt arm/annapurna/alpine/alpine_nb_service.c optional al_nb_service fdt arm/annapurna/alpine/alpine_pci.c optional al_pci fdt arm/annapurna/alpine/alpine_pci_msix.c optional al_pci fdt arm/annapurna/alpine/alpine_serdes.c optional al_serdes fdt \ no-depend \ compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}" diff --git a/sys/conf/files.x86 b/sys/conf/files.x86 index 292389ac312f..7df289375cd3 100644 --- a/sys/conf/files.x86 +++ b/sys/conf/files.x86 @@ -1,355 +1,355 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # # $FreeBSD$ # # This file contains all the x86 devices and such that are # common between i386 and amd64, but aren't applicable to # any other architecture we support. # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # atkbdmap.h optional atkbd_dflt_keymap \ compile-with "${KEYMAP} -L ${ATKBD_DFLT_KEYMAP} | ${KEYMAP_FIX} > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "atkbdmap.h" cddl/dev/fbt/x86/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" cddl/dev/dtrace/x86/dis_tables.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" cddl/dev/dtrace/x86/instr_size.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" compat/ndis/kern_ndis.c optional ndisapi pci compat/ndis/kern_windrv.c optional ndisapi pci compat/ndis/subr_hal.c optional ndisapi pci compat/ndis/subr_ndis.c optional ndisapi pci compat/ndis/subr_ntoskrnl.c optional ndisapi pci compat/ndis/subr_pe.c optional ndisapi pci compat/ndis/subr_usbd.c optional ndisapi pci crypto/aesni/aesni.c optional aesni aesni_ghash.o optional aesni \ dependency "$S/crypto/aesni/aesni_ghash.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_ghash.o" aesni_ccm.o optional aesni \ dependency "$S/crypto/aesni/aesni_ccm.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_ccm.o" aesni_wrap.o optional aesni \ dependency "$S/crypto/aesni/aesni_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_wrap.o" intel_sha1.o optional aesni \ dependency "$S/crypto/aesni/intel_sha1.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ no-implicit-rule \ clean "intel_sha1.o" intel_sha256.o optional aesni \ dependency "$S/crypto/aesni/intel_sha256.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ no-implicit-rule \ clean "intel_sha256.o" crypto/openssl/ossl_x86.c optional ossl crypto/via/padlock.c optional padlock crypto/via/padlock_cipher.c optional padlock crypto/via/padlock_hash.c optional padlock dev/acpica/acpi_hpet.c optional acpi dev/acpica/acpi_if.m standard dev/acpica/acpi_pci.c optional acpi pci dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pcib_acpi.c optional acpi pci dev/acpica/acpi_pcib_pci.c optional acpi pci dev/acpica/acpi_pxm.c optional acpi dev/acpica/acpi_timer.c optional acpi dev/amdsbwd/amdsbwd.c optional amdsbwd dev/amdsmn/amdsmn.c optional amdsmn | amdtemp dev/amdtemp/amdtemp.c optional amdtemp dev/arcmsr/arcmsr.c optional arcmsr pci dev/asmc/asmc.c optional asmc isa dev/atkbdc/atkbd.c optional atkbd atkbdc dev/atkbdc/atkbd_atkbdc.c optional atkbd atkbdc dev/atkbdc/atkbdc.c optional atkbdc dev/atkbdc/atkbdc_isa.c optional atkbdc isa dev/atkbdc/atkbdc_subr.c optional atkbdc dev/atkbdc/psm.c optional psm atkbdc dev/bxe/bxe.c optional bxe pci dev/bxe/bxe_stats.c optional bxe pci dev/bxe/bxe_debug.c optional bxe pci dev/bxe/ecore_sp.c optional bxe pci dev/bxe/bxe_elink.c optional bxe pci dev/bxe/57710_init_values.c optional bxe pci dev/bxe/57711_init_values.c optional bxe pci dev/bxe/57712_init_values.c optional bxe pci dev/coretemp/coretemp.c optional coretemp dev/cpuctl/cpuctl.c optional cpuctl dev/dpms/dpms.c optional dpms dev/fb/fb.c optional fb | vga dev/fb/s3_pci.c optional s3pci dev/fb/vesa.c optional vga vesa dev/fb/vga.c optional vga dev/fdc/fdc.c optional fdc dev/fdc/fdc_acpi.c optional fdc dev/fdc/fdc_isa.c optional fdc isa dev/gpio/bytgpio.c optional bytgpio dev/gpio/chvgpio.c optional chvgpio dev/hpt27xx/hpt27xx_os_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_osm_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_config.c optional hpt27xx hpt27xx_lib.o optional hpt27xx \ dependency "$S/dev/hpt27xx/$M-elf.hpt27xx_lib.o.uu" \ compile-with "uudecode < $S/dev/hpt27xx/$M-elf.hpt27xx_lib.o.uu" \ no-implicit-rule dev/hptmv/entry.c optional hptmv dev/hptmv/mv.c optional hptmv dev/hptmv/gui_lib.c optional hptmv dev/hptmv/hptproc.c optional hptmv dev/hptmv/ioctl.c optional hptmv hptmvraid.o optional hptmv \ dependency "$S/dev/hptmv/$M-elf.raid.o.uu" \ compile-with "uudecode < $S/dev/hptmv/$M-elf.raid.o.uu" \ no-implicit-rule dev/hptnr/hptnr_os_bsd.c optional hptnr dev/hptnr/hptnr_osm_bsd.c optional hptnr dev/hptnr/hptnr_config.c optional hptnr hptnr_lib.o optional hptnr \ dependency "$S/dev/hptnr/$M-elf.hptnr_lib.o.uu" \ compile-with "uudecode < $S/dev/hptnr/$M-elf.hptnr_lib.o.uu" \ no-implicit-rule dev/hptrr/hptrr_os_bsd.c optional hptrr dev/hptrr/hptrr_osm_bsd.c optional hptrr dev/hptrr/hptrr_config.c optional hptrr hptrr_lib.o optional hptrr \ dependency "$S/dev/hptrr/$M-elf.hptrr_lib.o.uu" \ compile-with "uudecode < $S/dev/hptrr/$M-elf.hptrr_lib.o.uu" \ no-implicit-rule dev/hwpmc/hwpmc_amd.c optional hwpmc dev/hwpmc/hwpmc_intel.c optional hwpmc dev/hwpmc/hwpmc_core.c optional hwpmc dev/hwpmc/hwpmc_uncore.c optional hwpmc dev/hwpmc/hwpmc_tsc.c optional hwpmc dev/hwpmc/hwpmc_x86.c optional hwpmc dev/hyperv/hvsock/hv_sock.c optional hyperv dev/hyperv/input/hv_kbd.c optional hyperv dev/hyperv/input/hv_kbdc.c optional hyperv dev/hyperv/pcib/vmbus_pcib.c optional hyperv pci dev/hyperv/netvsc/hn_nvs.c optional hyperv dev/hyperv/netvsc/hn_rndis.c optional hyperv dev/hyperv/netvsc/if_hn.c optional hyperv dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c optional hyperv dev/hyperv/utilities/hv_kvp.c optional hyperv dev/hyperv/utilities/hv_snapshot.c optional hyperv dev/hyperv/utilities/vmbus_heartbeat.c optional hyperv dev/hyperv/utilities/vmbus_ic.c optional hyperv dev/hyperv/utilities/vmbus_shutdown.c optional hyperv dev/hyperv/utilities/vmbus_timesync.c optional hyperv dev/hyperv/vmbus/hyperv.c optional hyperv dev/hyperv/vmbus/hyperv_busdma.c optional hyperv dev/hyperv/vmbus/vmbus.c optional hyperv pci dev/hyperv/vmbus/vmbus_br.c optional hyperv dev/hyperv/vmbus/vmbus_chan.c optional hyperv dev/hyperv/vmbus/vmbus_et.c optional hyperv dev/hyperv/vmbus/vmbus_if.m optional hyperv dev/hyperv/vmbus/vmbus_res.c optional hyperv dev/hyperv/vmbus/vmbus_xact.c optional hyperv dev/ichwd/ichwd.c optional ichwd dev/if_ndis/if_ndis.c optional ndis dev/if_ndis/if_ndis_pci.c optional ndis cardbus | ndis pci dev/if_ndis/if_ndis_usb.c optional ndis usb dev/imcsmb/imcsmb.c optional imcsmb dev/imcsmb/imcsmb_pci.c optional imcsmb pci dev/intel/spi.c optional intelspi dev/io/iodev.c optional io dev/iommu/busdma_iommu.c optional acpi iommu pci dev/iommu/iommu_gas.c optional acpi iommu pci dev/ipmi/ipmi.c optional ipmi dev/ipmi/ipmi_acpi.c optional ipmi acpi dev/ipmi/ipmi_isa.c optional ipmi isa dev/ipmi/ipmi_kcs.c optional ipmi dev/ipmi/ipmi_smic.c optional ipmi dev/ipmi/ipmi_smbus.c optional ipmi smbus dev/ipmi/ipmi_smbios.c optional ipmi dev/ipmi/ipmi_ssif.c optional ipmi smbus dev/ipmi/ipmi_pci.c optional ipmi pci dev/ipmi/ipmi_linux.c optional ipmi compat_linux32 dev/isci/isci.c optional isci dev/isci/isci_controller.c optional isci dev/isci/isci_domain.c optional isci dev/isci/isci_interrupt.c optional isci dev/isci/isci_io_request.c optional isci dev/isci/isci_logger.c optional isci dev/isci/isci_oem_parameters.c optional isci dev/isci/isci_remote_device.c optional isci dev/isci/isci_sysctl.c optional isci dev/isci/isci_task_request.c optional isci dev/isci/isci_timer.c optional isci dev/isci/scil/sati.c optional isci dev/isci/scil/sati_abort_task_set.c optional isci dev/isci/scil/sati_atapi.c optional isci dev/isci/scil/sati_device.c optional isci dev/isci/scil/sati_inquiry.c optional isci dev/isci/scil/sati_log_sense.c optional isci dev/isci/scil/sati_lun_reset.c optional isci dev/isci/scil/sati_mode_pages.c optional isci dev/isci/scil/sati_mode_select.c optional isci dev/isci/scil/sati_mode_sense.c optional isci dev/isci/scil/sati_mode_sense_10.c optional isci dev/isci/scil/sati_mode_sense_6.c optional isci dev/isci/scil/sati_move.c optional isci dev/isci/scil/sati_passthrough.c optional isci dev/isci/scil/sati_read.c optional isci dev/isci/scil/sati_read_buffer.c optional isci dev/isci/scil/sati_read_capacity.c optional isci dev/isci/scil/sati_reassign_blocks.c optional isci dev/isci/scil/sati_report_luns.c optional isci dev/isci/scil/sati_request_sense.c optional isci dev/isci/scil/sati_start_stop_unit.c optional isci dev/isci/scil/sati_synchronize_cache.c optional isci dev/isci/scil/sati_test_unit_ready.c optional isci dev/isci/scil/sati_unmap.c optional isci dev/isci/scil/sati_util.c optional isci dev/isci/scil/sati_verify.c optional isci dev/isci/scil/sati_write.c optional isci dev/isci/scil/sati_write_and_verify.c optional isci dev/isci/scil/sati_write_buffer.c optional isci dev/isci/scil/sati_write_long.c optional isci dev/isci/scil/sci_abstract_list.c optional isci dev/isci/scil/sci_base_controller.c optional isci dev/isci/scil/sci_base_domain.c optional isci dev/isci/scil/sci_base_iterator.c optional isci dev/isci/scil/sci_base_library.c optional isci dev/isci/scil/sci_base_logger.c optional isci dev/isci/scil/sci_base_memory_descriptor_list.c optional isci dev/isci/scil/sci_base_memory_descriptor_list_decorator.c optional isci dev/isci/scil/sci_base_object.c optional isci dev/isci/scil/sci_base_observer.c optional isci dev/isci/scil/sci_base_phy.c optional isci dev/isci/scil/sci_base_port.c optional isci dev/isci/scil/sci_base_remote_device.c optional isci dev/isci/scil/sci_base_request.c optional isci dev/isci/scil/sci_base_state_machine.c optional isci dev/isci/scil/sci_base_state_machine_logger.c optional isci dev/isci/scil/sci_base_state_machine_observer.c optional isci dev/isci/scil/sci_base_subject.c optional isci dev/isci/scil/sci_util.c optional isci dev/isci/scil/scic_sds_controller.c optional isci dev/isci/scil/scic_sds_library.c optional isci dev/isci/scil/scic_sds_pci.c optional isci dev/isci/scil/scic_sds_phy.c optional isci dev/isci/scil/scic_sds_port.c optional isci dev/isci/scil/scic_sds_port_configuration_agent.c optional isci dev/isci/scil/scic_sds_remote_device.c optional isci dev/isci/scil/scic_sds_remote_node_context.c optional isci dev/isci/scil/scic_sds_remote_node_table.c optional isci dev/isci/scil/scic_sds_request.c optional isci dev/isci/scil/scic_sds_sgpio.c optional isci dev/isci/scil/scic_sds_smp_remote_device.c optional isci dev/isci/scil/scic_sds_smp_request.c optional isci dev/isci/scil/scic_sds_ssp_request.c optional isci dev/isci/scil/scic_sds_stp_packet_request.c optional isci dev/isci/scil/scic_sds_stp_remote_device.c optional isci dev/isci/scil/scic_sds_stp_request.c optional isci dev/isci/scil/scic_sds_unsolicited_frame_control.c optional isci dev/isci/scil/scif_sas_controller.c optional isci dev/isci/scil/scif_sas_controller_state_handlers.c optional isci dev/isci/scil/scif_sas_controller_states.c optional isci dev/isci/scil/scif_sas_domain.c optional isci dev/isci/scil/scif_sas_domain_state_handlers.c optional isci dev/isci/scil/scif_sas_domain_states.c optional isci dev/isci/scil/scif_sas_high_priority_request_queue.c optional isci dev/isci/scil/scif_sas_internal_io_request.c optional isci dev/isci/scil/scif_sas_io_request.c optional isci dev/isci/scil/scif_sas_io_request_state_handlers.c optional isci dev/isci/scil/scif_sas_io_request_states.c optional isci dev/isci/scil/scif_sas_library.c optional isci dev/isci/scil/scif_sas_remote_device.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substates.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substates.c optional isci dev/isci/scil/scif_sas_remote_device_state_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_states.c optional isci dev/isci/scil/scif_sas_request.c optional isci dev/isci/scil/scif_sas_smp_activity_clear_affiliation.c optional isci dev/isci/scil/scif_sas_smp_io_request.c optional isci dev/isci/scil/scif_sas_smp_phy.c optional isci dev/isci/scil/scif_sas_smp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_io_request.c optional isci dev/isci/scil/scif_sas_stp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_task_request.c optional isci dev/isci/scil/scif_sas_task_request.c optional isci dev/isci/scil/scif_sas_task_request_state_handlers.c optional isci dev/isci/scil/scif_sas_task_request_states.c optional isci dev/isci/scil/scif_sas_timer.c optional isci dev/itwd/itwd.c optional itwd dev/qat/qat.c optional qat dev/qat/qat_ae.c optional qat dev/qat/qat_c2xxx.c optional qat dev/qat/qat_c3xxx.c optional qat dev/qat/qat_c62x.c optional qat dev/qat/qat_d15xx.c optional qat dev/qat/qat_dh895xcc.c optional qat dev/qat/qat_hw15.c optional qat dev/qat/qat_hw17.c optional qat libkern/x86/crc32_sse42.c standard # # x86 shared code between IA32 and AMD64 architectures # x86/acpica/OsdEnvironment.c optional acpi x86/acpica/acpi_apm.c optional acpi x86/acpica/acpi_wakeup.c optional acpi x86/acpica/srat.c optional acpi x86/bios/smbios.c optional smbios x86/bios/vpd.c optional vpd x86/cpufreq/est.c optional cpufreq x86/cpufreq/hwpstate_amd.c optional cpufreq x86/cpufreq/hwpstate_intel.c optional cpufreq x86/cpufreq/p4tcc.c optional cpufreq x86/cpufreq/powernow.c optional cpufreq x86/iommu/intel_ctx.c optional acpi iommu pci x86/iommu/intel_drv.c optional acpi iommu pci x86/iommu/intel_fault.c optional acpi iommu pci x86/iommu/intel_idpgtbl.c optional acpi iommu pci x86/iommu/intel_intrmap.c optional acpi iommu pci x86/iommu/intel_qi.c optional acpi iommu pci x86/iommu/intel_quirks.c optional acpi iommu pci x86/iommu/intel_utils.c optional acpi iommu pci x86/isa/atrtc.c standard x86/isa/clock.c standard x86/isa/isa.c optional isa x86/isa/isa_dma.c optional isa x86/isa/nmi.c standard x86/isa/orm.c optional isa x86/pci/pci_bus.c optional pci x86/pci/qpi.c optional pci x86/x86/autoconf.c standard x86/x86/bus_machdep.c standard x86/x86/busdma_bounce.c standard x86/x86/busdma_machdep.c standard x86/x86/cpu_machdep.c standard -x86/x86/dbreg.c optional ddb +x86/x86/dbreg.c optional ddb | gdb x86/x86/dump_machdep.c standard x86/x86/fdt_machdep.c optional fdt x86/x86/identcpu.c standard x86/x86/intr_machdep.c standard x86/x86/legacy.c standard x86/x86/mca.c standard x86/x86/x86_mem.c optional mem x86/x86/mp_x86.c optional smp x86/x86/mp_watchdog.c optional mp_watchdog smp x86/x86/nexus.c standard x86/x86/pvclock.c standard x86/x86/stack_machdep.c optional ddb | stack x86/x86/tsc.c standard x86/x86/ucode.c standard x86/x86/delay.c standard x86/xen/hvm.c optional xenhvm x86/xen/xen_intr.c optional xenhvm x86/xen/xen_apic.c optional xenhvm x86/xen/xenpv.c optional xenhvm x86/xen/xen_msi.c optional xenhvm x86/xen/xen_nexus.c optional xenhvm diff --git a/sys/gdb/gdb_main.c b/sys/gdb/gdb_main.c index 6e0c9f21f947..d0dbdfa63cb7 100644 --- a/sys/gdb/gdb_main.c +++ b/sys/gdb/gdb_main.c @@ -1,882 +1,984 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2004 Marcel Moolenaar * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include SYSCTL_NODE(_debug, OID_AUTO, gdb, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "GDB settings"); static dbbe_init_f gdb_init; static dbbe_trap_f gdb_trap; KDB_BACKEND(gdb, gdb_init, NULL, NULL, gdb_trap); static struct gdb_dbgport null_gdb_dbgport; DATA_SET(gdb_dbgport_set, null_gdb_dbgport); SET_DECLARE(gdb_dbgport_set, struct gdb_dbgport); struct gdb_dbgport *gdb_cur = NULL; int gdb_listening = 0; bool gdb_ackmode = true; static unsigned char gdb_bindata[64]; #ifdef DDB bool gdb_return_to_ddb = false; #endif static int gdb_init(void) { struct gdb_dbgport *dp, **iter; int cur_pri, pri; gdb_cur = NULL; cur_pri = -1; SET_FOREACH(iter, gdb_dbgport_set) { dp = *iter; pri = (dp->gdb_probe != NULL) ? dp->gdb_probe() : -1; dp->gdb_active = (pri >= 0) ? 0 : -1; if (pri > cur_pri) { cur_pri = pri; gdb_cur = dp; } } if (gdb_cur != NULL) { printf("GDB: debug ports:"); SET_FOREACH(iter, gdb_dbgport_set) { dp = *iter; if (dp->gdb_active == 0) printf(" %s", dp->gdb_name); } printf("\n"); } else printf("GDB: no debug ports present\n"); if (gdb_cur != NULL) { gdb_cur->gdb_init(); printf("GDB: current port: %s\n", gdb_cur->gdb_name); } if (gdb_cur != NULL) { cur_pri = (boothowto & RB_GDB) ? 2 : 0; gdb_consinit(); } else cur_pri = -1; return (cur_pri); } static void gdb_do_mem_search(void) { size_t patlen; intmax_t addr, size; const unsigned char *found; if (gdb_rx_varhex(&addr) || gdb_rx_char() != ';' || gdb_rx_varhex(&size) || gdb_rx_char() != ';' || gdb_rx_bindata(gdb_bindata, sizeof(gdb_bindata), &patlen)) { gdb_tx_err(EINVAL); return; } if (gdb_search_mem((char *)(uintptr_t)addr, size, gdb_bindata, patlen, &found)) { if (found == 0ULL) gdb_tx_begin('0'); else { gdb_tx_begin('1'); gdb_tx_char(','); gdb_tx_hex((intmax_t)(uintptr_t)found, 8); } gdb_tx_end(); } else gdb_tx_err(EIO); } static void gdb_do_threadinfo(struct thread **thr_iter) { static struct thread * const done_sentinel = (void *)(uintptr_t)1; static const size_t tidsz_hex = sizeof(lwpid_t) * 2; size_t tds_sent; if (*thr_iter == NULL) { gdb_tx_err(ENXIO); return; } if (*thr_iter == done_sentinel) { gdb_tx_begin('l'); *thr_iter = NULL; goto sendit; } gdb_tx_begin('m'); for (tds_sent = 0; *thr_iter != NULL && gdb_txbuf_has_capacity(tidsz_hex + 1); *thr_iter = kdb_thr_next(*thr_iter), tds_sent++) { if (tds_sent > 0) gdb_tx_char(','); gdb_tx_varhex((*thr_iter)->td_tid); } /* * Can't send EOF and "some" in same packet, so set a sentinel to send * EOF when GDB asks us next. */ if (*thr_iter == NULL && tds_sent > 0) *thr_iter = done_sentinel; sendit: gdb_tx_end(); } #define BIT(n) (1ull << (n)) enum { GDB_MULTIPROCESS, GDB_SWBREAK, GDB_HWBREAK, GDB_QRELOCINSN, GDB_FORK_EVENTS, GDB_VFORK_EVENTS, GDB_EXEC_EVENTS, GDB_VCONT_SUPPORTED, GDB_QTHREADEVENTS, GDB_NO_RESUMED, }; static const char * const gdb_feature_names[] = { [GDB_MULTIPROCESS] = "multiprocess", [GDB_SWBREAK] = "swbreak", [GDB_HWBREAK] = "hwbreak", [GDB_QRELOCINSN] = "qRelocInsn", [GDB_FORK_EVENTS] = "fork-events", [GDB_VFORK_EVENTS] = "vfork-events", [GDB_EXEC_EVENTS] = "exec-events", [GDB_VCONT_SUPPORTED] = "vContSupported", [GDB_QTHREADEVENTS] = "QThreadEvents", [GDB_NO_RESUMED] = "no-resumed", }; static void gdb_do_qsupported(uint32_t *feat) { char *tok, *delim, ok; size_t i, toklen; /* Parse supported host features */ *feat = 0; switch (gdb_rx_char()) { case ':': break; case EOF: goto nofeatures; default: goto error; } while (gdb_rxsz > 0) { tok = gdb_rxp; delim = strchrnul(gdb_rxp, ';'); toklen = (delim - tok); gdb_rxp += toklen; gdb_rxsz -= toklen; if (*delim != '\0') { *delim = '\0'; gdb_rxp += 1; gdb_rxsz -= 1; } if (toklen < 2) goto error; ok = tok[toklen - 1]; if (ok != '-' && ok != '+') { /* * GDB only has one KV-pair feature, and we don't * support it, so ignore and move on. */ if (strchr(tok, '=') != NULL) continue; /* Not a KV-pair, and not a +/- flag? Malformed. */ goto error; } if (ok != '+') continue; tok[toklen - 1] = '\0'; for (i = 0; i < nitems(gdb_feature_names); i++) if (strcmp(gdb_feature_names[i], tok) == 0) break; if (i == nitems(gdb_feature_names)) { /* Unknown GDB feature. */ continue; } *feat |= BIT(i); } nofeatures: /* Send a supported feature list back */ gdb_tx_begin(0); gdb_tx_str("PacketSize"); gdb_tx_char('='); /* * We don't buffer framing bytes, but we do need to retain a byte for a * trailing nul. */ gdb_tx_varhex(GDB_BUFSZ + strlen("$#nn") - 1); gdb_tx_str(";qXfer:threads:read+"); /* * If the debugport is a reliable transport, request No Ack mode from * the server. The server may or may not choose to enter No Ack mode. * https://sourceware.org/gdb/onlinedocs/gdb/Packet-Acknowledgment.html */ if (gdb_cur->gdb_dbfeatures & GDB_DBGP_FEAT_RELIABLE) gdb_tx_str(";QStartNoAckMode+"); /* * Future consideration: * - vCont * - multiprocess */ gdb_tx_end(); return; error: *feat = 0; gdb_tx_err(EINVAL); } /* * A qXfer_context provides a vaguely generic way to generate a multi-packet * response on the fly, making some assumptions about the size of sbuf writes * vs actual packet length constraints. A non-byzantine gdb host should allow * hundreds of bytes per packet or more. * * Upper layers are considered responsible for escaping the four forbidden * characters '# $ } *'. */ struct qXfer_context { struct sbuf sb; size_t last_offset; bool flushed; bool lastmessage; char xfer_buf[GDB_BUFSZ]; }; static int qXfer_drain(void *v, const char *buf, int len) { struct qXfer_context *qx; if (len < 0) return (-EINVAL); qx = v; if (qx->flushed) { /* * Overflow. We lost some message. Maybe the packet size is * ridiculously small. */ printf("%s: Overflow in qXfer detected.\n", __func__); return (-ENOBUFS); } qx->last_offset += len; qx->flushed = true; if (qx->lastmessage) gdb_tx_begin('l'); else gdb_tx_begin('m'); memcpy(gdb_txp, buf, len); gdb_txp += len; gdb_tx_end(); return (len); } static int init_qXfer_ctx(struct qXfer_context *qx, uintmax_t len) { /* Protocol (max) length field includes framing overhead. */ if (len < sizeof("$m#nn")) return (ENOSPC); len -= 4; len = ummin(len, GDB_BUFSZ - 1); qx->last_offset = 0; qx->flushed = false; qx->lastmessage = false; sbuf_new(&qx->sb, qx->xfer_buf, len, SBUF_FIXEDLEN); sbuf_set_drain(&qx->sb, qXfer_drain, qx); return (0); } /* * Squashes special XML and GDB characters down to _. Sorry. */ static void qXfer_escape_xmlattr_str(char *dst, size_t dstlen, const char *src) { static const char *forbidden = "#$}*"; size_t i; char c; for (i = 0; i < dstlen - 1 && *src != 0; src++, i++) { c = *src; /* XML attr filter */ if (c < 32) c = '_'; /* We assume attributes will be "" quoted. */ if (c == '<' || c == '&' || c == '"') c = '_'; /* GDB escape. */ if (strchr(forbidden, c) != NULL) { /* * It would be nice to escape these properly, but to do * it correctly we need to escape them in the transmit * layer, potentially doubling our buffer requirements. * For now, avoid breaking the protocol by squashing * them to underscore. */ #if 0 *dst++ = '}'; c ^= 0x20; #endif c = '_'; } *dst++ = c; } if (*src != 0) printf("XXX%s: overflow; API misuse\n", __func__); *dst = 0; } /* * Dynamically generate qXfer:threads document, one packet at a time. * * The format is loosely described[0], although it does not seem that the * mentioned on that page is required. * * [0]: https://sourceware.org/gdb/current/onlinedocs/gdb/Thread-List-Format.html */ static void do_qXfer_threads_read(void) { /* Kludgy context */ static struct { struct qXfer_context qXfer; /* Kludgy state machine */ struct thread *iter; enum { XML_START_THREAD, /* ' ...' */ XML_END_THREAD, /* '' */ XML_SENT_END_THREADS, /* '' */ } next_step; } ctx; static char td_name_escape[MAXCOMLEN * 2 + 1]; const char *name_src; uintmax_t offset, len; int error; /* Annex part must be empty. */ if (gdb_rx_char() != ':') goto misformed_request; if (gdb_rx_varhex(&offset) != 0 || gdb_rx_char() != ',' || gdb_rx_varhex(&len) != 0) goto misformed_request; /* * Validate resume xfers. */ if (offset != 0) { if (offset != ctx.qXfer.last_offset) { printf("%s: Resumed offset %ju != expected %zu\n", __func__, offset, ctx.qXfer.last_offset); error = ESPIPE; goto request_error; } ctx.qXfer.flushed = false; } if (offset == 0) { ctx.iter = kdb_thr_first(); ctx.next_step = XML_START_THREAD; error = init_qXfer_ctx(&ctx.qXfer, len); if (error != 0) goto request_error; sbuf_cat(&ctx.qXfer.sb, ""); } while (!ctx.qXfer.flushed && ctx.iter != NULL) { switch (ctx.next_step) { case XML_START_THREAD: ctx.next_step = XML_THREAD_ID; sbuf_cat(&ctx.qXfer.sb, "td_tid); continue; case XML_THREAD_CORE: ctx.next_step = XML_THREAD_NAME; if (ctx.iter->td_oncpu != NOCPU) { sbuf_printf(&ctx.qXfer.sb, " core=\"%d\"", ctx.iter->td_oncpu); } continue; case XML_THREAD_NAME: ctx.next_step = XML_THREAD_EXTRA; if (ctx.iter->td_name[0] != 0) name_src = ctx.iter->td_name; else if (ctx.iter->td_proc != NULL && ctx.iter->td_proc->p_comm[0] != 0) name_src = ctx.iter->td_proc->p_comm; else continue; qXfer_escape_xmlattr_str(td_name_escape, sizeof(td_name_escape), name_src); sbuf_printf(&ctx.qXfer.sb, " name=\"%s\"", td_name_escape); continue; case XML_THREAD_EXTRA: ctx.next_step = XML_END_THREAD; sbuf_putc(&ctx.qXfer.sb, '>'); if (ctx.iter->td_state == TDS_RUNNING) sbuf_cat(&ctx.qXfer.sb, "Running"); else if (ctx.iter->td_state == TDS_RUNQ) sbuf_cat(&ctx.qXfer.sb, "RunQ"); else if (ctx.iter->td_state == TDS_CAN_RUN) sbuf_cat(&ctx.qXfer.sb, "CanRun"); else if (TD_ON_LOCK(ctx.iter)) sbuf_cat(&ctx.qXfer.sb, "Blocked"); else if (TD_IS_SLEEPING(ctx.iter)) sbuf_cat(&ctx.qXfer.sb, "Sleeping"); else if (TD_IS_SWAPPED(ctx.iter)) sbuf_cat(&ctx.qXfer.sb, "Swapped"); else if (TD_AWAITING_INTR(ctx.iter)) sbuf_cat(&ctx.qXfer.sb, "IthreadWait"); else if (TD_IS_SUSPENDED(ctx.iter)) sbuf_cat(&ctx.qXfer.sb, "Suspended"); else sbuf_cat(&ctx.qXfer.sb, "???"); continue; case XML_END_THREAD: ctx.next_step = XML_START_THREAD; sbuf_cat(&ctx.qXfer.sb, ""); ctx.iter = kdb_thr_next(ctx.iter); continue; /* * This one isn't part of the looping state machine, * but GCC complains if you leave an enum value out of the * select. */ case XML_SENT_END_THREADS: /* NOTREACHED */ break; } } if (ctx.qXfer.flushed) return; if (ctx.next_step != XML_SENT_END_THREADS) { ctx.next_step = XML_SENT_END_THREADS; sbuf_cat(&ctx.qXfer.sb, ""); } if (ctx.qXfer.flushed) return; ctx.qXfer.lastmessage = true; sbuf_finish(&ctx.qXfer.sb); sbuf_delete(&ctx.qXfer.sb); ctx.qXfer.last_offset = 0; return; misformed_request: /* * GDB "General-Query-Packets.html" qXfer-read anchor specifically * documents an E00 code for malformed requests or invalid annex. * Non-zero codes indicate invalid offset or "error reading the data." */ error = 0; request_error: gdb_tx_err(error); return; } /* * A set of standardized transfers from "special data areas." * * We've already matched on "qXfer:" and advanced the rx packet buffer past * that bit. Parse out the rest of the packet and generate an appropriate * response. */ static void do_qXfer(void) { if (!gdb_rx_equal("threads:")) goto unrecognized; if (!gdb_rx_equal("read:")) goto unrecognized; do_qXfer_threads_read(); return; unrecognized: gdb_tx_empty(); return; } static void gdb_handle_detach(void) { kdb_cpu_clear_singlestep(); gdb_listening = 0; if (gdb_cur->gdb_dbfeatures & GDB_DBGP_FEAT_WANTTERM) gdb_cur->gdb_term(); #ifdef DDB if (!gdb_return_to_ddb) return; gdb_return_to_ddb = false; if (kdb_dbbe_select("ddb") != 0) printf("The ddb backend could not be selected.\n"); #endif } +/* + * Handle a 'Z' packet: set a breakpoint or watchpoint. + * + * Currently, only watchpoints are supported. + */ +static void +gdb_z_insert(void) +{ + intmax_t addr, length; + char ztype; + int error; + + ztype = gdb_rx_char(); + if (gdb_rx_char() != ',' || gdb_rx_varhex(&addr) || + gdb_rx_char() != ',' || gdb_rx_varhex(&length)) { + error = EINVAL; + goto fail; + } + + switch (ztype) { + case '2': /* write watchpoint */ + error = kdb_cpu_set_watchpoint((vm_offset_t)addr, + (vm_size_t)length, KDB_DBG_ACCESS_W); + break; + case '3': /* read watchpoint */ + error = kdb_cpu_set_watchpoint((vm_offset_t)addr, + (vm_size_t)length, KDB_DBG_ACCESS_R); + break; + case '4': /* access (RW) watchpoint */ + error = kdb_cpu_set_watchpoint((vm_offset_t)addr, + (vm_size_t)length, KDB_DBG_ACCESS_RW); + break; + case '1': /* hardware breakpoint */ + case '0': /* software breakpoint */ + /* Not implemented. */ + gdb_tx_empty(); + return; + default: + error = EINVAL; + break; + } + if (error != 0) + goto fail; + gdb_tx_ok(); + return; +fail: + gdb_tx_err(error); + return; +} + +/* + * Handle a 'z' packet; clear a breakpoint or watchpoint. + * + * Currently, only watchpoints are supported. + */ +static void +gdb_z_remove(void) +{ + intmax_t addr, length; + char ztype; + int error; + + ztype = gdb_rx_char(); + if (gdb_rx_char() != ',' || gdb_rx_varhex(&addr) || + gdb_rx_char() != ',' || gdb_rx_varhex(&length)) { + error = EINVAL; + goto fail; + } + + switch (ztype) { + case '2': /* write watchpoint */ + case '3': /* read watchpoint */ + case '4': /* access (RW) watchpoint */ + error = kdb_cpu_clr_watchpoint((vm_offset_t)addr, + (vm_size_t)length); + break; + case '1': /* hardware breakpoint */ + case '0': /* software breakpoint */ + /* Not implemented. */ + gdb_tx_empty(); + return; + default: + error = EINVAL; + break; + } + if (error != 0) + goto fail; + gdb_tx_ok(); + return; +fail: + gdb_tx_err(error); + return; +} + static int gdb_trap(int type, int code) { jmp_buf jb; struct thread *thr_iter; void *prev_jb; uint32_t host_features; prev_jb = kdb_jmpbuf(jb); if (setjmp(jb) != 0) { printf("%s bailing, hopefully back to ddb!\n", __func__); gdb_listening = 0; (void)kdb_jmpbuf(prev_jb); return (1); } gdb_listening = 0; gdb_ackmode = true; /* * Send a T packet. We currently do not support watchpoints (the * awatch, rwatch or watch elements). */ gdb_tx_begin('T'); gdb_tx_hex(gdb_cpu_signal(type, code), 2); gdb_tx_varhex(GDB_REG_PC); gdb_tx_char(':'); gdb_tx_reg(GDB_REG_PC); gdb_tx_char(';'); gdb_tx_str("thread:"); gdb_tx_varhex((long)kdb_thread->td_tid); gdb_tx_char(';'); gdb_tx_end(); /* XXX check error condition. */ thr_iter = NULL; while (gdb_rx_begin() == 0) { /* printf("GDB: got '%s'\n", gdb_rxp); */ switch (gdb_rx_char()) { case '?': /* Last signal. */ gdb_tx_begin('T'); gdb_tx_hex(gdb_cpu_signal(type, code), 2); gdb_tx_str("thread:"); gdb_tx_varhex((long)kdb_thread->td_tid); gdb_tx_char(';'); gdb_tx_end(); break; case 'c': { /* Continue. */ uintmax_t addr; register_t pc; if (!gdb_rx_varhex(&addr)) { pc = addr; gdb_cpu_setreg(GDB_REG_PC, &pc); } kdb_cpu_clear_singlestep(); gdb_listening = 1; return (1); } case 'C': { /* Continue with signal. */ uintmax_t addr, sig; register_t pc; if (!gdb_rx_varhex(&sig) && gdb_rx_char() == ';' && !gdb_rx_varhex(&addr)) { pc = addr; gdb_cpu_setreg(GDB_REG_PC, &pc); } kdb_cpu_clear_singlestep(); gdb_listening = 1; return (1); } case 'D': { /* Detach */ gdb_tx_ok(); gdb_handle_detach(); return (1); } case 'g': { /* Read registers. */ size_t r; gdb_tx_begin(0); for (r = 0; r < GDB_NREGS; r++) gdb_tx_reg(r); gdb_tx_end(); break; } case 'G': { /* Write registers. */ char *val; bool success; size_t r; for (success = true, r = 0; r < GDB_NREGS; r++) { val = gdb_rxp; if (!gdb_rx_mem(val, gdb_cpu_regsz(r))) { gdb_tx_err(EINVAL); success = false; break; } gdb_cpu_setreg(r, val); } if (success) gdb_tx_ok(); break; } case 'H': { /* Set thread. */ intmax_t tid; struct thread *thr; /* Ignore 'g' (general) or 'c' (continue) flag. */ (void) gdb_rx_char(); if (gdb_rx_varhex(&tid)) { gdb_tx_err(EINVAL); break; } if (tid > 0) { thr = kdb_thr_lookup(tid); if (thr == NULL) { gdb_tx_err(ENOENT); break; } kdb_thr_select(thr); } gdb_tx_ok(); break; } case 'k': /* Kill request. */ gdb_handle_detach(); return (1); case 'm': { /* Read memory. */ uintmax_t addr, size; if (gdb_rx_varhex(&addr) || gdb_rx_char() != ',' || gdb_rx_varhex(&size)) { gdb_tx_err(EINVAL); break; } gdb_tx_begin(0); if (gdb_tx_mem((char *)(uintptr_t)addr, size)) gdb_tx_end(); else gdb_tx_err(EIO); break; } case 'M': { /* Write memory. */ uintmax_t addr, size; if (gdb_rx_varhex(&addr) || gdb_rx_char() != ',' || gdb_rx_varhex(&size) || gdb_rx_char() != ':') { gdb_tx_err(EINVAL); break; } if (gdb_rx_mem((char *)(uintptr_t)addr, size) == 0) gdb_tx_err(EIO); else gdb_tx_ok(); break; } case 'p': { /* Read register. */ uintmax_t reg; if (gdb_rx_varhex(®)) { gdb_tx_err(EINVAL); break; } gdb_tx_begin(0); gdb_tx_reg(reg); gdb_tx_end(); break; } case 'P': { /* Write register. */ char *val; uintmax_t reg; val = gdb_rxp; if (gdb_rx_varhex(®) || gdb_rx_char() != '=' || !gdb_rx_mem(val, gdb_cpu_regsz(reg))) { gdb_tx_err(EINVAL); break; } gdb_cpu_setreg(reg, val); gdb_tx_ok(); break; } case 'q': /* General query. */ if (gdb_rx_equal("C")) { gdb_tx_begin('Q'); gdb_tx_char('C'); gdb_tx_varhex((long)kdb_thread->td_tid); gdb_tx_end(); } else if (gdb_rx_equal("Supported")) { gdb_do_qsupported(&host_features); } else if (gdb_rx_equal("fThreadInfo")) { thr_iter = kdb_thr_first(); gdb_do_threadinfo(&thr_iter); } else if (gdb_rx_equal("sThreadInfo")) { gdb_do_threadinfo(&thr_iter); } else if (gdb_rx_equal("Xfer:")) { do_qXfer(); } else if (gdb_rx_equal("Search:memory:")) { gdb_do_mem_search(); #ifdef __powerpc__ } else if (gdb_rx_equal("Offsets")) { gdb_cpu_do_offsets(); #endif } else if (!gdb_cpu_query()) gdb_tx_empty(); break; case 'Q': if (gdb_rx_equal("StartNoAckMode")) { if ((gdb_cur->gdb_dbfeatures & GDB_DBGP_FEAT_RELIABLE) == 0) { /* * Shouldn't happen if we didn't * advertise support. Reject. */ gdb_tx_empty(); break; } gdb_ackmode = false; gdb_tx_ok(); } else gdb_tx_empty(); break; case 's': { /* Step. */ uintmax_t addr; register_t pc; if (!gdb_rx_varhex(&addr)) { pc = addr; gdb_cpu_setreg(GDB_REG_PC, &pc); } kdb_cpu_set_singlestep(); gdb_listening = 1; return (1); } case 'S': { /* Step with signal. */ uintmax_t addr, sig; register_t pc; if (!gdb_rx_varhex(&sig) && gdb_rx_char() == ';' && !gdb_rx_varhex(&addr)) { pc = addr; gdb_cpu_setreg(GDB_REG_PC, &pc); } kdb_cpu_set_singlestep(); gdb_listening = 1; return (1); } case 'T': { /* Thread alive. */ intmax_t tid; if (gdb_rx_varhex(&tid)) { gdb_tx_err(EINVAL); break; } if (kdb_thr_lookup(tid) != NULL) gdb_tx_ok(); else gdb_tx_err(ENOENT); break; } + case 'z': { /* Remove watchpoint. */ + gdb_z_remove(); + break; + } + case 'Z': { /* Set watchpoint. */ + gdb_z_insert(); + break; + } case EOF: /* Empty command. Treat as unknown command. */ /* FALLTHROUGH */ default: /* Unknown command. Send empty response. */ gdb_tx_empty(); break; } } (void)kdb_jmpbuf(prev_jb); return (0); } diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c index 0f528f6567ee..2dcdf923c467 100644 --- a/sys/x86/x86/mp_x86.c +++ b/sys/x86/x86/mp_x86.c @@ -1,1679 +1,1680 @@ /*- * Copyright (c) 1996, by Steve Passe * Copyright (c) 2003, by Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. The name of the developer may NOT be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #ifdef __i386__ #include "opt_apic.h" #endif #include "opt_cpu.h" #include "opt_ddb.h" +#include "opt_gdb.h" #include "opt_kstack_pages.h" #include "opt_pmap.h" #include "opt_sched.h" #include "opt_smp.h" #include "opt_stack.h" #include #include #include #include /* cngetc() */ #include #include #ifdef GPROF #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static MALLOC_DEFINE(M_CPUS, "cpus", "CPU items"); /* lock region used by kernel profiling */ int mcount_lock; int mp_naps; /* # of Applications processors */ int boot_cpu_id = -1; /* designated BSP */ /* AP uses this during bootstrap. Do not staticize. */ char *bootSTK; int bootAP; /* Free these after use */ void *bootstacks[MAXCPU]; void *dpcpu; struct pcb stoppcbs[MAXCPU]; struct susppcb **susppcbs; #ifdef COUNT_IPIS /* Interrupt counts. */ static u_long *ipi_preempt_counts[MAXCPU]; static u_long *ipi_ast_counts[MAXCPU]; u_long *ipi_invltlb_counts[MAXCPU]; u_long *ipi_invlrng_counts[MAXCPU]; u_long *ipi_invlpg_counts[MAXCPU]; u_long *ipi_invlcache_counts[MAXCPU]; u_long *ipi_rendezvous_counts[MAXCPU]; static u_long *ipi_hardclock_counts[MAXCPU]; #endif /* Default cpu_ops implementation. */ struct cpu_ops cpu_ops; /* * Local data and functions. */ static volatile cpuset_t ipi_stop_nmi_pending; volatile cpuset_t resuming_cpus; volatile cpuset_t toresume_cpus; /* used to hold the AP's until we are ready to release them */ struct mtx ap_boot_mtx; /* Set to 1 once we're ready to let the APs out of the pen. */ volatile int aps_ready = 0; /* * Store data from cpu_add() until later in the boot when we actually setup * the APs. */ struct cpu_info *cpu_info; int *apic_cpuids; int cpu_apic_ids[MAXCPU]; _Static_assert(MAXCPU <= MAX_APIC_ID, "MAXCPU cannot be larger that MAX_APIC_ID"); _Static_assert(xAPIC_MAX_APIC_ID <= MAX_APIC_ID, "xAPIC_MAX_APIC_ID cannot be larger that MAX_APIC_ID"); static void release_aps(void *dummy); static void cpustop_handler_post(u_int cpu); static int hyperthreading_allowed = 1; SYSCTL_INT(_machdep, OID_AUTO, hyperthreading_allowed, CTLFLAG_RDTUN, &hyperthreading_allowed, 0, "Use Intel HTT logical CPUs"); static int hyperthreading_intr_allowed = 0; SYSCTL_INT(_machdep, OID_AUTO, hyperthreading_intr_allowed, CTLFLAG_RDTUN, &hyperthreading_intr_allowed, 0, "Allow interrupts on HTT logical CPUs"); static struct topo_node topo_root; static int pkg_id_shift; static int node_id_shift; static int core_id_shift; static int disabled_cpus; struct cache_info { int id_shift; int present; } static caches[MAX_CACHE_LEVELS]; unsigned int boot_address; static bool stop_mwait = false; SYSCTL_BOOL(_machdep, OID_AUTO, stop_mwait, CTLFLAG_RWTUN, &stop_mwait, 0, "Use MONITOR/MWAIT when stopping CPU, if available"); #define MiB(v) (v ## ULL << 20) void mem_range_AP_init(void) { if (mem_range_softc.mr_op && mem_range_softc.mr_op->initAP) mem_range_softc.mr_op->initAP(&mem_range_softc); } /* * Round up to the next power of two, if necessary, and then * take log2. * Returns -1 if argument is zero. */ static __inline int mask_width(u_int x) { return (fls(x << (1 - powerof2(x))) - 1); } /* * Add a cache level to the cache topology description. */ static int add_deterministic_cache(int type, int level, int share_count) { if (type == 0) return (0); if (type > 3) { printf("unexpected cache type %d\n", type); return (1); } if (type == 2) /* ignore instruction cache */ return (1); if (level == 0 || level > MAX_CACHE_LEVELS) { printf("unexpected cache level %d\n", type); return (1); } if (caches[level - 1].present) { printf("WARNING: multiple entries for L%u data cache\n", level); printf("%u => %u\n", caches[level - 1].id_shift, mask_width(share_count)); } caches[level - 1].id_shift = mask_width(share_count); caches[level - 1].present = 1; if (caches[level - 1].id_shift > pkg_id_shift) { printf("WARNING: L%u data cache covers more " "APIC IDs than a package (%u > %u)\n", level, caches[level - 1].id_shift, pkg_id_shift); caches[level - 1].id_shift = pkg_id_shift; } if (caches[level - 1].id_shift < core_id_shift) { printf("WARNING: L%u data cache covers fewer " "APIC IDs than a core (%u < %u)\n", level, caches[level - 1].id_shift, core_id_shift); caches[level - 1].id_shift = core_id_shift; } return (1); } /* * Determine topology of processing units and caches for AMD CPUs. * See: * - AMD CPUID Specification (Publication # 25481) * - BKDG for AMD NPT Family 0Fh Processors (Publication # 32559) * - BKDG For AMD Family 10h Processors (Publication # 31116) * - BKDG For AMD Family 15h Models 00h-0Fh Processors (Publication # 42301) * - BKDG For AMD Family 16h Models 00h-0Fh Processors (Publication # 48751) * - PPR For AMD Family 17h Models 00h-0Fh Processors (Publication # 54945) */ static void topo_probe_amd(void) { u_int p[4]; uint64_t v; int level; int nodes_per_socket; int share_count; int type; int i; /* No multi-core capability. */ if ((amd_feature2 & AMDID2_CMP) == 0) return; /* For families 10h and newer. */ pkg_id_shift = (cpu_procinfo2 & AMDID_COREID_SIZE) >> AMDID_COREID_SIZE_SHIFT; /* For 0Fh family. */ if (pkg_id_shift == 0) pkg_id_shift = mask_width((cpu_procinfo2 & AMDID_CMP_CORES) + 1); /* * Families prior to 16h define the following value as * cores per compute unit and we don't really care about the AMD * compute units at the moment. Perhaps we should treat them as * cores and cores within the compute units as hardware threads, * but that's up for debate. * Later families define the value as threads per compute unit, * so we are following AMD's nomenclature here. */ if ((amd_feature2 & AMDID2_TOPOLOGY) != 0 && CPUID_TO_FAMILY(cpu_id) >= 0x16) { cpuid_count(0x8000001e, 0, p); share_count = ((p[1] >> 8) & 0xff) + 1; core_id_shift = mask_width(share_count); /* * For Zen (17h), gather Nodes per Processor. Each node is a * Zeppelin die; TR and EPYC CPUs will have multiple dies per * package. Communication latency between dies is higher than * within them. */ nodes_per_socket = ((p[2] >> 8) & 0x7) + 1; node_id_shift = pkg_id_shift - mask_width(nodes_per_socket); } if ((amd_feature2 & AMDID2_TOPOLOGY) != 0) { for (i = 0; ; i++) { cpuid_count(0x8000001d, i, p); type = p[0] & 0x1f; level = (p[0] >> 5) & 0x7; share_count = 1 + ((p[0] >> 14) & 0xfff); if (!add_deterministic_cache(type, level, share_count)) break; } } else { if (cpu_exthigh >= 0x80000005) { cpuid_count(0x80000005, 0, p); if (((p[2] >> 24) & 0xff) != 0) { caches[0].id_shift = 0; caches[0].present = 1; } } if (cpu_exthigh >= 0x80000006) { cpuid_count(0x80000006, 0, p); if (((p[2] >> 16) & 0xffff) != 0) { caches[1].id_shift = 0; caches[1].present = 1; } if (((p[3] >> 18) & 0x3fff) != 0) { nodes_per_socket = 1; if ((amd_feature2 & AMDID2_NODE_ID) != 0) { /* * Handle multi-node processors that * have multiple chips, each with its * own L3 cache, on the same die. */ v = rdmsr(0xc001100c); nodes_per_socket = 1 + ((v >> 3) & 0x7); } caches[2].id_shift = pkg_id_shift - mask_width(nodes_per_socket); caches[2].present = 1; } } } } /* * Determine topology of processing units for Intel CPUs * using CPUID Leaf 1 and Leaf 4, if supported. * See: * - Intel 64 Architecture Processor Topology Enumeration * - Intel 64 and IA-32 ArchitecturesSoftware Developer’s Manual, * Volume 3A: System Programming Guide, PROGRAMMING CONSIDERATIONS * FOR HARDWARE MULTI-THREADING CAPABLE PROCESSORS */ static void topo_probe_intel_0x4(void) { u_int p[4]; int max_cores; int max_logical; /* Both zero and one here mean one logical processor per package. */ max_logical = (cpu_feature & CPUID_HTT) != 0 ? (cpu_procinfo & CPUID_HTT_CORES) >> 16 : 1; if (max_logical <= 1) return; if (cpu_high >= 0x4) { cpuid_count(0x04, 0, p); max_cores = ((p[0] >> 26) & 0x3f) + 1; } else max_cores = 1; core_id_shift = mask_width(max_logical/max_cores); KASSERT(core_id_shift >= 0, ("intel topo: max_cores > max_logical\n")); pkg_id_shift = core_id_shift + mask_width(max_cores); } /* * Determine topology of processing units for Intel CPUs * using CPUID Leaf 11, if supported. * See: * - Intel 64 Architecture Processor Topology Enumeration * - Intel 64 and IA-32 ArchitecturesSoftware Developer’s Manual, * Volume 3A: System Programming Guide, PROGRAMMING CONSIDERATIONS * FOR HARDWARE MULTI-THREADING CAPABLE PROCESSORS */ static void topo_probe_intel_0xb(void) { u_int p[4]; int bits; int type; int i; /* Fall back if CPU leaf 11 doesn't really exist. */ cpuid_count(0x0b, 0, p); if (p[1] == 0) { topo_probe_intel_0x4(); return; } /* We only support three levels for now. */ for (i = 0; ; i++) { cpuid_count(0x0b, i, p); bits = p[0] & 0x1f; type = (p[2] >> 8) & 0xff; if (type == 0) break; /* TODO: check for duplicate (re-)assignment */ if (type == CPUID_TYPE_SMT) core_id_shift = bits; else if (type == CPUID_TYPE_CORE) pkg_id_shift = bits; else printf("unknown CPU level type %d\n", type); } if (pkg_id_shift < core_id_shift) { printf("WARNING: core covers more APIC IDs than a package\n"); core_id_shift = pkg_id_shift; } } /* * Determine topology of caches for Intel CPUs. * See: * - Intel 64 Architecture Processor Topology Enumeration * - Intel 64 and IA-32 Architectures Software Developer’s Manual * Volume 2A: Instruction Set Reference, A-M, * CPUID instruction */ static void topo_probe_intel_caches(void) { u_int p[4]; int level; int share_count; int type; int i; if (cpu_high < 0x4) { /* * Available cache level and sizes can be determined * via CPUID leaf 2, but that requires a huge table of hardcoded * values, so for now just assume L1 and L2 caches potentially * shared only by HTT processing units, if HTT is present. */ caches[0].id_shift = pkg_id_shift; caches[0].present = 1; caches[1].id_shift = pkg_id_shift; caches[1].present = 1; return; } for (i = 0; ; i++) { cpuid_count(0x4, i, p); type = p[0] & 0x1f; level = (p[0] >> 5) & 0x7; share_count = 1 + ((p[0] >> 14) & 0xfff); if (!add_deterministic_cache(type, level, share_count)) break; } } /* * Determine topology of processing units and caches for Intel CPUs. * See: * - Intel 64 Architecture Processor Topology Enumeration */ static void topo_probe_intel(void) { /* * Note that 0x1 <= cpu_high < 4 case should be * compatible with topo_probe_intel_0x4() logic when * CPUID.1:EBX[23:16] > 0 (cpu_cores will be 1) * or it should trigger the fallback otherwise. */ if (cpu_high >= 0xb) topo_probe_intel_0xb(); else if (cpu_high >= 0x1) topo_probe_intel_0x4(); topo_probe_intel_caches(); } /* * Topology information is queried only on BSP, on which this * code runs and for which it can query CPUID information. * Then topology is extrapolated on all packages using an * assumption that APIC ID to hardware component ID mapping is * homogenious. * That doesn't necesserily imply that the topology is uniform. */ void topo_probe(void) { static int cpu_topo_probed = 0; struct x86_topo_layer { int type; int subtype; int id_shift; } topo_layers[MAX_CACHE_LEVELS + 4]; struct topo_node *parent; struct topo_node *node; int layer; int nlayers; int node_id; int i; if (cpu_topo_probed) return; CPU_ZERO(&logical_cpus_mask); if (mp_ncpus <= 1) ; /* nothing */ else if (cpu_vendor_id == CPU_VENDOR_AMD || cpu_vendor_id == CPU_VENDOR_HYGON) topo_probe_amd(); else if (cpu_vendor_id == CPU_VENDOR_INTEL) topo_probe_intel(); KASSERT(pkg_id_shift >= core_id_shift, ("bug in APIC topology discovery")); nlayers = 0; bzero(topo_layers, sizeof(topo_layers)); topo_layers[nlayers].type = TOPO_TYPE_PKG; topo_layers[nlayers].id_shift = pkg_id_shift; if (bootverbose) printf("Package ID shift: %u\n", topo_layers[nlayers].id_shift); nlayers++; if (pkg_id_shift > node_id_shift && node_id_shift != 0) { topo_layers[nlayers].type = TOPO_TYPE_GROUP; topo_layers[nlayers].id_shift = node_id_shift; if (bootverbose) printf("Node ID shift: %u\n", topo_layers[nlayers].id_shift); nlayers++; } /* * Consider all caches to be within a package/chip * and "in front" of all sub-components like * cores and hardware threads. */ for (i = MAX_CACHE_LEVELS - 1; i >= 0; --i) { if (caches[i].present) { if (node_id_shift != 0) KASSERT(caches[i].id_shift <= node_id_shift, ("bug in APIC topology discovery")); KASSERT(caches[i].id_shift <= pkg_id_shift, ("bug in APIC topology discovery")); KASSERT(caches[i].id_shift >= core_id_shift, ("bug in APIC topology discovery")); topo_layers[nlayers].type = TOPO_TYPE_CACHE; topo_layers[nlayers].subtype = i + 1; topo_layers[nlayers].id_shift = caches[i].id_shift; if (bootverbose) printf("L%u cache ID shift: %u\n", topo_layers[nlayers].subtype, topo_layers[nlayers].id_shift); nlayers++; } } if (pkg_id_shift > core_id_shift) { topo_layers[nlayers].type = TOPO_TYPE_CORE; topo_layers[nlayers].id_shift = core_id_shift; if (bootverbose) printf("Core ID shift: %u\n", topo_layers[nlayers].id_shift); nlayers++; } topo_layers[nlayers].type = TOPO_TYPE_PU; topo_layers[nlayers].id_shift = 0; nlayers++; topo_init_root(&topo_root); for (i = 0; i <= max_apic_id; ++i) { if (!cpu_info[i].cpu_present) continue; parent = &topo_root; for (layer = 0; layer < nlayers; ++layer) { node_id = i >> topo_layers[layer].id_shift; parent = topo_add_node_by_hwid(parent, node_id, topo_layers[layer].type, topo_layers[layer].subtype); } } parent = &topo_root; for (layer = 0; layer < nlayers; ++layer) { node_id = boot_cpu_id >> topo_layers[layer].id_shift; node = topo_find_node_by_hwid(parent, node_id, topo_layers[layer].type, topo_layers[layer].subtype); topo_promote_child(node); parent = node; } cpu_topo_probed = 1; } /* * Assign logical CPU IDs to local APICs. */ void assign_cpu_ids(void) { struct topo_node *node; u_int smt_mask; int nhyper; smt_mask = (1u << core_id_shift) - 1; /* * Assign CPU IDs to local APIC IDs and disable any CPUs * beyond MAXCPU. CPU 0 is always assigned to the BSP. */ mp_ncpus = 0; nhyper = 0; TOPO_FOREACH(node, &topo_root) { if (node->type != TOPO_TYPE_PU) continue; if ((node->hwid & smt_mask) != (boot_cpu_id & smt_mask)) cpu_info[node->hwid].cpu_hyperthread = 1; if (resource_disabled("lapic", node->hwid)) { if (node->hwid != boot_cpu_id) cpu_info[node->hwid].cpu_disabled = 1; else printf("Cannot disable BSP, APIC ID = %d\n", node->hwid); } if (!hyperthreading_allowed && cpu_info[node->hwid].cpu_hyperthread) cpu_info[node->hwid].cpu_disabled = 1; if (mp_ncpus >= MAXCPU) cpu_info[node->hwid].cpu_disabled = 1; if (cpu_info[node->hwid].cpu_disabled) { disabled_cpus++; continue; } if (cpu_info[node->hwid].cpu_hyperthread) nhyper++; cpu_apic_ids[mp_ncpus] = node->hwid; apic_cpuids[node->hwid] = mp_ncpus; topo_set_pu_id(node, mp_ncpus); mp_ncpus++; } KASSERT(mp_maxid >= mp_ncpus - 1, ("%s: counters out of sync: max %d, count %d", __func__, mp_maxid, mp_ncpus)); mp_ncores = mp_ncpus - nhyper; smp_threads_per_core = mp_ncpus / mp_ncores; } /* * Print various information about the SMP system hardware and setup. */ void cpu_mp_announce(void) { struct topo_node *node; const char *hyperthread; struct topo_analysis topology; printf("FreeBSD/SMP: "); if (topo_analyze(&topo_root, 1, &topology)) { printf("%d package(s)", topology.entities[TOPO_LEVEL_PKG]); if (topology.entities[TOPO_LEVEL_GROUP] > 1) printf(" x %d groups", topology.entities[TOPO_LEVEL_GROUP]); if (topology.entities[TOPO_LEVEL_CACHEGROUP] > 1) printf(" x %d cache groups", topology.entities[TOPO_LEVEL_CACHEGROUP]); if (topology.entities[TOPO_LEVEL_CORE] > 0) printf(" x %d core(s)", topology.entities[TOPO_LEVEL_CORE]); if (topology.entities[TOPO_LEVEL_THREAD] > 1) printf(" x %d hardware threads", topology.entities[TOPO_LEVEL_THREAD]); } else { printf("Non-uniform topology"); } printf("\n"); if (disabled_cpus) { printf("FreeBSD/SMP Online: "); if (topo_analyze(&topo_root, 0, &topology)) { printf("%d package(s)", topology.entities[TOPO_LEVEL_PKG]); if (topology.entities[TOPO_LEVEL_GROUP] > 1) printf(" x %d groups", topology.entities[TOPO_LEVEL_GROUP]); if (topology.entities[TOPO_LEVEL_CACHEGROUP] > 1) printf(" x %d cache groups", topology.entities[TOPO_LEVEL_CACHEGROUP]); if (topology.entities[TOPO_LEVEL_CORE] > 0) printf(" x %d core(s)", topology.entities[TOPO_LEVEL_CORE]); if (topology.entities[TOPO_LEVEL_THREAD] > 1) printf(" x %d hardware threads", topology.entities[TOPO_LEVEL_THREAD]); } else { printf("Non-uniform topology"); } printf("\n"); } if (!bootverbose) return; TOPO_FOREACH(node, &topo_root) { switch (node->type) { case TOPO_TYPE_PKG: printf("Package HW ID = %u\n", node->hwid); break; case TOPO_TYPE_CORE: printf("\tCore HW ID = %u\n", node->hwid); break; case TOPO_TYPE_PU: if (cpu_info[node->hwid].cpu_hyperthread) hyperthread = "/HT"; else hyperthread = ""; if (node->subtype == 0) printf("\t\tCPU (AP%s): APIC ID: %u" "(disabled)\n", hyperthread, node->hwid); else if (node->id == 0) printf("\t\tCPU0 (BSP): APIC ID: %u\n", node->hwid); else printf("\t\tCPU%u (AP%s): APIC ID: %u\n", node->id, hyperthread, node->hwid); break; default: /* ignored */ break; } } } /* * Add a scheduling group, a group of logical processors sharing * a particular cache (and, thus having an affinity), to the scheduling * topology. * This function recursively works on lower level caches. */ static void x86topo_add_sched_group(struct topo_node *root, struct cpu_group *cg_root) { struct topo_node *node; int nchildren; int ncores; int i; KASSERT(root->type == TOPO_TYPE_SYSTEM || root->type == TOPO_TYPE_CACHE || root->type == TOPO_TYPE_GROUP, ("x86topo_add_sched_group: bad type: %u", root->type)); CPU_COPY(&root->cpuset, &cg_root->cg_mask); cg_root->cg_count = root->cpu_count; if (root->type == TOPO_TYPE_SYSTEM) cg_root->cg_level = CG_SHARE_NONE; else cg_root->cg_level = root->subtype; /* * Check how many core nodes we have under the given root node. * If we have multiple logical processors, but not multiple * cores, then those processors must be hardware threads. */ ncores = 0; node = root; while (node != NULL) { if (node->type != TOPO_TYPE_CORE) { node = topo_next_node(root, node); continue; } ncores++; node = topo_next_nonchild_node(root, node); } if (cg_root->cg_level != CG_SHARE_NONE && root->cpu_count > 1 && ncores < 2) cg_root->cg_flags = CG_FLAG_SMT; /* * Find out how many cache nodes we have under the given root node. * We ignore cache nodes that cover all the same processors as the * root node. Also, we do not descend below found cache nodes. * That is, we count top-level "non-redundant" caches under the root * node. */ nchildren = 0; node = root; while (node != NULL) { if ((node->type != TOPO_TYPE_GROUP && node->type != TOPO_TYPE_CACHE) || (root->type != TOPO_TYPE_SYSTEM && CPU_CMP(&node->cpuset, &root->cpuset) == 0)) { node = topo_next_node(root, node); continue; } nchildren++; node = topo_next_nonchild_node(root, node); } cg_root->cg_child = smp_topo_alloc(nchildren); cg_root->cg_children = nchildren; /* * Now find again the same cache nodes as above and recursively * build scheduling topologies for them. */ node = root; i = 0; while (node != NULL) { if ((node->type != TOPO_TYPE_GROUP && node->type != TOPO_TYPE_CACHE) || (root->type != TOPO_TYPE_SYSTEM && CPU_CMP(&node->cpuset, &root->cpuset) == 0)) { node = topo_next_node(root, node); continue; } cg_root->cg_child[i].cg_parent = cg_root; x86topo_add_sched_group(node, &cg_root->cg_child[i]); i++; node = topo_next_nonchild_node(root, node); } } /* * Build the MI scheduling topology from the discovered hardware topology. */ struct cpu_group * cpu_topo(void) { struct cpu_group *cg_root; if (mp_ncpus <= 1) return (smp_topo_none()); cg_root = smp_topo_alloc(1); x86topo_add_sched_group(&topo_root, cg_root); return (cg_root); } static void cpu_alloc(void *dummy __unused) { /* * Dynamically allocate the arrays that depend on the * maximum APIC ID. */ cpu_info = malloc(sizeof(*cpu_info) * (max_apic_id + 1), M_CPUS, M_WAITOK | M_ZERO); apic_cpuids = malloc(sizeof(*apic_cpuids) * (max_apic_id + 1), M_CPUS, M_WAITOK | M_ZERO); } SYSINIT(cpu_alloc, SI_SUB_CPU, SI_ORDER_FIRST, cpu_alloc, NULL); /* * Add a logical CPU to the topology. */ void cpu_add(u_int apic_id, char boot_cpu) { if (apic_id > max_apic_id) { panic("SMP: APIC ID %d too high", apic_id); return; } KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %u added twice", apic_id)); cpu_info[apic_id].cpu_present = 1; if (boot_cpu) { KASSERT(boot_cpu_id == -1, ("CPU %u claims to be BSP, but CPU %u already is", apic_id, boot_cpu_id)); boot_cpu_id = apic_id; cpu_info[apic_id].cpu_bsp = 1; } if (bootverbose) printf("SMP: Added CPU %u (%s)\n", apic_id, boot_cpu ? "BSP" : "AP"); } void cpu_mp_setmaxid(void) { /* * mp_ncpus and mp_maxid should be already set by calls to cpu_add(). * If there were no calls to cpu_add() assume this is a UP system. */ if (mp_ncpus == 0) mp_ncpus = 1; } int cpu_mp_probe(void) { /* * Always record BSP in CPU map so that the mbuf init code works * correctly. */ CPU_SETOF(0, &all_cpus); return (mp_ncpus > 1); } /* Allocate memory for the AP trampoline. */ void alloc_ap_trampoline(vm_paddr_t *physmap, unsigned int *physmap_idx) { unsigned int i; bool allocated; allocated = false; for (i = *physmap_idx; i <= *physmap_idx; i -= 2) { /* * Find a memory region big enough and below the 1MB boundary * for the trampoline code. * NB: needs to be page aligned. */ if (physmap[i] >= MiB(1) || (trunc_page(physmap[i + 1]) - round_page(physmap[i])) < round_page(bootMP_size)) continue; allocated = true; /* * Try to steal from the end of the region to mimic previous * behaviour, else fallback to steal from the start. */ if (physmap[i + 1] < MiB(1)) { boot_address = trunc_page(physmap[i + 1]); if ((physmap[i + 1] - boot_address) < bootMP_size) boot_address -= round_page(bootMP_size); physmap[i + 1] = boot_address; } else { boot_address = round_page(physmap[i]); physmap[i] = boot_address + round_page(bootMP_size); } if (physmap[i] == physmap[i + 1] && *physmap_idx != 0) { memmove(&physmap[i], &physmap[i + 2], sizeof(*physmap) * (*physmap_idx - i + 2)); *physmap_idx -= 2; } break; } if (!allocated) { boot_address = basemem * 1024 - bootMP_size; if (bootverbose) printf( "Cannot find enough space for the boot trampoline, placing it at %#x", boot_address); } } /* * AP CPU's call this to initialize themselves. */ void init_secondary_tail(void) { u_int cpuid; pmap_activate_boot(vmspace_pmap(proc0.p_vmspace)); /* * On real hardware, switch to x2apic mode if possible. Do it * after aps_ready was signalled, to avoid manipulating the * mode while BSP might still want to send some IPI to us * (second startup IPI is ignored on modern hardware etc). */ lapic_xapic_mode(); /* Initialize the PAT MSR. */ pmap_init_pat(); /* set up CPU registers and state */ cpu_setregs(); /* set up SSE/NX */ initializecpu(); /* set up FPU state on the AP */ #ifdef __amd64__ fpuinit(); #else npxinit(false); #endif if (cpu_ops.cpu_init) cpu_ops.cpu_init(); /* A quick check from sanity claus */ cpuid = PCPU_GET(cpuid); if (PCPU_GET(apic_id) != lapic_id()) { printf("SMP: cpuid = %d\n", cpuid); printf("SMP: actual apic_id = %d\n", lapic_id()); printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id)); panic("cpuid mismatch! boom!!"); } /* Initialize curthread. */ KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); PCPU_SET(curthread, PCPU_GET(idlethread)); mtx_lock_spin(&ap_boot_mtx); mca_init(); /* Init local apic for irq's */ lapic_setup(1); /* Set memory range attributes for this CPU to match the BSP */ mem_range_AP_init(); smp_cpus++; CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid); if (bootverbose) printf("SMP: AP CPU #%d Launched!\n", cpuid); else printf("%s%d%s", smp_cpus == 2 ? "Launching APs: " : "", cpuid, smp_cpus == mp_ncpus ? "\n" : " "); /* Determine if we are a logical CPU. */ if (cpu_info[PCPU_GET(apic_id)].cpu_hyperthread) CPU_SET(cpuid, &logical_cpus_mask); if (bootverbose) lapic_dump("AP"); if (smp_cpus == mp_ncpus) { /* enable IPI's, tlb shootdown, freezes etc */ atomic_store_rel_int(&smp_started, 1); } #ifdef __amd64__ /* * Enable global pages TLB extension * This also implicitly flushes the TLB */ load_cr4(rcr4() | CR4_PGE); if (pmap_pcid_enabled) load_cr4(rcr4() | CR4_PCIDE); load_ds(_udatasel); load_es(_udatasel); load_fs(_ufssel); #endif mtx_unlock_spin(&ap_boot_mtx); /* Wait until all the AP's are up. */ while (atomic_load_acq_int(&smp_started) == 0) ia32_pause(); #ifndef EARLY_AP_STARTUP /* Start per-CPU event timers. */ cpu_initclocks_ap(); #endif kcsan_cpu_init(cpuid); /* * Assert that smp_after_idle_runnable condition is reasonable. */ MPASS(PCPU_GET(curpcb) == NULL); sched_throw(NULL); panic("scheduler returned us to %s", __func__); /* NOTREACHED */ } static void smp_after_idle_runnable(void *arg __unused) { struct pcpu *pc; int cpu; for (cpu = 1; cpu < mp_ncpus; cpu++) { pc = pcpu_find(cpu); while (atomic_load_ptr(&pc->pc_curpcb) == NULL) cpu_spinwait(); kmem_free((vm_offset_t)bootstacks[cpu], kstack_pages * PAGE_SIZE); } } SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, smp_after_idle_runnable, NULL); /* * We tell the I/O APIC code about all the CPUs we want to receive * interrupts. If we don't want certain CPUs to receive IRQs we * can simply not tell the I/O APIC code about them in this function. * We also do not tell it about the BSP since it tells itself about * the BSP internally to work with UP kernels and on UP machines. */ void set_interrupt_apic_ids(void) { u_int i, apic_id; for (i = 0; i < MAXCPU; i++) { apic_id = cpu_apic_ids[i]; if (apic_id == -1) continue; if (cpu_info[apic_id].cpu_bsp) continue; if (cpu_info[apic_id].cpu_disabled) continue; /* Don't let hyperthreads service interrupts. */ if (cpu_info[apic_id].cpu_hyperthread && !hyperthreading_intr_allowed) continue; intr_add_cpu(i); } } #ifdef COUNT_XINVLTLB_HITS u_int xhits_gbl[MAXCPU]; u_int xhits_pg[MAXCPU]; u_int xhits_rng[MAXCPU]; static SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, ""); SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl, sizeof(xhits_gbl), "IU", ""); SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg, sizeof(xhits_pg), "IU", ""); SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng, sizeof(xhits_rng), "IU", ""); u_int ipi_global; u_int ipi_page; u_int ipi_range; u_int ipi_range_size; SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, ""); SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, ""); SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, ""); SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size, 0, ""); #endif /* COUNT_XINVLTLB_HITS */ /* * Init and startup IPI. */ void ipi_startup(int apic_id, int vector) { /* * This attempts to follow the algorithm described in the * Intel Multiprocessor Specification v1.4 in section B.4. * For each IPI, we allow the local APIC ~20us to deliver the * IPI. If that times out, we panic. */ /* * first we do an INIT IPI: this INIT IPI might be run, resetting * and running the target CPU. OR this INIT IPI might be latched (P5 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be * ignored. */ lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_LEVEL | APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id); lapic_ipi_wait(100); /* Explicitly deassert the INIT IPI. */ lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_LEVEL | APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id); DELAY(10000); /* wait ~10mS */ /* * next we do a STARTUP IPI: the previous INIT IPI might still be * latched, (P5 bug) this 1st STARTUP would then terminate * immediately, and the previously started INIT IPI would continue. OR * the previous INIT IPI has already run. and this STARTUP IPI will * run. OR the previous INIT IPI was ignored. and this STARTUP IPI * will run. */ lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | vector, apic_id); if (!lapic_ipi_wait(100)) panic("Failed to deliver first STARTUP IPI to APIC %d", apic_id); DELAY(200); /* wait ~200uS */ /* * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is * recognized after hardware RESET or INIT IPI. */ lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | vector, apic_id); if (!lapic_ipi_wait(100)) panic("Failed to deliver second STARTUP IPI to APIC %d", apic_id); DELAY(200); /* wait ~200uS */ } static bool ipi_bitmap_set(int cpu, u_int ipi) { u_int bitmap, old, new; u_int *cpu_bitmap; bitmap = 1 << ipi; cpu_bitmap = &cpuid_to_pcpu[cpu]->pc_ipi_bitmap; old = *cpu_bitmap; for (;;) { if ((old & bitmap) != 0) break; new = old | bitmap; if (atomic_fcmpset_int(cpu_bitmap, &old, new)) break; } return (old != 0); } /* * Send an IPI to specified CPU handling the bitmap logic. */ static void ipi_send_cpu(int cpu, u_int ipi) { KASSERT((u_int)cpu < MAXCPU && cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu)); if (IPI_IS_BITMAPED(ipi)) { if (ipi_bitmap_set(cpu, ipi)) return; ipi = IPI_BITMAP_VECTOR; } lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); } void ipi_bitmap_handler(struct trapframe frame) { struct trapframe *oldframe; struct thread *td; int cpu = PCPU_GET(cpuid); u_int ipi_bitmap; td = curthread; ipi_bitmap = atomic_readandclear_int(&cpuid_to_pcpu[cpu]-> pc_ipi_bitmap); /* * sched_preempt() must be called to clear the pending preempt * IPI to enable delivery of further preempts. However, the * critical section will cause extra scheduler lock thrashing * when used unconditionally. Only critical_enter() if * hardclock must also run, which requires the section entry. */ if (ipi_bitmap & (1 << IPI_HARDCLOCK)) critical_enter(); td->td_intr_nesting_level++; oldframe = td->td_intr_frame; td->td_intr_frame = &frame; #if defined(STACK) || defined(DDB) if (ipi_bitmap & (1 << IPI_TRACE)) stack_capture_intr(); #endif if (ipi_bitmap & (1 << IPI_PREEMPT)) { #ifdef COUNT_IPIS (*ipi_preempt_counts[cpu])++; #endif sched_preempt(td); } if (ipi_bitmap & (1 << IPI_AST)) { #ifdef COUNT_IPIS (*ipi_ast_counts[cpu])++; #endif /* Nothing to do for AST */ } if (ipi_bitmap & (1 << IPI_HARDCLOCK)) { #ifdef COUNT_IPIS (*ipi_hardclock_counts[cpu])++; #endif hardclockintr(); } td->td_intr_frame = oldframe; td->td_intr_nesting_level--; if (ipi_bitmap & (1 << IPI_HARDCLOCK)) critical_exit(); } /* * send an IPI to a set of cpus. */ void ipi_selected(cpuset_t cpus, u_int ipi) { int cpu; /* * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit * of help in order to understand what is the source. * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) CPU_OR_ATOMIC(&ipi_stop_nmi_pending, &cpus); while ((cpu = CPU_FFS(&cpus)) != 0) { cpu--; CPU_CLR(cpu, &cpus); CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi); ipi_send_cpu(cpu, ipi); } } /* * send an IPI to a specific CPU. */ void ipi_cpu(int cpu, u_int ipi) { /* * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit * of help in order to understand what is the source. * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) CPU_SET_ATOMIC(cpu, &ipi_stop_nmi_pending); CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, ipi); ipi_send_cpu(cpu, ipi); } /* * send an IPI to all CPUs EXCEPT myself */ void ipi_all_but_self(u_int ipi) { cpuset_t other_cpus; int cpu, c; /* * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit * of help in order to understand what is the source. * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) { other_cpus = all_cpus; CPU_CLR(PCPU_GET(cpuid), &other_cpus); CPU_OR_ATOMIC(&ipi_stop_nmi_pending, &other_cpus); } CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); if (IPI_IS_BITMAPED(ipi)) { cpu = PCPU_GET(cpuid); CPU_FOREACH(c) { if (c != cpu) ipi_bitmap_set(c, ipi); } ipi = IPI_BITMAP_VECTOR; } lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); } void ipi_self_from_nmi(u_int vector) { lapic_ipi_vectored(vector, APIC_IPI_DEST_SELF); /* Wait for IPI to finish. */ if (!lapic_ipi_wait(50000)) { if (KERNEL_PANICKED()) return; else panic("APIC: IPI is stuck"); } } int ipi_nmi_handler(void) { u_int cpuid; /* * As long as there is not a simple way to know about a NMI's * source, if the bitmask for the current CPU is present in * the global pending bitword an IPI_STOP_HARD has been issued * and should be handled. */ cpuid = PCPU_GET(cpuid); if (!CPU_ISSET(cpuid, &ipi_stop_nmi_pending)) return (1); CPU_CLR_ATOMIC(cpuid, &ipi_stop_nmi_pending); cpustop_handler(); return (0); } int nmi_kdb_lock; void nmi_call_kdb_smp(u_int type, struct trapframe *frame) { int cpu; bool call_post; cpu = PCPU_GET(cpuid); if (atomic_cmpset_acq_int(&nmi_kdb_lock, 0, 1)) { nmi_call_kdb(cpu, type, frame); call_post = false; } else { savectx(&stoppcbs[cpu]); CPU_SET_ATOMIC(cpu, &stopped_cpus); while (!atomic_cmpset_acq_int(&nmi_kdb_lock, 0, 1)) ia32_pause(); call_post = true; } atomic_store_rel_int(&nmi_kdb_lock, 0); if (call_post) cpustop_handler_post(cpu); } /* * Handle an IPI_STOP by saving our current context and spinning (or mwaiting, * if available) until we are resumed. */ void cpustop_handler(void) { struct monitorbuf *mb; u_int cpu; bool use_mwait; cpu = PCPU_GET(cpuid); savectx(&stoppcbs[cpu]); use_mwait = (stop_mwait && (cpu_feature2 & CPUID2_MON) != 0 && !mwait_cpustop_broken); if (use_mwait) { mb = PCPU_PTR(monitorbuf); atomic_store_int(&mb->stop_state, MONITOR_STOPSTATE_STOPPED); } /* Indicate that we are stopped */ CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ while (!CPU_ISSET(cpu, &started_cpus)) { if (use_mwait) { cpu_monitor(mb, 0, 0); if (atomic_load_int(&mb->stop_state) == MONITOR_STOPSTATE_STOPPED) cpu_mwait(0, MWAIT_C1); continue; } ia32_pause(); /* * Halt non-BSP CPUs on panic -- we're never going to need them * again, and might as well save power / release resources * (e.g., overprovisioned VM infrastructure). */ while (__predict_false(!IS_BSP() && KERNEL_PANICKED())) halt(); } cpustop_handler_post(cpu); } static void cpustop_handler_post(u_int cpu) { CPU_CLR_ATOMIC(cpu, &started_cpus); CPU_CLR_ATOMIC(cpu, &stopped_cpus); /* * We don't broadcast TLB invalidations to other CPUs when they are * stopped. Hence, we clear the TLB before resuming. */ invltlb_glob(); -#if defined(__amd64__) && defined(DDB) +#if defined(__amd64__) && (defined(DDB) || defined(GDB)) amd64_db_resume_dbreg(); #endif if (cpu == 0 && cpustop_restartfunc != NULL) { cpustop_restartfunc(); cpustop_restartfunc = NULL; } } /* * Handle an IPI_SUSPEND by saving our current context and spinning until we * are resumed. */ void cpususpend_handler(void) { u_int cpu; mtx_assert(&smp_ipi_mtx, MA_NOTOWNED); cpu = PCPU_GET(cpuid); if (savectx(&susppcbs[cpu]->sp_pcb)) { #ifdef __amd64__ fpususpend(susppcbs[cpu]->sp_fpususpend); #else npxsuspend(susppcbs[cpu]->sp_fpususpend); #endif /* * suspended_cpus is cleared shortly after each AP is restarted * by a Startup IPI, so that the BSP can proceed to restarting * the next AP. * * resuming_cpus gets cleared when the AP completes * initialization after having been released by the BSP. * resuming_cpus is probably not the best name for the * variable, because it is actually a set of processors that * haven't resumed yet and haven't necessarily started resuming. * * Note that suspended_cpus is meaningful only for ACPI suspend * as it's not really used for Xen suspend since the APs are * automatically restored to the running state and the correct * context. For the same reason resumectx is never called in * that case. */ CPU_SET_ATOMIC(cpu, &suspended_cpus); CPU_SET_ATOMIC(cpu, &resuming_cpus); /* * Invalidate the cache after setting the global status bits. * The last AP to set its bit may end up being an Owner of the * corresponding cache line in MOESI protocol. The AP may be * stopped before the cache line is written to the main memory. */ wbinvd(); } else { #ifdef __amd64__ fpuresume(susppcbs[cpu]->sp_fpususpend); #else npxresume(susppcbs[cpu]->sp_fpususpend); #endif pmap_init_pat(); initializecpu(); PCPU_SET(switchtime, 0); PCPU_SET(switchticks, ticks); /* Indicate that we have restarted and restored the context. */ CPU_CLR_ATOMIC(cpu, &suspended_cpus); } /* Wait for resume directive */ while (!CPU_ISSET(cpu, &toresume_cpus)) ia32_pause(); /* Re-apply microcode updates. */ ucode_reload(); #ifdef __i386__ /* Finish removing the identity mapping of low memory for this AP. */ invltlb_glob(); #endif if (cpu_ops.cpu_resume) cpu_ops.cpu_resume(); #ifdef __amd64__ if (vmm_resume_p) vmm_resume_p(); #endif /* Resume MCA and local APIC */ lapic_xapic_mode(); mca_resume(); lapic_setup(0); /* Indicate that we are resumed */ CPU_CLR_ATOMIC(cpu, &resuming_cpus); CPU_CLR_ATOMIC(cpu, &suspended_cpus); CPU_CLR_ATOMIC(cpu, &toresume_cpus); } /* * Handle an IPI_SWI by waking delayed SWI thread. */ void ipi_swi_handler(struct trapframe frame) { intr_event_handle(clk_intr_event, &frame); } /* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. */ static void release_aps(void *dummy __unused) { if (mp_ncpus == 1) return; atomic_store_rel_int(&aps_ready, 1); while (smp_started == 0) ia32_pause(); } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); #ifdef COUNT_IPIS /* * Setup interrupt counters for IPI handlers. */ static void mp_ipi_intrcnt(void *dummy) { char buf[64]; int i; CPU_FOREACH(i) { snprintf(buf, sizeof(buf), "cpu%d:invltlb", i); intrcnt_add(buf, &ipi_invltlb_counts[i]); snprintf(buf, sizeof(buf), "cpu%d:invlrng", i); intrcnt_add(buf, &ipi_invlrng_counts[i]); snprintf(buf, sizeof(buf), "cpu%d:invlpg", i); intrcnt_add(buf, &ipi_invlpg_counts[i]); snprintf(buf, sizeof(buf), "cpu%d:invlcache", i); intrcnt_add(buf, &ipi_invlcache_counts[i]); snprintf(buf, sizeof(buf), "cpu%d:preempt", i); intrcnt_add(buf, &ipi_preempt_counts[i]); snprintf(buf, sizeof(buf), "cpu%d:ast", i); intrcnt_add(buf, &ipi_ast_counts[i]); snprintf(buf, sizeof(buf), "cpu%d:rendezvous", i); intrcnt_add(buf, &ipi_rendezvous_counts[i]); snprintf(buf, sizeof(buf), "cpu%d:hardclock", i); intrcnt_add(buf, &ipi_hardclock_counts[i]); } } SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL); #endif