Page MenuHomeFreeBSD
Files F172354561

D59648.id186791.diff
No OneTemporary
Actions

D59648.id186791.diff
View Options

diff --git a/sys/arm64/arm64/db_trace.c b/sys/arm64/arm64/db_trace.c
--- a/sys/arm64/arm64/db_trace.c
+++ b/sys/arm64/arm64/db_trace.c
@@ -132,7 +132,8 @@
if (!INKERNEL(frame->fp))
break;
} else {
- if (strcmp(name, "fork_trampoline") == 0)
+ if (strcmp(name, "fork_trampoline") == 0 ||
+ strcmp(name, "fork_trampoline_kthread") == 0)
break;
if (!unwind_frame(td, frame))
diff --git a/sys/arm64/arm64/exception.S b/sys/arm64/arm64/exception.S
--- a/sys/arm64/arm64/exception.S
+++ b/sys/arm64/arm64/exception.S
@@ -32,9 +32,21 @@
#include "assym.inc"
#include <sys/intr.h>
+#include <machine/trap_cfi.h>
.text
+.macro trap_entry name
+ .globl \name
+ .text
+ .align 2
+ .type \name, #function
+\name:
+ trap_cfi_entry \name, full
+ BTI_C
+ DTRACE_NOP
+.endm
+
/*
* This is limited to 28 instructions as it's placed in the exception vector
* slot that is 32 instructions long. We need one for the branch, and three
@@ -46,20 +58,51 @@
.else
stp x0, x1, [sp, #-(TF_SIZE - TF_X)]!
.endif
+ .cfi_def_cfa_offset (TF_SIZE - TF_X)
+ trap_cfi_saved x0, TF_X
+ trap_cfi_saved x1, TF_X + 8
stp x2, x3, [sp, #(2 * 8)]
+ trap_cfi_saved x2, TF_X + 2 * 8
+ trap_cfi_saved x3, TF_X + 3 * 8
stp x4, x5, [sp, #(4 * 8)]
+ trap_cfi_saved x4, TF_X + 4 * 8
+ trap_cfi_saved x5, TF_X + 5 * 8
stp x6, x7, [sp, #(6 * 8)]
+ trap_cfi_saved x6, TF_X + 6 * 8
+ trap_cfi_saved x7, TF_X + 7 * 8
stp x8, x9, [sp, #(8 * 8)]
+ trap_cfi_saved x8, TF_X + 8 * 8
+ trap_cfi_saved x9, TF_X + 9 * 8
stp x10, x11, [sp, #(10 * 8)]
+ trap_cfi_saved x10, TF_X + 10 * 8
+ trap_cfi_saved x11, TF_X + 11 * 8
stp x12, x13, [sp, #(12 * 8)]
+ trap_cfi_saved x12, TF_X + 12 * 8
+ trap_cfi_saved x13, TF_X + 13 * 8
stp x14, x15, [sp, #(14 * 8)]
+ trap_cfi_saved x14, TF_X + 14 * 8
+ trap_cfi_saved x15, TF_X + 15 * 8
stp x16, x17, [sp, #(16 * 8)]
+ trap_cfi_saved x16, TF_X + 16 * 8
+ trap_cfi_saved x17, TF_X + 17 * 8
stp x18, x19, [sp, #(18 * 8)]
+ trap_cfi_saved x18, TF_X + 18 * 8
+ trap_cfi_saved x19, TF_X + 19 * 8
stp x20, x21, [sp, #(20 * 8)]
+ trap_cfi_saved x20, TF_X + 20 * 8
+ trap_cfi_saved x21, TF_X + 21 * 8
stp x22, x23, [sp, #(22 * 8)]
+ trap_cfi_saved x22, TF_X + 22 * 8
+ trap_cfi_saved x23, TF_X + 23 * 8
stp x24, x25, [sp, #(24 * 8)]
+ trap_cfi_saved x24, TF_X + 24 * 8
+ trap_cfi_saved x25, TF_X + 25 * 8
stp x26, x27, [sp, #(26 * 8)]
+ trap_cfi_saved x26, TF_X + 26 * 8
+ trap_cfi_saved x27, TF_X + 27 * 8
stp x28, x29, [sp, #(28 * 8)]
+ trap_cfi_saved x28, TF_X + 28 * 8
+ trap_cfi_saved x29, TF_X + 29 * 8
.if \el == 1
add x18, sp, #(TF_SIZE - TF_X + 128)
.else
@@ -70,7 +113,12 @@
mrs x12, esr_el1
mrs x13, far_el1
stp x18, lr, [sp, #(TF_SP - TF_X)]!
+ .cfi_def_cfa_offset TF_SIZE
+ trap_cfi_saved 31, TF_SP
+ trap_cfi_saved x30, TF_LR
stp x10, x11, [sp, #(TF_ELR)]
+ trap_cfi_saved CFI_PC, TF_ELR
+ trap_cfi_saved CFI_CPSR, TF_SPSR
stp x12, x13, [sp, #(TF_ESR)]
mrs x18, tpidr_el1
.endm
@@ -195,8 +243,10 @@
.endif
.if \el == 0
add sp, sp, #(TF_SIZE)
+ .cfi_def_cfa_offset 0
.else
mov sp, x18
+ .cfi_def_cfa sp, -128
mrs x18, tpidr_el1
.endif
.endm
@@ -252,7 +302,7 @@
#define KMSAN_LEAVE
#endif
-ENTRY(handle_el1h_sync)
+trap_entry handle_el1h_sync
save_registers 1
KMSAN_ENTER
ldr x0, [x18, #PC_CURTHREAD]
@@ -261,9 +311,9 @@
KMSAN_LEAVE
restore_registers 1
ERET
-END(handle_el1h_sync)
+ trap_cfi_end handle_el1h_sync
-ENTRY(handle_el1h_irq)
+trap_entry handle_el1h_irq
save_registers 1
KMSAN_ENTER
mov x0, sp
@@ -272,9 +322,9 @@
KMSAN_LEAVE
restore_registers 1
ERET
-END(handle_el1h_irq)
+ trap_cfi_end handle_el1h_irq
-ENTRY(handle_el1h_fiq)
+trap_entry handle_el1h_fiq
save_registers 1
KMSAN_ENTER
mov x0, sp
@@ -283,18 +333,18 @@
KMSAN_LEAVE
restore_registers 1
ERET
-END(handle_el1h_fiq)
+ trap_cfi_end handle_el1h_fiq
-ENTRY(handle_el1h_serror)
+trap_entry handle_el1h_serror
save_registers 1
KMSAN_ENTER
mov x0, sp
1: bl do_serror
b 1b
KMSAN_LEAVE
-END(handle_el1h_serror)
+ trap_cfi_end handle_el1h_serror
-ENTRY(handle_el0_sync)
+trap_entry handle_el0_sync
save_registers 0
KMSAN_ENTER
ldr x0, [x18, #PC_CURTHREAD]
@@ -307,9 +357,9 @@
KMSAN_LEAVE
restore_registers 0
ERET
-END(handle_el0_sync)
+ trap_cfi_end handle_el0_sync
-ENTRY(handle_el0_irq)
+trap_entry handle_el0_irq
save_registers 0
KMSAN_ENTER
mov x0, sp
@@ -319,9 +369,9 @@
KMSAN_LEAVE
restore_registers 0
ERET
-END(handle_el0_irq)
+ trap_cfi_end handle_el0_irq
-ENTRY(handle_el0_fiq)
+trap_entry handle_el0_fiq
save_registers 0
KMSAN_ENTER
mov x0, sp
@@ -331,34 +381,42 @@
KMSAN_LEAVE
restore_registers 0
ERET
-END(handle_el0_fiq)
+ trap_cfi_end handle_el0_fiq
-ENTRY(handle_el0_serror)
+trap_entry handle_el0_serror
save_registers 0
KMSAN_ENTER
mov x0, sp
1: bl do_serror
b 1b
KMSAN_LEAVE
-END(handle_el0_serror)
+ trap_cfi_end handle_el0_serror
-ENTRY(handle_empty_exception)
+trap_entry handle_empty_exception
save_registers 0
KMSAN_ENTER
mov x0, sp
1: bl unhandled_exception
b 1b
KMSAN_LEAVE
-END(handle_empty_exception)
+ trap_cfi_end handle_empty_exception
.macro vector name, el
.align 7
+ trap_cfi_entry exception_vector_\@, empty
+.if \el == 1
+ .cfi_def_cfa sp, -128
+.else
+ .cfi_def_cfa sp, 0
+.endif
+ .cfi_undefined CFI_PC
save_registers_head \el
b handle_\name
dsb sy
isb
/* Break instruction to ensure we aren't executing code here. */
brk 0x42
+ trap_cfi_end
.endm
.macro vempty el
diff --git a/sys/arm64/arm64/swtch.S b/sys/arm64/arm64/swtch.S
--- a/sys/arm64/arm64/swtch.S
+++ b/sys/arm64/arm64/swtch.S
@@ -30,6 +30,7 @@
*/
#include "assym.inc"
+
#include "opt_kstack_pages.h"
#include <sys/elf_common.h>
@@ -37,6 +38,7 @@
#include <machine/asm.h>
#include <machine/armreg.h>
#include <machine/proc.h>
+#include <machine/trap_cfi.h>
.macro clear_step_flag pcbflags, tmp
tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
@@ -227,7 +229,10 @@
ret
END(cpu_switch)
-ENTRY(fork_trampoline)
+EENTRY(fork_trampoline)
+ trap_cfi_entry fork_trampoline, full
+ BTI_C
+ DTRACE_NOP
mov x0, x19
mov x1, x20
mov x2, sp
@@ -279,14 +284,28 @@
ldp x28, x29, [sp, #TF_X + 28 * 8]
add sp, sp, #(TF_SIZE)
+ .cfi_def_cfa_offset 0
/*
* No need for interrupts reenabling since PSR
* will be set to the desired value anyway.
*/
ERET
-
-END(fork_trampoline)
+
+ trap_cfi_end fork_trampoline
+
+EENTRY(fork_trampoline_kthread)
+ trap_cfi_entry fork_trampoline_kthread, terminal
+ BTI_C
+ DTRACE_NOP
+ mov x0, x19
+ mov x1, x20
+ mov x2, sp
+ mov fp, #0
+ bl _C_LABEL(fork_exit)
+ brk #0
+1: b 1b
+ trap_cfi_end fork_trampoline_kthread
ENTRY(savectx)
/* Store the callee-saved registers */
diff --git a/sys/arm64/arm64/vm_machdep.c b/sys/arm64/arm64/vm_machdep.c
--- a/sys/arm64/arm64/vm_machdep.c
+++ b/sys/arm64/arm64/vm_machdep.c
@@ -182,7 +182,10 @@
td->td_pcb->pcb_x[PCB_X19] = (uintptr_t)fork_return;
td->td_pcb->pcb_x[PCB_X20] = (uintptr_t)td;
- td->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline;
+ if ((td->td_pflags & TDP_KTHREAD) != 0)
+ td->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline_kthread;
+ else
+ td->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline;
td->td_pcb->pcb_sp = (uintptr_t)td->td_frame;
/* Update VFP state for the new thread */
diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h
--- a/sys/arm64/include/cpu.h
+++ b/sys/arm64/include/cpu.h
@@ -262,6 +262,7 @@
void cpu_halt(void) __dead2;
void cpu_reset(void) __dead2;
void fork_trampoline(void);
+void fork_trampoline_kthread(void);
void identify_cache(uint64_t);
void identify_cpu(u_int);
void install_cpu_errata(void);
diff --git a/sys/arm64/include/frame.h b/sys/arm64/include/frame.h
--- a/sys/arm64/include/frame.h
+++ b/sys/arm64/include/frame.h
@@ -41,7 +41,8 @@
#include <sys/ucontext.h>
/*
- * NOTE: keep this structure in sync with struct reg and struct mcontext.
+ * NOTE: keep this structure in sync with struct reg, struct mcontext, and the
+ * trap frame CFI information.
*/
struct trapframe {
uint64_t tf_sp;
diff --git a/sys/arm64/include/trap_cfi.h b/sys/arm64/include/trap_cfi.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/include/trap_cfi.h
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2026 FreeBSD Foundation
+ *
+ * This software was developed by Minsoo Choo under sponsorship from the
+ * FreeBSD Foundation.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifndef _MACHINE_TRAP_CFI_H_
+#define _MACHINE_TRAP_CFI_H_
+
+#include "assym.inc"
+
+ .cfi_sections .eh_frame
+
+#define CFI_PC 32
+#define CFI_CPSR 33
+
+.macro trap_cfi_entry name, state=empty
+ .cfi_startproc
+ .cfi_signal_frame
+ .cfi_return_column CFI_PC
+ .cfi_def_cfa sp, 0
+ .cfi_undefined CFI_PC
+.ifc \state,full
+ trap_cfi_full
+.endif
+.ifc \state,terminal
+ .cfi_def_cfa sp, TF_SIZE
+.endif
+.endm
+
+.macro trap_cfi_end name
+ .ifnb \name
+ .ltorg
+ .endif
+ .cfi_endproc
+ .ifnb \name
+ .size \name, . - \name
+ .endif
+.endm
+
+.macro trap_cfi_saved reg, off
+ .cfi_offset \reg, \off - TF_SIZE
+.endm
+
+.macro trap_cfi_full
+ .cfi_def_cfa sp, TF_SIZE
+ trap_cfi_saved 31, TF_SP
+ trap_cfi_saved x30, TF_LR
+ trap_cfi_saved CFI_PC, TF_ELR
+ trap_cfi_saved CFI_CPSR, TF_SPSR
+ .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29
+ trap_cfi_saved x\n, TF_X + \n * 8
+ .endr
+.endm
+
+#endif /* _MACHINE_TRAP_CFI_H_ */
diff --git a/sys/conf/ldscript.arm64 b/sys/conf/ldscript.arm64
--- a/sys/conf/ldscript.arm64
+++ b/sys/conf/ldscript.arm64
@@ -45,6 +45,7 @@
*(.note.gnu.build-id)
PROVIDE (__build_id_end = .);
}
+ .eh_frame : { KEEP (*(.eh_frame)) }
.rel.text :
{ *(.rel.text) *(.rel.gnu.linkonce.t*) }
.rela.text :

File Metadata

Mime Type
text/plain
Expires
Fri, Sep 18, 10:18 PM (1 h, 51 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
39177198
Default Alt Text
D59648.id186791.diff (9 KB)

Event Timeline