Page MenuHomeFreeBSD
Files F172499325

D59648.diff
No OneTemporary
Actions

D59648.diff
View Options

diff --git a/sys/arm64/arm64/db_trace.c b/sys/arm64/arm64/db_trace.c
--- a/sys/arm64/arm64/db_trace.c
+++ b/sys/arm64/arm64/db_trace.c
@@ -132,7 +132,8 @@
if (!INKERNEL(frame->fp))
break;
} else {
- if (strcmp(name, "fork_trampoline") == 0)
+ if (strcmp(name, "fork_trampoline") == 0 ||
+ strcmp(name, "fork_trampoline_kthread") == 0)
break;
if (!unwind_frame(td, frame))
diff --git a/sys/arm64/arm64/exception.S b/sys/arm64/arm64/exception.S
--- a/sys/arm64/arm64/exception.S
+++ b/sys/arm64/arm64/exception.S
@@ -32,6 +32,7 @@
#include "assym.inc"
#include <sys/intr.h>
+#include <machine/trap_cfi.h>
.text
@@ -46,20 +47,51 @@
.else
stp x0, x1, [sp, #-(TF_SIZE - TF_X)]!
.endif
+ .cfi_def_cfa_offset (TF_SIZE - TF_X)
+ trap_cfi_saved x0, TF_X
+ trap_cfi_saved x1, TF_X + 8
stp x2, x3, [sp, #(2 * 8)]
+ trap_cfi_saved x2, TF_X + 2 * 8
+ trap_cfi_saved x3, TF_X + 3 * 8
stp x4, x5, [sp, #(4 * 8)]
+ trap_cfi_saved x4, TF_X + 4 * 8
+ trap_cfi_saved x5, TF_X + 5 * 8
stp x6, x7, [sp, #(6 * 8)]
+ trap_cfi_saved x6, TF_X + 6 * 8
+ trap_cfi_saved x7, TF_X + 7 * 8
stp x8, x9, [sp, #(8 * 8)]
+ trap_cfi_saved x8, TF_X + 8 * 8
+ trap_cfi_saved x9, TF_X + 9 * 8
stp x10, x11, [sp, #(10 * 8)]
+ trap_cfi_saved x10, TF_X + 10 * 8
+ trap_cfi_saved x11, TF_X + 11 * 8
stp x12, x13, [sp, #(12 * 8)]
+ trap_cfi_saved x12, TF_X + 12 * 8
+ trap_cfi_saved x13, TF_X + 13 * 8
stp x14, x15, [sp, #(14 * 8)]
+ trap_cfi_saved x14, TF_X + 14 * 8
+ trap_cfi_saved x15, TF_X + 15 * 8
stp x16, x17, [sp, #(16 * 8)]
+ trap_cfi_saved x16, TF_X + 16 * 8
+ trap_cfi_saved x17, TF_X + 17 * 8
stp x18, x19, [sp, #(18 * 8)]
+ trap_cfi_saved x18, TF_X + 18 * 8
+ trap_cfi_saved x19, TF_X + 19 * 8
stp x20, x21, [sp, #(20 * 8)]
+ trap_cfi_saved x20, TF_X + 20 * 8
+ trap_cfi_saved x21, TF_X + 21 * 8
stp x22, x23, [sp, #(22 * 8)]
+ trap_cfi_saved x22, TF_X + 22 * 8
+ trap_cfi_saved x23, TF_X + 23 * 8
stp x24, x25, [sp, #(24 * 8)]
+ trap_cfi_saved x24, TF_X + 24 * 8
+ trap_cfi_saved x25, TF_X + 25 * 8
stp x26, x27, [sp, #(26 * 8)]
+ trap_cfi_saved x26, TF_X + 26 * 8
+ trap_cfi_saved x27, TF_X + 27 * 8
stp x28, x29, [sp, #(28 * 8)]
+ trap_cfi_saved x28, TF_X + 28 * 8
+ trap_cfi_saved x29, TF_X + 29 * 8
.if \el == 1
add x18, sp, #(TF_SIZE - TF_X + 128)
.else
@@ -70,7 +102,12 @@
mrs x12, esr_el1
mrs x13, far_el1
stp x18, lr, [sp, #(TF_SP - TF_X)]!
+ .cfi_def_cfa_offset TF_SIZE
+ trap_cfi_saved 31, TF_SP
+ trap_cfi_saved x30, TF_LR
stp x10, x11, [sp, #(TF_ELR)]
+ trap_cfi_saved CFI_PC, TF_ELR
+ trap_cfi_saved CFI_CPSR, TF_SPSR
stp x12, x13, [sp, #(TF_ESR)]
mrs x18, tpidr_el1
.endm
@@ -195,8 +232,10 @@
.endif
.if \el == 0
add sp, sp, #(TF_SIZE)
+ .cfi_def_cfa_offset 0
.else
mov sp, x18
+ .cfi_def_cfa sp, -128
mrs x18, tpidr_el1
.endif
.endm
@@ -252,7 +291,7 @@
#define KMSAN_LEAVE
#endif
-ENTRY(handle_el1h_sync)
+ENTRY(handle_el1h_sync, trap_cfi_entry full)
save_registers 1
KMSAN_ENTER
ldr x0, [x18, #PC_CURTHREAD]
@@ -263,7 +302,7 @@
ERET
END(handle_el1h_sync)
-ENTRY(handle_el1h_irq)
+ENTRY(handle_el1h_irq, trap_cfi_entry full)
save_registers 1
KMSAN_ENTER
mov x0, sp
@@ -274,7 +313,7 @@
ERET
END(handle_el1h_irq)
-ENTRY(handle_el1h_fiq)
+ENTRY(handle_el1h_fiq, trap_cfi_entry full)
save_registers 1
KMSAN_ENTER
mov x0, sp
@@ -285,7 +324,7 @@
ERET
END(handle_el1h_fiq)
-ENTRY(handle_el1h_serror)
+ENTRY(handle_el1h_serror, trap_cfi_entry full)
save_registers 1
KMSAN_ENTER
mov x0, sp
@@ -294,7 +333,7 @@
KMSAN_LEAVE
END(handle_el1h_serror)
-ENTRY(handle_el0_sync)
+ENTRY(handle_el0_sync, trap_cfi_entry full)
save_registers 0
KMSAN_ENTER
ldr x0, [x18, #PC_CURTHREAD]
@@ -309,7 +348,7 @@
ERET
END(handle_el0_sync)
-ENTRY(handle_el0_irq)
+ENTRY(handle_el0_irq, trap_cfi_entry full)
save_registers 0
KMSAN_ENTER
mov x0, sp
@@ -321,7 +360,7 @@
ERET
END(handle_el0_irq)
-ENTRY(handle_el0_fiq)
+ENTRY(handle_el0_fiq, trap_cfi_entry full)
save_registers 0
KMSAN_ENTER
mov x0, sp
@@ -333,7 +372,7 @@
ERET
END(handle_el0_fiq)
-ENTRY(handle_el0_serror)
+ENTRY(handle_el0_serror, trap_cfi_entry full)
save_registers 0
KMSAN_ENTER
mov x0, sp
@@ -342,7 +381,7 @@
KMSAN_LEAVE
END(handle_el0_serror)
-ENTRY(handle_empty_exception)
+ENTRY(handle_empty_exception, trap_cfi_entry full)
save_registers 0
KMSAN_ENTER
mov x0, sp
@@ -353,12 +392,21 @@
.macro vector name, el
.align 7
+ .cfi_startproc
+ trap_cfi_entry empty
+.if \el == 1
+ .cfi_def_cfa sp, -128
+.else
+ .cfi_def_cfa sp, 0
+.endif
+ .cfi_undefined CFI_PC
save_registers_head \el
b handle_\name
dsb sy
isb
/* Break instruction to ensure we aren't executing code here. */
brk 0x42
+ .cfi_endproc
.endm
.macro vempty el
diff --git a/sys/arm64/arm64/swtch.S b/sys/arm64/arm64/swtch.S
--- a/sys/arm64/arm64/swtch.S
+++ b/sys/arm64/arm64/swtch.S
@@ -37,6 +37,7 @@
#include <machine/asm.h>
#include <machine/armreg.h>
#include <machine/proc.h>
+#include <machine/trap_cfi.h>
.macro clear_step_flag pcbflags, tmp
tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
@@ -227,7 +228,7 @@
ret
END(cpu_switch)
-ENTRY(fork_trampoline)
+ENTRY(fork_trampoline, trap_cfi_entry full)
mov x0, x19
mov x1, x20
mov x2, sp
@@ -279,6 +280,7 @@
ldp x28, x29, [sp, #TF_X + 28 * 8]
add sp, sp, #(TF_SIZE)
+ .cfi_def_cfa_offset 0
/*
* No need for interrupts reenabling since PSR
@@ -288,6 +290,16 @@
END(fork_trampoline)
+ENTRY(fork_trampoline_kthread, trap_cfi_entry terminal)
+ mov x0, x19
+ mov x1, x20
+ mov x2, sp
+ mov fp, #0
+ bl _C_LABEL(fork_exit)
+ brk #0
+1: b 1b
+END(fork_trampoline_kthread)
+
ENTRY(savectx)
/* Store the callee-saved registers */
stp x19, x20, [x0, #PCB_REGS + (PCB_X19 + 0) * 8]
diff --git a/sys/arm64/arm64/vm_machdep.c b/sys/arm64/arm64/vm_machdep.c
--- a/sys/arm64/arm64/vm_machdep.c
+++ b/sys/arm64/arm64/vm_machdep.c
@@ -182,7 +182,10 @@
td->td_pcb->pcb_x[PCB_X19] = (uintptr_t)fork_return;
td->td_pcb->pcb_x[PCB_X20] = (uintptr_t)td;
- td->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline;
+ if ((td->td_pflags & TDP_KTHREAD) != 0)
+ td->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline_kthread;
+ else
+ td->td_pcb->pcb_x[PCB_LR] = (uintptr_t)fork_trampoline;
td->td_pcb->pcb_sp = (uintptr_t)td->td_frame;
/* Update VFP state for the new thread */
diff --git a/sys/arm64/include/asm.h b/sys/arm64/include/asm.h
--- a/sys/arm64/include/asm.h
+++ b/sys/arm64/include/asm.h
@@ -46,11 +46,11 @@
#define DTRACE_NOP
#endif
-#define LENTRY(sym) \
+#define LENTRY(sym, ...) \
.text; .align 2; .type sym,#function; sym: \
- .cfi_startproc; BTI_C; DTRACE_NOP
-#define ENTRY(sym) \
- .globl sym; LENTRY(sym)
+ .cfi_startproc; __VA_ARGS__; BTI_C; DTRACE_NOP
+#define ENTRY(sym, ...) \
+ .globl sym; LENTRY(sym, __VA_ARGS__)
#define EENTRY(sym) \
.globl sym; .text; .align 2; .type sym,#function; sym:
#define LEND(sym) .ltorg; .cfi_endproc; .size sym, . - sym
diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h
--- a/sys/arm64/include/cpu.h
+++ b/sys/arm64/include/cpu.h
@@ -262,6 +262,7 @@
void cpu_halt(void) __dead2;
void cpu_reset(void) __dead2;
void fork_trampoline(void);
+void fork_trampoline_kthread(void);
void identify_cache(uint64_t);
void identify_cpu(u_int);
void install_cpu_errata(void);
diff --git a/sys/arm64/include/frame.h b/sys/arm64/include/frame.h
--- a/sys/arm64/include/frame.h
+++ b/sys/arm64/include/frame.h
@@ -41,7 +41,8 @@
#include <sys/ucontext.h>
/*
- * NOTE: keep this structure in sync with struct reg and struct mcontext.
+ * NOTE: keep this structure in sync with struct reg, struct mcontext, and the
+ * trap frame CFI information.
*/
struct trapframe {
uint64_t tf_sp;
diff --git a/sys/arm64/include/trap_cfi.h b/sys/arm64/include/trap_cfi.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/include/trap_cfi.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2026 FreeBSD Foundation
+ *
+ * This software was developed by Minsoo Choo under sponsorship from the
+ * FreeBSD Foundation.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifndef _MACHINE_TRAP_CFI_H_
+#define _MACHINE_TRAP_CFI_H_
+
+#include "assym.inc"
+
+ .cfi_sections .eh_frame
+
+#define CFI_PC 32
+#define CFI_CPSR 33
+
+.macro trap_cfi_entry state=empty
+ .cfi_signal_frame
+ .cfi_return_column CFI_PC
+ .cfi_def_cfa sp, 0
+ .cfi_undefined CFI_PC
+.ifc \state,full
+ trap_cfi_full
+.endif
+.ifc \state,terminal
+ .cfi_def_cfa sp, TF_SIZE
+.endif
+.endm
+
+.macro trap_cfi_saved reg, off
+ .cfi_offset \reg, \off - TF_SIZE
+.endm
+
+.macro trap_cfi_full
+ .cfi_def_cfa sp, TF_SIZE
+ trap_cfi_saved 31, TF_SP
+ trap_cfi_saved x30, TF_LR
+ trap_cfi_saved CFI_PC, TF_ELR
+ trap_cfi_saved CFI_CPSR, TF_SPSR
+ .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29
+ trap_cfi_saved x\n, TF_X + \n * 8
+ .endr
+.endm
+
+#endif /* _MACHINE_TRAP_CFI_H_ */
diff --git a/sys/conf/ldscript.arm64 b/sys/conf/ldscript.arm64
--- a/sys/conf/ldscript.arm64
+++ b/sys/conf/ldscript.arm64
@@ -45,6 +45,7 @@
*(.note.gnu.build-id)
PROVIDE (__build_id_end = .);
}
+ .eh_frame : { KEEP (*(.eh_frame)) }
.rel.text :
{ *(.rel.text) *(.rel.gnu.linkonce.t*) }
.rela.text :

File Metadata

Mime Type
text/plain
Expires
Sat, Sep 19, 9:24 PM (4 h, 10 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
39035340
Default Alt Text
D59648.diff (9 KB)

Event Timeline