Page MenuHomeFreeBSD
Files F173601222

D59649.id186696.diff
No OneTemporary
Actions

D59649.id186696.diff
View Options

diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S
--- a/sys/amd64/amd64/apic_vector.S
+++ b/sys/amd64/amd64/apic_vector.S
@@ -93,6 +93,7 @@
3:
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
.endm
/*
@@ -105,8 +106,11 @@
.text
SUPERALIGN_TEXT
IDTVEC(spuriousint)
+ trap_cfi_entry ., empty
+ trap_cfi_machine
/* No EOI cycle used here */
jmp doreti_iret
+ trap_cfi_end .
ISR_VEC 1, apic_isr1
ISR_VEC 2, apic_isr2
@@ -125,6 +129,7 @@
call lapic_handle_timer
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
/*
* Local APIC CMCI handler.
@@ -134,6 +139,7 @@
call lapic_handle_cmc
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
/*
* Local APIC error interrupt handler.
@@ -143,6 +149,7 @@
call lapic_handle_error
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
/*
* Local APIC thermal interrupt handler.
@@ -152,6 +159,7 @@
call lapic_handle_thermal
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
#ifdef XENHVM
/*
@@ -164,6 +172,7 @@
call xen_arch_intr_handle_upcall
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
#endif
/*
@@ -181,6 +190,7 @@
call as_lapic_eoi
KMSAN_LEAVE
jmp ld_regs
+ trap_cfi_end .
/*
* Handler for IPIs sent via the per-cpu IPI bitmap.
@@ -192,6 +202,7 @@
call ipi_bitmap_handler
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
/*
* Executed by a CPU when it receives an IPI_STOP from another CPU.
@@ -202,6 +213,7 @@
call cpustop_handler
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
/*
* Executed by a CPU when it receives an IPI_SUSPEND from another CPU.
@@ -212,6 +224,7 @@
call as_lapic_eoi
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
/*
* Executed by a CPU when it receives an IPI_OFF from another CPU.
@@ -223,6 +236,7 @@
call as_lapic_eoi
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
/*
* Executed by a CPU when it receives an IPI_SWI.
@@ -234,6 +248,7 @@
call ipi_swi_handler
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
/*
* Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
@@ -251,6 +266,7 @@
call as_lapic_eoi
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
/*
* IPI handler whose purpose is to interrupt the CPU with minimum overhead.
@@ -273,15 +289,31 @@
.text
SUPERALIGN_TEXT
IDTVEC(justreturn)
+ trap_cfi_entry ., empty
+ trap_cfi_machine
pushq %rax
+ .cfi_adjust_cfa_offset 8
+ .cfi_offset %rax, -48
pushq %rcx
+ .cfi_adjust_cfa_offset 8
+ .cfi_offset %rcx, -56
pushq %rdx
+ .cfi_adjust_cfa_offset 8
+ .cfi_offset %rdx, -64
call as_lapic_eoi
popq %rdx
+ .cfi_adjust_cfa_offset -8
+ .cfi_same_value %rdx
popq %rcx
+ .cfi_adjust_cfa_offset -8
+ .cfi_same_value %rcx
popq %rax
+ .cfi_adjust_cfa_offset -8
+ .cfi_same_value %rax
jmp doreti_iret
+ trap_cfi_end .
INTR_HANDLER justreturn1
call as_lapic_eoi
jmp doreti
+ trap_cfi_end .
diff --git a/sys/amd64/amd64/atpic_vector.S b/sys/amd64/amd64/atpic_vector.S
--- a/sys/amd64/amd64/atpic_vector.S
+++ b/sys/amd64/amd64/atpic_vector.S
@@ -49,6 +49,7 @@
call atpic_handle_intr
KMSAN_LEAVE
jmp doreti
+ trap_cfi_end .
.endm
INTR 0, atpic_intr0
diff --git a/sys/amd64/amd64/db_trace.c b/sys/amd64/amd64/db_trace.c
--- a/sys/amd64/amd64/db_trace.c
+++ b/sys/amd64/amd64/db_trace.c
@@ -306,7 +306,8 @@
actframe = tf->tf_rsp - 8;
}
} else if (name != NULL &&
- strcmp(name, "fork_trampoline") == 0) {
+ (strcmp(name, "fork_trampoline") == 0 ||
+ strcmp(name, "fork_trampoline_kthread") == 0)) {
/*
* Don't try to walk back on a stack for a
* process that hasn't actually been run yet.
diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -103,22 +103,33 @@
.macro TRAP_NOEN l, trapno
PTI_ENTRY \l,\l\()_pti_k,\l\()_pti_u
\l\()_pti_k:
+ trap_cfi_entry ., empty
+ trap_cfi_machine
subq $TF_RIP,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $\trapno,TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
movq $0,TF_ERR(%rsp)
jmp alltraps_noen_k
+ trap_cfi_end .
\l\()_pti_u:
+ trap_cfi_entry ., empty
+ trap_cfi_machine
subq $TF_RIP,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $\trapno,TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
movq $0,TF_ERR(%rsp)
jmp alltraps_noen_u
+ trap_cfi_end .
.globl X\l
.type X\l,@function
X\l:
+ trap_cfi_entry ., empty
+ trap_cfi_machine
subq $TF_RIP,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $\trapno,TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
movq $0,TF_ERR(%rsp)
@@ -127,6 +138,7 @@
swapgs
lfence
jmp alltraps_noen_u
+ trap_cfi_end .
.endm
TRAP_NOEN bpt, T_BPTFLT
@@ -138,22 +150,33 @@
.macro TRAP l, trapno
PTI_ENTRY \l,\l\()_pti_k,\l\()_pti_u
\l\()_pti_k:
+ trap_cfi_entry ., empty
+ trap_cfi_machine
subq $TF_RIP,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $\trapno,TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
movq $0,TF_ERR(%rsp)
jmp alltraps_k
+ trap_cfi_end .
\l\()_pti_u:
+ trap_cfi_entry ., empty
+ trap_cfi_machine
subq $TF_RIP,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $\trapno,TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
movq $0,TF_ERR(%rsp)
jmp alltraps_u
+ trap_cfi_end .
.globl X\l
.type X\l,@function
X\l:
+ trap_cfi_entry ., empty
+ trap_cfi_machine
subq $TF_RIP,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $\trapno,TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
movq $0,TF_ERR(%rsp)
@@ -162,6 +185,7 @@
swapgs
lfence
jmp alltraps_u
+ trap_cfi_end .
.endm
TRAP div, T_DIVIDE
@@ -178,19 +202,30 @@
.macro TRAP_ERR l, trapno
PTI_ENTRY \l,\l\()_pti_k,\l\()_pti_u,has_err=1
\l\()_pti_k:
+ trap_cfi_entry ., empty
+ trap_cfi_machine 1
subq $TF_ERR,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $\trapno,TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
jmp alltraps_k
+ trap_cfi_end .
\l\()_pti_u:
+ trap_cfi_entry ., empty
+ trap_cfi_machine 1
subq $TF_ERR,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $\trapno,TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
jmp alltraps_u
+ trap_cfi_end .
.globl X\l
.type X\l,@function
X\l:
+ trap_cfi_entry ., empty
+ trap_cfi_machine 1
subq $TF_ERR,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $\trapno,TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
testb $SEL_RPL_MASK,TF_CS(%rsp)
@@ -198,6 +233,7 @@
swapgs
lfence
jmp alltraps_u
+ trap_cfi_end .
.endm
TRAP_ERR tss, T_TSSFLT
@@ -214,23 +250,46 @@
.globl alltraps_u
.type alltraps_u,@function
alltraps_u:
+ trap_cfi_entry ., empty
+ .cfi_def_cfa %rsp, TF_SIZE
+ trap_cfi_saved %rip, TF_RIP
+ trap_cfi_saved %cs, TF_CS
+ trap_cfi_saved 49, TF_RFLAGS
+ trap_cfi_saved %rsp, TF_RSP
+ trap_cfi_saved %ss, TF_SS
movq %rdi,TF_RDI(%rsp)
+ trap_cfi_saved %rdi, TF_RDI
movq %rdx,TF_RDX(%rsp)
+ trap_cfi_saved %rdx, TF_RDX
movq %rax,TF_RAX(%rsp)
+ trap_cfi_saved %rax, TF_RAX
movq %rcx,TF_RCX(%rsp)
+ trap_cfi_saved %rcx, TF_RCX
movq PCPU(CURPCB),%rdi
andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
call handle_ibrs_entry
jmp alltraps_save_segs
+ trap_cfi_end .
SUPERALIGN_TEXT
.globl alltraps_k
.type alltraps_k,@function
alltraps_k:
+ trap_cfi_entry ., empty
+ .cfi_def_cfa %rsp, TF_SIZE
+ trap_cfi_saved %rip, TF_RIP
+ trap_cfi_saved %cs, TF_CS
+ trap_cfi_saved 49, TF_RFLAGS
+ trap_cfi_saved %rsp, TF_RSP
+ trap_cfi_saved %ss, TF_SS
lfence
movq %rdi,TF_RDI(%rsp)
+ trap_cfi_saved %rdi, TF_RDI
movq %rdx,TF_RDX(%rsp)
+ trap_cfi_saved %rdx, TF_RDX
movq %rax,TF_RAX(%rsp)
+ trap_cfi_saved %rax, TF_RAX
movq %rcx,TF_RCX(%rsp)
+ trap_cfi_saved %rcx, TF_RCX
alltraps_save_segs:
SAVE_SEGS
testl $PSL_I,TF_RFLAGS(%rsp)
@@ -238,16 +297,27 @@
sti
alltraps_pushregs_no_rax:
movq %rsi,TF_RSI(%rsp)
+ trap_cfi_saved %rsi, TF_RSI
movq %r8,TF_R8(%rsp)
+ trap_cfi_saved %r8, TF_R8
movq %r9,TF_R9(%rsp)
+ trap_cfi_saved %r9, TF_R9
movq %rbx,TF_RBX(%rsp)
+ trap_cfi_saved %rbx, TF_RBX
movq %rbp,TF_RBP(%rsp)
+ trap_cfi_saved %rbp, TF_RBP
movq %r10,TF_R10(%rsp)
+ trap_cfi_saved %r10, TF_R10
movq %r11,TF_R11(%rsp)
+ trap_cfi_saved %r11, TF_R11
movq %r12,TF_R12(%rsp)
+ trap_cfi_saved %r12, TF_R12
movq %r13,TF_R13(%rsp)
+ trap_cfi_saved %r13, TF_R13
movq %r14,TF_R14(%rsp)
+ trap_cfi_saved %r14, TF_R14
movq %r15,TF_R15(%rsp)
+ trap_cfi_saved %r15, TF_R15
movl $TF_HASSEGS,TF_FLAGS(%rsp)
pushfq
andq $~(PSL_D | PSL_AC),(%rsp)
@@ -277,14 +347,17 @@
/* Jump to the code hooked in by DTrace. */
jmpq *dtrace_invop_jump_addr
#endif
+ trap_cfi_end .
.globl calltrap
.type calltrap,@function
calltrap:
+ trap_cfi_entry ., full
KMSAN_ENTER
movq %rsp, %rdi
call trap_check
KMSAN_LEAVE
jmp doreti /* Handle any pending ASTs */
+ trap_cfi_end .
/*
* alltraps_noen_u/k entry points.
@@ -296,46 +369,85 @@
.globl alltraps_noen_u
.type alltraps_noen_u,@function
alltraps_noen_u:
+ trap_cfi_entry ., empty
+ .cfi_def_cfa %rsp, TF_SIZE
+ trap_cfi_saved %rip, TF_RIP
+ trap_cfi_saved %cs, TF_CS
+ trap_cfi_saved 49, TF_RFLAGS
+ trap_cfi_saved %rsp, TF_RSP
+ trap_cfi_saved %ss, TF_SS
movq %rdi,TF_RDI(%rsp)
+ trap_cfi_saved %rdi, TF_RDI
movq PCPU(CURPCB),%rdi
andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
jmp alltraps_noen_save_segs
+ trap_cfi_end .
SUPERALIGN_TEXT
.globl alltraps_noen_k
.type alltraps_noen_k,@function
alltraps_noen_k:
+ trap_cfi_entry ., empty
+ .cfi_def_cfa %rsp, TF_SIZE
+ trap_cfi_saved %rip, TF_RIP
+ trap_cfi_saved %cs, TF_CS
+ trap_cfi_saved 49, TF_RFLAGS
+ trap_cfi_saved %rsp, TF_RSP
+ trap_cfi_saved %ss, TF_SS
lfence
movq %rdi,TF_RDI(%rsp)
+ trap_cfi_saved %rdi, TF_RDI
alltraps_noen_save_segs:
SAVE_SEGS
movq %rdx,TF_RDX(%rsp)
+ trap_cfi_saved %rdx, TF_RDX
movq %rax,TF_RAX(%rsp)
+ trap_cfi_saved %rax, TF_RAX
movq %rcx,TF_RCX(%rsp)
+ trap_cfi_saved %rcx, TF_RCX
testb $SEL_RPL_MASK,TF_CS(%rsp)
jz alltraps_pushregs_no_rax
call handle_ibrs_entry
jmp alltraps_pushregs_no_rax
+ trap_cfi_end .
IDTVEC(dblfault)
+ trap_cfi_entry ., empty
+ trap_cfi_machine 1
subq $TF_ERR,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $T_DOUBLEFLT,TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
movq $0,TF_ERR(%rsp)
movq %rdi,TF_RDI(%rsp)
+ trap_cfi_saved %rdi, TF_RDI
movq %rsi,TF_RSI(%rsp)
+ trap_cfi_saved %rsi, TF_RSI
movq %rdx,TF_RDX(%rsp)
+ trap_cfi_saved %rdx, TF_RDX
movq %rcx,TF_RCX(%rsp)
+ trap_cfi_saved %rcx, TF_RCX
movq %r8,TF_R8(%rsp)
+ trap_cfi_saved %r8, TF_R8
movq %r9,TF_R9(%rsp)
+ trap_cfi_saved %r9, TF_R9
movq %rax,TF_RAX(%rsp)
+ trap_cfi_saved %rax, TF_RAX
movq %rbx,TF_RBX(%rsp)
+ trap_cfi_saved %rbx, TF_RBX
movq %rbp,TF_RBP(%rsp)
+ trap_cfi_saved %rbp, TF_RBP
movq %r10,TF_R10(%rsp)
+ trap_cfi_saved %r10, TF_R10
movq %r11,TF_R11(%rsp)
+ trap_cfi_saved %r11, TF_R11
movq %r12,TF_R12(%rsp)
+ trap_cfi_saved %r12, TF_R12
movq %r13,TF_R13(%rsp)
+ trap_cfi_saved %r13, TF_R13
movq %r14,TF_R14(%rsp)
+ trap_cfi_saved %r14, TF_R14
movq %r15,TF_R15(%rsp)
+ trap_cfi_saved %r15, TF_R15
SAVE_SEGS
movl $TF_HASSEGS,TF_FLAGS(%rsp)
pushfq
@@ -358,45 +470,75 @@
KMSAN_LEAVE
3: hlt
jmp 3b
+ trap_cfi_end .
ALIGN_TEXT
IDTVEC(page_pti)
+ trap_cfi_entry ., empty
+ trap_cfi_machine 1
testb $SEL_RPL_MASK,PTI_CS-PTI_ERR(%rsp)
jz page_k
swapgs
lfence
pushq %rax
+ .cfi_adjust_cfa_offset 8
+ .cfi_offset %rax, PTI_RAX - PTI_SIZE
movq %cr3,%rax
movq %rax,PCPU(SAVED_UCR3)
cmpq $~0,PCPU(UCR3)
jne 1f
+ .cfi_remember_state
popq %rax
+ .cfi_adjust_cfa_offset -8
+ .cfi_same_value %rax
jmp page_u
+ .cfi_restore_state
1: pushq %rdx
- PTI_UUENTRY has_err=1
+ .cfi_adjust_cfa_offset 8
+ .cfi_offset %rdx, PTI_RDX - PTI_SIZE
+ PTI_UUENTRY has_err=1, cfi=1
jmp page_u
+ trap_cfi_end .
ALIGN_TEXT
IDTVEC(page)
+ trap_cfi_entry ., empty
+ trap_cfi_machine 1
testb $SEL_RPL_MASK,TF_CS-TF_ERR(%rsp) /* Did we come from kernel? */
jnz page_u_swapgs /* already running with kernel GS.base */
+ trap_cfi_end .
page_k:
+ trap_cfi_entry ., empty
+ trap_cfi_machine 1
lfence
subq $TF_ERR,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movq %rdi,TF_RDI(%rsp) /* free up GP registers */
+ trap_cfi_saved %rdi, TF_RDI
movq %rax,TF_RAX(%rsp)
+ trap_cfi_saved %rax, TF_RAX
movq %rdx,TF_RDX(%rsp)
+ trap_cfi_saved %rdx, TF_RDX
movq %rcx,TF_RCX(%rsp)
+ trap_cfi_saved %rcx, TF_RCX
jmp page_cr2
+ trap_cfi_end .
ALIGN_TEXT
page_u_swapgs:
+ trap_cfi_entry ., empty
+ trap_cfi_machine 1
swapgs
lfence
page_u:
subq $TF_ERR,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movq %rdi,TF_RDI(%rsp)
+ trap_cfi_saved %rdi, TF_RDI
movq %rax,TF_RAX(%rsp)
+ trap_cfi_saved %rax, TF_RAX
movq %rdx,TF_RDX(%rsp)
+ trap_cfi_saved %rdx, TF_RDX
movq %rcx,TF_RCX(%rsp)
+ trap_cfi_saved %rcx, TF_RCX
movq PCPU(CURPCB),%rdi
andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
movq PCPU(SAVED_UCR3),%rax
@@ -411,6 +553,7 @@
jz alltraps_pushregs_no_rax
sti
jmp alltraps_pushregs_no_rax
+ trap_cfi_end .
/*
* We have to special-case this one. If we get a trap in doreti() at
@@ -448,16 +591,25 @@
1: swapgs
jmp X\name
IDTVEC(\name\()_pti)
+ trap_cfi_entry ., empty
+ trap_cfi_machine 1
cmpq $doreti_iret,PTI_RIP-2*8(%rsp)
je \name\()_pti_doreti
testb $SEL_RPL_MASK,PTI_CS-2*8(%rsp) /* %rax, %rdx not yet pushed */
jz X\name /* lfence is not needed until %gs: use */
- PTI_UENTRY has_err=1
+ PTI_UENTRY has_err=1, cfi=1
swapgs /* fence provided by PTI_UENTRY */
+ trap_cfi_end .
IDTVEC(\name)
+ trap_cfi_entry ., empty
+ trap_cfi_machine 1
subq $TF_ERR,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $\trapno,TF_TRAPNO(%rsp)
jmp prot_addrf
+.ifnc \name,prot
+ trap_cfi_end .
+.endif
.endm
PROTF_ENTRY missing, T_SEGNPFLT
@@ -467,9 +619,13 @@
prot_addrf:
movq $0,TF_ADDR(%rsp)
movq %rdi,TF_RDI(%rsp) /* free up a GP register */
+ trap_cfi_saved %rdi, TF_RDI
movq %rax,TF_RAX(%rsp)
+ trap_cfi_saved %rax, TF_RAX
movq %rdx,TF_RDX(%rsp)
+ trap_cfi_saved %rdx, TF_RDX
movq %rcx,TF_RCX(%rsp)
+ trap_cfi_saved %rcx, TF_RCX
movw %fs,TF_FS(%rsp)
movw %gs,TF_GS(%rsp)
leaq doreti_iret(%rip),%rdi
@@ -501,6 +657,7 @@
6: lfence
movq PCPU(CURPCB),%rdi
jmp 4b
+ trap_cfi_end .
/*
* Fast syscall entry point. We enter here with just our new %cs/%ss set,
@@ -513,6 +670,10 @@
*/
SUPERALIGN_TEXT
IDTVEC(fast_syscall_pti)
+ trap_cfi_entry ., empty
+ .cfi_register %rip, %rcx
+ .cfi_register 49, %r11
+ .cfi_val_offset %rsp, 0
swapgs
cmpq $~0,PCPU(UCR3)
je fast_syscall_common
@@ -521,19 +682,34 @@
movq %rax,%cr3
movq PCPU(SCRATCH_RAX),%rax
jmp fast_syscall_common
+ trap_cfi_end .
SUPERALIGN_TEXT
IDTVEC(fast_syscall)
+ trap_cfi_entry ., empty
+ .cfi_register %rip, %rcx
+ .cfi_register 49, %r11
+ .cfi_val_offset %rsp, 0
swapgs
+ trap_cfi_end .
fast_syscall_common:
+ trap_cfi_entry ., empty
+ .cfi_register %rip, %rcx
+ .cfi_register 49, %r11
+ .cfi_val_offset %rsp, 0
movq %rsp,PCPU(SCRATCH_RSP)
movq PCPU(RSP0),%rsp
+ .cfi_undefined %rsp
/* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
subq $TF_SIZE,%rsp
+ .cfi_def_cfa_offset TF_SIZE
/* defer TF_RSP till we have a spare register */
movq %r11,TF_RFLAGS(%rsp)
+ trap_cfi_saved 49, TF_RFLAGS
movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */
+ trap_cfi_saved %rip, TF_RIP
movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */
movq %r11,TF_RSP(%rsp) /* user stack pointer */
+ trap_cfi_saved %rsp, TF_RSP
/*
* Save a few arg registers early to free them for use in
* handle_ibrs_entry(). %r10 is especially tricky. It is not an
@@ -542,26 +718,41 @@
* clobber %r11, but %r11 (original %eflags) has been saved.
*/
movq %rax,TF_RAX(%rsp) /* syscall number */
+ trap_cfi_saved %rax, TF_RAX
movq %rdx,TF_RDX(%rsp) /* arg 3 */
+ trap_cfi_saved %rdx, TF_RDX
movq %r10,TF_RCX(%rsp) /* arg 4 */
+ trap_cfi_saved %rcx, TF_RCX
SAVE_SEGS
call handle_ibrs_entry
movq PCPU(CURPCB),%r11
andl $~PCB_FULL_IRET,PCB_FLAGS(%r11)
sti
movq $KUDSEL,TF_SS(%rsp)
+ trap_cfi_saved %ss, TF_SS
movq $KUCSEL,TF_CS(%rsp)
+ trap_cfi_saved %cs, TF_CS
movq $2,TF_ERR(%rsp)
movq %rdi,TF_RDI(%rsp) /* arg 1 */
+ trap_cfi_saved %rdi, TF_RDI
movq %rsi,TF_RSI(%rsp) /* arg 2 */
+ trap_cfi_saved %rsi, TF_RSI
movq %r8,TF_R8(%rsp) /* arg 5 */
+ trap_cfi_saved %r8, TF_R8
movq %r9,TF_R9(%rsp) /* arg 6 */
+ trap_cfi_saved %r9, TF_R9
movq %rbx,TF_RBX(%rsp) /* C preserved */
+ trap_cfi_saved %rbx, TF_RBX
movq %rbp,TF_RBP(%rsp) /* C preserved */
+ trap_cfi_saved %rbp, TF_RBP
movq %r12,TF_R12(%rsp) /* C preserved */
+ trap_cfi_saved %r12, TF_R12
movq %r13,TF_R13(%rsp) /* C preserved */
+ trap_cfi_saved %r13, TF_R13
movq %r14,TF_R14(%rsp) /* C preserved */
+ trap_cfi_saved %r14, TF_R14
movq %r15,TF_R15(%rsp) /* C preserved */
+ trap_cfi_saved %r15, TF_R15
movl $TF_HASSEGS,TF_FLAGS(%rsp)
movq PCPU(CURTHREAD),%rdi
movq %rsp,TD_FRAME(%rdi)
@@ -587,6 +778,11 @@
movq TF_RFLAGS(%rsp),%r11 /* original %rflags */
movq TF_RIP(%rsp),%rcx /* original %rip */
movq TF_RSP(%rsp),%rsp /* user stack pointer */
+ .cfi_def_cfa %rsp, 0
+ trap_cfi_gprs_live
+ .cfi_register %rip, %rcx
+ .cfi_register 49, %r11
+ .cfi_val_offset %rsp, 0
xorl %r8d,%r8d /* zero the rest of GPRs */
xorl %r10d,%r10d
cmpq $~0,PCPU(UCR3)
@@ -607,6 +803,7 @@
4: /* Requested full context restore, use doreti for that. */
jmp doreti
+ trap_cfi_end .
/*
* Here for CYA insurance, in case a "syscall" instruction gets
@@ -614,7 +811,12 @@
* to *something* if EFER_SCE is enabled.
*/
IDTVEC(fast_syscall32)
+ trap_cfi_entry ., empty
+ .cfi_register %rip, %rcx
+ .cfi_register 49, %r11
+ .cfi_val_offset %rsp, 0
sysret
+ trap_cfi_end .
/*
* DB# handler is very similar to NM#, because 'mov/pop %ss' delay
@@ -623,25 +825,43 @@
* on IST stack and be ready for non-kernel GSBASE.
*/
IDTVEC(dbg)
+ trap_cfi_entry ., empty
+ trap_cfi_machine
subq $TF_RIP,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $(T_TRCTRAP),TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
movq $0,TF_ERR(%rsp)
movq %rdi,TF_RDI(%rsp)
+ trap_cfi_saved %rdi, TF_RDI
movq %rsi,TF_RSI(%rsp)
+ trap_cfi_saved %rsi, TF_RSI
movq %rdx,TF_RDX(%rsp)
+ trap_cfi_saved %rdx, TF_RDX
movq %rcx,TF_RCX(%rsp)
+ trap_cfi_saved %rcx, TF_RCX
movq %r8,TF_R8(%rsp)
+ trap_cfi_saved %r8, TF_R8
movq %r9,TF_R9(%rsp)
+ trap_cfi_saved %r9, TF_R9
movq %rax,TF_RAX(%rsp)
+ trap_cfi_saved %rax, TF_RAX
movq %rbx,TF_RBX(%rsp)
+ trap_cfi_saved %rbx, TF_RBX
movq %rbp,TF_RBP(%rsp)
+ trap_cfi_saved %rbp, TF_RBP
movq %r10,TF_R10(%rsp)
+ trap_cfi_saved %r10, TF_R10
movq %r11,TF_R11(%rsp)
+ trap_cfi_saved %r11, TF_R11
movq %r12,TF_R12(%rsp)
+ trap_cfi_saved %r12, TF_R12
movq %r13,TF_R13(%rsp)
+ trap_cfi_saved %r13, TF_R13
movq %r14,TF_R14(%rsp)
+ trap_cfi_saved %r14, TF_R14
movq %r15,TF_R15(%rsp)
+ trap_cfi_saved %r15, TF_R15
SAVE_SEGS
movl $TF_HASSEGS,TF_FLAGS(%rsp)
pushfq
@@ -717,6 +937,7 @@
movq %rsp,%rsi
rep;movsb
movq %rax,%rsp
+ .cfi_def_cfa %rsp, TF_SIZE
call handle_ibrs_entry
movq PCPU(CURPCB),%rdi
orl $PCB_FULL_IRET,PCB_FLAGS(%rdi)
@@ -730,6 +951,7 @@
orq %rdx,%rax
movq %rax,PCB_GSBASE(%rdi)
3: jmp calltrap
+ trap_cfi_end .
/*
* NMI handling is special.
@@ -756,25 +978,43 @@
*/
IDTVEC(nmi)
+ trap_cfi_entry ., empty
+ trap_cfi_machine
subq $TF_RIP,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $(T_NMI),TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
movq $0,TF_ERR(%rsp)
movq %rdi,TF_RDI(%rsp)
+ trap_cfi_saved %rdi, TF_RDI
movq %rsi,TF_RSI(%rsp)
+ trap_cfi_saved %rsi, TF_RSI
movq %rdx,TF_RDX(%rsp)
+ trap_cfi_saved %rdx, TF_RDX
movq %rcx,TF_RCX(%rsp)
+ trap_cfi_saved %rcx, TF_RCX
movq %r8,TF_R8(%rsp)
+ trap_cfi_saved %r8, TF_R8
movq %r9,TF_R9(%rsp)
+ trap_cfi_saved %r9, TF_R9
movq %rax,TF_RAX(%rsp)
+ trap_cfi_saved %rax, TF_RAX
movq %rbx,TF_RBX(%rsp)
+ trap_cfi_saved %rbx, TF_RBX
movq %rbp,TF_RBP(%rsp)
+ trap_cfi_saved %rbp, TF_RBP
movq %r10,TF_R10(%rsp)
+ trap_cfi_saved %r10, TF_R10
movq %r11,TF_R11(%rsp)
+ trap_cfi_saved %r11, TF_R11
movq %r12,TF_R12(%rsp)
+ trap_cfi_saved %r12, TF_R12
movq %r13,TF_R13(%rsp)
+ trap_cfi_saved %r13, TF_R13
movq %r14,TF_R14(%rsp)
+ trap_cfi_saved %r14, TF_R14
movq %r15,TF_R15(%rsp)
+ trap_cfi_saved %r15, TF_R15
SAVE_SEGS
movl $TF_HASSEGS,TF_FLAGS(%rsp)
pushfq
@@ -839,7 +1079,9 @@
movq %rax,PCB_GSBASE(%rdi)
3:
/* Note: this label is also used by ddb and gdb: */
+ trap_cfi_end .
nmi_calltrap:
+ trap_cfi_entry ., full
KMSAN_ENTER
movq %rsp,%rdi
call trap
@@ -941,6 +1183,7 @@
RESTORE_REGS
addq $TF_RIP,%rsp
jmp doreti_iret
+ trap_cfi_end .
/*
* MC# handling is similar to NMI.
@@ -964,25 +1207,43 @@
*/
IDTVEC(mchk)
+ trap_cfi_entry ., empty
+ trap_cfi_machine
subq $TF_RIP,%rsp
+ .cfi_def_cfa_offset TF_SIZE
movl $(T_MCHK),TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)
movq $0,TF_ERR(%rsp)
movq %rdi,TF_RDI(%rsp)
+ trap_cfi_saved %rdi, TF_RDI
movq %rsi,TF_RSI(%rsp)
+ trap_cfi_saved %rsi, TF_RSI
movq %rdx,TF_RDX(%rsp)
+ trap_cfi_saved %rdx, TF_RDX
movq %rcx,TF_RCX(%rsp)
+ trap_cfi_saved %rcx, TF_RCX
movq %r8,TF_R8(%rsp)
+ trap_cfi_saved %r8, TF_R8
movq %r9,TF_R9(%rsp)
+ trap_cfi_saved %r9, TF_R9
movq %rax,TF_RAX(%rsp)
+ trap_cfi_saved %rax, TF_RAX
movq %rbx,TF_RBX(%rsp)
+ trap_cfi_saved %rbx, TF_RBX
movq %rbp,TF_RBP(%rsp)
+ trap_cfi_saved %rbp, TF_RBP
movq %r10,TF_R10(%rsp)
+ trap_cfi_saved %r10, TF_R10
movq %r11,TF_R11(%rsp)
+ trap_cfi_saved %r11, TF_R11
movq %r12,TF_R12(%rsp)
+ trap_cfi_saved %r12, TF_R12
movq %r13,TF_R13(%rsp)
+ trap_cfi_saved %r13, TF_R13
movq %r14,TF_R14(%rsp)
+ trap_cfi_saved %r14, TF_R14
movq %r15,TF_R15(%rsp)
+ trap_cfi_saved %r15, TF_R15
SAVE_SEGS
movl $TF_HASSEGS,TF_FLAGS(%rsp)
pushfq
@@ -1028,7 +1289,9 @@
movq %rax,%cr3
1: call handle_ibrs_entry
/* Note: this label is also used by ddb and gdb: */
+ trap_cfi_end .
mchk_calltrap:
+ trap_cfi_entry ., full
KMSAN_ENTER
movq %rsp,%rdi
call mca_intr
@@ -1057,8 +1320,10 @@
RESTORE_REGS
addq $TF_RIP,%rsp
jmp doreti_iret
+ trap_cfi_end .
ENTRY(fork_trampoline)
+ trap_cfi_entry fork_trampoline, full
movq %r12,%rdi /* function */
movq %rbx,%rsi /* arg1 */
movq %rsp,%rdx /* trapframe pointer */
@@ -1066,6 +1331,17 @@
cmpb $0, fred
jne fred_checkast
jmp doreti /* Handle any ASTs */
+ trap_cfi_end fork_trampoline
+
+ENTRY(fork_trampoline_kthread)
+ trap_cfi_entry fork_trampoline_kthread, terminal
+ movq %r12,%rdi
+ movq %rbx,%rsi
+ movq %rsp,%rdx
+ call fork_exit
+ ud2
+1: jmp 1b
+ trap_cfi_end fork_trampoline_kthread
/*
* To efficiently implement classification of trap and interrupt handlers
@@ -1111,6 +1387,7 @@
.type doreti,@function
.globl doreti
doreti:
+ trap_cfi_entry ., full
/*
* Check if ASTs can be handled now.
*/
@@ -1246,9 +1523,13 @@
jmp doreti_iret
1: swapgs
2: addq $TF_RIP,%rsp
+ .cfi_def_cfa_offset 40
.globl doreti_iret
doreti_iret:
+ trap_cfi_machine
+ trap_cfi_gprs_live
iretq
+ trap_cfi_end .
/*
* 32bit gsbase path.
diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c
--- a/sys/amd64/amd64/vm_machdep.c
+++ b/sys/amd64/amd64/vm_machdep.c
@@ -168,7 +168,10 @@
pcb2->pcb_rbp = 0;
pcb2->pcb_rsp = (register_t)td2->td_frame - sizeof(void *);
pcb2->pcb_rbx = (register_t)td2; /* fork_trampoline argument */
- pcb2->pcb_rip = (register_t)fork_trampoline;
+ if ((td2->td_pflags & TDP_KTHREAD) != 0)
+ pcb2->pcb_rip = (register_t)fork_trampoline_kthread;
+ else
+ pcb2->pcb_rip = (register_t)fork_trampoline;
/*-
* pcb2->pcb_dr*: cloned above.
* pcb2->pcb_savefpu: cloned above.
diff --git a/sys/amd64/ia32/ia32_exception.S b/sys/amd64/ia32/ia32_exception.S
--- a/sys/amd64/ia32/ia32_exception.S
+++ b/sys/amd64/ia32/ia32_exception.S
@@ -39,34 +39,59 @@
*/
SUPERALIGN_TEXT
IDTVEC(int0x80_syscall_pti)
- PTI_UENTRY has_err=0
+ trap_cfi_entry ., empty
+ trap_cfi_machine
+ PTI_UENTRY has_err=0, cfi=1
jmp int0x80_syscall_common
+ trap_cfi_end .
SUPERALIGN_TEXT
IDTVEC(int0x80_syscall)
+ trap_cfi_entry ., empty
+ trap_cfi_machine
swapgs
+ trap_cfi_end .
int0x80_syscall_common:
+ trap_cfi_entry ., empty
+ trap_cfi_machine
pushq $2 /* sizeof "int 0x80" */
+ .cfi_adjust_cfa_offset 8
subq $TF_ERR,%rsp /* skip over tf_trapno */
+ .cfi_def_cfa_offset TF_SIZE
movq %rdi,TF_RDI(%rsp)
+ trap_cfi_saved %rdi, TF_RDI
movq PCPU(CURPCB),%rdi
andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi)
SAVE_SEGS
movq %rax,TF_RAX(%rsp)
+ trap_cfi_saved %rax, TF_RAX
movq %rdx,TF_RDX(%rsp)
+ trap_cfi_saved %rdx, TF_RDX
movq %rcx,TF_RCX(%rsp)
+ trap_cfi_saved %rcx, TF_RCX
call handle_ibrs_entry
sti
movq %rsi,TF_RSI(%rsp)
+ trap_cfi_saved %rsi, TF_RSI
movq %r8,TF_R8(%rsp)
+ trap_cfi_saved %r8, TF_R8
movq %r9,TF_R9(%rsp)
+ trap_cfi_saved %r9, TF_R9
movq %rbx,TF_RBX(%rsp)
+ trap_cfi_saved %rbx, TF_RBX
movq %rbp,TF_RBP(%rsp)
+ trap_cfi_saved %rbp, TF_RBP
movq %r10,TF_R10(%rsp)
+ trap_cfi_saved %r10, TF_R10
movq %r11,TF_R11(%rsp)
+ trap_cfi_saved %r11, TF_R11
movq %r12,TF_R12(%rsp)
+ trap_cfi_saved %r12, TF_R12
movq %r13,TF_R13(%rsp)
+ trap_cfi_saved %r13, TF_R13
movq %r14,TF_R14(%rsp)
+ trap_cfi_saved %r14, TF_R14
movq %r15,TF_R15(%rsp)
+ trap_cfi_saved %r15, TF_R15
movl $TF_HASSEGS,TF_FLAGS(%rsp)
pushfq
andq $~(PSL_D | PSL_AC),(%rsp)
@@ -74,3 +99,4 @@
movq %rsp, %rdi
call ia32_syscall
jmp doreti
+ trap_cfi_end .
diff --git a/sys/amd64/include/asmacros.h b/sys/amd64/include/asmacros.h
--- a/sys/amd64/include/asmacros.h
+++ b/sys/amd64/include/asmacros.h
@@ -77,6 +77,8 @@
popq %rbp
#ifdef LOCORE
+#include <machine/trap_cfi.h>
+
/*
* Access per-CPU data.
*/
@@ -107,25 +109,49 @@
.endr
.endm
- .macro PTI_UUENTRY has_err
+ .macro PTI_UUENTRY has_err, cfi=0
movq PCPU(KCR3),%rax
movq %rax,%cr3
movq PCPU(RSP0),%rax
subq $PTI_SIZE - 8 * (1 - \has_err),%rax
MOVE_STACKS ((PTI_SIZE / 8) - 1 + \has_err)
+.if \cfi
+ .cfi_def_cfa %rax, PTI_SIZE - 8 * (1 - \has_err)
+ .cfi_offset %rax, PTI_RAX - (PTI_SIZE - 8 * (1 - \has_err))
+ .cfi_offset %rdx, PTI_RDX - (PTI_SIZE - 8 * (1 - \has_err))
+.endif
movq %rax,%rsp
+.if \cfi
+ .cfi_def_cfa_register %rsp
+.endif
popq %rdx
+.if \cfi
+ .cfi_adjust_cfa_offset -8
+ .cfi_same_value %rdx
+.endif
popq %rax
+.if \cfi
+ .cfi_adjust_cfa_offset -8
+ .cfi_same_value %rax
+.endif
.endm
- .macro PTI_UENTRY has_err
+ .macro PTI_UENTRY has_err, cfi=0
swapgs
lfence
cmpq $~0,PCPU(UCR3)
je 1f
pushq %rax
+.if \cfi
+ .cfi_adjust_cfa_offset 8
+ .cfi_offset %rax, -(PTI_SIZE - 8 * (2 - \has_err))
+.endif
pushq %rdx
- PTI_UUENTRY \has_err
+.if \cfi
+ .cfi_adjust_cfa_offset 8
+ .cfi_offset %rdx, -(PTI_SIZE - 8 * (1 - \has_err))
+.endif
+ PTI_UUENTRY \has_err, \cfi
1:
.endm
@@ -134,11 +160,14 @@
.globl X\name\()_pti
.type X\name\()_pti,@function
X\name\()_pti:
+ trap_cfi_entry X\name\()_pti, empty
+ trap_cfi_machine \has_err
/* %rax, %rdx, and possibly err are not yet pushed */
testb $SEL_RPL_MASK,PTI_CS-PTI_ERR-((1-\has_err)*8)(%rsp)
jz \contk
- PTI_UENTRY \has_err
+ PTI_UENTRY \has_err, cfi=1
jmp \contu
+ trap_cfi_end X\name\()_pti
.endm
.macro PTI_INTRENTRY vec_name
@@ -146,10 +175,13 @@
.globl X\vec_name\()_pti
.type X\vec_name\()_pti,@function
X\vec_name\()_pti:
+ trap_cfi_entry X\vec_name\()_pti, empty
+ trap_cfi_machine
testb $SEL_RPL_MASK,PTI_CS-3*8(%rsp) /* err, %rax, %rdx not pushed */
jz .L\vec_name\()_u
- PTI_UENTRY has_err=0
+ PTI_UENTRY has_err=0, cfi=1
jmp .L\vec_name\()_u
+ trap_cfi_end X\vec_name\()_pti
.endm
.macro INTR_PUSH_FRAME vec_name
@@ -157,27 +189,45 @@
.globl X\vec_name
.type X\vec_name,@function
X\vec_name:
+ trap_cfi_entry X\vec_name, empty
+ trap_cfi_machine
testb $SEL_RPL_MASK,PTI_CS-3*8(%rsp) /* come from kernel? */
jz .L\vec_name\()_u /* Yes, dont swapgs again */
swapgs
.L\vec_name\()_u:
lfence
subq $TF_RIP,%rsp /* skip dummy tf_err and tf_trapno */
+ .cfi_def_cfa_offset TF_SIZE
movq %rdi,TF_RDI(%rsp)
+ trap_cfi_saved %rdi, TF_RDI
movq %rsi,TF_RSI(%rsp)
+ trap_cfi_saved %rsi, TF_RSI
movq %rdx,TF_RDX(%rsp)
+ trap_cfi_saved %rdx, TF_RDX
movq %rcx,TF_RCX(%rsp)
+ trap_cfi_saved %rcx, TF_RCX
movq %r8,TF_R8(%rsp)
+ trap_cfi_saved %r8, TF_R8
movq %r9,TF_R9(%rsp)
+ trap_cfi_saved %r9, TF_R9
movq %rax,TF_RAX(%rsp)
+ trap_cfi_saved %rax, TF_RAX
movq %rbx,TF_RBX(%rsp)
+ trap_cfi_saved %rbx, TF_RBX
movq %rbp,TF_RBP(%rsp)
+ trap_cfi_saved %rbp, TF_RBP
movq %r10,TF_R10(%rsp)
+ trap_cfi_saved %r10, TF_R10
movq %r11,TF_R11(%rsp)
+ trap_cfi_saved %r11, TF_R11
movq %r12,TF_R12(%rsp)
+ trap_cfi_saved %r12, TF_R12
movq %r13,TF_R13(%rsp)
+ trap_cfi_saved %r13, TF_R13
movq %r14,TF_R14(%rsp)
+ trap_cfi_saved %r14, TF_R14
movq %r15,TF_R15(%rsp)
+ trap_cfi_saved %r15, TF_R15
SAVE_SEGS
movl $TF_HASSEGS,TF_FLAGS(%rsp)
pushfq
diff --git a/sys/amd64/include/cpu.h b/sys/amd64/include/cpu.h
--- a/sys/amd64/include/cpu.h
+++ b/sys/amd64/include/cpu.h
@@ -78,6 +78,7 @@
void cpu_lock_delay(void);
void cpu_reset(void);
void fork_trampoline(void);
+void fork_trampoline_kthread(void);
/*
* Return contents of in-cpu fast counter as a sort of "bogo-time"
diff --git a/sys/amd64/include/trap_cfi.h b/sys/amd64/include/trap_cfi.h
new file mode 100644
--- /dev/null
+++ b/sys/amd64/include/trap_cfi.h
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2026 FreeBSD Foundation
+ *
+ * This software was developed by Minsoo Choo under sponsorship from the
+ * FreeBSD Foundation.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifndef _MACHINE_TRAP_CFI_H_
+#define _MACHINE_TRAP_CFI_H_
+
+#include "assym.inc"
+
+ .macro trap_cfi_entry name, state=empty
+ .cfi_sections .eh_frame
+ .cfi_startproc
+ .cfi_signal_frame
+ .cfi_return_column %rip
+ .cfi_def_cfa %rsp, 0
+ .cfi_undefined %rip
+.ifc \state,full
+ trap_cfi_full
+.endif
+.ifc \state,terminal
+ .cfi_def_cfa %rsp, TF_SIZE
+.endif
+ .endm
+
+ .macro trap_cfi_end name
+ .cfi_endproc
+ .endm
+
+ .macro trap_cfi_machine has_err=0
+.if \has_err
+ .cfi_def_cfa %rsp, 48
+.else
+ .cfi_def_cfa %rsp, 40
+.endif
+ .cfi_offset %rip, -40
+ .cfi_offset %cs, -32
+ .cfi_offset 49, -24 /* rflags */
+ .cfi_offset %rsp, -16
+ .cfi_offset %ss, -8
+ .endm
+
+ .macro trap_cfi_saved reg, off
+ .cfi_offset \reg, \off - TF_SIZE
+ .endm
+
+ .macro trap_cfi_full
+ .cfi_def_cfa %rsp, TF_SIZE
+ trap_cfi_saved %rdi, TF_RDI
+ trap_cfi_saved %rsi, TF_RSI
+ trap_cfi_saved %rdx, TF_RDX
+ trap_cfi_saved %rcx, TF_RCX
+ trap_cfi_saved %r8, TF_R8
+ trap_cfi_saved %r9, TF_R9
+ trap_cfi_saved %rax, TF_RAX
+ trap_cfi_saved %rbx, TF_RBX
+ trap_cfi_saved %rbp, TF_RBP
+ trap_cfi_saved %r10, TF_R10
+ trap_cfi_saved %r11, TF_R11
+ trap_cfi_saved %r12, TF_R12
+ trap_cfi_saved %r13, TF_R13
+ trap_cfi_saved %r14, TF_R14
+ trap_cfi_saved %r15, TF_R15
+ trap_cfi_saved %rip, TF_RIP
+ trap_cfi_saved %cs, TF_CS
+ trap_cfi_saved 49, TF_RFLAGS
+ trap_cfi_saved %rsp, TF_RSP
+ trap_cfi_saved %ss, TF_SS
+ .endm
+
+ .macro trap_cfi_gprs_live
+ .irp reg,%rdi,%rsi,%rdx,%rcx,%r8,%r9,%rax,%rbx,%rbp,%r10,%r11,%r12,%r13,%r14,%r15
+ .cfi_same_value \reg
+ .endr
+ .endm
+
+#endif /* _MACHINE_TRAP_CFI_H_ */
diff --git a/sys/dev/hyperv/vmbus/amd64/vmbus_vector.S b/sys/dev/hyperv/vmbus/amd64/vmbus_vector.S
--- a/sys/dev/hyperv/vmbus/amd64/vmbus_vector.S
+++ b/sys/dev/hyperv/vmbus/amd64/vmbus_vector.S
@@ -40,5 +40,6 @@
movq %rsp, %rdi
call vmbus_handle_intr
jmp doreti
+ trap_cfi_end .
.section .note.GNU-stack,"",%progbits
diff --git a/sys/x86/include/frame.h b/sys/x86/include/frame.h
--- a/sys/x86/include/frame.h
+++ b/sys/x86/include/frame.h
@@ -115,6 +115,8 @@
*
* The ordering of this is specifically so that we can take first 6
* the syscall arguments directly from the beginning of the frame.
+ *
+ * NOTE: keep this structure in sync with the trap frame CFI information.
*/
struct trapframe {

File Metadata

Mime Type
text/plain
Expires
Mon, Sep 28, 2:54 AM (14 h, 28 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
39761025
Default Alt Text
D59649.id186696.diff (30 KB)

Event Timeline