Differential D29529 Diff 86708 sys/amd64/amd64/exception.S

Changeset View

Standalone View

sys/amd64/amd64/exception.S

Show First 20 Lines • Show All 95 Lines • ▼ Show 20 Lines
* but does not mess with %ds, %es, %gs or %fs. We swap the %gs base for		* but does not mess with %ds, %es, %gs or %fs. We swap the %gs base for
* for the kernel mode operation shortly, without changes to the selector		* for the kernel mode operation shortly, without changes to the selector
* loaded. Since superuser long mode works with any selectors loaded into		* loaded. Since superuser long mode works with any selectors loaded into
* segment registers other then %cs, which makes them mostly unused in long		* segment registers other then %cs, which makes them mostly unused in long
* mode, and kernel does not reference %fs, leave them alone. The segment		* mode, and kernel does not reference %fs, leave them alone. The segment
* registers are reloaded on return to the usermode.		* registers are reloaded on return to the usermode.
*/		*/

MCOUNT_LABEL(user)
MCOUNT_LABEL(btrap)

/* Traps that we leave interrupts disabled for. */		/* Traps that we leave interrupts disabled for. */
.macro TRAP_NOEN l, trapno		.macro TRAP_NOEN l, trapno
PTI_ENTRY \l,\l\()_pti_k,\l\()_pti_u		PTI_ENTRY \l,\l\()_pti_k,\l\()_pti_u
\l\()_pti_k:		\l\()_pti_k:
subq $TF_RIP,%rsp		subq $TF_RIP,%rsp
movl $\trapno,TF_TRAPNO(%rsp)		movl $\trapno,TF_TRAPNO(%rsp)
movq $0,TF_ADDR(%rsp)		movq $0,TF_ADDR(%rsp)
movq $0,TF_ERR(%rsp)		movq $0,TF_ERR(%rsp)
▲ Show 20 Lines • Show All 137 Lines • ▼ Show 20 Lines	alltraps_pushregs_no_rax:
movq %r12,TF_R12(%rsp)		movq %r12,TF_R12(%rsp)
movq %r13,TF_R13(%rsp)		movq %r13,TF_R13(%rsp)
movq %r14,TF_R14(%rsp)		movq %r14,TF_R14(%rsp)
movq %r15,TF_R15(%rsp)		movq %r15,TF_R15(%rsp)
movl $TF_HASSEGS,TF_FLAGS(%rsp)		movl $TF_HASSEGS,TF_FLAGS(%rsp)
pushfq		pushfq
andq $~(PSL_D \| PSL_AC),(%rsp)		andq $~(PSL_D \| PSL_AC),(%rsp)
popfq		popfq
FAKE_MCOUNT(TF_RIP(%rsp))
#ifdef KDTRACE_HOOKS		#ifdef KDTRACE_HOOKS
/*		/*
* DTrace Function Boundary Trace (fbt) probes are triggered		* DTrace Function Boundary Trace (fbt) probes are triggered
* by int3 (0xcc) which causes the #BP (T_BPTFLT) breakpoint		* by int3 (0xcc) which causes the #BP (T_BPTFLT) breakpoint
* interrupt. For all other trap types, just handle them in		* interrupt. For all other trap types, just handle them in
* the usual way.		* the usual way.
*/		*/
testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */		testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
Show All 14 Lines	#ifdef KDTRACE_HOOKS
/* Jump to the code hooked in by DTrace. */		/* Jump to the code hooked in by DTrace. */
jmpq *dtrace_invop_jump_addr		jmpq *dtrace_invop_jump_addr
#endif		#endif
.globl calltrap		.globl calltrap
.type calltrap,@function		.type calltrap,@function
calltrap:		calltrap:
movq %rsp,%rdi		movq %rsp,%rdi
call trap_check		call trap_check
MEXITCOUNT
jmp doreti /* Handle any pending ASTs */		jmp doreti /* Handle any pending ASTs */

/*		/*
* alltraps_noen_u/k entry points.		* alltraps_noen_u/k entry points.
* Again, SWAPGS must be already performed by prologue, if needed.		* Again, SWAPGS must be already performed by prologue, if needed.
* Unlike alltraps above, we want to leave the interrupts disabled.		* Unlike alltraps above, we want to leave the interrupts disabled.
* This corresponds to SDT_SYS386IGT on the i386 port.		* This corresponds to SDT_SYS386IGT on the i386 port.
*/		*/
▲ Show 20 Lines • Show All 271 Lines • ▼ Show 20 Lines	fast_syscall_common:
movq %r9,TF_R9(%rsp) /* arg 6 */		movq %r9,TF_R9(%rsp) /* arg 6 */
movq %rbx,TF_RBX(%rsp) /* C preserved */		movq %rbx,TF_RBX(%rsp) /* C preserved */
movq %rbp,TF_RBP(%rsp) /* C preserved */		movq %rbp,TF_RBP(%rsp) /* C preserved */
movq %r12,TF_R12(%rsp) /* C preserved */		movq %r12,TF_R12(%rsp) /* C preserved */
movq %r13,TF_R13(%rsp) /* C preserved */		movq %r13,TF_R13(%rsp) /* C preserved */
movq %r14,TF_R14(%rsp) /* C preserved */		movq %r14,TF_R14(%rsp) /* C preserved */
movq %r15,TF_R15(%rsp) /* C preserved */		movq %r15,TF_R15(%rsp) /* C preserved */
movl $TF_HASSEGS,TF_FLAGS(%rsp)		movl $TF_HASSEGS,TF_FLAGS(%rsp)
FAKE_MCOUNT(TF_RIP(%rsp))
movq PCPU(CURTHREAD),%rdi		movq PCPU(CURTHREAD),%rdi
movq %rsp,TD_FRAME(%rdi)		movq %rsp,TD_FRAME(%rdi)
movl TF_RFLAGS(%rsp),%esi		movl TF_RFLAGS(%rsp),%esi
andl $PSL_T,%esi		andl $PSL_T,%esi
call amd64_syscall		call amd64_syscall
1: movq PCPU(CURPCB),%rax		1: movq PCPU(CURPCB),%rax
/* Disable interrupts before testing PCB_FULL_IRET. */		/* Disable interrupts before testing PCB_FULL_IRET. */
cli		cli
testl $PCB_FULL_IRET,PCB_FLAGS(%rax)		testl $PCB_FULL_IRET,PCB_FLAGS(%rax)
jnz 4f		jnz 4f
/* Check for and handle AST's on return to userland. */		/* Check for and handle AST's on return to userland. */
movq PCPU(CURTHREAD),%rax		movq PCPU(CURTHREAD),%rax
testl $TDF_ASTPENDING \| TDF_NEEDRESCHED,TD_FLAGS(%rax)		testl $TDF_ASTPENDING \| TDF_NEEDRESCHED,TD_FLAGS(%rax)
jne 3f		jne 3f
call handle_ibrs_exit		call handle_ibrs_exit
callq *mds_handler		callq *mds_handler
/* Restore preserved registers. */		/* Restore preserved registers. */
MEXITCOUNT
movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */		movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */
movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */		movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */
movq TF_RDX(%rsp),%rdx /* return value 2 */		movq TF_RDX(%rsp),%rdx /* return value 2 */
movq TF_RAX(%rsp),%rax /* return value 1 */		movq TF_RAX(%rsp),%rax /* return value 1 */
movq TF_RFLAGS(%rsp),%r11 /* original %rflags */		movq TF_RFLAGS(%rsp),%r11 /* original %rflags */
movq TF_RIP(%rsp),%rcx /* original %rip */		movq TF_RIP(%rsp),%rcx /* original %rip */
movq TF_RSP(%rsp),%rsp /* user stack pointer */		movq TF_RSP(%rsp),%rsp /* user stack pointer */
xorl %r8d,%r8d /* zero the rest of GPRs */		xorl %r8d,%r8d /* zero the rest of GPRs */
Show All 10 Lines

3: /* AST scheduled. */		3: /* AST scheduled. */
sti		sti
movq %rsp,%rdi		movq %rsp,%rdi
call ast		call ast
jmp 1b		jmp 1b

4: /* Requested full context restore, use doreti for that. */		4: /* Requested full context restore, use doreti for that. */
MEXITCOUNT
jmp doreti		jmp doreti

/*		/*
* Here for CYA insurance, in case a "syscall" instruction gets		* Here for CYA insurance, in case a "syscall" instruction gets
* issued from 32 bit compatibility mode. MSR_CSTAR has to point		* issued from 32 bit compatibility mode. MSR_CSTAR has to point
* to something if EFER_SCE is enabled.		* to something if EFER_SCE is enabled.
*/		*/
IDTVEC(fast_syscall32)		IDTVEC(fast_syscall32)
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines	IDTVEC(dbg)
je 1f		je 1f
movq %rax,%cr3		movq %rax,%cr3
1: testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip)		1: testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip)
je 2f		je 2f
movl $MSR_IA32_SPEC_CTRL,%ecx		movl $MSR_IA32_SPEC_CTRL,%ecx
rdmsr		rdmsr
movl %eax,%r14d		movl %eax,%r14d
call handle_ibrs_entry		call handle_ibrs_entry
2: FAKE_MCOUNT(TF_RIP(%rsp))		2: movq %rsp,%rdi
movq %rsp,%rdi
call trap		call trap
MEXITCOUNT
testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip)		testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip)
je 3f		je 3f
movl %r14d,%eax		movl %r14d,%eax
xorl %edx,%edx		xorl %edx,%edx
movl $MSR_IA32_SPEC_CTRL,%ecx		movl $MSR_IA32_SPEC_CTRL,%ecx
wrmsr		wrmsr
/*		/*
* Put back the preserved MSR_GSBASE value.		* Put back the preserved MSR_GSBASE value.
▲ Show 20 Lines • Show All 154 Lines • ▼ Show 20 Lines	2: cmpw $KUG32SEL,TF_GS(%rsp)
movl $MSR_KGSBASE,%ecx		movl $MSR_KGSBASE,%ecx
rdmsr		rdmsr
shlq $32,%rdx		shlq $32,%rdx
orq %rdx,%rax		orq %rdx,%rax
movq %rax,PCB_GSBASE(%rdi)		movq %rax,PCB_GSBASE(%rdi)
3:		3:
/* Note: this label is also used by ddb and gdb: */		/* Note: this label is also used by ddb and gdb: */
nmi_calltrap:		nmi_calltrap:
FAKE_MCOUNT(TF_RIP(%rsp))
movq %rsp,%rdi		movq %rsp,%rdi
call trap		call trap
MEXITCOUNT
#ifdef HWPMC_HOOKS		#ifdef HWPMC_HOOKS
/*		/*
* Capture a userspace callchain if needed.		* Capture a userspace callchain if needed.
*		*
* - Check if the current trap was from user mode.		* - Check if the current trap was from user mode.
* - Check if the current thread is valid.		* - Check if the current thread is valid.
* - Check if the thread requires a user call chain to be		* - Check if the thread requires a user call chain to be
* captured.		* captured.
▲ Show 20 Lines • Show All 169 Lines • ▼ Show 20 Lines	mchk_fromuserspace:
movq %cr3,%r13		movq %cr3,%r13
movq PCPU(KCR3),%rax		movq PCPU(KCR3),%rax
cmpq $~0,%rax		cmpq $~0,%rax
je 1f		je 1f
movq %rax,%cr3		movq %rax,%cr3
1: call handle_ibrs_entry		1: call handle_ibrs_entry
/* Note: this label is also used by ddb and gdb: */		/* Note: this label is also used by ddb and gdb: */
mchk_calltrap:		mchk_calltrap:
FAKE_MCOUNT(TF_RIP(%rsp))
movq %rsp,%rdi		movq %rsp,%rdi
call mca_intr		call mca_intr
MEXITCOUNT
testl %ebx,%ebx /* %ebx != 0 => return to userland */		testl %ebx,%ebx /* %ebx != 0 => return to userland */
jnz doreti_exit		jnz doreti_exit
/*		/*
* Restore speculation control MSR, if preserved.		* Restore speculation control MSR, if preserved.
*/		*/
testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip)		testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip)
je 1f		je 1f
movl %r14d,%eax		movl %r14d,%eax
Show All 14 Lines	1: movl $MSR_GSBASE,%ecx
addq $TF_RIP,%rsp		addq $TF_RIP,%rsp
jmp doreti_iret		jmp doreti_iret

ENTRY(fork_trampoline)		ENTRY(fork_trampoline)
movq %r12,%rdi /* function */		movq %r12,%rdi /* function */
movq %rbx,%rsi /* arg1 */		movq %rbx,%rsi /* arg1 */
movq %rsp,%rdx /* trapframe pointer */		movq %rsp,%rdx /* trapframe pointer */
call fork_exit		call fork_exit
MEXITCOUNT
jmp doreti /* Handle any ASTs */		jmp doreti /* Handle any ASTs */

/*		/*
* To efficiently implement classification of trap and interrupt handlers		* To efficiently implement classification of trap and interrupt handlers
* for profiling, there must be only trap handlers between the labels btrap		* for profiling, there must be only trap handlers between the labels btrap
* and bintr, and only interrupt handlers between the labels bintr and		* and bintr, and only interrupt handlers between the labels bintr and
* eintr. This is implemented (partly) by including files that contain		* eintr. This is implemented (partly) by including files that contain
* some of the handlers. Before including the files, set up a normal asm		* some of the handlers. Before including the files, set up a normal asm
Show All 9 Lines

#include <amd64/ia32/ia32_exception.S>		#include <amd64/ia32/ia32_exception.S>
#endif		#endif

.data		.data
.p2align 4		.p2align 4
.text		.text
SUPERALIGN_TEXT		SUPERALIGN_TEXT
MCOUNT_LABEL(bintr)

#include <amd64/amd64/apic_vector.S>		#include <amd64/amd64/apic_vector.S>

#ifdef DEV_ATPIC		#ifdef DEV_ATPIC
.data		.data
.p2align 4		.p2align 4
.text		.text
SUPERALIGN_TEXT		SUPERALIGN_TEXT

#include <amd64/amd64/atpic_vector.S>		#include <amd64/amd64/atpic_vector.S>
#endif		#endif

.text
MCOUNT_LABEL(eintr)

/*		/*
* void doreti(struct trapframe)		* void doreti(struct trapframe)
*		*
* Handle return from interrupts, traps and syscalls.		* Handle return from interrupts, traps and syscalls.
*/		*/
.text		.text
SUPERALIGN_TEXT		SUPERALIGN_TEXT
.type doreti,@function		.type doreti,@function
.globl doreti		.globl doreti
doreti:		doreti:
FAKE_MCOUNT($bintr) /* init "from" bintr -> doreti */
/*		/*
* Check if ASTs can be handled now.		* Check if ASTs can be handled now.
*/		*/
testb $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */		testb $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */
jz doreti_exit /* can't handle ASTs now if not */		jz doreti_exit /* can't handle ASTs now if not */

doreti_ast:		doreti_ast:
/*		/*
Show All 13 Lines	doreti_ast:
/*		/*
* doreti_exit: pop registers, iret.		* doreti_exit: pop registers, iret.
*		*
* The segment register pop is a special case, since it may		* The segment register pop is a special case, since it may
* fault if (for example) a sigreturn specifies bad segment		* fault if (for example) a sigreturn specifies bad segment
* registers. The fault is handled in trap.c.		* registers. The fault is handled in trap.c.
*/		*/
doreti_exit:		doreti_exit:
MEXITCOUNT
movq PCPU(CURPCB),%r8		movq PCPU(CURPCB),%r8

/*		/*
* Do not reload segment registers for kernel.		* Do not reload segment registers for kernel.
* Since we do not reload segments registers with sane		* Since we do not reload segments registers with sane
* values on kernel entry, descriptors referenced by		* values on kernel entry, descriptors referenced by
* segments registers might be not valid. This is fatal		* segments registers might be not valid. This is fatal
* for user mode, but is not a problem for the kernel.		* for user mode, but is not a problem for the kernel.
▲ Show 20 Lines • Show All 144 Lines • ▼ Show 20 Lines	1:
movq %r11,TF_R11(%rsp)		movq %r11,TF_R11(%rsp)
movq %r12,TF_R12(%rsp)		movq %r12,TF_R12(%rsp)
movq %r13,TF_R13(%rsp)		movq %r13,TF_R13(%rsp)
movq %r14,TF_R14(%rsp)		movq %r14,TF_R14(%rsp)
movq %r15,TF_R15(%rsp)		movq %r15,TF_R15(%rsp)
movl $T_PROTFLT,TF_TRAPNO(%rsp)		movl $T_PROTFLT,TF_TRAPNO(%rsp)
movq $0,TF_ERR(%rsp) /* XXX should be the error code */		movq $0,TF_ERR(%rsp) /* XXX should be the error code */
movq $0,TF_ADDR(%rsp)		movq $0,TF_ADDR(%rsp)
FAKE_MCOUNT(TF_RIP(%rsp))
jmp calltrap		jmp calltrap

ALIGN_TEXT		ALIGN_TEXT
.globl ds_load_fault		.globl ds_load_fault
ds_load_fault:		ds_load_fault:
movl $T_PROTFLT,TF_TRAPNO(%rsp)		movl $T_PROTFLT,TF_TRAPNO(%rsp)
testb $SEL_RPL_MASK,TF_CS(%rsp)		testb $SEL_RPL_MASK,TF_CS(%rsp)
jz 1f		jz 1f
▲ Show 20 Lines • Show All 80 Lines • Show Last 20 Lines