Changeset View
Changeset View
Standalone View
Standalone View
head/sys/amd64/amd64/exception.S
Show First 20 Lines • Show All 165 Lines • ▼ Show 20 Lines | X\l: | ||||
* before the trap. This approximates SDT_SYS386TGT on the i386 port. | * before the trap. This approximates SDT_SYS386TGT on the i386 port. | ||||
*/ | */ | ||||
SUPERALIGN_TEXT | SUPERALIGN_TEXT | ||||
.globl alltraps | .globl alltraps | ||||
.type alltraps,@function | .type alltraps,@function | ||||
alltraps: | alltraps: | ||||
movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
jz alltraps_segs /* already running with kernel GS.base */ | jz 1f /* already running with kernel GS.base */ | ||||
swapgs | swapgs | ||||
movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
alltraps_segs: | 1: SAVE_SEGS | ||||
SAVE_SEGS | |||||
testl $PSL_I,TF_RFLAGS(%rsp) | |||||
jz alltraps_pushregs_no_rdi | |||||
sti | |||||
alltraps_pushregs_no_rdi: | |||||
movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
movq %rcx,TF_RCX(%rsp) | |||||
testb $SEL_RPL_MASK,TF_CS(%rsp) | |||||
jz 2f | |||||
call handle_ibrs_entry | |||||
2: testl $PSL_I,TF_RFLAGS(%rsp) | |||||
jz alltraps_pushregs_no_rax | |||||
sti | |||||
alltraps_pushregs_no_rax: | alltraps_pushregs_no_rax: | ||||
movq %rsi,TF_RSI(%rsp) | movq %rsi,TF_RSI(%rsp) | ||||
movq %rcx,TF_RCX(%rsp) | |||||
movq %r8,TF_R8(%rsp) | movq %r8,TF_R8(%rsp) | ||||
movq %r9,TF_R9(%rsp) | movq %r9,TF_R9(%rsp) | ||||
movq %rbx,TF_RBX(%rsp) | movq %rbx,TF_RBX(%rsp) | ||||
movq %rbp,TF_RBP(%rsp) | movq %rbp,TF_RBP(%rsp) | ||||
movq %r10,TF_R10(%rsp) | movq %r10,TF_R10(%rsp) | ||||
movq %r11,TF_R11(%rsp) | movq %r11,TF_R11(%rsp) | ||||
movq %r12,TF_R12(%rsp) | movq %r12,TF_R12(%rsp) | ||||
movq %r13,TF_R13(%rsp) | movq %r13,TF_R13(%rsp) | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | calltrap: | ||||
* SDT_SYS386IGT on the i386 port. | * SDT_SYS386IGT on the i386 port. | ||||
*/ | */ | ||||
SUPERALIGN_TEXT | SUPERALIGN_TEXT | ||||
.globl alltraps_noen | .globl alltraps_noen | ||||
.type alltraps_noen,@function | .type alltraps_noen,@function | ||||
alltraps_noen: | alltraps_noen: | ||||
movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
jz alltraps_noen_segs /* already running with kernel GS.base */ | jz 1f /* already running with kernel GS.base */ | ||||
swapgs | swapgs | ||||
movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
alltraps_noen_segs: | 1: SAVE_SEGS | ||||
SAVE_SEGS | movq %rdx,TF_RDX(%rsp) | ||||
jmp alltraps_pushregs_no_rdi | movq %rax,TF_RAX(%rsp) | ||||
movq %rcx,TF_RCX(%rsp) | |||||
testb $SEL_RPL_MASK,TF_CS(%rsp) | |||||
jz alltraps_pushregs_no_rax | |||||
call handle_ibrs_entry | |||||
jmp alltraps_pushregs_no_rax | |||||
IDTVEC(dblfault) | IDTVEC(dblfault) | ||||
subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) | movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) | ||||
movq $0,TF_ADDR(%rsp) | movq $0,TF_ADDR(%rsp) | ||||
movq $0,TF_ERR(%rsp) | movq $0,TF_ERR(%rsp) | ||||
movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
movq %rsi,TF_RSI(%rsp) | movq %rsi,TF_RSI(%rsp) | ||||
Show All 35 Lines | IDTVEC(page_pti) | ||||
pushq %rdx | pushq %rdx | ||||
movq %cr3,%rax | movq %cr3,%rax | ||||
movq %rax,PCPU(SAVED_UCR3) | movq %rax,PCPU(SAVED_UCR3) | ||||
PTI_UUENTRY has_err=1 | PTI_UUENTRY has_err=1 | ||||
subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
movq %rcx,TF_RCX(%rsp) | |||||
jmp page_u | jmp page_u | ||||
IDTVEC(page) | IDTVEC(page) | ||||
subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
movq %rdi,TF_RDI(%rsp) /* free up GP registers */ | movq %rdi,TF_RDI(%rsp) /* free up GP registers */ | ||||
movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
movq %rcx,TF_RCX(%rsp) | |||||
testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
jz page_cr2 /* already running with kernel GS.base */ | jz page_cr2 /* already running with kernel GS.base */ | ||||
swapgs | swapgs | ||||
page_u: movq PCPU(CURPCB),%rdi | page_u: movq PCPU(CURPCB),%rdi | ||||
andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
movq PCPU(SAVED_UCR3),%rax | movq PCPU(SAVED_UCR3),%rax | ||||
movq %rax,PCB_SAVED_UCR3(%rdi) | movq %rax,PCB_SAVED_UCR3(%rdi) | ||||
call handle_ibrs_entry | |||||
page_cr2: | page_cr2: | ||||
movq %cr2,%rdi /* preserve %cr2 before .. */ | movq %cr2,%rdi /* preserve %cr2 before .. */ | ||||
movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ | movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ | ||||
SAVE_SEGS | SAVE_SEGS | ||||
movl $T_PAGEFLT,TF_TRAPNO(%rsp) | movl $T_PAGEFLT,TF_TRAPNO(%rsp) | ||||
testl $PSL_I,TF_RFLAGS(%rsp) | testl $PSL_I,TF_RFLAGS(%rsp) | ||||
jz alltraps_pushregs_no_rax | jz alltraps_pushregs_no_rax | ||||
sti | sti | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | IDTVEC(\name) | ||||
PROTF_ENTRY stk, T_STKFLT | PROTF_ENTRY stk, T_STKFLT | ||||
PROTF_ENTRY prot, T_PROTFLT | PROTF_ENTRY prot, T_PROTFLT | ||||
prot_addrf: | prot_addrf: | ||||
movq $0,TF_ADDR(%rsp) | movq $0,TF_ADDR(%rsp) | ||||
movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | ||||
movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
movq %rcx,TF_RCX(%rsp) | |||||
movw %fs,TF_FS(%rsp) | movw %fs,TF_FS(%rsp) | ||||
movw %gs,TF_GS(%rsp) | movw %gs,TF_GS(%rsp) | ||||
leaq doreti_iret(%rip),%rdi | leaq doreti_iret(%rip),%rdi | ||||
cmpq %rdi,TF_RIP(%rsp) | cmpq %rdi,TF_RIP(%rsp) | ||||
je 5f /* kernel but with user gsbase!! */ | je 5f /* kernel but with user gsbase!! */ | ||||
testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
jz 6f /* already running with kernel GS.base */ | jz 6f /* already running with kernel GS.base */ | ||||
testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) | testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) | ||||
Show All 9 Lines | 2: swapgs | ||||
testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) | testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) | ||||
jz 4f | jz 4f | ||||
cmpw $KUF32SEL,TF_FS(%rsp) | cmpw $KUF32SEL,TF_FS(%rsp) | ||||
jne 3f | jne 3f | ||||
movq %rax,PCB_FSBASE(%rdi) | movq %rax,PCB_FSBASE(%rdi) | ||||
3: cmpw $KUG32SEL,TF_GS(%rsp) | 3: cmpw $KUG32SEL,TF_GS(%rsp) | ||||
jne 4f | jne 4f | ||||
movq %rdx,PCB_GSBASE(%rdi) | movq %rdx,PCB_GSBASE(%rdi) | ||||
4: orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) /* always full iret from GPF */ | 4: call handle_ibrs_entry | ||||
orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) /* always full iret from GPF */ | |||||
movw %es,TF_ES(%rsp) | movw %es,TF_ES(%rsp) | ||||
movw %ds,TF_DS(%rsp) | movw %ds,TF_DS(%rsp) | ||||
testl $PSL_I,TF_RFLAGS(%rsp) | testl $PSL_I,TF_RFLAGS(%rsp) | ||||
jz alltraps_pushregs_no_rax | jz alltraps_pushregs_no_rax | ||||
sti | sti | ||||
jmp alltraps_pushregs_no_rax | jmp alltraps_pushregs_no_rax | ||||
5: swapgs | 5: swapgs | ||||
Show All 27 Lines | fast_syscall_common: | ||||
subq $TF_SIZE,%rsp | subq $TF_SIZE,%rsp | ||||
/* defer TF_RSP till we have a spare register */ | /* defer TF_RSP till we have a spare register */ | ||||
movq %r11,TF_RFLAGS(%rsp) | movq %r11,TF_RFLAGS(%rsp) | ||||
movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ | movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ | ||||
movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ | movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ | ||||
movq %r11,TF_RSP(%rsp) /* user stack pointer */ | movq %r11,TF_RSP(%rsp) /* user stack pointer */ | ||||
movq PCPU(SCRATCH_RAX),%rax | movq PCPU(SCRATCH_RAX),%rax | ||||
movq %rax,TF_RAX(%rsp) /* syscall number */ | movq %rax,TF_RAX(%rsp) /* syscall number */ | ||||
movq %rdx,TF_RDX(%rsp) /* arg 3 */ | |||||
SAVE_SEGS | SAVE_SEGS | ||||
call handle_ibrs_entry | |||||
movq PCPU(CURPCB),%r11 | movq PCPU(CURPCB),%r11 | ||||
andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) | andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) | ||||
sti | sti | ||||
movq $KUDSEL,TF_SS(%rsp) | movq $KUDSEL,TF_SS(%rsp) | ||||
movq $KUCSEL,TF_CS(%rsp) | movq $KUCSEL,TF_CS(%rsp) | ||||
movq $2,TF_ERR(%rsp) | movq $2,TF_ERR(%rsp) | ||||
movq %rdi,TF_RDI(%rsp) /* arg 1 */ | movq %rdi,TF_RDI(%rsp) /* arg 1 */ | ||||
movq %rsi,TF_RSI(%rsp) /* arg 2 */ | movq %rsi,TF_RSI(%rsp) /* arg 2 */ | ||||
movq %rdx,TF_RDX(%rsp) /* arg 3 */ | |||||
movq %r10,TF_RCX(%rsp) /* arg 4 */ | movq %r10,TF_RCX(%rsp) /* arg 4 */ | ||||
movq %r8,TF_R8(%rsp) /* arg 5 */ | movq %r8,TF_R8(%rsp) /* arg 5 */ | ||||
movq %r9,TF_R9(%rsp) /* arg 6 */ | movq %r9,TF_R9(%rsp) /* arg 6 */ | ||||
movq %rbx,TF_RBX(%rsp) /* C preserved */ | movq %rbx,TF_RBX(%rsp) /* C preserved */ | ||||
movq %rbp,TF_RBP(%rsp) /* C preserved */ | movq %rbp,TF_RBP(%rsp) /* C preserved */ | ||||
movq %r12,TF_R12(%rsp) /* C preserved */ | movq %r12,TF_R12(%rsp) /* C preserved */ | ||||
movq %r13,TF_R13(%rsp) /* C preserved */ | movq %r13,TF_R13(%rsp) /* C preserved */ | ||||
movq %r14,TF_R14(%rsp) /* C preserved */ | movq %r14,TF_R14(%rsp) /* C preserved */ | ||||
Show All 9 Lines | 1: movq PCPU(CURPCB),%rax | ||||
/* Disable interrupts before testing PCB_FULL_IRET. */ | /* Disable interrupts before testing PCB_FULL_IRET. */ | ||||
cli | cli | ||||
testl $PCB_FULL_IRET,PCB_FLAGS(%rax) | testl $PCB_FULL_IRET,PCB_FLAGS(%rax) | ||||
jnz 4f | jnz 4f | ||||
/* Check for and handle AST's on return to userland. */ | /* Check for and handle AST's on return to userland. */ | ||||
movq PCPU(CURTHREAD),%rax | movq PCPU(CURTHREAD),%rax | ||||
testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) | testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) | ||||
jne 3f | jne 3f | ||||
call handle_ibrs_exit | |||||
/* Restore preserved registers. */ | /* Restore preserved registers. */ | ||||
MEXITCOUNT | MEXITCOUNT | ||||
movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ | movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ | ||||
movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ | movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ | ||||
movq TF_RDX(%rsp),%rdx /* return value 2 */ | movq TF_RDX(%rsp),%rdx /* return value 2 */ | ||||
movq TF_RAX(%rsp),%rax /* return value 1 */ | movq TF_RAX(%rsp),%rax /* return value 1 */ | ||||
movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ | movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ | ||||
movq TF_RIP(%rsp),%rcx /* original %rip */ | movq TF_RIP(%rsp),%rcx /* original %rip */ | ||||
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | IDTVEC(nmi) | ||||
movq %r15,TF_R15(%rsp) | movq %r15,TF_R15(%rsp) | ||||
SAVE_SEGS | SAVE_SEGS | ||||
movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
cld | cld | ||||
xorl %ebx,%ebx | xorl %ebx,%ebx | ||||
testb $SEL_RPL_MASK,TF_CS(%rsp) | testb $SEL_RPL_MASK,TF_CS(%rsp) | ||||
jnz nmi_fromuserspace | jnz nmi_fromuserspace | ||||
/* | /* | ||||
* We've interrupted the kernel. Preserve GS.base in %r12 | * We've interrupted the kernel. Preserve GS.base in %r12, | ||||
* and %cr3 in %r13. | * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d. | ||||
*/ | */ | ||||
movl $MSR_GSBASE,%ecx | movl $MSR_GSBASE,%ecx | ||||
rdmsr | rdmsr | ||||
movq %rax,%r12 | movq %rax,%r12 | ||||
shlq $32,%rdx | shlq $32,%rdx | ||||
orq %rdx,%r12 | orq %rdx,%r12 | ||||
/* Retrieve and load the canonical value for GS.base. */ | /* Retrieve and load the canonical value for GS.base. */ | ||||
movq TF_SIZE(%rsp),%rdx | movq TF_SIZE(%rsp),%rdx | ||||
movl %edx,%eax | movl %edx,%eax | ||||
shrq $32,%rdx | shrq $32,%rdx | ||||
wrmsr | wrmsr | ||||
movq %cr3,%r13 | movq %cr3,%r13 | ||||
movq PCPU(KCR3),%rax | movq PCPU(KCR3),%rax | ||||
cmpq $~0,%rax | cmpq $~0,%rax | ||||
je nmi_calltrap | je 1f | ||||
movq %rax,%cr3 | movq %rax,%cr3 | ||||
1: testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip) | |||||
je nmi_calltrap | |||||
movl $MSR_IA32_SPEC_CTRL,%ecx | |||||
rdmsr | |||||
movl %eax,%r14d | |||||
call handle_ibrs_entry | |||||
jmp nmi_calltrap | jmp nmi_calltrap | ||||
nmi_fromuserspace: | nmi_fromuserspace: | ||||
incl %ebx | incl %ebx | ||||
swapgs | swapgs | ||||
movq %cr3,%r13 | movq %cr3,%r13 | ||||
movq PCPU(KCR3),%rax | movq PCPU(KCR3),%rax | ||||
cmpq $~0,%rax | cmpq $~0,%rax | ||||
je 1f | je 1f | ||||
movq %rax,%cr3 | movq %rax,%cr3 | ||||
1: movq PCPU(CURPCB),%rdi | 1: call handle_ibrs_entry | ||||
movq PCPU(CURPCB),%rdi | |||||
testq %rdi,%rdi | testq %rdi,%rdi | ||||
jz 3f | jz 3f | ||||
orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) | orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) | testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) | ||||
jz 3f | jz 3f | ||||
cmpw $KUF32SEL,TF_FS(%rsp) | cmpw $KUF32SEL,TF_FS(%rsp) | ||||
jne 2f | jne 2f | ||||
rdfsbase %rax | rdfsbase %rax | ||||
▲ Show 20 Lines • Show All 78 Lines • ▼ Show 20 Lines | outofnmi: | ||||
sti | sti | ||||
call *%rax | call *%rax | ||||
cli | cli | ||||
nocallchain: | nocallchain: | ||||
#endif | #endif | ||||
testl %ebx,%ebx /* %ebx == 0 => return to userland */ | testl %ebx,%ebx /* %ebx == 0 => return to userland */ | ||||
jnz doreti_exit | jnz doreti_exit | ||||
/* | /* | ||||
* Restore speculation control MSR, if preserved. | |||||
*/ | |||||
testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip) | |||||
je 1f | |||||
movl %r14d,%eax | |||||
xorl %edx,%edx | |||||
movl $MSR_IA32_SPEC_CTRL,%ecx | |||||
wrmsr | |||||
/* | |||||
* Put back the preserved MSR_GSBASE value. | * Put back the preserved MSR_GSBASE value. | ||||
*/ | */ | ||||
movl $MSR_GSBASE,%ecx | 1: movl $MSR_GSBASE,%ecx | ||||
movq %r12,%rdx | movq %r12,%rdx | ||||
movl %edx,%eax | movl %edx,%eax | ||||
shrq $32,%rdx | shrq $32,%rdx | ||||
wrmsr | wrmsr | ||||
movq %r13,%cr3 | movq %r13,%cr3 | ||||
RESTORE_REGS | RESTORE_REGS | ||||
addq $TF_RIP,%rsp | addq $TF_RIP,%rsp | ||||
jmp doreti_iret | jmp doreti_iret | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | IDTVEC(mchk) | ||||
movq %r15,TF_R15(%rsp) | movq %r15,TF_R15(%rsp) | ||||
SAVE_SEGS | SAVE_SEGS | ||||
movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
cld | cld | ||||
xorl %ebx,%ebx | xorl %ebx,%ebx | ||||
testb $SEL_RPL_MASK,TF_CS(%rsp) | testb $SEL_RPL_MASK,TF_CS(%rsp) | ||||
jnz mchk_fromuserspace | jnz mchk_fromuserspace | ||||
/* | /* | ||||
* We've interrupted the kernel. Preserve GS.base in %r12 | * We've interrupted the kernel. Preserve GS.base in %r12, | ||||
* and %cr3 in %r13. | * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d. | ||||
*/ | */ | ||||
movl $MSR_GSBASE,%ecx | movl $MSR_GSBASE,%ecx | ||||
rdmsr | rdmsr | ||||
movq %rax,%r12 | movq %rax,%r12 | ||||
shlq $32,%rdx | shlq $32,%rdx | ||||
orq %rdx,%r12 | orq %rdx,%r12 | ||||
/* Retrieve and load the canonical value for GS.base. */ | /* Retrieve and load the canonical value for GS.base. */ | ||||
movq TF_SIZE(%rsp),%rdx | movq TF_SIZE(%rsp),%rdx | ||||
movl %edx,%eax | movl %edx,%eax | ||||
shrq $32,%rdx | shrq $32,%rdx | ||||
wrmsr | wrmsr | ||||
movq %cr3,%r13 | movq %cr3,%r13 | ||||
movq PCPU(KCR3),%rax | movq PCPU(KCR3),%rax | ||||
cmpq $~0,%rax | cmpq $~0,%rax | ||||
je mchk_calltrap | je 1f | ||||
movq %rax,%cr3 | movq %rax,%cr3 | ||||
1: testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip) | |||||
je mchk_calltrap | |||||
movl $MSR_IA32_SPEC_CTRL,%ecx | |||||
rdmsr | |||||
movl %eax,%r14d | |||||
call handle_ibrs_entry | |||||
jmp mchk_calltrap | jmp mchk_calltrap | ||||
mchk_fromuserspace: | mchk_fromuserspace: | ||||
incl %ebx | incl %ebx | ||||
swapgs | swapgs | ||||
movq %cr3,%r13 | movq %cr3,%r13 | ||||
movq PCPU(KCR3),%rax | movq PCPU(KCR3),%rax | ||||
cmpq $~0,%rax | cmpq $~0,%rax | ||||
je 1f | je 1f | ||||
movq %rax,%cr3 | movq %rax,%cr3 | ||||
1: | 1: call handle_ibrs_entry | ||||
/* Note: this label is also used by ddb and gdb: */ | /* Note: this label is also used by ddb and gdb: */ | ||||
mchk_calltrap: | mchk_calltrap: | ||||
FAKE_MCOUNT(TF_RIP(%rsp)) | FAKE_MCOUNT(TF_RIP(%rsp)) | ||||
movq %rsp,%rdi | movq %rsp,%rdi | ||||
call mca_intr | call mca_intr | ||||
MEXITCOUNT | MEXITCOUNT | ||||
testl %ebx,%ebx /* %ebx == 0 => return to userland */ | testl %ebx,%ebx /* %ebx == 0 => return to userland */ | ||||
jnz doreti_exit | jnz doreti_exit | ||||
/* | /* | ||||
* Restore speculation control MSR, if preserved. | |||||
*/ | |||||
testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip) | |||||
je 1f | |||||
movl %r14d,%eax | |||||
xorl %edx,%edx | |||||
movl $MSR_IA32_SPEC_CTRL,%ecx | |||||
wrmsr | |||||
/* | |||||
* Put back the preserved MSR_GSBASE value. | * Put back the preserved MSR_GSBASE value. | ||||
*/ | */ | ||||
movl $MSR_GSBASE,%ecx | 1: movl $MSR_GSBASE,%ecx | ||||
movq %r12,%rdx | movq %r12,%rdx | ||||
movl %edx,%eax | movl %edx,%eax | ||||
shrq $32,%rdx | shrq $32,%rdx | ||||
wrmsr | wrmsr | ||||
movq %r13,%cr3 | movq %r13,%cr3 | ||||
RESTORE_REGS | RESTORE_REGS | ||||
addq $TF_RIP,%rsp | addq $TF_RIP,%rsp | ||||
jmp doreti_iret | jmp doreti_iret | ||||
▲ Show 20 Lines • Show All 161 Lines • ▼ Show 20 Lines | ld_es: | ||||
.globl ld_ds | .globl ld_ds | ||||
ld_ds: | ld_ds: | ||||
movw TF_DS(%rsp),%ds | movw TF_DS(%rsp),%ds | ||||
ld_regs: | ld_regs: | ||||
RESTORE_REGS | RESTORE_REGS | ||||
testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
jz 2f /* keep running with kernel GS.base */ | jz 2f /* keep running with kernel GS.base */ | ||||
cli | cli | ||||
call handle_ibrs_exit_rs | |||||
cmpb $0,pti | cmpb $0,pti | ||||
je 1f | je 1f | ||||
pushq %rdx | pushq %rdx | ||||
movq PCPU(PRVSPACE),%rdx | movq PCPU(PRVSPACE),%rdx | ||||
addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8-PTI_SIZE,%rdx | addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8-PTI_SIZE,%rdx | ||||
movq %rax,PTI_RAX(%rdx) | movq %rax,PTI_RAX(%rdx) | ||||
popq %rax | popq %rax | ||||
movq %rax,PTI_RDX(%rdx) | movq %rax,PTI_RDX(%rdx) | ||||
Show All 35 Lines | set_segs: | ||||
* above. trap() (amd64/amd64/trap.c) catches this specific | * above. trap() (amd64/amd64/trap.c) catches this specific | ||||
* case, sends the process a signal and continues in the | * case, sends the process a signal and continues in the | ||||
* corresponding place in the code below. | * corresponding place in the code below. | ||||
*/ | */ | ||||
ALIGN_TEXT | ALIGN_TEXT | ||||
.globl doreti_iret_fault | .globl doreti_iret_fault | ||||
doreti_iret_fault: | doreti_iret_fault: | ||||
subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ | subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ | ||||
movq %rax,TF_RAX(%rsp) | |||||
movq %rdx,TF_RDX(%rsp) | |||||
movq %rcx,TF_RCX(%rsp) | |||||
call handle_ibrs_entry | |||||
testb $SEL_RPL_MASK,TF_CS(%rsp) | testb $SEL_RPL_MASK,TF_CS(%rsp) | ||||
jz 1f | jz 1f | ||||
sti | sti | ||||
1: | 1: | ||||
SAVE_SEGS | SAVE_SEGS | ||||
movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
movq %rsi,TF_RSI(%rsp) | movq %rsi,TF_RSI(%rsp) | ||||
movq %rdx,TF_RDX(%rsp) | |||||
movq %rcx,TF_RCX(%rsp) | |||||
movq %r8,TF_R8(%rsp) | movq %r8,TF_R8(%rsp) | ||||
movq %r9,TF_R9(%rsp) | movq %r9,TF_R9(%rsp) | ||||
movq %rax,TF_RAX(%rsp) | |||||
movq %rbx,TF_RBX(%rsp) | movq %rbx,TF_RBX(%rsp) | ||||
movq %rbp,TF_RBP(%rsp) | movq %rbp,TF_RBP(%rsp) | ||||
movq %r10,TF_R10(%rsp) | movq %r10,TF_R10(%rsp) | ||||
movq %r11,TF_R11(%rsp) | movq %r11,TF_R11(%rsp) | ||||
movq %r12,TF_R12(%rsp) | movq %r12,TF_R12(%rsp) | ||||
movq %r13,TF_R13(%rsp) | movq %r13,TF_R13(%rsp) | ||||
movq %r14,TF_R14(%rsp) | movq %r14,TF_R14(%rsp) | ||||
movq %r15,TF_R15(%rsp) | movq %r15,TF_R15(%rsp) | ||||
▲ Show 20 Lines • Show All 92 Lines • Show Last 20 Lines |