Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/exception.S
| Show All 32 Lines | |||||
| * | * | ||||
| * $FreeBSD$ | * $FreeBSD$ | ||||
| */ | */ | ||||
| #include "opt_atpic.h" | #include "opt_atpic.h" | ||||
| #include "opt_compat.h" | #include "opt_compat.h" | ||||
| #include "opt_hwpmc_hooks.h" | #include "opt_hwpmc_hooks.h" | ||||
| #include "assym.s" | |||||
| #include <machine/asmacros.h> | #include <machine/asmacros.h> | ||||
| #include <machine/psl.h> | #include <machine/psl.h> | ||||
| #include <machine/trap.h> | #include <machine/trap.h> | ||||
| #include <machine/specialreg.h> | #include <machine/specialreg.h> | ||||
| #include "assym.s" | |||||
| #ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
| .bss | .bss | ||||
| .globl dtrace_invop_jump_addr | .globl dtrace_invop_jump_addr | ||||
| .align 8 | .align 8 | ||||
| .type dtrace_invop_jump_addr,@object | .type dtrace_invop_jump_addr,@object | ||||
| .size dtrace_invop_jump_addr,8 | .size dtrace_invop_jump_addr,8 | ||||
| dtrace_invop_jump_addr: | dtrace_invop_jump_addr: | ||||
| .zero 8 | .zero 8 | ||||
| Show All 39 Lines | |||||
| * segment registers other then %cs, which makes them mostly unused in long | * segment registers other then %cs, which makes them mostly unused in long | ||||
| * mode, and kernel does not reference %fs, leave them alone. The segment | * mode, and kernel does not reference %fs, leave them alone. The segment | ||||
| * registers are reloaded on return to the usermode. | * registers are reloaded on return to the usermode. | ||||
| */ | */ | ||||
| MCOUNT_LABEL(user) | MCOUNT_LABEL(user) | ||||
| MCOUNT_LABEL(btrap) | MCOUNT_LABEL(btrap) | ||||
| /* Traps that we leave interrupts disabled for.. */ | /* Traps that we leave interrupts disabled for. */ | ||||
| #define TRAP_NOEN(a) \ | .macro TRAP_NOEN l, trapno | ||||
| subq $TF_RIP,%rsp; \ | PTI_ENTRY \l,X\l | ||||
| movl $(a),TF_TRAPNO(%rsp) ; \ | .globl X\l | ||||
| movq $0,TF_ADDR(%rsp) ; \ | .type X\l,@function | ||||
| movq $0,TF_ERR(%rsp) ; \ | X\l: subq $TF_RIP,%rsp | ||||
| movl $\trapno,TF_TRAPNO(%rsp) | |||||
| movq $0,TF_ADDR(%rsp) | |||||
| movq $0,TF_ERR(%rsp) | |||||
| jmp alltraps_noen | jmp alltraps_noen | ||||
| IDTVEC(dbg) | .endm | ||||
| TRAP_NOEN(T_TRCTRAP) | |||||
| IDTVEC(bpt) | TRAP_NOEN dbg, T_TRCTRAP | ||||
| TRAP_NOEN(T_BPTFLT) | TRAP_NOEN bpt, T_BPTFLT | ||||
| #ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
| IDTVEC(dtrace_ret) | TRAP_NOEN dtrace_ret, T_DTRACE_RET | ||||
| TRAP_NOEN(T_DTRACE_RET) | |||||
| #endif | #endif | ||||
| /* Regular traps; The cpu does not supply tf_err for these. */ | /* Regular traps; The cpu does not supply tf_err for these. */ | ||||
| #define TRAP(a) \ | .macro TRAP l, trapno | ||||
| subq $TF_RIP,%rsp; \ | PTI_ENTRY \l,X\l | ||||
| movl $(a),TF_TRAPNO(%rsp) ; \ | .globl X\l | ||||
| movq $0,TF_ADDR(%rsp) ; \ | .type X\l,@function | ||||
| movq $0,TF_ERR(%rsp) ; \ | X\l: | ||||
| subq $TF_RIP,%rsp | |||||
| movl $\trapno,TF_TRAPNO(%rsp) | |||||
| movq $0,TF_ADDR(%rsp) | |||||
| movq $0,TF_ERR(%rsp) | |||||
| jmp alltraps | jmp alltraps | ||||
| IDTVEC(div) | .endm | ||||
| TRAP(T_DIVIDE) | |||||
| IDTVEC(ofl) | |||||
| TRAP(T_OFLOW) | |||||
| IDTVEC(bnd) | |||||
| TRAP(T_BOUND) | |||||
| IDTVEC(ill) | |||||
| TRAP(T_PRIVINFLT) | |||||
| IDTVEC(dna) | |||||
| TRAP(T_DNA) | |||||
| IDTVEC(fpusegm) | |||||
| TRAP(T_FPOPFLT) | |||||
| IDTVEC(mchk) | |||||
| TRAP(T_MCHK) | |||||
| IDTVEC(rsvd) | |||||
| TRAP(T_RESERVED) | |||||
| IDTVEC(fpu) | |||||
| TRAP(T_ARITHTRAP) | |||||
| IDTVEC(xmm) | |||||
| TRAP(T_XMMFLT) | |||||
| /* This group of traps have tf_err already pushed by the cpu */ | TRAP div, T_DIVIDE | ||||
| #define TRAP_ERR(a) \ | TRAP ofl, T_OFLOW | ||||
| subq $TF_ERR,%rsp; \ | TRAP bnd, T_BOUND | ||||
| movl $(a),TF_TRAPNO(%rsp) ; \ | TRAP ill, T_PRIVINFLT | ||||
| movq $0,TF_ADDR(%rsp) ; \ | TRAP dna, T_DNA | ||||
| jmp alltraps | TRAP fpusegm, T_FPOPFLT | ||||
| IDTVEC(tss) | TRAP mchk, T_MCHK | ||||
| TRAP_ERR(T_TSSFLT) | TRAP rsvd, T_RESERVED | ||||
| IDTVEC(missing) | TRAP fpu, T_ARITHTRAP | ||||
| TRAP xmm, T_XMMFLT | |||||
| /* This group of traps have tf_err already pushed by the cpu. */ | |||||
| .macro TRAP_ERR l, trapno | |||||
| PTI_ENTRY \l,X\l,has_err=1 | |||||
| .globl X\l | |||||
| .type X\l,@function | |||||
| X\l: | |||||
| subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
| movl $T_SEGNPFLT,TF_TRAPNO(%rsp) | movl $\trapno,TF_TRAPNO(%rsp) | ||||
| jmp prot_addrf | movq $0,TF_ADDR(%rsp) | ||||
| IDTVEC(stk) | jmp alltraps | ||||
| subq $TF_ERR,%rsp | .endm | ||||
| movl $T_STKFLT,TF_TRAPNO(%rsp) | |||||
| jmp prot_addrf | |||||
| IDTVEC(align) | |||||
| TRAP_ERR(T_ALIGNFLT) | |||||
| TRAP_ERR tss, T_TSSFLT | |||||
| TRAP_ERR align, T_ALIGNFLT | |||||
| /* | /* | ||||
| * alltraps entry point. Use swapgs if this is the first time in the | * alltraps entry point. Use swapgs if this is the first time in the | ||||
| * kernel from userland. Reenable interrupts if they were enabled | * kernel from userland. Reenable interrupts if they were enabled | ||||
| * before the trap. This approximates SDT_SYS386TGT on the i386 port. | * before the trap. This approximates SDT_SYS386TGT on the i386 port. | ||||
| */ | */ | ||||
| SUPERALIGN_TEXT | SUPERALIGN_TEXT | ||||
| .globl alltraps | .globl alltraps | ||||
| .type alltraps,@function | .type alltraps,@function | ||||
| alltraps: | alltraps: | ||||
| movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
| testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
| jz alltraps_testi /* already running with kernel GS.base */ | jz alltraps_segs /* already running with kernel GS.base */ | ||||
| swapgs | swapgs | ||||
| movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
| andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
| movw %fs,TF_FS(%rsp) | alltraps_segs: | ||||
| movw %gs,TF_GS(%rsp) | SAVE_SEGS | ||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| alltraps_testi: | |||||
| testl $PSL_I,TF_RFLAGS(%rsp) | testl $PSL_I,TF_RFLAGS(%rsp) | ||||
| jz alltraps_pushregs_no_rdi | jz alltraps_pushregs_no_rdi | ||||
| sti | sti | ||||
| alltraps_pushregs_no_rdi: | alltraps_pushregs_no_rdi: | ||||
| movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
| movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
| alltraps_pushregs_no_rax: | alltraps_pushregs_no_rax: | ||||
| movq %rsi,TF_RSI(%rsp) | movq %rsi,TF_RSI(%rsp) | ||||
| ▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | calltrap: | ||||
| * SDT_SYS386IGT on the i386 port. | * SDT_SYS386IGT on the i386 port. | ||||
| */ | */ | ||||
| SUPERALIGN_TEXT | SUPERALIGN_TEXT | ||||
| .globl alltraps_noen | .globl alltraps_noen | ||||
| .type alltraps_noen,@function | .type alltraps_noen,@function | ||||
| alltraps_noen: | alltraps_noen: | ||||
| movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
| testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
| jz 1f /* already running with kernel GS.base */ | jz alltraps_noen_segs /* already running with kernel GS.base */ | ||||
| swapgs | swapgs | ||||
| movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
| andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
| 1: movw %fs,TF_FS(%rsp) | alltraps_noen_segs: | ||||
| movw %gs,TF_GS(%rsp) | SAVE_SEGS | ||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| jmp alltraps_pushregs_no_rdi | jmp alltraps_pushregs_no_rdi | ||||
| IDTVEC(dblfault) | IDTVEC(dblfault) | ||||
| subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
| movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) | movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) | ||||
| movq $0,TF_ADDR(%rsp) | movq $0,TF_ADDR(%rsp) | ||||
| movq $0,TF_ERR(%rsp) | movq $0,TF_ERR(%rsp) | ||||
| movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
| movq %rsi,TF_RSI(%rsp) | movq %rsi,TF_RSI(%rsp) | ||||
| movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
| movq %rcx,TF_RCX(%rsp) | movq %rcx,TF_RCX(%rsp) | ||||
| movq %r8,TF_R8(%rsp) | movq %r8,TF_R8(%rsp) | ||||
| movq %r9,TF_R9(%rsp) | movq %r9,TF_R9(%rsp) | ||||
| movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
| movq %rbx,TF_RBX(%rsp) | movq %rbx,TF_RBX(%rsp) | ||||
| movq %rbp,TF_RBP(%rsp) | movq %rbp,TF_RBP(%rsp) | ||||
| movq %r10,TF_R10(%rsp) | movq %r10,TF_R10(%rsp) | ||||
| movq %r11,TF_R11(%rsp) | movq %r11,TF_R11(%rsp) | ||||
| movq %r12,TF_R12(%rsp) | movq %r12,TF_R12(%rsp) | ||||
| movq %r13,TF_R13(%rsp) | movq %r13,TF_R13(%rsp) | ||||
| movq %r14,TF_R14(%rsp) | movq %r14,TF_R14(%rsp) | ||||
| movq %r15,TF_R15(%rsp) | movq %r15,TF_R15(%rsp) | ||||
| movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
| movw %gs,TF_GS(%rsp) | |||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
| cld | cld | ||||
| testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
| jz 1f /* already running with kernel GS.base */ | jz 1f /* already running with kernel GS.base */ | ||||
| swapgs | swapgs | ||||
| 1: | 1: | ||||
| movq %rsp,%rdi | movq PCPU(KCR3),%rax | ||||
jtl: If I'm reading the code correctly, PCPU(KCR3) is only set when pti != 0; therefore, it seems… | |||||
Not Done Inline ActionsYeah, I already fixed it since I tested the non-PTI case as well. Thank you for noting. kib: Yeah, I already fixed it since I tested the non-PTI case as well. Thank you for noting. | |||||
| cmpq $~0,%rax | |||||
| jne 2f | |||||
| movq %rax,%cr3 | |||||
| 2: movq %rsp,%rdi | |||||
| call dblfault_handler | call dblfault_handler | ||||
| 2: | 3: hlt | ||||
| hlt | jmp 3b | ||||
| jmp 2b | |||||
| PTI_ENTRY page, Xpage, has_err=1 | |||||
| IDTVEC(page) | IDTVEC(page) | ||||
| subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
| movl $T_PAGEFLT,TF_TRAPNO(%rsp) | |||||
| movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | ||||
| testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
| jz 1f /* already running with kernel GS.base */ | jz page_cr2 /* already running with kernel GS.base */ | ||||
| swapgs | swapgs | ||||
| movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
| andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
| 1: movq %cr2,%rdi /* preserve %cr2 before .. */ | page_cr2: | ||||
| movq %cr2,%rdi /* preserve %cr2 before .. */ | |||||
| movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ | movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ | ||||
| movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
| movw %gs,TF_GS(%rsp) | movl $T_PAGEFLT,TF_TRAPNO(%rsp) | ||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| testl $PSL_I,TF_RFLAGS(%rsp) | testl $PSL_I,TF_RFLAGS(%rsp) | ||||
| jz alltraps_pushregs_no_rdi | jz alltraps_pushregs_no_rdi | ||||
| sti | sti | ||||
| jmp alltraps_pushregs_no_rdi | jmp alltraps_pushregs_no_rdi | ||||
| /* | /* | ||||
| * We have to special-case this one. If we get a trap in doreti() at | * We have to special-case this one. If we get a trap in doreti() at | ||||
| * the iretq stage, we'll reenter with the wrong gs state. We'll have | * the iretq stage, we'll reenter with the wrong gs state. We'll have | ||||
| * to do a special the swapgs in this case even coming from the kernel. | * to do a special the swapgs in this case even coming from the kernel. | ||||
| * XXX linux has a trap handler for their equivalent of load_gs(). | * XXX linux has a trap handler for their equivalent of load_gs(). | ||||
| * | |||||
| * On the stack, we have the hardware interrupt frame to return | |||||
| * to usermode (faulted) and another frame with error code, for | |||||
| * fault. For PTI, copy both frames to the main thread stack. | |||||
| */ | */ | ||||
| IDTVEC(prot) | .macro PROTF_ENTRY name,trapno | ||||
| \name\()_pti_doreti: | |||||
| pushq %rax | |||||
| pushq %rdx | |||||
| swapgs | |||||
| movq PCPU(KCR3),%rax | |||||
| movq %rax,%cr3 | |||||
| movq PCPU(RSP0),%rax | |||||
| subq $2*PTI_SIZE-3*8,%rax | |||||
| MOVE_STACKS (PTI_SIZE / 4 - 3) | |||||
| movq %rax,%rsp | |||||
| popq %rdx | |||||
| popq %rax | |||||
| swapgs | |||||
| jmp X\name | |||||
| IDTVEC(\name\()_pti) | |||||
| cmpq $doreti_iret,PTI_RIP-2*8(%rsp) | |||||
| je \name\()_pti_doreti | |||||
| testb $SEL_RPL_MASK,PTI_CS-2*8(%rsp) /* %rax, %rdx not yet pushed */ | |||||
| jz X\name | |||||
| PTI_UENTRY has_err=1 | |||||
| swapgs | |||||
| IDTVEC(\name) | |||||
| subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
| movl $T_PROTFLT,TF_TRAPNO(%rsp) | movl $\trapno,TF_TRAPNO(%rsp) | ||||
| jmp prot_addrf | |||||
| .endm | |||||
| PROTF_ENTRY missing, T_SEGNPFLT | |||||
| PROTF_ENTRY stk, T_STKFLT | |||||
| PROTF_ENTRY prot, T_PROTFLT | |||||
| prot_addrf: | prot_addrf: | ||||
| movq $0,TF_ADDR(%rsp) | movq $0,TF_ADDR(%rsp) | ||||
| movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | ||||
| movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
| movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
| movw %fs,TF_FS(%rsp) | movw %fs,TF_FS(%rsp) | ||||
| movw %gs,TF_GS(%rsp) | movw %gs,TF_GS(%rsp) | ||||
| leaq doreti_iret(%rip),%rdi | leaq doreti_iret(%rip),%rdi | ||||
| Show All 35 Lines | |||||
| * Fast syscall entry point. We enter here with just our new %cs/%ss set, | * Fast syscall entry point. We enter here with just our new %cs/%ss set, | ||||
| * and the new privilige level. We are still running on the old user stack | * and the new privilige level. We are still running on the old user stack | ||||
| * pointer. We have to juggle a few things around to find our stack etc. | * pointer. We have to juggle a few things around to find our stack etc. | ||||
| * swapgs gives us access to our PCPU space only. | * swapgs gives us access to our PCPU space only. | ||||
| * | * | ||||
| * We do not support invoking this from a custom segment registers, | * We do not support invoking this from a custom segment registers, | ||||
| * esp. %cs, %ss, %fs, %gs, e.g. using entries from an LDT. | * esp. %cs, %ss, %fs, %gs, e.g. using entries from an LDT. | ||||
| */ | */ | ||||
| SUPERALIGN_TEXT | |||||
| IDTVEC(fast_syscall_pti) | |||||
| swapgs | |||||
| movq %rax,PCPU(SCRATCH_RAX) | |||||
| movq PCPU(KCR3),%rax | |||||
| movq %rax,%cr3 | |||||
| jmp fast_syscall_common | |||||
| SUPERALIGN_TEXT | |||||
| IDTVEC(fast_syscall) | IDTVEC(fast_syscall) | ||||
| swapgs | swapgs | ||||
| movq %rax,PCPU(SCRATCH_RAX) | |||||
| fast_syscall_common: | |||||
| movq %rsp,PCPU(SCRATCH_RSP) | movq %rsp,PCPU(SCRATCH_RSP) | ||||
| movq PCPU(RSP0),%rsp | movq PCPU(RSP0),%rsp | ||||
| /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ | /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ | ||||
| subq $TF_SIZE,%rsp | subq $TF_SIZE,%rsp | ||||
| /* defer TF_RSP till we have a spare register */ | /* defer TF_RSP till we have a spare register */ | ||||
| movq %r11,TF_RFLAGS(%rsp) | movq %r11,TF_RFLAGS(%rsp) | ||||
| movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ | movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ | ||||
| movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ | movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ | ||||
| movq %r11,TF_RSP(%rsp) /* user stack pointer */ | movq %r11,TF_RSP(%rsp) /* user stack pointer */ | ||||
| movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
| movw %gs,TF_GS(%rsp) | |||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| movq PCPU(CURPCB),%r11 | movq PCPU(CURPCB),%r11 | ||||
| andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) | andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) | ||||
| sti | sti | ||||
| movq $KUDSEL,TF_SS(%rsp) | movq $KUDSEL,TF_SS(%rsp) | ||||
| movq $KUCSEL,TF_CS(%rsp) | movq $KUCSEL,TF_CS(%rsp) | ||||
| movq $2,TF_ERR(%rsp) | movq $2,TF_ERR(%rsp) | ||||
| movq %rdi,TF_RDI(%rsp) /* arg 1 */ | movq %rdi,TF_RDI(%rsp) /* arg 1 */ | ||||
| movq %rsi,TF_RSI(%rsp) /* arg 2 */ | movq %rsi,TF_RSI(%rsp) /* arg 2 */ | ||||
| movq %rdx,TF_RDX(%rsp) /* arg 3 */ | movq %rdx,TF_RDX(%rsp) /* arg 3 */ | ||||
| movq %r10,TF_RCX(%rsp) /* arg 4 */ | movq %r10,TF_RCX(%rsp) /* arg 4 */ | ||||
| movq %r8,TF_R8(%rsp) /* arg 5 */ | movq %r8,TF_R8(%rsp) /* arg 5 */ | ||||
| movq %r9,TF_R9(%rsp) /* arg 6 */ | movq %r9,TF_R9(%rsp) /* arg 6 */ | ||||
| movq PCPU(SCRATCH_RAX),%rax | |||||
| movq %rax,TF_RAX(%rsp) /* syscall number */ | movq %rax,TF_RAX(%rsp) /* syscall number */ | ||||
| movq %rbx,TF_RBX(%rsp) /* C preserved */ | movq %rbx,TF_RBX(%rsp) /* C preserved */ | ||||
| movq %rbp,TF_RBP(%rsp) /* C preserved */ | movq %rbp,TF_RBP(%rsp) /* C preserved */ | ||||
| movq %r12,TF_R12(%rsp) /* C preserved */ | movq %r12,TF_R12(%rsp) /* C preserved */ | ||||
| movq %r13,TF_R13(%rsp) /* C preserved */ | movq %r13,TF_R13(%rsp) /* C preserved */ | ||||
| movq %r14,TF_R14(%rsp) /* C preserved */ | movq %r14,TF_R14(%rsp) /* C preserved */ | ||||
| movq %r15,TF_R15(%rsp) /* C preserved */ | movq %r15,TF_R15(%rsp) /* C preserved */ | ||||
| movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
| cld | cld | ||||
| FAKE_MCOUNT(TF_RIP(%rsp)) | FAKE_MCOUNT(TF_RIP(%rsp)) | ||||
| movq PCPU(CURTHREAD),%rdi | movq PCPU(CURTHREAD),%rdi | ||||
| movq %rsp,TD_FRAME(%rdi) | movq %rsp,TD_FRAME(%rdi) | ||||
| movl TF_RFLAGS(%rsp),%esi | movl TF_RFLAGS(%rsp),%esi | ||||
| andl $PSL_T,%esi | andl $PSL_T,%esi | ||||
| call amd64_syscall | call amd64_syscall | ||||
| 1: movq PCPU(CURPCB),%rax | 2: movq PCPU(CURPCB),%rax | ||||
| /* Disable interrupts before testing PCB_FULL_IRET. */ | /* Disable interrupts before testing PCB_FULL_IRET. */ | ||||
| cli | cli | ||||
| testl $PCB_FULL_IRET,PCB_FLAGS(%rax) | testl $PCB_FULL_IRET,PCB_FLAGS(%rax) | ||||
| jnz 3f | jnz 5f | ||||
| /* Check for and handle AST's on return to userland. */ | /* Check for and handle AST's on return to userland. */ | ||||
| movq PCPU(CURTHREAD),%rax | movq PCPU(CURTHREAD),%rax | ||||
| testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) | testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) | ||||
| jne 2f | jne 4f | ||||
| /* Restore preserved registers. */ | /* Restore preserved registers. */ | ||||
| MEXITCOUNT | MEXITCOUNT | ||||
| movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ | movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ | ||||
| movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ | movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ | ||||
| movq TF_RDX(%rsp),%rdx /* return value 2 */ | movq TF_RDX(%rsp),%rdx /* return value 2 */ | ||||
| movq TF_RAX(%rsp),%rax /* return value 1 */ | movq TF_RAX(%rsp),%rax /* return value 1 */ | ||||
| movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ | movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ | ||||
| movq TF_RIP(%rsp),%rcx /* original %rip */ | movq TF_RIP(%rsp),%rcx /* original %rip */ | ||||
| movq TF_RSP(%rsp),%rsp /* user stack pointer */ | movq TF_RSP(%rsp),%rsp /* user stack pointer */ | ||||
| swapgs | cmpb $0,pti | ||||
| je 3f | |||||
| movq PCPU(UCR3),%r9 | |||||
| movq %r9,%cr3 | |||||
| xorl %r9d,%r9d | |||||
| 3: swapgs | |||||
| sysretq | sysretq | ||||
| 2: /* AST scheduled. */ | 4: /* AST scheduled. */ | ||||
| sti | sti | ||||
| movq %rsp,%rdi | movq %rsp,%rdi | ||||
| call ast | call ast | ||||
| jmp 1b | jmp 2b | ||||
| 3: /* Requested full context restore, use doreti for that. */ | 5: /* Requested full context restore, use doreti for that. */ | ||||
| MEXITCOUNT | MEXITCOUNT | ||||
| jmp doreti | jmp doreti | ||||
| /* | /* | ||||
| * Here for CYA insurance, in case a "syscall" instruction gets | * Here for CYA insurance, in case a "syscall" instruction gets | ||||
| * issued from 32 bit compatibility mode. MSR_CSTAR has to point | * issued from 32 bit compatibility mode. MSR_CSTAR has to point | ||||
| * to *something* if EFER_SCE is enabled. | * to *something* if EFER_SCE is enabled. | ||||
| */ | */ | ||||
| Show All 39 Lines | IDTVEC(nmi) | ||||
| movq %rbx,TF_RBX(%rsp) | movq %rbx,TF_RBX(%rsp) | ||||
| movq %rbp,TF_RBP(%rsp) | movq %rbp,TF_RBP(%rsp) | ||||
| movq %r10,TF_R10(%rsp) | movq %r10,TF_R10(%rsp) | ||||
| movq %r11,TF_R11(%rsp) | movq %r11,TF_R11(%rsp) | ||||
| movq %r12,TF_R12(%rsp) | movq %r12,TF_R12(%rsp) | ||||
| movq %r13,TF_R13(%rsp) | movq %r13,TF_R13(%rsp) | ||||
| movq %r14,TF_R14(%rsp) | movq %r14,TF_R14(%rsp) | ||||
| movq %r15,TF_R15(%rsp) | movq %r15,TF_R15(%rsp) | ||||
| movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
| movw %gs,TF_GS(%rsp) | |||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
| cld | cld | ||||
| xorl %ebx,%ebx | xorl %ebx,%ebx | ||||
| testb $SEL_RPL_MASK,TF_CS(%rsp) | testb $SEL_RPL_MASK,TF_CS(%rsp) | ||||
| jnz nmi_fromuserspace | jnz nmi_fromuserspace | ||||
| /* | /* | ||||
| * We've interrupted the kernel. Preserve GS.base in %r12. | * We've interrupted the kernel. Preserve GS.base in %r12. | ||||
| */ | */ | ||||
| movl $MSR_GSBASE,%ecx | movl $MSR_GSBASE,%ecx | ||||
| rdmsr | rdmsr | ||||
| movq %rax,%r12 | movq %rax,%r12 | ||||
| shlq $32,%rdx | shlq $32,%rdx | ||||
| orq %rdx,%r12 | orq %rdx,%r12 | ||||
| /* Retrieve and load the canonical value for GS.base. */ | /* Retrieve and load the canonical value for GS.base. */ | ||||
| movq TF_SIZE(%rsp),%rdx | movq TF_SIZE(%rsp),%rdx | ||||
| movl %edx,%eax | movl %edx,%eax | ||||
| shrq $32,%rdx | shrq $32,%rdx | ||||
| wrmsr | wrmsr | ||||
| movq %cr3,%rax | |||||
| movq %rax,TF_ADDR(%rsp) | |||||
| movq PCPU(KCR3),%rax | |||||
Not Done Inline ActionsIf I'm reading the code correctly, PCPU(KCR3) is only set when pti != 0; therefore, it seems like these lines should be conditioned on pti != 0. (But, I could be missing something.) jtl: If I'm reading the code correctly, PCPU(KCR3) is only set when pti != 0; therefore, it seems… | |||||
Not Done Inline ActionsIt looks to me like PCPU(KCR3) is always set, it just only changes in the pti!=0 case. This seems to avoid a compare & branch. kithrup_mac.com: It looks to me like PCPU(KCR3) is always set, it just only changes in the pti!=0 case. This… | |||||
Not Done Inline ActionsIt is not for !pti case. For now wrapping accesses with tests is the right thing to do. kib: It is not for !pti case. For now wrapping accesses with tests is the right thing to do. | |||||
| cmpq $~0,%rax | |||||
| je nmi_calltrap | |||||
| movq %rax,%cr3 | |||||
| jmp nmi_calltrap | jmp nmi_calltrap | ||||
| nmi_fromuserspace: | nmi_fromuserspace: | ||||
| incl %ebx | incl %ebx | ||||
| swapgs | swapgs | ||||
| testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) | movq PCPU(KCR3),%rax | ||||
Not Done Inline ActionsDitto. jtl: Ditto. | |||||
Not Done Inline ActionsThanks for fixing these (and your overall work on the bug)! FYI, I think this one (nmi_fromuserspace) still needs the change to only set cr3 when pti != 0. jtl: Thanks for fixing these (and your overall work on the bug)! FYI, I think this one… | |||||
| jz 2f | cmpq $~0,%rax | ||||
| je 1f | |||||
| movq %rax,%cr3 | |||||
| 1: testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) | |||||
| jz 3f | |||||
| movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
| testq %rdi,%rdi | testq %rdi,%rdi | ||||
| jz 2f | jz 3f | ||||
| cmpw $KUF32SEL,TF_FS(%rsp) | cmpw $KUF32SEL,TF_FS(%rsp) | ||||
| jne 1f | jne 2f | ||||
| rdfsbase %rax | rdfsbase %rax | ||||
| movq %rax,PCB_FSBASE(%rdi) | movq %rax,PCB_FSBASE(%rdi) | ||||
| 1: cmpw $KUG32SEL,TF_GS(%rsp) | 2: cmpw $KUG32SEL,TF_GS(%rsp) | ||||
| jne 2f | jne 3f | ||||
| movl $MSR_KGSBASE,%ecx | movl $MSR_KGSBASE,%ecx | ||||
| rdmsr | rdmsr | ||||
| shlq $32,%rdx | shlq $32,%rdx | ||||
| orq %rdx,%rax | orq %rdx,%rax | ||||
| movq %rax,PCB_GSBASE(%rdi) | movq %rax,PCB_GSBASE(%rdi) | ||||
| 2: | 3: | ||||
| /* Note: this label is also used by ddb and gdb: */ | /* Note: this label is also used by ddb and gdb: */ | ||||
| nmi_calltrap: | nmi_calltrap: | ||||
| FAKE_MCOUNT(TF_RIP(%rsp)) | FAKE_MCOUNT(TF_RIP(%rsp)) | ||||
| movq %rsp,%rdi | movq %rsp,%rdi | ||||
| call trap | call trap | ||||
| MEXITCOUNT | MEXITCOUNT | ||||
| #ifdef HWPMC_HOOKS | #ifdef HWPMC_HOOKS | ||||
| /* | /* | ||||
| ▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | nmi_kernelexit: | ||||
| /* | /* | ||||
| * Put back the preserved MSR_GSBASE value. | * Put back the preserved MSR_GSBASE value. | ||||
| */ | */ | ||||
| movl $MSR_GSBASE,%ecx | movl $MSR_GSBASE,%ecx | ||||
| movq %r12,%rdx | movq %r12,%rdx | ||||
| movl %edx,%eax | movl %edx,%eax | ||||
| shrq $32,%rdx | shrq $32,%rdx | ||||
| wrmsr | wrmsr | ||||
| movq TF_ADDR(%rsp),%rax | |||||
| movq %rax,%cr3 | |||||
| nmi_restoreregs: | nmi_restoreregs: | ||||
| movq TF_RDI(%rsp),%rdi | movq TF_RDI(%rsp),%rdi | ||||
| movq TF_RSI(%rsp),%rsi | movq TF_RSI(%rsp),%rsi | ||||
| movq TF_RDX(%rsp),%rdx | movq TF_RDX(%rsp),%rdx | ||||
| movq TF_RCX(%rsp),%rcx | movq TF_RCX(%rsp),%rcx | ||||
| movq TF_R8(%rsp),%r8 | movq TF_R8(%rsp),%r8 | ||||
| movq TF_R9(%rsp),%r9 | movq TF_R9(%rsp),%r9 | ||||
| movq TF_RAX(%rsp),%rax | movq TF_RAX(%rsp),%rax | ||||
| ▲ Show 20 Lines • Show All 183 Lines • ▼ Show 20 Lines | ld_regs: | ||||
| movq TF_RBP(%rsp),%rbp | movq TF_RBP(%rsp),%rbp | ||||
| movq TF_R10(%rsp),%r10 | movq TF_R10(%rsp),%r10 | ||||
| movq TF_R11(%rsp),%r11 | movq TF_R11(%rsp),%r11 | ||||
| movq TF_R12(%rsp),%r12 | movq TF_R12(%rsp),%r12 | ||||
| movq TF_R13(%rsp),%r13 | movq TF_R13(%rsp),%r13 | ||||
| movq TF_R14(%rsp),%r14 | movq TF_R14(%rsp),%r14 | ||||
| movq TF_R15(%rsp),%r15 | movq TF_R15(%rsp),%r15 | ||||
| testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
| jz 1f /* keep running with kernel GS.base */ | jz 2f /* keep running with kernel GS.base */ | ||||
| cli | cli | ||||
| cmpb $0,pti | |||||
| je 1f | |||||
| pushq %rdx | |||||
| movq PCPU(PRVSPACE),%rdx | |||||
| addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8-PTI_SIZE,%rdx | |||||
| movq %rax,PTI_RAX(%rdx) | |||||
| popq %rax | |||||
| movq %rax,PTI_RDX(%rdx) | |||||
| movq TF_RIP(%rsp),%rax | |||||
| movq %rax,PTI_RIP(%rdx) | |||||
| movq TF_CS(%rsp),%rax | |||||
| movq %rax,PTI_CS(%rdx) | |||||
| movq TF_RFLAGS(%rsp),%rax | |||||
| movq %rax,PTI_RFLAGS(%rdx) | |||||
| movq TF_RSP(%rsp),%rax | |||||
| movq %rax,PTI_RSP(%rdx) | |||||
| movq TF_SS(%rsp),%rax | |||||
| movq %rax,PTI_SS(%rdx) | |||||
| movq PCPU(UCR3),%rax | |||||
| swapgs | swapgs | ||||
| 1: | movq %rdx,%rsp | ||||
| addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */ | movq %rax,%cr3 | ||||
| popq %rdx | |||||
| popq %rax | |||||
| addq $8,%rsp | |||||
| jmp doreti_iret | |||||
| 1: swapgs | |||||
| 2: addq $TF_RIP,%rsp | |||||
| .globl doreti_iret | .globl doreti_iret | ||||
| doreti_iret: | doreti_iret: | ||||
| iretq | iretq | ||||
| set_segs: | set_segs: | ||||
| movw $KUDSEL,%ax | movw $KUDSEL,%ax | ||||
| movw %ax,TF_DS(%rsp) | movw %ax,TF_DS(%rsp) | ||||
| movw %ax,TF_ES(%rsp) | movw %ax,TF_ES(%rsp) | ||||
| movw $KUF32SEL,TF_FS(%rsp) | movw $KUF32SEL,TF_FS(%rsp) | ||||
| movw $KUG32SEL,TF_GS(%rsp) | movw $KUG32SEL,TF_GS(%rsp) | ||||
| jmp do_segs | jmp do_segs | ||||
| /* | /* | ||||
| * doreti_iret_fault. Alternative return code for | * doreti_iret_fault. Alternative return code for | ||||
| * the case where we get a fault in the doreti_exit code | * the case where we get a fault in the doreti_exit code | ||||
| * above. trap() (amd64/amd64/trap.c) catches this specific | * above. trap() (amd64/amd64/trap.c) catches this specific | ||||
| * case, sends the process a signal and continues in the | * case, sends the process a signal and continues in the | ||||
| * corresponding place in the code below. | * corresponding place in the code below. | ||||
| */ | */ | ||||
| ALIGN_TEXT | ALIGN_TEXT | ||||
| .globl doreti_iret_fault | .globl doreti_iret_fault | ||||
| doreti_iret_fault: | doreti_iret_fault: | ||||
| subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ | subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ | ||||
| testl $PSL_I,TF_RFLAGS(%rsp) | testb $SEL_RPL_MASK,TF_CS(%rsp) | ||||
| jz 1f | jz 1f | ||||
| sti | sti | ||||
| 1: | 1: | ||||
| movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
| movw %gs,TF_GS(%rsp) | |||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
| movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
| movq %rsi,TF_RSI(%rsp) | movq %rsi,TF_RSI(%rsp) | ||||
| movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
| movq %rcx,TF_RCX(%rsp) | movq %rcx,TF_RCX(%rsp) | ||||
| movq %r8,TF_R8(%rsp) | movq %r8,TF_R8(%rsp) | ||||
| movq %r9,TF_R9(%rsp) | movq %r9,TF_R9(%rsp) | ||||
| movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
| Show All 10 Lines | 1: | ||||
| movq $0,TF_ADDR(%rsp) | movq $0,TF_ADDR(%rsp) | ||||
| FAKE_MCOUNT(TF_RIP(%rsp)) | FAKE_MCOUNT(TF_RIP(%rsp)) | ||||
| jmp calltrap | jmp calltrap | ||||
| ALIGN_TEXT | ALIGN_TEXT | ||||
| .globl ds_load_fault | .globl ds_load_fault | ||||
| ds_load_fault: | ds_load_fault: | ||||
| movl $T_PROTFLT,TF_TRAPNO(%rsp) | movl $T_PROTFLT,TF_TRAPNO(%rsp) | ||||
| testl $PSL_I,TF_RFLAGS(%rsp) | testb $SEL_RPL_MASK,TF_CS(%rsp) | ||||
| jz 1f | jz 1f | ||||
| sti | sti | ||||
| 1: | 1: | ||||
| movq %rsp,%rdi | movq %rsp,%rdi | ||||
| call trap | call trap | ||||
| movw $KUDSEL,TF_DS(%rsp) | movw $KUDSEL,TF_DS(%rsp) | ||||
| jmp doreti | jmp doreti | ||||
| ▲ Show 20 Lines • Show All 73 Lines • Show Last 20 Lines | |||||
If I'm reading the code correctly, PCPU(KCR3) is only set when pti != 0; therefore, it seems like these lines should be conditioned on pti != 0. (But, I could be missing something.)