Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/exception.S
| /*- | /*- | ||||
| * Copyright (c) 1989, 1990 William F. Jolitz. | * Copyright (c) 1989, 1990 William F. Jolitz. | ||||
| * Copyright (c) 1990 The Regents of the University of California. | * Copyright (c) 1990 The Regents of the University of California. | ||||
| * Copyright (c) 2007 The FreeBSD Foundation | * Copyright (c) 2007-2018 The FreeBSD Foundation | ||||
| * All rights reserved. | * All rights reserved. | ||||
| * | * | ||||
| * Portions of this software were developed by A. Joseph Koshy under | * Portions of this software were developed by A. Joseph Koshy under | ||||
| * sponsorship from the FreeBSD Foundation and Google, Inc. | * sponsorship from the FreeBSD Foundation and Google, Inc. | ||||
| * | * | ||||
| * Portions of this software were developed by | |||||
| * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from | |||||
| * the FreeBSD Foundation. | |||||
| * | |||||
| * Redistribution and use in source and binary forms, with or without | * Redistribution and use in source and binary forms, with or without | ||||
| * modification, are permitted provided that the following conditions | * modification, are permitted provided that the following conditions | ||||
| * are met: | * are met: | ||||
| * 1. Redistributions of source code must retain the above copyright | * 1. Redistributions of source code must retain the above copyright | ||||
| * notice, this list of conditions and the following disclaimer. | * notice, this list of conditions and the following disclaimer. | ||||
| * 2. Redistributions in binary form must reproduce the above copyright | * 2. Redistributions in binary form must reproduce the above copyright | ||||
| * notice, this list of conditions and the following disclaimer in the | * notice, this list of conditions and the following disclaimer in the | ||||
| * documentation and/or other materials provided with the distribution. | * documentation and/or other materials provided with the distribution. | ||||
| Show All 15 Lines | |||||
| * | * | ||||
| * $FreeBSD$ | * $FreeBSD$ | ||||
| */ | */ | ||||
| #include "opt_atpic.h" | #include "opt_atpic.h" | ||||
| #include "opt_compat.h" | #include "opt_compat.h" | ||||
| #include "opt_hwpmc_hooks.h" | #include "opt_hwpmc_hooks.h" | ||||
| #include "assym.s" | |||||
| #include <machine/asmacros.h> | #include <machine/asmacros.h> | ||||
| #include <machine/psl.h> | #include <machine/psl.h> | ||||
| #include <machine/trap.h> | #include <machine/trap.h> | ||||
| #include <machine/specialreg.h> | #include <machine/specialreg.h> | ||||
| #include "assym.s" | |||||
| #ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
| .bss | .bss | ||||
| .globl dtrace_invop_jump_addr | .globl dtrace_invop_jump_addr | ||||
| .align 8 | .align 8 | ||||
| .type dtrace_invop_jump_addr,@object | .type dtrace_invop_jump_addr,@object | ||||
| .size dtrace_invop_jump_addr,8 | .size dtrace_invop_jump_addr,8 | ||||
| dtrace_invop_jump_addr: | dtrace_invop_jump_addr: | ||||
| .zero 8 | .zero 8 | ||||
| Show All 39 Lines | |||||
| * segment registers other then %cs, which makes them mostly unused in long | * segment registers other then %cs, which makes them mostly unused in long | ||||
| * mode, and kernel does not reference %fs, leave them alone. The segment | * mode, and kernel does not reference %fs, leave them alone. The segment | ||||
| * registers are reloaded on return to the usermode. | * registers are reloaded on return to the usermode. | ||||
| */ | */ | ||||
| MCOUNT_LABEL(user) | MCOUNT_LABEL(user) | ||||
| MCOUNT_LABEL(btrap) | MCOUNT_LABEL(btrap) | ||||
| /* Traps that we leave interrupts disabled for.. */ | /* Traps that we leave interrupts disabled for. */ | ||||
| #define TRAP_NOEN(a) \ | .macro TRAP_NOEN l, trapno | ||||
| subq $TF_RIP,%rsp; \ | PTI_ENTRY \l,X\l | ||||
| movl $(a),TF_TRAPNO(%rsp) ; \ | .globl X\l | ||||
| movq $0,TF_ADDR(%rsp) ; \ | .type X\l,@function | ||||
| movq $0,TF_ERR(%rsp) ; \ | X\l: subq $TF_RIP,%rsp | ||||
| movl $\trapno,TF_TRAPNO(%rsp) | |||||
| movq $0,TF_ADDR(%rsp) | |||||
| movq $0,TF_ERR(%rsp) | |||||
| jmp alltraps_noen | jmp alltraps_noen | ||||
| IDTVEC(dbg) | .endm | ||||
| TRAP_NOEN(T_TRCTRAP) | |||||
| IDTVEC(bpt) | TRAP_NOEN dbg, T_TRCTRAP | ||||
| TRAP_NOEN(T_BPTFLT) | TRAP_NOEN bpt, T_BPTFLT | ||||
| #ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
| IDTVEC(dtrace_ret) | TRAP_NOEN dtrace_ret, T_DTRACE_RET | ||||
| TRAP_NOEN(T_DTRACE_RET) | |||||
| #endif | #endif | ||||
| /* Regular traps; The cpu does not supply tf_err for these. */ | /* Regular traps; The cpu does not supply tf_err for these. */ | ||||
| #define TRAP(a) \ | .macro TRAP l, trapno | ||||
| subq $TF_RIP,%rsp; \ | PTI_ENTRY \l,X\l | ||||
| movl $(a),TF_TRAPNO(%rsp) ; \ | .globl X\l | ||||
| movq $0,TF_ADDR(%rsp) ; \ | .type X\l,@function | ||||
| movq $0,TF_ERR(%rsp) ; \ | X\l: | ||||
| subq $TF_RIP,%rsp | |||||
| movl $\trapno,TF_TRAPNO(%rsp) | |||||
| movq $0,TF_ADDR(%rsp) | |||||
| movq $0,TF_ERR(%rsp) | |||||
| jmp alltraps | jmp alltraps | ||||
| IDTVEC(div) | .endm | ||||
| TRAP(T_DIVIDE) | |||||
| IDTVEC(ofl) | |||||
| TRAP(T_OFLOW) | |||||
| IDTVEC(bnd) | |||||
| TRAP(T_BOUND) | |||||
| IDTVEC(ill) | |||||
| TRAP(T_PRIVINFLT) | |||||
| IDTVEC(dna) | |||||
| TRAP(T_DNA) | |||||
| IDTVEC(fpusegm) | |||||
| TRAP(T_FPOPFLT) | |||||
| IDTVEC(mchk) | |||||
| TRAP(T_MCHK) | |||||
| IDTVEC(rsvd) | |||||
| TRAP(T_RESERVED) | |||||
| IDTVEC(fpu) | |||||
| TRAP(T_ARITHTRAP) | |||||
| IDTVEC(xmm) | |||||
| TRAP(T_XMMFLT) | |||||
| /* This group of traps have tf_err already pushed by the cpu */ | TRAP div, T_DIVIDE | ||||
| #define TRAP_ERR(a) \ | TRAP ofl, T_OFLOW | ||||
| subq $TF_ERR,%rsp; \ | TRAP bnd, T_BOUND | ||||
| movl $(a),TF_TRAPNO(%rsp) ; \ | TRAP ill, T_PRIVINFLT | ||||
| movq $0,TF_ADDR(%rsp) ; \ | TRAP dna, T_DNA | ||||
| jmp alltraps | TRAP fpusegm, T_FPOPFLT | ||||
| IDTVEC(tss) | TRAP mchk, T_MCHK | ||||
| TRAP_ERR(T_TSSFLT) | TRAP rsvd, T_RESERVED | ||||
| IDTVEC(missing) | TRAP fpu, T_ARITHTRAP | ||||
| TRAP xmm, T_XMMFLT | |||||
| /* This group of traps have tf_err already pushed by the cpu. */ | |||||
| .macro TRAP_ERR l, trapno | |||||
| PTI_ENTRY \l,X\l,has_err=1 | |||||
| .globl X\l | |||||
| .type X\l,@function | |||||
| X\l: | |||||
| subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
| movl $T_SEGNPFLT,TF_TRAPNO(%rsp) | movl $\trapno,TF_TRAPNO(%rsp) | ||||
| jmp prot_addrf | movq $0,TF_ADDR(%rsp) | ||||
| IDTVEC(stk) | jmp alltraps | ||||
| subq $TF_ERR,%rsp | .endm | ||||
| movl $T_STKFLT,TF_TRAPNO(%rsp) | |||||
| jmp prot_addrf | |||||
| IDTVEC(align) | |||||
| TRAP_ERR(T_ALIGNFLT) | |||||
| TRAP_ERR tss, T_TSSFLT | |||||
| TRAP_ERR align, T_ALIGNFLT | |||||
| /* | /* | ||||
| * alltraps entry point. Use swapgs if this is the first time in the | * alltraps entry point. Use swapgs if this is the first time in the | ||||
| * kernel from userland. Reenable interrupts if they were enabled | * kernel from userland. Reenable interrupts if they were enabled | ||||
| * before the trap. This approximates SDT_SYS386TGT on the i386 port. | * before the trap. This approximates SDT_SYS386TGT on the i386 port. | ||||
| */ | */ | ||||
| SUPERALIGN_TEXT | SUPERALIGN_TEXT | ||||
| .globl alltraps | .globl alltraps | ||||
| .type alltraps,@function | .type alltraps,@function | ||||
| alltraps: | alltraps: | ||||
| movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
| testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
| jz alltraps_testi /* already running with kernel GS.base */ | jz alltraps_segs /* already running with kernel GS.base */ | ||||
| swapgs | swapgs | ||||
| movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
| andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
| movw %fs,TF_FS(%rsp) | alltraps_segs: | ||||
| movw %gs,TF_GS(%rsp) | SAVE_SEGS | ||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| alltraps_testi: | |||||
| testl $PSL_I,TF_RFLAGS(%rsp) | testl $PSL_I,TF_RFLAGS(%rsp) | ||||
| jz alltraps_pushregs_no_rdi | jz alltraps_pushregs_no_rdi | ||||
| sti | sti | ||||
| alltraps_pushregs_no_rdi: | alltraps_pushregs_no_rdi: | ||||
| movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
| movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
| alltraps_pushregs_no_rax: | alltraps_pushregs_no_rax: | ||||
| movq %rsi,TF_RSI(%rsp) | movq %rsi,TF_RSI(%rsp) | ||||
| ▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | calltrap: | ||||
| * SDT_SYS386IGT on the i386 port. | * SDT_SYS386IGT on the i386 port. | ||||
| */ | */ | ||||
| SUPERALIGN_TEXT | SUPERALIGN_TEXT | ||||
| .globl alltraps_noen | .globl alltraps_noen | ||||
| .type alltraps_noen,@function | .type alltraps_noen,@function | ||||
| alltraps_noen: | alltraps_noen: | ||||
| movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
| testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
| jz 1f /* already running with kernel GS.base */ | jz alltraps_noen_segs /* already running with kernel GS.base */ | ||||
| swapgs | swapgs | ||||
| movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
| andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
| 1: movw %fs,TF_FS(%rsp) | alltraps_noen_segs: | ||||
| movw %gs,TF_GS(%rsp) | SAVE_SEGS | ||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| jmp alltraps_pushregs_no_rdi | jmp alltraps_pushregs_no_rdi | ||||
| IDTVEC(dblfault) | IDTVEC(dblfault) | ||||
| subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
| movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) | movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) | ||||
| movq $0,TF_ADDR(%rsp) | movq $0,TF_ADDR(%rsp) | ||||
| movq $0,TF_ERR(%rsp) | movq $0,TF_ERR(%rsp) | ||||
| movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
| movq %rsi,TF_RSI(%rsp) | movq %rsi,TF_RSI(%rsp) | ||||
| movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
| movq %rcx,TF_RCX(%rsp) | movq %rcx,TF_RCX(%rsp) | ||||
| movq %r8,TF_R8(%rsp) | movq %r8,TF_R8(%rsp) | ||||
| movq %r9,TF_R9(%rsp) | movq %r9,TF_R9(%rsp) | ||||
| movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
| movq %rbx,TF_RBX(%rsp) | movq %rbx,TF_RBX(%rsp) | ||||
| movq %rbp,TF_RBP(%rsp) | movq %rbp,TF_RBP(%rsp) | ||||
| movq %r10,TF_R10(%rsp) | movq %r10,TF_R10(%rsp) | ||||
| movq %r11,TF_R11(%rsp) | movq %r11,TF_R11(%rsp) | ||||
| movq %r12,TF_R12(%rsp) | movq %r12,TF_R12(%rsp) | ||||
| movq %r13,TF_R13(%rsp) | movq %r13,TF_R13(%rsp) | ||||
| movq %r14,TF_R14(%rsp) | movq %r14,TF_R14(%rsp) | ||||
| movq %r15,TF_R15(%rsp) | movq %r15,TF_R15(%rsp) | ||||
| movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
| movw %gs,TF_GS(%rsp) | |||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
| cld | cld | ||||
| testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
| jz 1f /* already running with kernel GS.base */ | jz 1f /* already running with kernel GS.base */ | ||||
| swapgs | swapgs | ||||
| 1: | 1: | ||||
| movq %rsp,%rdi | movq PCPU(KCR3),%rax | ||||
jtl: If I'm reading the code correctly, PCPU(KCR3) is only set when pti != 0; therefore, it seems… | |||||
Not Done Inline ActionsYeah, I already fixed it since I tested the non-PTI case as well. Thank you for noting. kib: Yeah, I already fixed it since I tested the non-PTI case as well. Thank you for noting. | |||||
| cmpq $~0,%rax | |||||
| je 2f | |||||
| movq %rax,%cr3 | |||||
| 2: movq %rsp,%rdi | |||||
| call dblfault_handler | call dblfault_handler | ||||
| 2: | 3: hlt | ||||
| hlt | jmp 3b | ||||
| jmp 2b | |||||
| PTI_ENTRY page, Xpage, has_err=1 | |||||
| IDTVEC(page) | IDTVEC(page) | ||||
| subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
| movl $T_PAGEFLT,TF_TRAPNO(%rsp) | |||||
| movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | ||||
| testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
| jz 1f /* already running with kernel GS.base */ | jz page_cr2 /* already running with kernel GS.base */ | ||||
| swapgs | swapgs | ||||
| movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
| andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
| 1: movq %cr2,%rdi /* preserve %cr2 before .. */ | page_cr2: | ||||
| movq %cr2,%rdi /* preserve %cr2 before .. */ | |||||
| movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ | movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ | ||||
| movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
| movw %gs,TF_GS(%rsp) | movl $T_PAGEFLT,TF_TRAPNO(%rsp) | ||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| testl $PSL_I,TF_RFLAGS(%rsp) | testl $PSL_I,TF_RFLAGS(%rsp) | ||||
| jz alltraps_pushregs_no_rdi | jz alltraps_pushregs_no_rdi | ||||
| sti | sti | ||||
| jmp alltraps_pushregs_no_rdi | jmp alltraps_pushregs_no_rdi | ||||
| /* | /* | ||||
| * We have to special-case this one. If we get a trap in doreti() at | * We have to special-case this one. If we get a trap in doreti() at | ||||
| * the iretq stage, we'll reenter with the wrong gs state. We'll have | * the iretq stage, we'll reenter with the wrong gs state. We'll have | ||||
| * to do a special the swapgs in this case even coming from the kernel. | * to do a special the swapgs in this case even coming from the kernel. | ||||
| * XXX linux has a trap handler for their equivalent of load_gs(). | * XXX linux has a trap handler for their equivalent of load_gs(). | ||||
| * | |||||
| * On the stack, we have the hardware interrupt frame to return | |||||
| * to usermode (faulted) and another frame with error code, for | |||||
| * fault. For PTI, copy both frames to the main thread stack. | |||||
| */ | */ | ||||
| IDTVEC(prot) | .macro PROTF_ENTRY name,trapno | ||||
| \name\()_pti_doreti: | |||||
| pushq %rax | |||||
| pushq %rdx | |||||
| swapgs | |||||
| movq PCPU(KCR3),%rax | |||||
| movq %rax,%cr3 | |||||
| movq PCPU(RSP0),%rax | |||||
| subq $2*PTI_SIZE-3*8,%rax | |||||
| MOVE_STACKS (PTI_SIZE / 4 - 3) | |||||
| movq %rax,%rsp | |||||
| popq %rdx | |||||
| popq %rax | |||||
| swapgs | |||||
| jmp X\name | |||||
| IDTVEC(\name\()_pti) | |||||
| cmpq $doreti_iret,PTI_RIP-2*8(%rsp) | |||||
| je \name\()_pti_doreti | |||||
| testb $SEL_RPL_MASK,PTI_CS-2*8(%rsp) /* %rax, %rdx not yet pushed */ | |||||
| jz X\name | |||||
| PTI_UENTRY has_err=1 | |||||
| swapgs | |||||
| IDTVEC(\name) | |||||
| subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
| movl $T_PROTFLT,TF_TRAPNO(%rsp) | movl $\trapno,TF_TRAPNO(%rsp) | ||||
| jmp prot_addrf | |||||
| .endm | |||||
| PROTF_ENTRY missing, T_SEGNPFLT | |||||
| PROTF_ENTRY stk, T_STKFLT | |||||
| PROTF_ENTRY prot, T_PROTFLT | |||||
| prot_addrf: | prot_addrf: | ||||
| movq $0,TF_ADDR(%rsp) | movq $0,TF_ADDR(%rsp) | ||||
| movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | ||||
| movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
| movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
| movw %fs,TF_FS(%rsp) | movw %fs,TF_FS(%rsp) | ||||
| movw %gs,TF_GS(%rsp) | movw %gs,TF_GS(%rsp) | ||||
| leaq doreti_iret(%rip),%rdi | leaq doreti_iret(%rip),%rdi | ||||
| Show All 35 Lines | |||||
| * Fast syscall entry point. We enter here with just our new %cs/%ss set, | * Fast syscall entry point. We enter here with just our new %cs/%ss set, | ||||
| * and the new privilige level. We are still running on the old user stack | * and the new privilige level. We are still running on the old user stack | ||||
| * pointer. We have to juggle a few things around to find our stack etc. | * pointer. We have to juggle a few things around to find our stack etc. | ||||
| * swapgs gives us access to our PCPU space only. | * swapgs gives us access to our PCPU space only. | ||||
| * | * | ||||
| * We do not support invoking this from a custom segment registers, | * We do not support invoking this from a custom segment registers, | ||||
| * esp. %cs, %ss, %fs, %gs, e.g. using entries from an LDT. | * esp. %cs, %ss, %fs, %gs, e.g. using entries from an LDT. | ||||
| */ | */ | ||||
| SUPERALIGN_TEXT | |||||
| IDTVEC(fast_syscall_pti) | |||||
| swapgs | |||||
| movq %rax,PCPU(SCRATCH_RAX) | |||||
| movq PCPU(KCR3),%rax | |||||
| movq %rax,%cr3 | |||||
| jmp fast_syscall_common | |||||
| SUPERALIGN_TEXT | |||||
| IDTVEC(fast_syscall) | IDTVEC(fast_syscall) | ||||
| swapgs | swapgs | ||||
| movq %rax,PCPU(SCRATCH_RAX) | |||||
| fast_syscall_common: | |||||
| movq %rsp,PCPU(SCRATCH_RSP) | movq %rsp,PCPU(SCRATCH_RSP) | ||||
| movq PCPU(RSP0),%rsp | movq PCPU(RSP0),%rsp | ||||
| /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ | /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ | ||||
| subq $TF_SIZE,%rsp | subq $TF_SIZE,%rsp | ||||
| /* defer TF_RSP till we have a spare register */ | /* defer TF_RSP till we have a spare register */ | ||||
| movq %r11,TF_RFLAGS(%rsp) | movq %r11,TF_RFLAGS(%rsp) | ||||
| movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ | movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ | ||||
| movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ | movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ | ||||
| movq %r11,TF_RSP(%rsp) /* user stack pointer */ | movq %r11,TF_RSP(%rsp) /* user stack pointer */ | ||||
| movw %fs,TF_FS(%rsp) | movq PCPU(SCRATCH_RAX),%rax | ||||
| movw %gs,TF_GS(%rsp) | movq %rax,TF_RAX(%rsp) /* syscall number */ | ||||
| movw %es,TF_ES(%rsp) | SAVE_SEGS | ||||
| movw %ds,TF_DS(%rsp) | |||||
| movq PCPU(CURPCB),%r11 | movq PCPU(CURPCB),%r11 | ||||
| andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) | andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) | ||||
| sti | sti | ||||
| movq $KUDSEL,TF_SS(%rsp) | movq $KUDSEL,TF_SS(%rsp) | ||||
| movq $KUCSEL,TF_CS(%rsp) | movq $KUCSEL,TF_CS(%rsp) | ||||
| movq $2,TF_ERR(%rsp) | movq $2,TF_ERR(%rsp) | ||||
| movq %rdi,TF_RDI(%rsp) /* arg 1 */ | movq %rdi,TF_RDI(%rsp) /* arg 1 */ | ||||
| movq %rsi,TF_RSI(%rsp) /* arg 2 */ | movq %rsi,TF_RSI(%rsp) /* arg 2 */ | ||||
| movq %rdx,TF_RDX(%rsp) /* arg 3 */ | movq %rdx,TF_RDX(%rsp) /* arg 3 */ | ||||
| movq %r10,TF_RCX(%rsp) /* arg 4 */ | movq %r10,TF_RCX(%rsp) /* arg 4 */ | ||||
| movq %r8,TF_R8(%rsp) /* arg 5 */ | movq %r8,TF_R8(%rsp) /* arg 5 */ | ||||
| movq %r9,TF_R9(%rsp) /* arg 6 */ | movq %r9,TF_R9(%rsp) /* arg 6 */ | ||||
| movq %rax,TF_RAX(%rsp) /* syscall number */ | |||||
| movq %rbx,TF_RBX(%rsp) /* C preserved */ | movq %rbx,TF_RBX(%rsp) /* C preserved */ | ||||
| movq %rbp,TF_RBP(%rsp) /* C preserved */ | movq %rbp,TF_RBP(%rsp) /* C preserved */ | ||||
| movq %r12,TF_R12(%rsp) /* C preserved */ | movq %r12,TF_R12(%rsp) /* C preserved */ | ||||
| movq %r13,TF_R13(%rsp) /* C preserved */ | movq %r13,TF_R13(%rsp) /* C preserved */ | ||||
| movq %r14,TF_R14(%rsp) /* C preserved */ | movq %r14,TF_R14(%rsp) /* C preserved */ | ||||
| movq %r15,TF_R15(%rsp) /* C preserved */ | movq %r15,TF_R15(%rsp) /* C preserved */ | ||||
| movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
| FAKE_MCOUNT(TF_RIP(%rsp)) | FAKE_MCOUNT(TF_RIP(%rsp)) | ||||
| movq PCPU(CURTHREAD),%rdi | movq PCPU(CURTHREAD),%rdi | ||||
| movq %rsp,TD_FRAME(%rdi) | movq %rsp,TD_FRAME(%rdi) | ||||
| movl TF_RFLAGS(%rsp),%esi | movl TF_RFLAGS(%rsp),%esi | ||||
| andl $PSL_T,%esi | andl $PSL_T,%esi | ||||
| call amd64_syscall | call amd64_syscall | ||||
| 1: movq PCPU(CURPCB),%rax | 1: movq PCPU(CURPCB),%rax | ||||
| /* Disable interrupts before testing PCB_FULL_IRET. */ | /* Disable interrupts before testing PCB_FULL_IRET. */ | ||||
| cli | cli | ||||
| testl $PCB_FULL_IRET,PCB_FLAGS(%rax) | testl $PCB_FULL_IRET,PCB_FLAGS(%rax) | ||||
| jnz 3f | jnz 4f | ||||
| /* Check for and handle AST's on return to userland. */ | /* Check for and handle AST's on return to userland. */ | ||||
| movq PCPU(CURTHREAD),%rax | movq PCPU(CURTHREAD),%rax | ||||
| testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) | testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) | ||||
| jne 2f | jne 3f | ||||
| /* Restore preserved registers. */ | /* Restore preserved registers. */ | ||||
| MEXITCOUNT | MEXITCOUNT | ||||
| movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ | movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ | ||||
| movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ | movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ | ||||
| movq TF_RDX(%rsp),%rdx /* return value 2 */ | movq TF_RDX(%rsp),%rdx /* return value 2 */ | ||||
| movq TF_RAX(%rsp),%rax /* return value 1 */ | movq TF_RAX(%rsp),%rax /* return value 1 */ | ||||
| movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ | movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ | ||||
| movq TF_RIP(%rsp),%rcx /* original %rip */ | movq TF_RIP(%rsp),%rcx /* original %rip */ | ||||
| movq TF_RSP(%rsp),%rsp /* user stack pointer */ | movq TF_RSP(%rsp),%rsp /* user stack pointer */ | ||||
| swapgs | cmpb $0,pti | ||||
| je 2f | |||||
| movq PCPU(UCR3),%r9 | |||||
| movq %r9,%cr3 | |||||
| xorl %r9d,%r9d | |||||
| 2: swapgs | |||||
| sysretq | sysretq | ||||
| 2: /* AST scheduled. */ | 3: /* AST scheduled. */ | ||||
| sti | sti | ||||
| movq %rsp,%rdi | movq %rsp,%rdi | ||||
| call ast | call ast | ||||
| jmp 1b | jmp 1b | ||||
| 3: /* Requested full context restore, use doreti for that. */ | 4: /* Requested full context restore, use doreti for that. */ | ||||
| MEXITCOUNT | MEXITCOUNT | ||||
| jmp doreti | jmp doreti | ||||
| /* | /* | ||||
| * Here for CYA insurance, in case a "syscall" instruction gets | * Here for CYA insurance, in case a "syscall" instruction gets | ||||
| * issued from 32 bit compatibility mode. MSR_CSTAR has to point | * issued from 32 bit compatibility mode. MSR_CSTAR has to point | ||||
| * to *something* if EFER_SCE is enabled. | * to *something* if EFER_SCE is enabled. | ||||
| */ | */ | ||||
| Show All 39 Lines | IDTVEC(nmi) | ||||
| movq %rbx,TF_RBX(%rsp) | movq %rbx,TF_RBX(%rsp) | ||||
| movq %rbp,TF_RBP(%rsp) | movq %rbp,TF_RBP(%rsp) | ||||
| movq %r10,TF_R10(%rsp) | movq %r10,TF_R10(%rsp) | ||||
| movq %r11,TF_R11(%rsp) | movq %r11,TF_R11(%rsp) | ||||
| movq %r12,TF_R12(%rsp) | movq %r12,TF_R12(%rsp) | ||||
| movq %r13,TF_R13(%rsp) | movq %r13,TF_R13(%rsp) | ||||
| movq %r14,TF_R14(%rsp) | movq %r14,TF_R14(%rsp) | ||||
| movq %r15,TF_R15(%rsp) | movq %r15,TF_R15(%rsp) | ||||
| movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
| movw %gs,TF_GS(%rsp) | |||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
| cld | cld | ||||
| xorl %ebx,%ebx | xorl %ebx,%ebx | ||||
| testb $SEL_RPL_MASK,TF_CS(%rsp) | testb $SEL_RPL_MASK,TF_CS(%rsp) | ||||
| jnz nmi_fromuserspace | jnz nmi_fromuserspace | ||||
| /* | /* | ||||
| * We've interrupted the kernel. Preserve GS.base in %r12. | * We've interrupted the kernel. Preserve GS.base in %r12 | ||||
| * and %cr3 in %r13. | |||||
| */ | */ | ||||
| movl $MSR_GSBASE,%ecx | movl $MSR_GSBASE,%ecx | ||||
| rdmsr | rdmsr | ||||
| movq %rax,%r12 | movq %rax,%r12 | ||||
| shlq $32,%rdx | shlq $32,%rdx | ||||
| orq %rdx,%r12 | orq %rdx,%r12 | ||||
| /* Retrieve and load the canonical value for GS.base. */ | /* Retrieve and load the canonical value for GS.base. */ | ||||
| movq TF_SIZE(%rsp),%rdx | movq TF_SIZE(%rsp),%rdx | ||||
| movl %edx,%eax | movl %edx,%eax | ||||
| shrq $32,%rdx | shrq $32,%rdx | ||||
| wrmsr | wrmsr | ||||
| movq %cr3,%r13 | |||||
| movq PCPU(KCR3),%rax | |||||
| cmpq $~0,%rax | |||||
Not Done Inline ActionsIf I'm reading the code correctly, PCPU(KCR3) is only set when pti != 0; therefore, it seems like these lines should be conditioned on pti != 0. (But, I could be missing something.) jtl: If I'm reading the code correctly, PCPU(KCR3) is only set when pti != 0; therefore, it seems… | |||||
Not Done Inline ActionsIt looks to me like PCPU(KCR3) is always set, it just only changes in the pti!=0 case. This seems to avoid a compare & branch. kithrup_mac.com: It looks to me like PCPU(KCR3) is always set, it just only changes in the pti!=0 case. This… | |||||
Not Done Inline ActionsIt is not for !pti case. For now wrapping accesses with tests is the right thing to do. kib: It is not for !pti case. For now wrapping accesses with tests is the right thing to do. | |||||
| je nmi_calltrap | |||||
| movq %rax,%cr3 | |||||
| jmp nmi_calltrap | jmp nmi_calltrap | ||||
| nmi_fromuserspace: | nmi_fromuserspace: | ||||
| incl %ebx | incl %ebx | ||||
| swapgs | swapgs | ||||
| testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) | movq %cr3,%r13 | ||||
Not Done Inline ActionsDitto. jtl: Ditto. | |||||
Not Done Inline ActionsThanks for fixing these (and your overall work on the bug)! FYI, I think this one (nmi_fromuserspace) still needs the change to only set cr3 when pti != 0. jtl: Thanks for fixing these (and your overall work on the bug)! FYI, I think this one… | |||||
| jz 2f | movq PCPU(KCR3),%rax | ||||
| cmpq $~0,%rax | |||||
| je 1f | |||||
| movq %rax,%cr3 | |||||
| movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
| testq %rdi,%rdi | testq %rdi,%rdi | ||||
| jz 2f | jz 3f | ||||
| orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) | |||||
| 1: testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) | |||||
| jz 3f | |||||
| cmpw $KUF32SEL,TF_FS(%rsp) | cmpw $KUF32SEL,TF_FS(%rsp) | ||||
| jne 1f | jne 2f | ||||
| rdfsbase %rax | rdfsbase %rax | ||||
| movq %rax,PCB_FSBASE(%rdi) | movq %rax,PCB_FSBASE(%rdi) | ||||
| 1: cmpw $KUG32SEL,TF_GS(%rsp) | 2: cmpw $KUG32SEL,TF_GS(%rsp) | ||||
| jne 2f | jne 3f | ||||
| movl $MSR_KGSBASE,%ecx | movl $MSR_KGSBASE,%ecx | ||||
| rdmsr | rdmsr | ||||
| shlq $32,%rdx | shlq $32,%rdx | ||||
| orq %rdx,%rax | orq %rdx,%rax | ||||
| movq %rax,PCB_GSBASE(%rdi) | movq %rax,PCB_GSBASE(%rdi) | ||||
| 2: | 3: | ||||
| /* Note: this label is also used by ddb and gdb: */ | /* Note: this label is also used by ddb and gdb: */ | ||||
| nmi_calltrap: | nmi_calltrap: | ||||
| FAKE_MCOUNT(TF_RIP(%rsp)) | FAKE_MCOUNT(TF_RIP(%rsp)) | ||||
| movq %rsp,%rdi | movq %rsp,%rdi | ||||
| call trap | call trap | ||||
| MEXITCOUNT | MEXITCOUNT | ||||
| #ifdef HWPMC_HOOKS | #ifdef HWPMC_HOOKS | ||||
| /* | /* | ||||
| * Capture a userspace callchain if needed. | * Capture a userspace callchain if needed. | ||||
| * | * | ||||
| * - Check if the current trap was from user mode. | * - Check if the current trap was from user mode. | ||||
| * - Check if the current thread is valid. | * - Check if the current thread is valid. | ||||
| * - Check if the thread requires a user call chain to be | * - Check if the thread requires a user call chain to be | ||||
| * captured. | * captured. | ||||
| * | * | ||||
| * We are still in NMI mode at this point. | * We are still in NMI mode at this point. | ||||
| */ | */ | ||||
| testl %ebx,%ebx | testl %ebx,%ebx | ||||
| jz nocallchain /* not from userspace */ | jz nocallchain /* not from userspace */ | ||||
| movq PCPU(CURTHREAD),%rax | movq PCPU(CURTHREAD),%rax | ||||
| orq %rax,%rax /* curthread present? */ | orq %rax,%rax /* curthread present? */ | ||||
| jz nocallchain | jz nocallchain | ||||
| testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */ | |||||
| jz nocallchain | |||||
| /* | /* | ||||
| * A user callchain is to be captured, so: | * Move execution to the regular kernel stack, because we | ||||
| * - Move execution to the regular kernel stack, to allow for | * committed to return through doreti. | ||||
| * nested NMI interrupts. | |||||
| * - Take the processor out of "NMI" mode by faking an "iret". | |||||
| * - Enable interrupts, so that copyin() can work. | |||||
| */ | */ | ||||
| movq %rsp,%rsi /* source stack pointer */ | movq %rsp,%rsi /* source stack pointer */ | ||||
| movq $TF_SIZE,%rcx | movq $TF_SIZE,%rcx | ||||
| movq PCPU(RSP0),%rdx | movq PCPU(RSP0),%rdx | ||||
| subq %rcx,%rdx | subq %rcx,%rdx | ||||
| movq %rdx,%rdi /* destination stack pointer */ | movq %rdx,%rdi /* destination stack pointer */ | ||||
| shrq $3,%rcx /* trap frame size in long words */ | shrq $3,%rcx /* trap frame size in long words */ | ||||
| cld | cld | ||||
| rep | rep | ||||
| movsq /* copy trapframe */ | movsq /* copy trapframe */ | ||||
| movq %rdx,%rsp /* we are on the regular kstack */ | |||||
| testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */ | |||||
| jz nocallchain | |||||
| /* | |||||
| * A user callchain is to be captured, so: | |||||
| * - Take the processor out of "NMI" mode by faking an "iret", | |||||
| * to allow for nested NMI interrupts. | |||||
| * - Enable interrupts, so that copyin() can work. | |||||
| */ | |||||
| movl %ss,%eax | movl %ss,%eax | ||||
| pushq %rax /* tf_ss */ | pushq %rax /* tf_ss */ | ||||
| pushq %rdx /* tf_rsp (on kernel stack) */ | pushq %rdx /* tf_rsp (on kernel stack) */ | ||||
| pushfq /* tf_rflags */ | pushfq /* tf_rflags */ | ||||
| movl %cs,%eax | movl %cs,%eax | ||||
| pushq %rax /* tf_cs */ | pushq %rax /* tf_cs */ | ||||
| pushq $outofnmi /* tf_rip */ | pushq $outofnmi /* tf_rip */ | ||||
| iretq | iretq | ||||
| Show All 24 Lines | nmi_kernelexit: | ||||
| /* | /* | ||||
| * Put back the preserved MSR_GSBASE value. | * Put back the preserved MSR_GSBASE value. | ||||
| */ | */ | ||||
| movl $MSR_GSBASE,%ecx | movl $MSR_GSBASE,%ecx | ||||
| movq %r12,%rdx | movq %r12,%rdx | ||||
| movl %edx,%eax | movl %edx,%eax | ||||
| shrq $32,%rdx | shrq $32,%rdx | ||||
| wrmsr | wrmsr | ||||
| movq %r13,%cr3 | |||||
| nmi_restoreregs: | nmi_restoreregs: | ||||
| movq TF_RDI(%rsp),%rdi | RESTORE_REGS | ||||
| movq TF_RSI(%rsp),%rsi | |||||
| movq TF_RDX(%rsp),%rdx | |||||
| movq TF_RCX(%rsp),%rcx | |||||
| movq TF_R8(%rsp),%r8 | |||||
| movq TF_R9(%rsp),%r9 | |||||
| movq TF_RAX(%rsp),%rax | |||||
| movq TF_RBX(%rsp),%rbx | |||||
| movq TF_RBP(%rsp),%rbp | |||||
| movq TF_R10(%rsp),%r10 | |||||
| movq TF_R11(%rsp),%r11 | |||||
| movq TF_R12(%rsp),%r12 | |||||
| movq TF_R13(%rsp),%r13 | |||||
| movq TF_R14(%rsp),%r14 | |||||
| movq TF_R15(%rsp),%r15 | |||||
| addq $TF_RIP,%rsp | addq $TF_RIP,%rsp | ||||
| jmp doreti_iret | jmp doreti_iret | ||||
| ENTRY(fork_trampoline) | ENTRY(fork_trampoline) | ||||
| movq %r12,%rdi /* function */ | movq %r12,%rdi /* function */ | ||||
| movq %rbx,%rsi /* arg1 */ | movq %rbx,%rsi /* arg1 */ | ||||
| movq %rsp,%rdx /* trapframe pointer */ | movq %rsp,%rdx /* trapframe pointer */ | ||||
| call fork_exit | call fork_exit | ||||
| ▲ Show 20 Lines • Show All 151 Lines • ▼ Show 20 Lines | ld_gsbase: | ||||
| wrmsr /* May trap if non-canonical, but only for TLS. */ | wrmsr /* May trap if non-canonical, but only for TLS. */ | ||||
| .globl ld_es | .globl ld_es | ||||
| ld_es: | ld_es: | ||||
| movw TF_ES(%rsp),%es | movw TF_ES(%rsp),%es | ||||
| .globl ld_ds | .globl ld_ds | ||||
| ld_ds: | ld_ds: | ||||
| movw TF_DS(%rsp),%ds | movw TF_DS(%rsp),%ds | ||||
| ld_regs: | ld_regs: | ||||
| movq TF_RDI(%rsp),%rdi | RESTORE_REGS | ||||
| movq TF_RSI(%rsp),%rsi | |||||
| movq TF_RDX(%rsp),%rdx | |||||
| movq TF_RCX(%rsp),%rcx | |||||
| movq TF_R8(%rsp),%r8 | |||||
| movq TF_R9(%rsp),%r9 | |||||
| movq TF_RAX(%rsp),%rax | |||||
| movq TF_RBX(%rsp),%rbx | |||||
| movq TF_RBP(%rsp),%rbp | |||||
| movq TF_R10(%rsp),%r10 | |||||
| movq TF_R11(%rsp),%r11 | |||||
| movq TF_R12(%rsp),%r12 | |||||
| movq TF_R13(%rsp),%r13 | |||||
| movq TF_R14(%rsp),%r14 | |||||
| movq TF_R15(%rsp),%r15 | |||||
| testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
| jz 1f /* keep running with kernel GS.base */ | jz 2f /* keep running with kernel GS.base */ | ||||
| cli | cli | ||||
| cmpb $0,pti | |||||
| je 1f | |||||
| pushq %rdx | |||||
| movq PCPU(PRVSPACE),%rdx | |||||
| addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8-PTI_SIZE,%rdx | |||||
| movq %rax,PTI_RAX(%rdx) | |||||
| popq %rax | |||||
| movq %rax,PTI_RDX(%rdx) | |||||
| movq TF_RIP(%rsp),%rax | |||||
| movq %rax,PTI_RIP(%rdx) | |||||
| movq TF_CS(%rsp),%rax | |||||
| movq %rax,PTI_CS(%rdx) | |||||
| movq TF_RFLAGS(%rsp),%rax | |||||
| movq %rax,PTI_RFLAGS(%rdx) | |||||
| movq TF_RSP(%rsp),%rax | |||||
| movq %rax,PTI_RSP(%rdx) | |||||
| movq TF_SS(%rsp),%rax | |||||
| movq %rax,PTI_SS(%rdx) | |||||
| movq PCPU(UCR3),%rax | |||||
| swapgs | swapgs | ||||
| 1: | movq %rdx,%rsp | ||||
| addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */ | movq %rax,%cr3 | ||||
| popq %rdx | |||||
| popq %rax | |||||
| addq $8,%rsp | |||||
| jmp doreti_iret | |||||
| 1: swapgs | |||||
| 2: addq $TF_RIP,%rsp | |||||
| .globl doreti_iret | .globl doreti_iret | ||||
| doreti_iret: | doreti_iret: | ||||
| iretq | iretq | ||||
| set_segs: | set_segs: | ||||
| movw $KUDSEL,%ax | movw $KUDSEL,%ax | ||||
| movw %ax,TF_DS(%rsp) | movw %ax,TF_DS(%rsp) | ||||
| movw %ax,TF_ES(%rsp) | movw %ax,TF_ES(%rsp) | ||||
| movw $KUF32SEL,TF_FS(%rsp) | movw $KUF32SEL,TF_FS(%rsp) | ||||
| movw $KUG32SEL,TF_GS(%rsp) | movw $KUG32SEL,TF_GS(%rsp) | ||||
| jmp do_segs | jmp do_segs | ||||
| /* | /* | ||||
| * doreti_iret_fault. Alternative return code for | * doreti_iret_fault. Alternative return code for | ||||
| * the case where we get a fault in the doreti_exit code | * the case where we get a fault in the doreti_exit code | ||||
| * above. trap() (amd64/amd64/trap.c) catches this specific | * above. trap() (amd64/amd64/trap.c) catches this specific | ||||
| * case, sends the process a signal and continues in the | * case, sends the process a signal and continues in the | ||||
| * corresponding place in the code below. | * corresponding place in the code below. | ||||
| */ | */ | ||||
| ALIGN_TEXT | ALIGN_TEXT | ||||
| .globl doreti_iret_fault | .globl doreti_iret_fault | ||||
| doreti_iret_fault: | doreti_iret_fault: | ||||
| subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ | subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ | ||||
| testl $PSL_I,TF_RFLAGS(%rsp) | testb $SEL_RPL_MASK,TF_CS(%rsp) | ||||
| jz 1f | jz 1f | ||||
| sti | sti | ||||
| 1: | 1: | ||||
| movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
| movw %gs,TF_GS(%rsp) | |||||
| movw %es,TF_ES(%rsp) | |||||
| movw %ds,TF_DS(%rsp) | |||||
| movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
| movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
| movq %rsi,TF_RSI(%rsp) | movq %rsi,TF_RSI(%rsp) | ||||
| movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
| movq %rcx,TF_RCX(%rsp) | movq %rcx,TF_RCX(%rsp) | ||||
| movq %r8,TF_R8(%rsp) | movq %r8,TF_R8(%rsp) | ||||
| movq %r9,TF_R9(%rsp) | movq %r9,TF_R9(%rsp) | ||||
| movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
| Show All 10 Lines | 1: | ||||
| movq $0,TF_ADDR(%rsp) | movq $0,TF_ADDR(%rsp) | ||||
| FAKE_MCOUNT(TF_RIP(%rsp)) | FAKE_MCOUNT(TF_RIP(%rsp)) | ||||
| jmp calltrap | jmp calltrap | ||||
| ALIGN_TEXT | ALIGN_TEXT | ||||
| .globl ds_load_fault | .globl ds_load_fault | ||||
| ds_load_fault: | ds_load_fault: | ||||
| movl $T_PROTFLT,TF_TRAPNO(%rsp) | movl $T_PROTFLT,TF_TRAPNO(%rsp) | ||||
| testl $PSL_I,TF_RFLAGS(%rsp) | testb $SEL_RPL_MASK,TF_CS(%rsp) | ||||
| jz 1f | jz 1f | ||||
| sti | sti | ||||
| 1: | 1: | ||||
| movq %rsp,%rdi | movq %rsp,%rdi | ||||
| call trap | call trap | ||||
| movw $KUDSEL,TF_DS(%rsp) | movw $KUDSEL,TF_DS(%rsp) | ||||
| jmp doreti | jmp doreti | ||||
| ▲ Show 20 Lines • Show All 73 Lines • Show Last 20 Lines | |||||
If I'm reading the code correctly, PCPU(KCR3) is only set when pti != 0; therefore, it seems like these lines should be conditioned on pti != 0. (But, I could be missing something.)