Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/exception.S
/*- | /*- | ||||
* Copyright (c) 1989, 1990 William F. Jolitz. | * Copyright (c) 1989, 1990 William F. Jolitz. | ||||
* Copyright (c) 1990 The Regents of the University of California. | * Copyright (c) 1990 The Regents of the University of California. | ||||
* Copyright (c) 2007 The FreeBSD Foundation | * Copyright (c) 2007-2018 The FreeBSD Foundation | ||||
* All rights reserved. | * All rights reserved. | ||||
* | * | ||||
* Portions of this software were developed by A. Joseph Koshy under | * Portions of this software were developed by A. Joseph Koshy under | ||||
* sponsorship from the FreeBSD Foundation and Google, Inc. | * sponsorship from the FreeBSD Foundation and Google, Inc. | ||||
* | * | ||||
* Portions of this software were developed by | |||||
* Konstantin Belousov <kib@FreeBSD.org> under sponsorship from | |||||
* the FreeBSD Foundation. | |||||
* | |||||
* Redistribution and use in source and binary forms, with or without | * Redistribution and use in source and binary forms, with or without | ||||
* modification, are permitted provided that the following conditions | * modification, are permitted provided that the following conditions | ||||
* are met: | * are met: | ||||
* 1. Redistributions of source code must retain the above copyright | * 1. Redistributions of source code must retain the above copyright | ||||
* notice, this list of conditions and the following disclaimer. | * notice, this list of conditions and the following disclaimer. | ||||
* 2. Redistributions in binary form must reproduce the above copyright | * 2. Redistributions in binary form must reproduce the above copyright | ||||
* notice, this list of conditions and the following disclaimer in the | * notice, this list of conditions and the following disclaimer in the | ||||
* documentation and/or other materials provided with the distribution. | * documentation and/or other materials provided with the distribution. | ||||
Show All 15 Lines | |||||
* | * | ||||
* $FreeBSD$ | * $FreeBSD$ | ||||
*/ | */ | ||||
#include "opt_atpic.h" | #include "opt_atpic.h" | ||||
#include "opt_compat.h" | #include "opt_compat.h" | ||||
#include "opt_hwpmc_hooks.h" | #include "opt_hwpmc_hooks.h" | ||||
#include "assym.s" | |||||
#include <machine/asmacros.h> | #include <machine/asmacros.h> | ||||
#include <machine/psl.h> | #include <machine/psl.h> | ||||
#include <machine/trap.h> | #include <machine/trap.h> | ||||
#include <machine/specialreg.h> | #include <machine/specialreg.h> | ||||
#include "assym.s" | |||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
.bss | .bss | ||||
.globl dtrace_invop_jump_addr | .globl dtrace_invop_jump_addr | ||||
.align 8 | .align 8 | ||||
.type dtrace_invop_jump_addr,@object | .type dtrace_invop_jump_addr,@object | ||||
.size dtrace_invop_jump_addr,8 | .size dtrace_invop_jump_addr,8 | ||||
dtrace_invop_jump_addr: | dtrace_invop_jump_addr: | ||||
.zero 8 | .zero 8 | ||||
Show All 39 Lines | |||||
* segment registers other then %cs, which makes them mostly unused in long | * segment registers other then %cs, which makes them mostly unused in long | ||||
* mode, and kernel does not reference %fs, leave them alone. The segment | * mode, and kernel does not reference %fs, leave them alone. The segment | ||||
* registers are reloaded on return to the usermode. | * registers are reloaded on return to the usermode. | ||||
*/ | */ | ||||
MCOUNT_LABEL(user) | MCOUNT_LABEL(user) | ||||
MCOUNT_LABEL(btrap) | MCOUNT_LABEL(btrap) | ||||
/* Traps that we leave interrupts disabled for.. */ | /* Traps that we leave interrupts disabled for. */ | ||||
#define TRAP_NOEN(a) \ | .macro TRAP_NOEN l, trapno | ||||
subq $TF_RIP,%rsp; \ | PTI_ENTRY \l,X\l | ||||
movl $(a),TF_TRAPNO(%rsp) ; \ | .globl X\l | ||||
movq $0,TF_ADDR(%rsp) ; \ | .type X\l,@function | ||||
movq $0,TF_ERR(%rsp) ; \ | X\l: subq $TF_RIP,%rsp | ||||
movl $\trapno,TF_TRAPNO(%rsp) | |||||
movq $0,TF_ADDR(%rsp) | |||||
movq $0,TF_ERR(%rsp) | |||||
jmp alltraps_noen | jmp alltraps_noen | ||||
IDTVEC(dbg) | .endm | ||||
TRAP_NOEN(T_TRCTRAP) | |||||
IDTVEC(bpt) | TRAP_NOEN dbg, T_TRCTRAP | ||||
TRAP_NOEN(T_BPTFLT) | TRAP_NOEN bpt, T_BPTFLT | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
IDTVEC(dtrace_ret) | TRAP_NOEN dtrace_ret, T_DTRACE_RET | ||||
TRAP_NOEN(T_DTRACE_RET) | |||||
#endif | #endif | ||||
/* Regular traps; The cpu does not supply tf_err for these. */ | /* Regular traps; The cpu does not supply tf_err for these. */ | ||||
#define TRAP(a) \ | .macro TRAP l, trapno | ||||
subq $TF_RIP,%rsp; \ | PTI_ENTRY \l,X\l | ||||
movl $(a),TF_TRAPNO(%rsp) ; \ | .globl X\l | ||||
movq $0,TF_ADDR(%rsp) ; \ | .type X\l,@function | ||||
movq $0,TF_ERR(%rsp) ; \ | X\l: | ||||
subq $TF_RIP,%rsp | |||||
movl $\trapno,TF_TRAPNO(%rsp) | |||||
movq $0,TF_ADDR(%rsp) | |||||
movq $0,TF_ERR(%rsp) | |||||
jmp alltraps | jmp alltraps | ||||
IDTVEC(div) | .endm | ||||
TRAP(T_DIVIDE) | |||||
IDTVEC(ofl) | |||||
TRAP(T_OFLOW) | |||||
IDTVEC(bnd) | |||||
TRAP(T_BOUND) | |||||
IDTVEC(ill) | |||||
TRAP(T_PRIVINFLT) | |||||
IDTVEC(dna) | |||||
TRAP(T_DNA) | |||||
IDTVEC(fpusegm) | |||||
TRAP(T_FPOPFLT) | |||||
IDTVEC(mchk) | |||||
TRAP(T_MCHK) | |||||
IDTVEC(rsvd) | |||||
TRAP(T_RESERVED) | |||||
IDTVEC(fpu) | |||||
TRAP(T_ARITHTRAP) | |||||
IDTVEC(xmm) | |||||
TRAP(T_XMMFLT) | |||||
/* This group of traps have tf_err already pushed by the cpu */ | TRAP div, T_DIVIDE | ||||
#define TRAP_ERR(a) \ | TRAP ofl, T_OFLOW | ||||
subq $TF_ERR,%rsp; \ | TRAP bnd, T_BOUND | ||||
movl $(a),TF_TRAPNO(%rsp) ; \ | TRAP ill, T_PRIVINFLT | ||||
movq $0,TF_ADDR(%rsp) ; \ | TRAP dna, T_DNA | ||||
jmp alltraps | TRAP fpusegm, T_FPOPFLT | ||||
IDTVEC(tss) | TRAP mchk, T_MCHK | ||||
TRAP_ERR(T_TSSFLT) | TRAP rsvd, T_RESERVED | ||||
IDTVEC(missing) | TRAP fpu, T_ARITHTRAP | ||||
TRAP xmm, T_XMMFLT | |||||
/* This group of traps have tf_err already pushed by the cpu. */ | |||||
.macro TRAP_ERR l, trapno | |||||
PTI_ENTRY \l,X\l,has_err=1 | |||||
.globl X\l | |||||
.type X\l,@function | |||||
X\l: | |||||
subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
movl $T_SEGNPFLT,TF_TRAPNO(%rsp) | movl $\trapno,TF_TRAPNO(%rsp) | ||||
jmp prot_addrf | movq $0,TF_ADDR(%rsp) | ||||
IDTVEC(stk) | jmp alltraps | ||||
subq $TF_ERR,%rsp | .endm | ||||
movl $T_STKFLT,TF_TRAPNO(%rsp) | |||||
jmp prot_addrf | |||||
IDTVEC(align) | |||||
TRAP_ERR(T_ALIGNFLT) | |||||
TRAP_ERR tss, T_TSSFLT | |||||
TRAP_ERR align, T_ALIGNFLT | |||||
/* | /* | ||||
* alltraps entry point. Use swapgs if this is the first time in the | * alltraps entry point. Use swapgs if this is the first time in the | ||||
* kernel from userland. Reenable interrupts if they were enabled | * kernel from userland. Reenable interrupts if they were enabled | ||||
* before the trap. This approximates SDT_SYS386TGT on the i386 port. | * before the trap. This approximates SDT_SYS386TGT on the i386 port. | ||||
*/ | */ | ||||
SUPERALIGN_TEXT | SUPERALIGN_TEXT | ||||
.globl alltraps | .globl alltraps | ||||
.type alltraps,@function | .type alltraps,@function | ||||
alltraps: | alltraps: | ||||
movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
jz alltraps_testi /* already running with kernel GS.base */ | jz alltraps_segs /* already running with kernel GS.base */ | ||||
swapgs | swapgs | ||||
movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
movw %fs,TF_FS(%rsp) | alltraps_segs: | ||||
movw %gs,TF_GS(%rsp) | SAVE_SEGS | ||||
movw %es,TF_ES(%rsp) | |||||
movw %ds,TF_DS(%rsp) | |||||
alltraps_testi: | |||||
testl $PSL_I,TF_RFLAGS(%rsp) | testl $PSL_I,TF_RFLAGS(%rsp) | ||||
jz alltraps_pushregs_no_rdi | jz alltraps_pushregs_no_rdi | ||||
sti | sti | ||||
alltraps_pushregs_no_rdi: | alltraps_pushregs_no_rdi: | ||||
movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
alltraps_pushregs_no_rax: | alltraps_pushregs_no_rax: | ||||
movq %rsi,TF_RSI(%rsp) | movq %rsi,TF_RSI(%rsp) | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | calltrap: | ||||
* SDT_SYS386IGT on the i386 port. | * SDT_SYS386IGT on the i386 port. | ||||
*/ | */ | ||||
SUPERALIGN_TEXT | SUPERALIGN_TEXT | ||||
.globl alltraps_noen | .globl alltraps_noen | ||||
.type alltraps_noen,@function | .type alltraps_noen,@function | ||||
alltraps_noen: | alltraps_noen: | ||||
movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
jz 1f /* already running with kernel GS.base */ | jz alltraps_noen_segs /* already running with kernel GS.base */ | ||||
swapgs | swapgs | ||||
movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
1: movw %fs,TF_FS(%rsp) | alltraps_noen_segs: | ||||
movw %gs,TF_GS(%rsp) | SAVE_SEGS | ||||
movw %es,TF_ES(%rsp) | |||||
movw %ds,TF_DS(%rsp) | |||||
jmp alltraps_pushregs_no_rdi | jmp alltraps_pushregs_no_rdi | ||||
IDTVEC(dblfault) | IDTVEC(dblfault) | ||||
subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) | movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) | ||||
movq $0,TF_ADDR(%rsp) | movq $0,TF_ADDR(%rsp) | ||||
movq $0,TF_ERR(%rsp) | movq $0,TF_ERR(%rsp) | ||||
movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
movq %rsi,TF_RSI(%rsp) | movq %rsi,TF_RSI(%rsp) | ||||
movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
movq %rcx,TF_RCX(%rsp) | movq %rcx,TF_RCX(%rsp) | ||||
movq %r8,TF_R8(%rsp) | movq %r8,TF_R8(%rsp) | ||||
movq %r9,TF_R9(%rsp) | movq %r9,TF_R9(%rsp) | ||||
movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
movq %rbx,TF_RBX(%rsp) | movq %rbx,TF_RBX(%rsp) | ||||
movq %rbp,TF_RBP(%rsp) | movq %rbp,TF_RBP(%rsp) | ||||
movq %r10,TF_R10(%rsp) | movq %r10,TF_R10(%rsp) | ||||
movq %r11,TF_R11(%rsp) | movq %r11,TF_R11(%rsp) | ||||
movq %r12,TF_R12(%rsp) | movq %r12,TF_R12(%rsp) | ||||
movq %r13,TF_R13(%rsp) | movq %r13,TF_R13(%rsp) | ||||
movq %r14,TF_R14(%rsp) | movq %r14,TF_R14(%rsp) | ||||
movq %r15,TF_R15(%rsp) | movq %r15,TF_R15(%rsp) | ||||
movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
movw %gs,TF_GS(%rsp) | |||||
movw %es,TF_ES(%rsp) | |||||
movw %ds,TF_DS(%rsp) | |||||
movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
cld | cld | ||||
testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
jz 1f /* already running with kernel GS.base */ | jz 1f /* already running with kernel GS.base */ | ||||
swapgs | swapgs | ||||
1: | 1: | ||||
movq %rsp,%rdi | movq PCPU(KCR3),%rax | ||||
jtl: If I'm reading the code correctly, PCPU(KCR3) is only set when pti != 0; therefore, it seems… | |||||
Not Done Inline ActionsYeah, I already fixed it since I tested the non-PTI case as well. Thank you for noting. kib: Yeah, I already fixed it since I tested the non-PTI case as well. Thank you for noting. | |||||
cmpq $~0,%rax | |||||
je 2f | |||||
movq %rax,%cr3 | |||||
2: movq %rsp,%rdi | |||||
call dblfault_handler | call dblfault_handler | ||||
2: | 3: hlt | ||||
hlt | jmp 3b | ||||
jmp 2b | |||||
PTI_ENTRY page, Xpage, has_err=1 | |||||
IDTVEC(page) | IDTVEC(page) | ||||
subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
movl $T_PAGEFLT,TF_TRAPNO(%rsp) | |||||
movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | ||||
testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
jz 1f /* already running with kernel GS.base */ | jz page_cr2 /* already running with kernel GS.base */ | ||||
swapgs | swapgs | ||||
movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) | ||||
1: movq %cr2,%rdi /* preserve %cr2 before .. */ | page_cr2: | ||||
movq %cr2,%rdi /* preserve %cr2 before .. */ | |||||
movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ | movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ | ||||
movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
movw %gs,TF_GS(%rsp) | movl $T_PAGEFLT,TF_TRAPNO(%rsp) | ||||
movw %es,TF_ES(%rsp) | |||||
movw %ds,TF_DS(%rsp) | |||||
testl $PSL_I,TF_RFLAGS(%rsp) | testl $PSL_I,TF_RFLAGS(%rsp) | ||||
jz alltraps_pushregs_no_rdi | jz alltraps_pushregs_no_rdi | ||||
sti | sti | ||||
jmp alltraps_pushregs_no_rdi | jmp alltraps_pushregs_no_rdi | ||||
/* | /* | ||||
* We have to special-case this one. If we get a trap in doreti() at | * We have to special-case this one. If we get a trap in doreti() at | ||||
* the iretq stage, we'll reenter with the wrong gs state. We'll have | * the iretq stage, we'll reenter with the wrong gs state. We'll have | ||||
* to do a special the swapgs in this case even coming from the kernel. | * to do a special the swapgs in this case even coming from the kernel. | ||||
* XXX linux has a trap handler for their equivalent of load_gs(). | * XXX linux has a trap handler for their equivalent of load_gs(). | ||||
* | |||||
* On the stack, we have the hardware interrupt frame to return | |||||
* to usermode (faulted) and another frame with error code, for | |||||
* fault. For PTI, copy both frames to the main thread stack. | |||||
*/ | */ | ||||
IDTVEC(prot) | .macro PROTF_ENTRY name,trapno | ||||
\name\()_pti_doreti: | |||||
pushq %rax | |||||
pushq %rdx | |||||
swapgs | |||||
movq PCPU(KCR3),%rax | |||||
movq %rax,%cr3 | |||||
movq PCPU(RSP0),%rax | |||||
subq $2*PTI_SIZE-3*8,%rax | |||||
MOVE_STACKS (PTI_SIZE / 4 - 3) | |||||
movq %rax,%rsp | |||||
popq %rdx | |||||
popq %rax | |||||
swapgs | |||||
jmp X\name | |||||
IDTVEC(\name\()_pti) | |||||
cmpq $doreti_iret,PTI_RIP-2*8(%rsp) | |||||
je \name\()_pti_doreti | |||||
testb $SEL_RPL_MASK,PTI_CS-2*8(%rsp) /* %rax, %rdx not yet pushed */ | |||||
jz X\name | |||||
PTI_UENTRY has_err=1 | |||||
swapgs | |||||
IDTVEC(\name) | |||||
subq $TF_ERR,%rsp | subq $TF_ERR,%rsp | ||||
movl $T_PROTFLT,TF_TRAPNO(%rsp) | movl $\trapno,TF_TRAPNO(%rsp) | ||||
jmp prot_addrf | |||||
.endm | |||||
PROTF_ENTRY missing, T_SEGNPFLT | |||||
PROTF_ENTRY stk, T_STKFLT | |||||
PROTF_ENTRY prot, T_PROTFLT | |||||
prot_addrf: | prot_addrf: | ||||
movq $0,TF_ADDR(%rsp) | movq $0,TF_ADDR(%rsp) | ||||
movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | movq %rdi,TF_RDI(%rsp) /* free up a GP register */ | ||||
movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
movw %fs,TF_FS(%rsp) | movw %fs,TF_FS(%rsp) | ||||
movw %gs,TF_GS(%rsp) | movw %gs,TF_GS(%rsp) | ||||
leaq doreti_iret(%rip),%rdi | leaq doreti_iret(%rip),%rdi | ||||
Show All 35 Lines | |||||
* Fast syscall entry point. We enter here with just our new %cs/%ss set, | * Fast syscall entry point. We enter here with just our new %cs/%ss set, | ||||
* and the new privilige level. We are still running on the old user stack | * and the new privilige level. We are still running on the old user stack | ||||
* pointer. We have to juggle a few things around to find our stack etc. | * pointer. We have to juggle a few things around to find our stack etc. | ||||
* swapgs gives us access to our PCPU space only. | * swapgs gives us access to our PCPU space only. | ||||
* | * | ||||
* We do not support invoking this from a custom segment registers, | * We do not support invoking this from a custom segment registers, | ||||
* esp. %cs, %ss, %fs, %gs, e.g. using entries from an LDT. | * esp. %cs, %ss, %fs, %gs, e.g. using entries from an LDT. | ||||
*/ | */ | ||||
SUPERALIGN_TEXT | |||||
IDTVEC(fast_syscall_pti) | |||||
swapgs | |||||
movq %rax,PCPU(SCRATCH_RAX) | |||||
movq PCPU(KCR3),%rax | |||||
movq %rax,%cr3 | |||||
jmp fast_syscall_common | |||||
SUPERALIGN_TEXT | |||||
IDTVEC(fast_syscall) | IDTVEC(fast_syscall) | ||||
swapgs | swapgs | ||||
movq %rax,PCPU(SCRATCH_RAX) | |||||
fast_syscall_common: | |||||
movq %rsp,PCPU(SCRATCH_RSP) | movq %rsp,PCPU(SCRATCH_RSP) | ||||
movq PCPU(RSP0),%rsp | movq PCPU(RSP0),%rsp | ||||
/* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ | /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ | ||||
subq $TF_SIZE,%rsp | subq $TF_SIZE,%rsp | ||||
/* defer TF_RSP till we have a spare register */ | /* defer TF_RSP till we have a spare register */ | ||||
movq %r11,TF_RFLAGS(%rsp) | movq %r11,TF_RFLAGS(%rsp) | ||||
movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ | movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ | ||||
movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ | movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ | ||||
movq %r11,TF_RSP(%rsp) /* user stack pointer */ | movq %r11,TF_RSP(%rsp) /* user stack pointer */ | ||||
movw %fs,TF_FS(%rsp) | movq PCPU(SCRATCH_RAX),%rax | ||||
movw %gs,TF_GS(%rsp) | movq %rax,TF_RAX(%rsp) /* syscall number */ | ||||
movw %es,TF_ES(%rsp) | SAVE_SEGS | ||||
movw %ds,TF_DS(%rsp) | |||||
movq PCPU(CURPCB),%r11 | movq PCPU(CURPCB),%r11 | ||||
andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) | andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) | ||||
sti | sti | ||||
movq $KUDSEL,TF_SS(%rsp) | movq $KUDSEL,TF_SS(%rsp) | ||||
movq $KUCSEL,TF_CS(%rsp) | movq $KUCSEL,TF_CS(%rsp) | ||||
movq $2,TF_ERR(%rsp) | movq $2,TF_ERR(%rsp) | ||||
movq %rdi,TF_RDI(%rsp) /* arg 1 */ | movq %rdi,TF_RDI(%rsp) /* arg 1 */ | ||||
movq %rsi,TF_RSI(%rsp) /* arg 2 */ | movq %rsi,TF_RSI(%rsp) /* arg 2 */ | ||||
movq %rdx,TF_RDX(%rsp) /* arg 3 */ | movq %rdx,TF_RDX(%rsp) /* arg 3 */ | ||||
movq %r10,TF_RCX(%rsp) /* arg 4 */ | movq %r10,TF_RCX(%rsp) /* arg 4 */ | ||||
movq %r8,TF_R8(%rsp) /* arg 5 */ | movq %r8,TF_R8(%rsp) /* arg 5 */ | ||||
movq %r9,TF_R9(%rsp) /* arg 6 */ | movq %r9,TF_R9(%rsp) /* arg 6 */ | ||||
movq %rax,TF_RAX(%rsp) /* syscall number */ | |||||
movq %rbx,TF_RBX(%rsp) /* C preserved */ | movq %rbx,TF_RBX(%rsp) /* C preserved */ | ||||
movq %rbp,TF_RBP(%rsp) /* C preserved */ | movq %rbp,TF_RBP(%rsp) /* C preserved */ | ||||
movq %r12,TF_R12(%rsp) /* C preserved */ | movq %r12,TF_R12(%rsp) /* C preserved */ | ||||
movq %r13,TF_R13(%rsp) /* C preserved */ | movq %r13,TF_R13(%rsp) /* C preserved */ | ||||
movq %r14,TF_R14(%rsp) /* C preserved */ | movq %r14,TF_R14(%rsp) /* C preserved */ | ||||
movq %r15,TF_R15(%rsp) /* C preserved */ | movq %r15,TF_R15(%rsp) /* C preserved */ | ||||
movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
FAKE_MCOUNT(TF_RIP(%rsp)) | FAKE_MCOUNT(TF_RIP(%rsp)) | ||||
movq PCPU(CURTHREAD),%rdi | movq PCPU(CURTHREAD),%rdi | ||||
movq %rsp,TD_FRAME(%rdi) | movq %rsp,TD_FRAME(%rdi) | ||||
movl TF_RFLAGS(%rsp),%esi | movl TF_RFLAGS(%rsp),%esi | ||||
andl $PSL_T,%esi | andl $PSL_T,%esi | ||||
call amd64_syscall | call amd64_syscall | ||||
1: movq PCPU(CURPCB),%rax | 1: movq PCPU(CURPCB),%rax | ||||
/* Disable interrupts before testing PCB_FULL_IRET. */ | /* Disable interrupts before testing PCB_FULL_IRET. */ | ||||
cli | cli | ||||
testl $PCB_FULL_IRET,PCB_FLAGS(%rax) | testl $PCB_FULL_IRET,PCB_FLAGS(%rax) | ||||
jnz 3f | jnz 4f | ||||
/* Check for and handle AST's on return to userland. */ | /* Check for and handle AST's on return to userland. */ | ||||
movq PCPU(CURTHREAD),%rax | movq PCPU(CURTHREAD),%rax | ||||
testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) | testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) | ||||
jne 2f | jne 3f | ||||
/* Restore preserved registers. */ | /* Restore preserved registers. */ | ||||
MEXITCOUNT | MEXITCOUNT | ||||
movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ | movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ | ||||
movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ | movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ | ||||
movq TF_RDX(%rsp),%rdx /* return value 2 */ | movq TF_RDX(%rsp),%rdx /* return value 2 */ | ||||
movq TF_RAX(%rsp),%rax /* return value 1 */ | movq TF_RAX(%rsp),%rax /* return value 1 */ | ||||
movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ | movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ | ||||
movq TF_RIP(%rsp),%rcx /* original %rip */ | movq TF_RIP(%rsp),%rcx /* original %rip */ | ||||
movq TF_RSP(%rsp),%rsp /* user stack pointer */ | movq TF_RSP(%rsp),%rsp /* user stack pointer */ | ||||
swapgs | cmpb $0,pti | ||||
je 2f | |||||
movq PCPU(UCR3),%r9 | |||||
movq %r9,%cr3 | |||||
xorl %r9d,%r9d | |||||
2: swapgs | |||||
sysretq | sysretq | ||||
2: /* AST scheduled. */ | 3: /* AST scheduled. */ | ||||
sti | sti | ||||
movq %rsp,%rdi | movq %rsp,%rdi | ||||
call ast | call ast | ||||
jmp 1b | jmp 1b | ||||
3: /* Requested full context restore, use doreti for that. */ | 4: /* Requested full context restore, use doreti for that. */ | ||||
MEXITCOUNT | MEXITCOUNT | ||||
jmp doreti | jmp doreti | ||||
/* | /* | ||||
* Here for CYA insurance, in case a "syscall" instruction gets | * Here for CYA insurance, in case a "syscall" instruction gets | ||||
* issued from 32 bit compatibility mode. MSR_CSTAR has to point | * issued from 32 bit compatibility mode. MSR_CSTAR has to point | ||||
* to *something* if EFER_SCE is enabled. | * to *something* if EFER_SCE is enabled. | ||||
*/ | */ | ||||
Show All 39 Lines | IDTVEC(nmi) | ||||
movq %rbx,TF_RBX(%rsp) | movq %rbx,TF_RBX(%rsp) | ||||
movq %rbp,TF_RBP(%rsp) | movq %rbp,TF_RBP(%rsp) | ||||
movq %r10,TF_R10(%rsp) | movq %r10,TF_R10(%rsp) | ||||
movq %r11,TF_R11(%rsp) | movq %r11,TF_R11(%rsp) | ||||
movq %r12,TF_R12(%rsp) | movq %r12,TF_R12(%rsp) | ||||
movq %r13,TF_R13(%rsp) | movq %r13,TF_R13(%rsp) | ||||
movq %r14,TF_R14(%rsp) | movq %r14,TF_R14(%rsp) | ||||
movq %r15,TF_R15(%rsp) | movq %r15,TF_R15(%rsp) | ||||
movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
movw %gs,TF_GS(%rsp) | |||||
movw %es,TF_ES(%rsp) | |||||
movw %ds,TF_DS(%rsp) | |||||
movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
cld | cld | ||||
xorl %ebx,%ebx | xorl %ebx,%ebx | ||||
testb $SEL_RPL_MASK,TF_CS(%rsp) | testb $SEL_RPL_MASK,TF_CS(%rsp) | ||||
jnz nmi_fromuserspace | jnz nmi_fromuserspace | ||||
/* | /* | ||||
* We've interrupted the kernel. Preserve GS.base in %r12. | * We've interrupted the kernel. Preserve GS.base in %r12 | ||||
* and %cr3 in %r13. | |||||
*/ | */ | ||||
movl $MSR_GSBASE,%ecx | movl $MSR_GSBASE,%ecx | ||||
rdmsr | rdmsr | ||||
movq %rax,%r12 | movq %rax,%r12 | ||||
shlq $32,%rdx | shlq $32,%rdx | ||||
orq %rdx,%r12 | orq %rdx,%r12 | ||||
/* Retrieve and load the canonical value for GS.base. */ | /* Retrieve and load the canonical value for GS.base. */ | ||||
movq TF_SIZE(%rsp),%rdx | movq TF_SIZE(%rsp),%rdx | ||||
movl %edx,%eax | movl %edx,%eax | ||||
shrq $32,%rdx | shrq $32,%rdx | ||||
wrmsr | wrmsr | ||||
movq %cr3,%r13 | |||||
movq PCPU(KCR3),%rax | |||||
cmpq $~0,%rax | |||||
Not Done Inline ActionsIf I'm reading the code correctly, PCPU(KCR3) is only set when pti != 0; therefore, it seems like these lines should be conditioned on pti != 0. (But, I could be missing something.) jtl: If I'm reading the code correctly, PCPU(KCR3) is only set when pti != 0; therefore, it seems… | |||||
Not Done Inline ActionsIt looks to me like PCPU(KCR3) is always set, it just only changes in the pti!=0 case. This seems to avoid a compare & branch. kithrup_mac.com: It looks to me like PCPU(KCR3) is always set, it just only changes in the pti!=0 case. This… | |||||
Not Done Inline ActionsIt is not for !pti case. For now wrapping accesses with tests is the right thing to do. kib: It is not for !pti case. For now wrapping accesses with tests is the right thing to do. | |||||
je nmi_calltrap | |||||
movq %rax,%cr3 | |||||
jmp nmi_calltrap | jmp nmi_calltrap | ||||
nmi_fromuserspace: | nmi_fromuserspace: | ||||
incl %ebx | incl %ebx | ||||
swapgs | swapgs | ||||
testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) | movq %cr3,%r13 | ||||
Not Done Inline ActionsDitto. jtl: Ditto. | |||||
Not Done Inline ActionsThanks for fixing these (and your overall work on the bug)! FYI, I think this one (nmi_fromuserspace) still needs the change to only set cr3 when pti != 0. jtl: Thanks for fixing these (and your overall work on the bug)! FYI, I think this one… | |||||
jz 2f | movq PCPU(KCR3),%rax | ||||
cmpq $~0,%rax | |||||
je 1f | |||||
movq %rax,%cr3 | |||||
movq PCPU(CURPCB),%rdi | movq PCPU(CURPCB),%rdi | ||||
testq %rdi,%rdi | testq %rdi,%rdi | ||||
jz 2f | jz 3f | ||||
orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) | |||||
1: testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) | |||||
jz 3f | |||||
cmpw $KUF32SEL,TF_FS(%rsp) | cmpw $KUF32SEL,TF_FS(%rsp) | ||||
jne 1f | jne 2f | ||||
rdfsbase %rax | rdfsbase %rax | ||||
movq %rax,PCB_FSBASE(%rdi) | movq %rax,PCB_FSBASE(%rdi) | ||||
1: cmpw $KUG32SEL,TF_GS(%rsp) | 2: cmpw $KUG32SEL,TF_GS(%rsp) | ||||
jne 2f | jne 3f | ||||
movl $MSR_KGSBASE,%ecx | movl $MSR_KGSBASE,%ecx | ||||
rdmsr | rdmsr | ||||
shlq $32,%rdx | shlq $32,%rdx | ||||
orq %rdx,%rax | orq %rdx,%rax | ||||
movq %rax,PCB_GSBASE(%rdi) | movq %rax,PCB_GSBASE(%rdi) | ||||
2: | 3: | ||||
/* Note: this label is also used by ddb and gdb: */ | /* Note: this label is also used by ddb and gdb: */ | ||||
nmi_calltrap: | nmi_calltrap: | ||||
FAKE_MCOUNT(TF_RIP(%rsp)) | FAKE_MCOUNT(TF_RIP(%rsp)) | ||||
movq %rsp,%rdi | movq %rsp,%rdi | ||||
call trap | call trap | ||||
MEXITCOUNT | MEXITCOUNT | ||||
#ifdef HWPMC_HOOKS | #ifdef HWPMC_HOOKS | ||||
/* | /* | ||||
* Capture a userspace callchain if needed. | * Capture a userspace callchain if needed. | ||||
* | * | ||||
* - Check if the current trap was from user mode. | * - Check if the current trap was from user mode. | ||||
* - Check if the current thread is valid. | * - Check if the current thread is valid. | ||||
* - Check if the thread requires a user call chain to be | * - Check if the thread requires a user call chain to be | ||||
* captured. | * captured. | ||||
* | * | ||||
* We are still in NMI mode at this point. | * We are still in NMI mode at this point. | ||||
*/ | */ | ||||
testl %ebx,%ebx | testl %ebx,%ebx | ||||
jz nocallchain /* not from userspace */ | jz nocallchain /* not from userspace */ | ||||
movq PCPU(CURTHREAD),%rax | movq PCPU(CURTHREAD),%rax | ||||
orq %rax,%rax /* curthread present? */ | orq %rax,%rax /* curthread present? */ | ||||
jz nocallchain | jz nocallchain | ||||
testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */ | |||||
jz nocallchain | |||||
/* | /* | ||||
* A user callchain is to be captured, so: | * Move execution to the regular kernel stack, because we | ||||
* - Move execution to the regular kernel stack, to allow for | * committed to return through doreti. | ||||
* nested NMI interrupts. | |||||
* - Take the processor out of "NMI" mode by faking an "iret". | |||||
* - Enable interrupts, so that copyin() can work. | |||||
*/ | */ | ||||
movq %rsp,%rsi /* source stack pointer */ | movq %rsp,%rsi /* source stack pointer */ | ||||
movq $TF_SIZE,%rcx | movq $TF_SIZE,%rcx | ||||
movq PCPU(RSP0),%rdx | movq PCPU(RSP0),%rdx | ||||
subq %rcx,%rdx | subq %rcx,%rdx | ||||
movq %rdx,%rdi /* destination stack pointer */ | movq %rdx,%rdi /* destination stack pointer */ | ||||
shrq $3,%rcx /* trap frame size in long words */ | shrq $3,%rcx /* trap frame size in long words */ | ||||
cld | cld | ||||
rep | rep | ||||
movsq /* copy trapframe */ | movsq /* copy trapframe */ | ||||
movq %rdx,%rsp /* we are on the regular kstack */ | |||||
testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */ | |||||
jz nocallchain | |||||
/* | |||||
* A user callchain is to be captured, so: | |||||
* - Take the processor out of "NMI" mode by faking an "iret", | |||||
* to allow for nested NMI interrupts. | |||||
* - Enable interrupts, so that copyin() can work. | |||||
*/ | |||||
movl %ss,%eax | movl %ss,%eax | ||||
pushq %rax /* tf_ss */ | pushq %rax /* tf_ss */ | ||||
pushq %rdx /* tf_rsp (on kernel stack) */ | pushq %rdx /* tf_rsp (on kernel stack) */ | ||||
pushfq /* tf_rflags */ | pushfq /* tf_rflags */ | ||||
movl %cs,%eax | movl %cs,%eax | ||||
pushq %rax /* tf_cs */ | pushq %rax /* tf_cs */ | ||||
pushq $outofnmi /* tf_rip */ | pushq $outofnmi /* tf_rip */ | ||||
iretq | iretq | ||||
Show All 24 Lines | nmi_kernelexit: | ||||
/* | /* | ||||
* Put back the preserved MSR_GSBASE value. | * Put back the preserved MSR_GSBASE value. | ||||
*/ | */ | ||||
movl $MSR_GSBASE,%ecx | movl $MSR_GSBASE,%ecx | ||||
movq %r12,%rdx | movq %r12,%rdx | ||||
movl %edx,%eax | movl %edx,%eax | ||||
shrq $32,%rdx | shrq $32,%rdx | ||||
wrmsr | wrmsr | ||||
movq %r13,%cr3 | |||||
nmi_restoreregs: | nmi_restoreregs: | ||||
movq TF_RDI(%rsp),%rdi | RESTORE_REGS | ||||
movq TF_RSI(%rsp),%rsi | |||||
movq TF_RDX(%rsp),%rdx | |||||
movq TF_RCX(%rsp),%rcx | |||||
movq TF_R8(%rsp),%r8 | |||||
movq TF_R9(%rsp),%r9 | |||||
movq TF_RAX(%rsp),%rax | |||||
movq TF_RBX(%rsp),%rbx | |||||
movq TF_RBP(%rsp),%rbp | |||||
movq TF_R10(%rsp),%r10 | |||||
movq TF_R11(%rsp),%r11 | |||||
movq TF_R12(%rsp),%r12 | |||||
movq TF_R13(%rsp),%r13 | |||||
movq TF_R14(%rsp),%r14 | |||||
movq TF_R15(%rsp),%r15 | |||||
addq $TF_RIP,%rsp | addq $TF_RIP,%rsp | ||||
jmp doreti_iret | jmp doreti_iret | ||||
ENTRY(fork_trampoline) | ENTRY(fork_trampoline) | ||||
movq %r12,%rdi /* function */ | movq %r12,%rdi /* function */ | ||||
movq %rbx,%rsi /* arg1 */ | movq %rbx,%rsi /* arg1 */ | ||||
movq %rsp,%rdx /* trapframe pointer */ | movq %rsp,%rdx /* trapframe pointer */ | ||||
call fork_exit | call fork_exit | ||||
▲ Show 20 Lines • Show All 151 Lines • ▼ Show 20 Lines | ld_gsbase: | ||||
wrmsr /* May trap if non-canonical, but only for TLS. */ | wrmsr /* May trap if non-canonical, but only for TLS. */ | ||||
.globl ld_es | .globl ld_es | ||||
ld_es: | ld_es: | ||||
movw TF_ES(%rsp),%es | movw TF_ES(%rsp),%es | ||||
.globl ld_ds | .globl ld_ds | ||||
ld_ds: | ld_ds: | ||||
movw TF_DS(%rsp),%ds | movw TF_DS(%rsp),%ds | ||||
ld_regs: | ld_regs: | ||||
movq TF_RDI(%rsp),%rdi | RESTORE_REGS | ||||
movq TF_RSI(%rsp),%rsi | |||||
movq TF_RDX(%rsp),%rdx | |||||
movq TF_RCX(%rsp),%rcx | |||||
movq TF_R8(%rsp),%r8 | |||||
movq TF_R9(%rsp),%r9 | |||||
movq TF_RAX(%rsp),%rax | |||||
movq TF_RBX(%rsp),%rbx | |||||
movq TF_RBP(%rsp),%rbp | |||||
movq TF_R10(%rsp),%r10 | |||||
movq TF_R11(%rsp),%r11 | |||||
movq TF_R12(%rsp),%r12 | |||||
movq TF_R13(%rsp),%r13 | |||||
movq TF_R14(%rsp),%r14 | |||||
movq TF_R15(%rsp),%r15 | |||||
testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ | ||||
jz 1f /* keep running with kernel GS.base */ | jz 2f /* keep running with kernel GS.base */ | ||||
cli | cli | ||||
cmpb $0,pti | |||||
je 1f | |||||
pushq %rdx | |||||
movq PCPU(PRVSPACE),%rdx | |||||
addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8-PTI_SIZE,%rdx | |||||
movq %rax,PTI_RAX(%rdx) | |||||
popq %rax | |||||
movq %rax,PTI_RDX(%rdx) | |||||
movq TF_RIP(%rsp),%rax | |||||
movq %rax,PTI_RIP(%rdx) | |||||
movq TF_CS(%rsp),%rax | |||||
movq %rax,PTI_CS(%rdx) | |||||
movq TF_RFLAGS(%rsp),%rax | |||||
movq %rax,PTI_RFLAGS(%rdx) | |||||
movq TF_RSP(%rsp),%rax | |||||
movq %rax,PTI_RSP(%rdx) | |||||
movq TF_SS(%rsp),%rax | |||||
movq %rax,PTI_SS(%rdx) | |||||
movq PCPU(UCR3),%rax | |||||
swapgs | swapgs | ||||
1: | movq %rdx,%rsp | ||||
addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */ | movq %rax,%cr3 | ||||
popq %rdx | |||||
popq %rax | |||||
addq $8,%rsp | |||||
jmp doreti_iret | |||||
1: swapgs | |||||
2: addq $TF_RIP,%rsp | |||||
.globl doreti_iret | .globl doreti_iret | ||||
doreti_iret: | doreti_iret: | ||||
iretq | iretq | ||||
set_segs: | set_segs: | ||||
movw $KUDSEL,%ax | movw $KUDSEL,%ax | ||||
movw %ax,TF_DS(%rsp) | movw %ax,TF_DS(%rsp) | ||||
movw %ax,TF_ES(%rsp) | movw %ax,TF_ES(%rsp) | ||||
movw $KUF32SEL,TF_FS(%rsp) | movw $KUF32SEL,TF_FS(%rsp) | ||||
movw $KUG32SEL,TF_GS(%rsp) | movw $KUG32SEL,TF_GS(%rsp) | ||||
jmp do_segs | jmp do_segs | ||||
/* | /* | ||||
* doreti_iret_fault. Alternative return code for | * doreti_iret_fault. Alternative return code for | ||||
* the case where we get a fault in the doreti_exit code | * the case where we get a fault in the doreti_exit code | ||||
* above. trap() (amd64/amd64/trap.c) catches this specific | * above. trap() (amd64/amd64/trap.c) catches this specific | ||||
* case, sends the process a signal and continues in the | * case, sends the process a signal and continues in the | ||||
* corresponding place in the code below. | * corresponding place in the code below. | ||||
*/ | */ | ||||
ALIGN_TEXT | ALIGN_TEXT | ||||
.globl doreti_iret_fault | .globl doreti_iret_fault | ||||
doreti_iret_fault: | doreti_iret_fault: | ||||
subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ | subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ | ||||
testl $PSL_I,TF_RFLAGS(%rsp) | testb $SEL_RPL_MASK,TF_CS(%rsp) | ||||
jz 1f | jz 1f | ||||
sti | sti | ||||
1: | 1: | ||||
movw %fs,TF_FS(%rsp) | SAVE_SEGS | ||||
movw %gs,TF_GS(%rsp) | |||||
movw %es,TF_ES(%rsp) | |||||
movw %ds,TF_DS(%rsp) | |||||
movl $TF_HASSEGS,TF_FLAGS(%rsp) | movl $TF_HASSEGS,TF_FLAGS(%rsp) | ||||
movq %rdi,TF_RDI(%rsp) | movq %rdi,TF_RDI(%rsp) | ||||
movq %rsi,TF_RSI(%rsp) | movq %rsi,TF_RSI(%rsp) | ||||
movq %rdx,TF_RDX(%rsp) | movq %rdx,TF_RDX(%rsp) | ||||
movq %rcx,TF_RCX(%rsp) | movq %rcx,TF_RCX(%rsp) | ||||
movq %r8,TF_R8(%rsp) | movq %r8,TF_R8(%rsp) | ||||
movq %r9,TF_R9(%rsp) | movq %r9,TF_R9(%rsp) | ||||
movq %rax,TF_RAX(%rsp) | movq %rax,TF_RAX(%rsp) | ||||
Show All 10 Lines | 1: | ||||
movq $0,TF_ADDR(%rsp) | movq $0,TF_ADDR(%rsp) | ||||
FAKE_MCOUNT(TF_RIP(%rsp)) | FAKE_MCOUNT(TF_RIP(%rsp)) | ||||
jmp calltrap | jmp calltrap | ||||
ALIGN_TEXT | ALIGN_TEXT | ||||
.globl ds_load_fault | .globl ds_load_fault | ||||
ds_load_fault: | ds_load_fault: | ||||
movl $T_PROTFLT,TF_TRAPNO(%rsp) | movl $T_PROTFLT,TF_TRAPNO(%rsp) | ||||
testl $PSL_I,TF_RFLAGS(%rsp) | testb $SEL_RPL_MASK,TF_CS(%rsp) | ||||
jz 1f | jz 1f | ||||
sti | sti | ||||
1: | 1: | ||||
movq %rsp,%rdi | movq %rsp,%rdi | ||||
call trap | call trap | ||||
movw $KUDSEL,TF_DS(%rsp) | movw $KUDSEL,TF_DS(%rsp) | ||||
jmp doreti | jmp doreti | ||||
▲ Show 20 Lines • Show All 73 Lines • Show Last 20 Lines |
If I'm reading the code correctly, PCPU(KCR3) is only set when pti != 0; therefore, it seems like these lines should be conditioned on pti != 0. (But, I could be missing something.)