diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S --- a/sys/amd64/amd64/exception.S +++ b/sys/amd64/amd64/exception.S @@ -667,9 +667,10 @@ jnz dbg_fromuserspace lfence /* - * We've interrupted the kernel. Preserve GS.base in %r12, - * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d. + * We've interrupted the kernel. See comment in NMI handler about + * registers use. */ + movq %cr2,%r15 movl $MSR_GSBASE,%ecx rdmsr movq %rax,%r12 @@ -710,6 +711,7 @@ shrq $32,%rdx wrmsr movq %r13,%cr3 + movq %r15,%cr2 RESTORE_REGS addq $TF_RIP,%rsp jmp doreti_iret @@ -804,10 +806,14 @@ testb $SEL_RPL_MASK,TF_CS(%rsp) jnz nmi_fromuserspace /* - * We've interrupted the kernel. Preserve GS.base in %r12, - * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d. + * We've interrupted the kernel. Preserve in callee-saved regs: + * GS.base in %r12, + * %cr3 in %r13, + * possibly lower half of MSR_IA32_SPEC_CTL in %r14d, + * %cr2 in %r15. */ lfence + movq %cr2,%r15 movl $MSR_GSBASE,%ecx rdmsr movq %rax,%r12 @@ -957,6 +963,7 @@ je 2f call flush_l1d_sw /* bhyve L1TF assist */ 2: movq %r13,%cr3 + movq %r15,%cr2 RESTORE_REGS addq $TF_RIP,%rsp jmp doreti_iret @@ -1011,9 +1018,10 @@ testb $SEL_RPL_MASK,TF_CS(%rsp) jnz mchk_fromuserspace /* - * We've interrupted the kernel. Preserve GS.base in %r12, - * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d. + * We've interrupted the kernel. See comment in NMI handler about + * registers use. */ + movq %cr2,%r15 movl $MSR_GSBASE,%ecx rdmsr movq %rax,%r12 @@ -1071,6 +1079,7 @@ shrq $32,%rdx wrmsr movq %r13,%cr3 + movq %r15,%cr2 RESTORE_REGS addq $TF_RIP,%rsp jmp doreti_iret diff --git a/sys/amd64/amd64/support.S b/sys/amd64/amd64/support.S --- a/sys/amd64/amd64/support.S +++ b/sys/amd64/amd64/support.S @@ -1651,7 +1651,8 @@ * * N.B. The function does not follow ABI calling conventions, it corrupts %rbx. * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags - * registers are clobbered. The NMI handler caller only needs %r13 preserved. + * registers are clobbered. The NMI handler caller only needs %r13 and %r15 + * preserved. */ ENTRY(flush_l1d_sw) #define L1D_FLUSH_SIZE (64 * 1024)