Index: sys/amd64/amd64/cpu_switch.S =================================================================== --- sys/amd64/amd64/cpu_switch.S +++ sys/amd64/amd64/cpu_switch.S @@ -215,10 +215,14 @@ movq %r8,PCPU(RSP0) movq %r8,PCPU(CURPCB) /* Update the TSS_RSP0 pointer for the next interrupt */ - cmpb $0,pti(%rip) - jne 1f - movq %r8,TSS_RSP0(%rdx) -1: movq %r12,PCPU(CURTHREAD) /* into next thread */ + cmpq $~0,PCPU(UCR3) + je 1f + movq PCPU(PRVSPACE),%rax + addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8,%rax + movq %rax,TSS_RSP0(%rdx) + jmp 2f +1: movq %r8,TSS_RSP0(%rdx) +2: movq %r12,PCPU(CURTHREAD) /* into next thread */ /* Test if debug registers should be restored. */ testl $PCB_DBREGS,PCB_FLAGS(%r8) @@ -295,12 +299,7 @@ shrq $8,%rcx movl %ecx,8(%rax) movb $0x89,5(%rax) /* unset busy */ - cmpb $0,pti(%rip) - je 1f - movq PCPU(PRVSPACE),%rax - addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8,%rax - movq %rax,TSS_RSP0(%rdx) -1: movl $TSSSEL,%eax + movl $TSSSEL,%eax ltr %ax jmp done_tss Index: sys/amd64/amd64/exception.S =================================================================== --- sys/amd64/amd64/exception.S +++ sys/amd64/amd64/exception.S @@ -297,12 +297,14 @@ testb $SEL_RPL_MASK,PTI_CS-2*8(%rsp) jz Xpage swapgs + cmpq $~0,PCPU(UCR3) + je 1f pushq %rax pushq %rdx movq %cr3,%rax movq %rax,PCPU(SAVED_UCR3) PTI_UUENTRY has_err=1 - subq $TF_ERR,%rsp +1: subq $TF_ERR,%rsp movq %rdi,TF_RDI(%rsp) movq %rax,TF_RAX(%rsp) movq %rdx,TF_RDX(%rsp) @@ -347,13 +349,15 @@ pushq %rax pushq %rdx swapgs + cmpq $~0,PCPU(UCR3) + je 1f movq PCPU(KCR3),%rax movq %rax,%cr3 movq PCPU(RSP0),%rax subq $2*PTI_SIZE-3*8,%rax /* no err, %rax, %rdx in faulted frame */ MOVE_STACKS (PTI_SIZE / 4 - 3) movq %rax,%rsp - popq %rdx +1: popq %rdx popq %rax swapgs jmp X\name @@ -431,6 +435,8 @@ IDTVEC(fast_syscall_pti) swapgs movq %rax,PCPU(SCRATCH_RAX) + cmpq $~0,PCPU(UCR3) + je fast_syscall_common movq PCPU(KCR3),%rax movq %rax,%cr3 jmp fast_syscall_common @@ -498,6 +504,8 @@ movq TF_RSP(%rsp),%rsp /* user stack pointer */ cmpb $0,pti je 2f + cmpq $~0,PCPU(UCR3) + je 2f movq PCPU(UCR3),%r9 movq %r9,%cr3 xorl %r9d,%r9d @@ -1006,6 +1014,8 @@ call handle_ibrs_exit_rs cmpb $0,pti je 1f + cmpq $~0,PCPU(UCR3) + je 1f pushq %rdx movq PCPU(PRVSPACE),%rdx addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8-PTI_SIZE,%rdx Index: sys/amd64/amd64/machdep.c =================================================================== --- sys/amd64/amd64/machdep.c +++ sys/amd64/amd64/machdep.c @@ -1795,8 +1795,7 @@ rsp0 = (vm_offset_t)thread0.td_pcb; /* Ensure the stack is aligned to 16 bytes */ rsp0 &= ~0xFul; - common_tss[0].tss_rsp0 = pti ? ((vm_offset_t)PCPU_PTR(pti_stack) + - PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful : rsp0; + common_tss[0].tss_rsp0 = rsp0; PCPU_SET(rsp0, rsp0); PCPU_SET(curpcb, thread0.td_pcb); Index: sys/amd64/amd64/pmap.c =================================================================== --- sys/amd64/amd64/pmap.c +++ sys/amd64/amd64/pmap.c @@ -2607,8 +2607,10 @@ CPU_FOREACH(i) { pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; pmap->pm_pcids[i].pm_gen = 0; - if (!pti) + if (!pti) { __pcpu[i].pc_kcr3 = PMAP_NO_CR3; + __pcpu[i].pc_ucr3 = PMAP_NO_CR3; + } } PCPU_SET(curpmap, kernel_pmap); pmap_activate(curthread); @@ -2783,7 +2785,8 @@ * the kernel-mode page table active on return * to user space. */ - *pml4 |= pg_nx; + if (pmap->pm_ucr3 != PMAP_NO_CR3) + *pml4 |= pg_nx; pml4u = &pmap->pm_pml4u[pml4index]; *pml4u = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | @@ -7359,9 +7362,10 @@ { pmap_t oldpmap, pmap; struct invpcid_descr d; - uint64_t cached, cr3, kcr3, kern_pti_cached, ucr3; + uint64_t cached, cr3, kcr3, kern_pti_cached, rsp0, ucr3; register_t rflags; u_int cpuid; + struct amd64tss *tssp; oldpmap = PCPU_GET(curpmap); pmap = vmspace_pmap(td->td_proc->p_vmspace); @@ -7452,6 +7456,12 @@ PCPU_SET(ucr3, pmap->pm_ucr3); } } + if (pmap->pm_ucr3 != PMAP_NO_CR3) { + rsp0 = (vm_offset_t)PCPU_PTR(pti_stack) + + PC_PTI_STACK_SZ * sizeof(uint64_t); + tssp = PCPU_GET(tssp); + tssp->tss_rsp0 = rsp0; + } #ifdef SMP CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); #else Index: sys/amd64/include/asmacros.h =================================================================== --- sys/amd64/include/asmacros.h +++ sys/amd64/include/asmacros.h @@ -196,9 +196,12 @@ .macro PTI_UENTRY has_err swapgs + cmpq $~0,PCPU(UCR3) + je 1f pushq %rax pushq %rdx PTI_UUENTRY \has_err +1: .endm .macro PTI_ENTRY name, cont, has_err=0