Index: head/sys/amd64/vmm/intel/vmx.h =================================================================== --- head/sys/amd64/vmm/intel/vmx.h +++ head/sys/amd64/vmm/intel/vmx.h @@ -142,7 +142,6 @@ #define VMX_GUEST_VMEXIT 0 #define VMX_VMRESUME_ERROR 1 #define VMX_VMLAUNCH_ERROR 2 -#define VMX_INVEPT_ERROR 3 int vmx_enter_guest(struct vmxctx *ctx, struct vmx *vmx, int launched); void vmx_call_isr(uintptr_t entry); Index: head/sys/amd64/vmm/intel/vmx.c =================================================================== --- head/sys/amd64/vmm/intel/vmx.c +++ head/sys/amd64/vmm/intel/vmx.c @@ -2835,7 +2835,6 @@ switch (rc) { case VMX_VMRESUME_ERROR: case VMX_VMLAUNCH_ERROR: - case VMX_INVEPT_ERROR: vmexit->u.vmx.inst_type = rc; break; default: @@ -2940,6 +2939,29 @@ write_rflags(read_rflags() | vmxctx->host_tf); } +static __inline void +vmx_pmap_activate(struct vmx *vmx, pmap_t pmap) +{ + long eptgen; + int cpu; + + cpu = curcpu; + + CPU_SET_ATOMIC(cpu, &pmap->pm_active); + eptgen = atomic_load_long(&pmap->pm_eptgen); + if (eptgen != vmx->eptgen[cpu]) { + vmx->eptgen[cpu] = eptgen; + invept(INVEPT_TYPE_SINGLE_CONTEXT, + (struct invept_desc){ .eptp = vmx->eptp, ._res = 0 }); + } +} + +static __inline void +vmx_pmap_deactivate(struct vmx *vmx, pmap_t pmap) +{ + CPU_CLR_ATOMIC(curcpu, &pmap->pm_active); +} + static int vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap, struct vm_eventinfo *evinfo) @@ -3088,11 +3110,19 @@ */ vmx_msr_guest_enter_tsc_aux(vmx, vcpu); - vmx_run_trace(vmx, vcpu); vmx_dr_enter_guest(vmxctx); + + /* + * Mark the EPT as active on this host CPU and invalidate + * EPTP-tagged TLB entries if required. + */ + vmx_pmap_activate(vmx, pmap); + + vmx_run_trace(vmx, vcpu); rc = vmx_enter_guest(vmxctx, vmx, launched); - vmx_dr_leave_guest(vmxctx); + vmx_pmap_deactivate(vmx, pmap); + vmx_dr_leave_guest(vmxctx); vmx_msr_guest_exit_tsc_aux(vmx, vcpu); bare_lgdt(&gdtr); Index: head/sys/amd64/vmm/intel/vmx_genassym.c =================================================================== --- head/sys/amd64/vmm/intel/vmx_genassym.c +++ head/sys/amd64/vmm/intel/vmx_genassym.c @@ -70,17 +70,12 @@ ASSYM(VMXCTX_HOST_RBX, offsetof(struct vmxctx, host_rbx)); ASSYM(VMXCTX_INST_FAIL_STATUS, offsetof(struct vmxctx, inst_fail_status)); -ASSYM(VMXCTX_PMAP, offsetof(struct vmxctx, pmap)); -ASSYM(VMX_EPTGEN, offsetof(struct vmx, eptgen)); -ASSYM(VMX_EPTP, offsetof(struct vmx, eptp)); - ASSYM(VM_FAIL_INVALID, VM_FAIL_INVALID); ASSYM(VM_FAIL_VALID, VM_FAIL_VALID); ASSYM(VMX_GUEST_VMEXIT, VMX_GUEST_VMEXIT); ASSYM(VMX_VMRESUME_ERROR, VMX_VMRESUME_ERROR); ASSYM(VMX_VMLAUNCH_ERROR, VMX_VMLAUNCH_ERROR); -ASSYM(VMX_INVEPT_ERROR, VMX_INVEPT_ERROR); ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); Index: head/sys/amd64/vmm/intel/vmx_support.S =================================================================== --- head/sys/amd64/vmm/intel/vmx_support.S +++ head/sys/amd64/vmm/intel/vmx_support.S @@ -148,33 +148,6 @@ */ VMX_HOST_SAVE - /* - * Activate guest pmap on this cpu. - */ - movq VMXCTX_PMAP(%rdi), %r11 - movl PCPU(CPUID), %eax - LK btsl %eax, PM_ACTIVE(%r11) - - /* - * If 'vmx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen' - * then we must invalidate all mappings associated with this EPTP. - */ - movq PM_EPTGEN(%r11), %r10 - cmpq %r10, VMX_EPTGEN(%rsi, %rax, 8) - je guest_restore - - /* Refresh 'vmx->eptgen[curcpu]' */ - movq %r10, VMX_EPTGEN(%rsi, %rax, 8) - - /* Setup the invept descriptor on the host stack */ - mov %rsp, %r11 - movq VMX_EPTP(%rsi), %rax - movq %rax, -16(%r11) - movq $0x0, -8(%r11) - mov $0x1, %eax /* Single context invalidate */ - invept -16(%r11), %rax - jbe invept_error /* Check invept instruction error */ - guest_restore: movl %edx, %r8d cmpb $0, guest_l1d_flush_sw(%rip) @@ -208,10 +181,6 @@ movl $VMX_VMLAUNCH_ERROR, %eax jmp decode_inst_error -invept_error: - movl $VMX_INVEPT_ERROR, %eax - jmp decode_inst_error - decode_inst_error: movl $VM_FAIL_VALID, %r11d jz inst_error @@ -224,13 +193,6 @@ * it as a scratch register beyond this point. */ - /* - * Deactivate guest pmap from this cpu. - */ - movq VMXCTX_PMAP(%rdi), %r11 - movl PCPU(CPUID), %r10d - LK btrl %r10d, PM_ACTIVE(%r11) - VMX_HOST_RESTORE VLEAVE ret @@ -248,13 +210,6 @@ */ VMX_GUEST_SAVE - /* - * Deactivate guest pmap from this cpu. - */ - movq VMXCTX_PMAP(%rdi), %r11 - movl PCPU(CPUID), %r10d - LK btrl %r10d, PM_ACTIVE(%r11) - VMX_HOST_RESTORE VMX_GUEST_CLOBBER @@ -290,13 +245,6 @@ * Save guest state that is not automatically saved in the vmcs. */ VMX_GUEST_SAVE - - /* - * Deactivate guest pmap from this cpu. - */ - movq VMXCTX_PMAP(%rdi), %r11 - movl PCPU(CPUID), %r10d - LK btrl %r10d, PM_ACTIVE(%r11) VMX_HOST_RESTORE