Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/vmm/intel/vmx.c
| Show First 20 Lines • Show All 1,026 Lines • ▼ Show 20 Lines | |||||||||
| #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init)) | #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init)) | ||||||||
| static void * | static void * | ||||||||
| vmx_init(struct vm *vm, pmap_t pmap) | vmx_init(struct vm *vm, pmap_t pmap) | ||||||||
| { | { | ||||||||
| int i, error; | int i, error; | ||||||||
| struct vmx *vmx; | struct vmx *vmx; | ||||||||
| struct vmcs *vmcs; | struct vmcs *vmcs; | ||||||||
| struct vmx_vcpu *vcpu; | |||||||||
| uint32_t exc_bitmap; | uint32_t exc_bitmap; | ||||||||
| uint16_t maxcpus = vm_get_maxcpus(vm); | uint16_t maxcpus = vm_get_maxcpus(vm); | ||||||||
| uint16_t vpid[maxcpus]; | uint16_t vpid[maxcpus]; | ||||||||
| vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO); | vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO); | ||||||||
| if ((uintptr_t)vmx & PAGE_MASK) { | |||||||||
| panic("malloc of struct vmx not aligned on %d byte boundary", | |||||||||
| PAGE_SIZE); | |||||||||
| } | |||||||||
| vmx->vm = vm; | vmx->vm = vm; | ||||||||
| vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pmltop)); | vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pmltop)); | ||||||||
| /* | /* | ||||||||
| * Clean up EPTP-tagged guest physical and combined mappings | * Clean up EPTP-tagged guest physical and combined mappings | ||||||||
| * | * | ||||||||
| * VMX transitions are not required to invalidate any guest physical | * VMX transitions are not required to invalidate any guest physical | ||||||||
| * mappings. So, it may be possible for stale guest physical mappings | * mappings. So, it may be possible for stale guest physical mappings | ||||||||
| * to be present in the processor TLBs. | * to be present in the processor TLBs. | ||||||||
| * | * | ||||||||
| * Combined mappings for this EP4TA are also invalidated for all VPIDs. | * Combined mappings for this EP4TA are also invalidated for all VPIDs. | ||||||||
| */ | */ | ||||||||
| ept_invalidate_mappings(vmx->eptp); | ept_invalidate_mappings(vmx->eptp); | ||||||||
| vmx->msr_bitmap = malloc_aligned(PAGE_SIZE, PAGE_SIZE, M_VMX, | |||||||||
| M_WAITOK | M_ZERO); | |||||||||
| msr_bitmap_initialize(vmx->msr_bitmap); | msr_bitmap_initialize(vmx->msr_bitmap); | ||||||||
| /* | /* | ||||||||
| * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE. | * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE. | ||||||||
| * The guest FSBASE and GSBASE are saved and restored during | * The guest FSBASE and GSBASE are saved and restored during | ||||||||
| * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are | * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are | ||||||||
| * always restored from the vmcs host state area on vm-exit. | * always restored from the vmcs host state area on vm-exit. | ||||||||
| * | * | ||||||||
| Show All 35 Lines | vmx_init(struct vm *vm, pmap_t pmap) | ||||||||
| if (virtual_interrupt_delivery) { | if (virtual_interrupt_delivery) { | ||||||||
| error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE, | error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE, | ||||||||
| APIC_ACCESS_ADDRESS); | APIC_ACCESS_ADDRESS); | ||||||||
| /* XXX this should really return an error to the caller */ | /* XXX this should really return an error to the caller */ | ||||||||
| KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error)); | KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error)); | ||||||||
| } | } | ||||||||
| for (i = 0; i < maxcpus; i++) { | for (i = 0; i < maxcpus; i++) { | ||||||||
| vmcs = &vmx->vmcs[i]; | vcpu = &vmx->vcpus[i]; | ||||||||
| vcpu->vmcs = malloc_aligned(sizeof(*vmcs), PAGE_SIZE, M_VMX, | |||||||||
| M_WAITOK | M_ZERO); | |||||||||
markj: Shouldn't this use malloc_aligned() as well? If only to document the alignment requirement. | |||||||||
| vcpu->apic_page = malloc_aligned(PAGE_SIZE, PAGE_SIZE, M_VMX, | |||||||||
| M_WAITOK | M_ZERO); | |||||||||
| vcpu->pir_desc = malloc_aligned(sizeof(*vcpu->pir_desc), 64, | |||||||||
| M_VMX, M_WAITOK | M_ZERO); | |||||||||
| vmcs = vcpu->vmcs; | |||||||||
| vmcs->identifier = vmx_revision(); | vmcs->identifier = vmx_revision(); | ||||||||
| error = vmclear(vmcs); | error = vmclear(vmcs); | ||||||||
| if (error != 0) { | if (error != 0) { | ||||||||
| panic("vmx_init: vmclear error %d on vcpu %d\n", | panic("vmx_init: vmclear error %d on vcpu %d\n", | ||||||||
| error, i); | error, i); | ||||||||
| } | } | ||||||||
| vmx_msr_guest_init(vmx, i); | vmx_msr_guest_init(vmx, i); | ||||||||
| error = vmcs_init(vmcs); | error = vmcs_init(vmcs); | ||||||||
| KASSERT(error == 0, ("vmcs_init error %d", error)); | KASSERT(error == 0, ("vmcs_init error %d", error)); | ||||||||
| VMPTRLD(vmcs); | VMPTRLD(vmcs); | ||||||||
| error = 0; | error = 0; | ||||||||
| error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]); | error += vmwrite(VMCS_HOST_RSP, (u_long)&vcpu->ctx); | ||||||||
| error += vmwrite(VMCS_EPTP, vmx->eptp); | error += vmwrite(VMCS_EPTP, vmx->eptp); | ||||||||
| error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls); | error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls); | ||||||||
| error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls); | error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls); | ||||||||
| if (vcpu_trap_wbinvd(vm, i)) { | if (vcpu_trap_wbinvd(vm, i)) { | ||||||||
| KASSERT(cap_wbinvd_exit, ("WBINVD trap not available")); | KASSERT(cap_wbinvd_exit, ("WBINVD trap not available")); | ||||||||
| procbased_ctls2 |= PROCBASED2_WBINVD_EXITING; | procbased_ctls2 |= PROCBASED2_WBINVD_EXITING; | ||||||||
| } | } | ||||||||
| error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2); | error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2); | ||||||||
| Show All 13 Lines | for (i = 0; i < maxcpus; i++) { | ||||||||
| /* exception bitmap */ | /* exception bitmap */ | ||||||||
| if (vcpu_trace_exceptions(vm, i)) | if (vcpu_trace_exceptions(vm, i)) | ||||||||
| exc_bitmap = 0xffffffff; | exc_bitmap = 0xffffffff; | ||||||||
| else | else | ||||||||
| exc_bitmap = 1 << IDT_MC; | exc_bitmap = 1 << IDT_MC; | ||||||||
| error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap); | error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap); | ||||||||
| vmx->ctx[i].guest_dr6 = DBREG_DR6_RESERVED1; | vcpu->ctx.guest_dr6 = DBREG_DR6_RESERVED1; | ||||||||
| error += vmwrite(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1); | error += vmwrite(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1); | ||||||||
| if (tpr_shadowing) { | if (tpr_shadowing) { | ||||||||
| error += vmwrite(VMCS_VIRTUAL_APIC, | error += vmwrite(VMCS_VIRTUAL_APIC, | ||||||||
| vtophys(&vmx->apic_page[i])); | vtophys(vcpu->apic_page)); | ||||||||
| } | } | ||||||||
| if (virtual_interrupt_delivery) { | if (virtual_interrupt_delivery) { | ||||||||
| error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS); | error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS); | ||||||||
| error += vmwrite(VMCS_EOI_EXIT0, 0); | error += vmwrite(VMCS_EOI_EXIT0, 0); | ||||||||
| error += vmwrite(VMCS_EOI_EXIT1, 0); | error += vmwrite(VMCS_EOI_EXIT1, 0); | ||||||||
| error += vmwrite(VMCS_EOI_EXIT2, 0); | error += vmwrite(VMCS_EOI_EXIT2, 0); | ||||||||
| error += vmwrite(VMCS_EOI_EXIT3, 0); | error += vmwrite(VMCS_EOI_EXIT3, 0); | ||||||||
| } | } | ||||||||
| if (posted_interrupts) { | if (posted_interrupts) { | ||||||||
| error += vmwrite(VMCS_PIR_VECTOR, pirvec); | error += vmwrite(VMCS_PIR_VECTOR, pirvec); | ||||||||
| error += vmwrite(VMCS_PIR_DESC, | error += vmwrite(VMCS_PIR_DESC, | ||||||||
| vtophys(&vmx->pir_desc[i])); | vtophys(vcpu->pir_desc)); | ||||||||
| } | } | ||||||||
| VMCLEAR(vmcs); | VMCLEAR(vmcs); | ||||||||
| KASSERT(error == 0, ("vmx_init: error customizing the vmcs")); | KASSERT(error == 0, ("vmx_init: error customizing the vmcs")); | ||||||||
| vmx->cap[i].set = 0; | vcpu->cap.set = 0; | ||||||||
| vmx->cap[i].set |= cap_rdpid != 0 ? 1 << VM_CAP_RDPID : 0; | vcpu->cap.set |= cap_rdpid != 0 ? 1 << VM_CAP_RDPID : 0; | ||||||||
| vmx->cap[i].set |= cap_rdtscp != 0 ? 1 << VM_CAP_RDTSCP : 0; | vcpu->cap.set |= cap_rdtscp != 0 ? 1 << VM_CAP_RDTSCP : 0; | ||||||||
| vmx->cap[i].proc_ctls = procbased_ctls; | vcpu->cap.proc_ctls = procbased_ctls; | ||||||||
| vmx->cap[i].proc_ctls2 = procbased_ctls2; | vcpu->cap.proc_ctls2 = procbased_ctls2; | ||||||||
| vmx->cap[i].exc_bitmap = exc_bitmap; | vcpu->cap.exc_bitmap = exc_bitmap; | ||||||||
| vmx->state[i].nextrip = ~0; | vcpu->state.nextrip = ~0; | ||||||||
| vmx->state[i].lastcpu = NOCPU; | vcpu->state.lastcpu = NOCPU; | ||||||||
| vmx->state[i].vpid = vpid[i]; | vcpu->state.vpid = vpid[i]; | ||||||||
| /* | /* | ||||||||
| * Set up the CR0/4 shadows, and init the read shadow | * Set up the CR0/4 shadows, and init the read shadow | ||||||||
| * to the power-on register value from the Intel Sys Arch. | * to the power-on register value from the Intel Sys Arch. | ||||||||
| * CR0 - 0x60000010 | * CR0 - 0x60000010 | ||||||||
| * CR4 - 0 | * CR4 - 0 | ||||||||
| */ | */ | ||||||||
| error = vmx_setup_cr0_shadow(vmcs, 0x60000010); | error = vmx_setup_cr0_shadow(vmcs, 0x60000010); | ||||||||
| if (error != 0) | if (error != 0) | ||||||||
| panic("vmx_setup_cr0_shadow %d", error); | panic("vmx_setup_cr0_shadow %d", error); | ||||||||
| error = vmx_setup_cr4_shadow(vmcs, 0); | error = vmx_setup_cr4_shadow(vmcs, 0); | ||||||||
| if (error != 0) | if (error != 0) | ||||||||
| panic("vmx_setup_cr4_shadow %d", error); | panic("vmx_setup_cr4_shadow %d", error); | ||||||||
| vmx->ctx[i].pmap = pmap; | vcpu->ctx.pmap = pmap; | ||||||||
| } | } | ||||||||
| return (vmx); | return (vmx); | ||||||||
| } | } | ||||||||
| static int | static int | ||||||||
| vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx) | vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx) | ||||||||
| { | { | ||||||||
| Show All 39 Lines | |||||||||
| * Invalidate guest mappings identified by its vpid from the TLB. | * Invalidate guest mappings identified by its vpid from the TLB. | ||||||||
| */ | */ | ||||||||
| static __inline void | static __inline void | ||||||||
| vmx_invvpid(struct vmx *vmx, int vcpu, pmap_t pmap, int running) | vmx_invvpid(struct vmx *vmx, int vcpu, pmap_t pmap, int running) | ||||||||
| { | { | ||||||||
| struct vmxstate *vmxstate; | struct vmxstate *vmxstate; | ||||||||
| struct invvpid_desc invvpid_desc; | struct invvpid_desc invvpid_desc; | ||||||||
| vmxstate = &vmx->state[vcpu]; | vmxstate = &vmx->vcpus[vcpu].state; | ||||||||
| if (vmxstate->vpid == 0) | if (vmxstate->vpid == 0) | ||||||||
| return; | return; | ||||||||
| if (!running) { | if (!running) { | ||||||||
| /* | /* | ||||||||
| * Set the 'lastcpu' to an invalid host cpu. | * Set the 'lastcpu' to an invalid host cpu. | ||||||||
| * | * | ||||||||
| * This will invalidate TLB entries tagged with the vcpu's | * This will invalidate TLB entries tagged with the vcpu's | ||||||||
| Show All 39 Lines | vmx_invvpid(struct vmx *vmx, int vcpu, pmap_t pmap, int running) | ||||||||
| } | } | ||||||||
| } | } | ||||||||
| static void | static void | ||||||||
| vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap) | vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap) | ||||||||
| { | { | ||||||||
| struct vmxstate *vmxstate; | struct vmxstate *vmxstate; | ||||||||
| vmxstate = &vmx->state[vcpu]; | vmxstate = &vmx->vcpus[vcpu].state; | ||||||||
| if (vmxstate->lastcpu == curcpu) | if (vmxstate->lastcpu == curcpu) | ||||||||
| return; | return; | ||||||||
| vmxstate->lastcpu = curcpu; | vmxstate->lastcpu = curcpu; | ||||||||
| vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1); | vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1); | ||||||||
| vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase()); | vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase()); | ||||||||
| vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase()); | vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase()); | ||||||||
| vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase()); | vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase()); | ||||||||
| vmx_invvpid(vmx, vcpu, pmap, 1); | vmx_invvpid(vmx, vcpu, pmap, 1); | ||||||||
| } | } | ||||||||
| /* | /* | ||||||||
| * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set. | * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set. | ||||||||
| */ | */ | ||||||||
| CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0); | CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0); | ||||||||
| static void __inline | static void __inline | ||||||||
| vmx_set_int_window_exiting(struct vmx *vmx, int vcpu) | vmx_set_int_window_exiting(struct vmx *vmx, int vcpu) | ||||||||
| { | { | ||||||||
| struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; | |||||||||
| if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) { | if ((vmx_vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) { | ||||||||
| vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING; | vmx_vcpu->cap.proc_ctls |= PROCBASED_INT_WINDOW_EXITING; | ||||||||
| vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); | vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx_vcpu->cap.proc_ctls); | ||||||||
| VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting"); | VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting"); | ||||||||
| } | } | ||||||||
| } | } | ||||||||
| static void __inline | static void __inline | ||||||||
| vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu) | vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu) | ||||||||
| { | { | ||||||||
| struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; | |||||||||
| KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0, | KASSERT((vmx_vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0, | ||||||||
| ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls)); | ("intr_window_exiting not set: %#x", vmx_vcpu->cap.proc_ctls)); | ||||||||
| vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING; | vmx_vcpu->cap.proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING; | ||||||||
| vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); | vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx_vcpu->cap.proc_ctls); | ||||||||
| VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting"); | VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting"); | ||||||||
| } | } | ||||||||
| static void __inline | static void __inline | ||||||||
| vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu) | vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu) | ||||||||
| { | { | ||||||||
| struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; | |||||||||
| if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) { | if ((vmx_vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) { | ||||||||
| vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING; | vmx_vcpu->cap.proc_ctls |= PROCBASED_NMI_WINDOW_EXITING; | ||||||||
| vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); | vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx_vcpu->cap.proc_ctls); | ||||||||
| VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting"); | VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting"); | ||||||||
| } | } | ||||||||
| } | } | ||||||||
| static void __inline | static void __inline | ||||||||
| vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu) | vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu) | ||||||||
| { | { | ||||||||
| struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; | |||||||||
| KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0, | KASSERT((vmx_vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0, | ||||||||
| ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls)); | ("nmi_window_exiting not set %#x", vmx_vcpu->cap.proc_ctls)); | ||||||||
| vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING; | vmx_vcpu->cap.proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING; | ||||||||
| vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); | vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx_vcpu->cap.proc_ctls); | ||||||||
| VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting"); | VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting"); | ||||||||
| } | } | ||||||||
| int | int | ||||||||
| vmx_set_tsc_offset(struct vmx *vmx, int vcpu, uint64_t offset) | vmx_set_tsc_offset(struct vmx *vmx, int vcpu, uint64_t offset) | ||||||||
| { | { | ||||||||
| struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; | |||||||||
| int error; | int error; | ||||||||
| if ((vmx->cap[vcpu].proc_ctls & PROCBASED_TSC_OFFSET) == 0) { | if ((vmx_vcpu->cap.proc_ctls & PROCBASED_TSC_OFFSET) == 0) { | ||||||||
| vmx->cap[vcpu].proc_ctls |= PROCBASED_TSC_OFFSET; | vmx_vcpu->cap.proc_ctls |= PROCBASED_TSC_OFFSET; | ||||||||
| vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); | vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx_vcpu->cap.proc_ctls); | ||||||||
| VCPU_CTR0(vmx->vm, vcpu, "Enabling TSC offsetting"); | VCPU_CTR0(vmx->vm, vcpu, "Enabling TSC offsetting"); | ||||||||
| } | } | ||||||||
| error = vmwrite(VMCS_TSC_OFFSET, offset); | error = vmwrite(VMCS_TSC_OFFSET, offset); | ||||||||
| #ifdef BHYVE_SNAPSHOT | #ifdef BHYVE_SNAPSHOT | ||||||||
| if (error == 0) | if (error == 0) | ||||||||
| error = vm_set_tsc_offset(vmx->vm, vcpu, offset); | error = vm_set_tsc_offset(vmx->vm, vcpu, offset); | ||||||||
| #endif | #endif | ||||||||
| Show All 30 Lines | vmx_inject_nmi(struct vmx *vmx, int vcpu) | ||||||||
| /* Clear the request */ | /* Clear the request */ | ||||||||
| vm_nmi_clear(vmx->vm, vcpu); | vm_nmi_clear(vmx->vm, vcpu); | ||||||||
| } | } | ||||||||
| static void | static void | ||||||||
| vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic, | vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic, | ||||||||
| uint64_t guestrip) | uint64_t guestrip) | ||||||||
| { | { | ||||||||
| struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; | |||||||||
| int vector, need_nmi_exiting, extint_pending; | int vector, need_nmi_exiting, extint_pending; | ||||||||
| uint64_t rflags, entryinfo; | uint64_t rflags, entryinfo; | ||||||||
| uint32_t gi, info; | uint32_t gi, info; | ||||||||
| if (vmx->state[vcpu].nextrip != guestrip) { | if (vmx_vcpu->state.nextrip != guestrip) { | ||||||||
| gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); | gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); | ||||||||
| if (gi & HWINTR_BLOCKING) { | if (gi & HWINTR_BLOCKING) { | ||||||||
| VCPU_CTR2(vmx->vm, vcpu, "Guest interrupt blocking " | VCPU_CTR2(vmx->vm, vcpu, "Guest interrupt blocking " | ||||||||
| "cleared due to rip change: %#lx/%#lx", | "cleared due to rip change: %#lx/%#lx", | ||||||||
| vmx->state[vcpu].nextrip, guestrip); | vmx_vcpu->state.nextrip, guestrip); | ||||||||
Done Inline Actions
corvink: | |||||||||
| gi &= ~HWINTR_BLOCKING; | gi &= ~HWINTR_BLOCKING; | ||||||||
| vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); | vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); | ||||||||
| } | } | ||||||||
| } | } | ||||||||
| if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) { | if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) { | ||||||||
| KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry " | KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry " | ||||||||
| "intinfo is not valid: %#lx", __func__, entryinfo)); | "intinfo is not valid: %#lx", __func__, entryinfo)); | ||||||||
| ▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines | if (!extint_pending && virtual_interrupt_delivery) { | ||||||||
| return; | return; | ||||||||
| } | } | ||||||||
| /* | /* | ||||||||
| * If interrupt-window exiting is already in effect then don't bother | * If interrupt-window exiting is already in effect then don't bother | ||||||||
| * checking for pending interrupts. This is just an optimization and | * checking for pending interrupts. This is just an optimization and | ||||||||
| * not needed for correctness. | * not needed for correctness. | ||||||||
| */ | */ | ||||||||
| if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) { | if ((vmx_vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) { | ||||||||
| VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to " | VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to " | ||||||||
| "pending int_window_exiting"); | "pending int_window_exiting"); | ||||||||
| return; | return; | ||||||||
| } | } | ||||||||
| if (!extint_pending) { | if (!extint_pending) { | ||||||||
| /* Ask the local apic for a vector to inject */ | /* Ask the local apic for a vector to inject */ | ||||||||
| if (!vlapic_pending_intr(vlapic, &vector)) | if (!vlapic_pending_intr(vlapic, &vector)) | ||||||||
| ▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | |||||||||
| static int | static int | ||||||||
| vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) | vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) | ||||||||
| { | { | ||||||||
| struct vmxctx *vmxctx; | struct vmxctx *vmxctx; | ||||||||
| uint64_t xcrval; | uint64_t xcrval; | ||||||||
| const struct xsave_limits *limits; | const struct xsave_limits *limits; | ||||||||
| vmxctx = &vmx->ctx[vcpu]; | vmxctx = &vmx->vcpus[vcpu].ctx; | ||||||||
| limits = vmm_get_xsave_limits(); | limits = vmm_get_xsave_limits(); | ||||||||
| /* | /* | ||||||||
| * Note that the processor raises a GP# fault on its own if | * Note that the processor raises a GP# fault on its own if | ||||||||
| * xsetbv is executed for CPL != 0, so we do not have to | * xsetbv is executed for CPL != 0, so we do not have to | ||||||||
| * emulate that fault here. | * emulate that fault here. | ||||||||
| */ | */ | ||||||||
| ▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines | vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) | ||||||||
| return (HANDLED); | return (HANDLED); | ||||||||
| } | } | ||||||||
| static uint64_t | static uint64_t | ||||||||
| vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident) | vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident) | ||||||||
| { | { | ||||||||
| const struct vmxctx *vmxctx; | const struct vmxctx *vmxctx; | ||||||||
| vmxctx = &vmx->ctx[vcpu]; | vmxctx = &vmx->vcpus[vcpu].ctx; | ||||||||
| switch (ident) { | switch (ident) { | ||||||||
| case 0: | case 0: | ||||||||
| return (vmxctx->guest_rax); | return (vmxctx->guest_rax); | ||||||||
| case 1: | case 1: | ||||||||
| return (vmxctx->guest_rcx); | return (vmxctx->guest_rcx); | ||||||||
| case 2: | case 2: | ||||||||
| return (vmxctx->guest_rdx); | return (vmxctx->guest_rdx); | ||||||||
| Show All 28 Lines | vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident) | ||||||||
| } | } | ||||||||
| } | } | ||||||||
| static void | static void | ||||||||
| vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval) | vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval) | ||||||||
| { | { | ||||||||
| struct vmxctx *vmxctx; | struct vmxctx *vmxctx; | ||||||||
| vmxctx = &vmx->ctx[vcpu]; | vmxctx = &vmx->vcpus[vcpu].ctx; | ||||||||
| switch (ident) { | switch (ident) { | ||||||||
| case 0: | case 0: | ||||||||
| vmxctx->guest_rax = regval; | vmxctx->guest_rax = regval; | ||||||||
| break; | break; | ||||||||
| case 1: | case 1: | ||||||||
| vmxctx->guest_rcx = regval; | vmxctx->guest_rcx = regval; | ||||||||
| break; | break; | ||||||||
| ▲ Show 20 Lines • Show All 322 Lines • ▼ Show 20 Lines | ept_emulation_fault(uint64_t ept_qual) | ||||||||
| return (true); | return (true); | ||||||||
| } | } | ||||||||
| static __inline int | static __inline int | ||||||||
| apic_access_virtualization(struct vmx *vmx, int vcpuid) | apic_access_virtualization(struct vmx *vmx, int vcpuid) | ||||||||
| { | { | ||||||||
| uint32_t proc_ctls2; | uint32_t proc_ctls2; | ||||||||
| proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; | proc_ctls2 = vmx->vcpus[vcpuid].cap.proc_ctls2; | ||||||||
| return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0); | return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0); | ||||||||
| } | } | ||||||||
| static __inline int | static __inline int | ||||||||
| x2apic_virtualization(struct vmx *vmx, int vcpuid) | x2apic_virtualization(struct vmx *vmx, int vcpuid) | ||||||||
| { | { | ||||||||
| uint32_t proc_ctls2; | uint32_t proc_ctls2; | ||||||||
| proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; | proc_ctls2 = vmx->vcpus[vcpuid].cap.proc_ctls2; | ||||||||
| return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0); | return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0); | ||||||||
| } | } | ||||||||
| static int | static int | ||||||||
| vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic, | vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic, | ||||||||
| uint64_t qual) | uint64_t qual) | ||||||||
| { | { | ||||||||
| int error, handled, offset; | int error, handled, offset; | ||||||||
| ▲ Show 20 Lines • Show All 180 Lines • ▼ Show 20 Lines | emulate_rdmsr(struct vmx *vmx, int vcpuid, u_int num, bool *retu) | ||||||||
| if (lapic_msr(num)) | if (lapic_msr(num)) | ||||||||
| error = lapic_rdmsr(vmx->vm, vcpuid, num, &result, retu); | error = lapic_rdmsr(vmx->vm, vcpuid, num, &result, retu); | ||||||||
| else | else | ||||||||
| error = vmx_rdmsr(vmx, vcpuid, num, &result, retu); | error = vmx_rdmsr(vmx, vcpuid, num, &result, retu); | ||||||||
| if (error == 0) { | if (error == 0) { | ||||||||
| eax = result; | eax = result; | ||||||||
| vmxctx = &vmx->ctx[vcpuid]; | vmxctx = &vmx->vcpus[vcpuid].ctx; | ||||||||
| error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax); | error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax); | ||||||||
| KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error)); | KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error)); | ||||||||
| edx = result >> 32; | edx = result >> 32; | ||||||||
| error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx); | error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx); | ||||||||
| KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error)); | KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error)); | ||||||||
| } | } | ||||||||
| return (error); | return (error); | ||||||||
| } | } | ||||||||
| static int | static int | ||||||||
| vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) | vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) | ||||||||
| { | { | ||||||||
| int error, errcode, errcode_valid, handled, in; | int error, errcode, errcode_valid, handled, in; | ||||||||
| struct vmx_vcpu *vmx_vcpu; | |||||||||
| struct vmxctx *vmxctx; | struct vmxctx *vmxctx; | ||||||||
| struct vlapic *vlapic; | struct vlapic *vlapic; | ||||||||
| struct vm_inout_str *vis; | struct vm_inout_str *vis; | ||||||||
| struct vm_task_switch *ts; | struct vm_task_switch *ts; | ||||||||
| uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info; | uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info; | ||||||||
| uint32_t intr_type, intr_vec, reason; | uint32_t intr_type, intr_vec, reason; | ||||||||
| uint64_t exitintinfo, qual, gpa; | uint64_t exitintinfo, qual, gpa; | ||||||||
| bool retu; | bool retu; | ||||||||
| CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0); | CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0); | ||||||||
| CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0); | CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0); | ||||||||
| handled = UNHANDLED; | handled = UNHANDLED; | ||||||||
| vmxctx = &vmx->ctx[vcpu]; | vmx_vcpu = &vmx->vcpus[vcpu]; | ||||||||
| vmxctx = &vmx_vcpu->ctx; | |||||||||
| qual = vmexit->u.vmx.exit_qualification; | qual = vmexit->u.vmx.exit_qualification; | ||||||||
| reason = vmexit->u.vmx.exit_reason; | reason = vmexit->u.vmx.exit_reason; | ||||||||
| vmexit->exitcode = VM_EXITCODE_BOGUS; | vmexit->exitcode = VM_EXITCODE_BOGUS; | ||||||||
| vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1); | vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1); | ||||||||
| SDT_PROBE3(vmm, vmx, exit, entry, vmx, vcpu, vmexit); | SDT_PROBE3(vmm, vmx, exit, entry, vmx, vcpu, vmexit); | ||||||||
| ▲ Show 20 Lines • Show All 294 Lines • ▼ Show 20 Lines | if (intr_vec == IDT_MC) { | ||||||||
| return (1); | return (1); | ||||||||
| } | } | ||||||||
| /* | /* | ||||||||
| * If the hypervisor has requested user exits for | * If the hypervisor has requested user exits for | ||||||||
| * debug exceptions, bounce them out to userland. | * debug exceptions, bounce them out to userland. | ||||||||
| */ | */ | ||||||||
| if (intr_type == VMCS_INTR_T_SWEXCEPTION && intr_vec == IDT_BP && | if (intr_type == VMCS_INTR_T_SWEXCEPTION && intr_vec == IDT_BP && | ||||||||
| (vmx->cap[vcpu].set & (1 << VM_CAP_BPT_EXIT))) { | (vmx_vcpu->cap.set & (1 << VM_CAP_BPT_EXIT))) { | ||||||||
| vmexit->exitcode = VM_EXITCODE_BPT; | vmexit->exitcode = VM_EXITCODE_BPT; | ||||||||
| vmexit->u.bpt.inst_length = vmexit->inst_length; | vmexit->u.bpt.inst_length = vmexit->inst_length; | ||||||||
| vmexit->inst_length = 0; | vmexit->inst_length = 0; | ||||||||
| break; | break; | ||||||||
| } | } | ||||||||
| if (intr_vec == IDT_PF) { | if (intr_vec == IDT_PF) { | ||||||||
| error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual); | error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual); | ||||||||
| ▲ Show 20 Lines • Show All 309 Lines • ▼ Show 20 Lines | |||||||||
| } | } | ||||||||
| static int | static int | ||||||||
| vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap, | vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap, | ||||||||
| struct vm_eventinfo *evinfo) | struct vm_eventinfo *evinfo) | ||||||||
| { | { | ||||||||
| int rc, handled, launched; | int rc, handled, launched; | ||||||||
| struct vmx *vmx; | struct vmx *vmx; | ||||||||
| struct vmx_vcpu *vmx_vcpu; | |||||||||
| struct vm *vm; | struct vm *vm; | ||||||||
| struct vmxctx *vmxctx; | struct vmxctx *vmxctx; | ||||||||
| struct vmcs *vmcs; | struct vmcs *vmcs; | ||||||||
| struct vm_exit *vmexit; | struct vm_exit *vmexit; | ||||||||
| struct vlapic *vlapic; | struct vlapic *vlapic; | ||||||||
| uint32_t exit_reason; | uint32_t exit_reason; | ||||||||
| struct region_descriptor gdtr, idtr; | struct region_descriptor gdtr, idtr; | ||||||||
| uint16_t ldt_sel; | uint16_t ldt_sel; | ||||||||
| vmx = arg; | vmx = arg; | ||||||||
| vm = vmx->vm; | vm = vmx->vm; | ||||||||
| vmcs = &vmx->vmcs[vcpu]; | vmx_vcpu = &vmx->vcpus[vcpu]; | ||||||||
| vmxctx = &vmx->ctx[vcpu]; | vmcs = vmx_vcpu->vmcs; | ||||||||
| vmxctx = &vmx_vcpu->ctx; | |||||||||
| vlapic = vm_lapic(vm, vcpu); | vlapic = vm_lapic(vm, vcpu); | ||||||||
| vmexit = vm_exitinfo(vm, vcpu); | vmexit = vm_exitinfo(vm, vcpu); | ||||||||
| launched = 0; | launched = 0; | ||||||||
| KASSERT(vmxctx->pmap == pmap, | KASSERT(vmxctx->pmap == pmap, | ||||||||
| ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap)); | ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap)); | ||||||||
| vmx_msr_guest_enter(vmx, vcpu); | vmx_msr_guest_enter(vmx, vcpu); | ||||||||
| ▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Lines | if (vcpu_debugged(vm, vcpu)) { | ||||||||
| break; | break; | ||||||||
| } | } | ||||||||
| /* | /* | ||||||||
| * If TPR Shadowing is enabled, the TPR Threshold | * If TPR Shadowing is enabled, the TPR Threshold | ||||||||
| * must be updated right before entering the guest. | * must be updated right before entering the guest. | ||||||||
| */ | */ | ||||||||
| if (tpr_shadowing && !virtual_interrupt_delivery) { | if (tpr_shadowing && !virtual_interrupt_delivery) { | ||||||||
| if ((vmx->cap[vcpu].proc_ctls & PROCBASED_USE_TPR_SHADOW) != 0) { | if ((vmx_vcpu->cap.proc_ctls & PROCBASED_USE_TPR_SHADOW) != 0) { | ||||||||
| vmcs_write(VMCS_TPR_THRESHOLD, vlapic_get_cr8(vlapic)); | vmcs_write(VMCS_TPR_THRESHOLD, vlapic_get_cr8(vlapic)); | ||||||||
| } | } | ||||||||
| } | } | ||||||||
| /* | /* | ||||||||
| * VM exits restore the base address but not the | * VM exits restore the base address but not the | ||||||||
| * limits of GDTR and IDTR. The VMCS only stores the | * limits of GDTR and IDTR. The VMCS only stores the | ||||||||
| * base address, so VM exits set the limits to 0xffff. | * base address, so VM exits set the limits to 0xffff. | ||||||||
| ▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | do { | ||||||||
| /* Collect some information for VM exit processing */ | /* Collect some information for VM exit processing */ | ||||||||
| vmexit->rip = rip = vmcs_guest_rip(); | vmexit->rip = rip = vmcs_guest_rip(); | ||||||||
| vmexit->inst_length = vmexit_instruction_length(); | vmexit->inst_length = vmexit_instruction_length(); | ||||||||
| vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason(); | vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason(); | ||||||||
| vmexit->u.vmx.exit_qualification = vmcs_exit_qualification(); | vmexit->u.vmx.exit_qualification = vmcs_exit_qualification(); | ||||||||
| /* Update 'nextrip' */ | /* Update 'nextrip' */ | ||||||||
| vmx->state[vcpu].nextrip = rip; | vmx_vcpu->state.nextrip = rip; | ||||||||
| if (rc == VMX_GUEST_VMEXIT) { | if (rc == VMX_GUEST_VMEXIT) { | ||||||||
| vmx_exit_handle_nmi(vmx, vcpu, vmexit); | vmx_exit_handle_nmi(vmx, vcpu, vmexit); | ||||||||
| enable_intr(); | enable_intr(); | ||||||||
| handled = vmx_exit_process(vmx, vcpu, vmexit); | handled = vmx_exit_process(vmx, vcpu, vmexit); | ||||||||
| } else { | } else { | ||||||||
| enable_intr(); | enable_intr(); | ||||||||
| vmx_exit_inst_error(vmxctx, rc, vmexit); | vmx_exit_inst_error(vmxctx, rc, vmexit); | ||||||||
| Show All 21 Lines | vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap, | ||||||||
| return (0); | return (0); | ||||||||
| } | } | ||||||||
| static void | static void | ||||||||
| vmx_cleanup(void *arg) | vmx_cleanup(void *arg) | ||||||||
| { | { | ||||||||
| int i; | int i; | ||||||||
| struct vmx_vcpu *vcpu; | |||||||||
| struct vmx *vmx = arg; | struct vmx *vmx = arg; | ||||||||
| uint16_t maxcpus; | uint16_t maxcpus; | ||||||||
| if (apic_access_virtualization(vmx, 0)) | if (apic_access_virtualization(vmx, 0)) | ||||||||
| vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); | vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); | ||||||||
| maxcpus = vm_get_maxcpus(vmx->vm); | maxcpus = vm_get_maxcpus(vmx->vm); | ||||||||
| for (i = 0; i < maxcpus; i++) | for (i = 0; i < maxcpus; i++) { | ||||||||
| vpid_free(vmx->state[i].vpid); | vcpu = &vmx->vcpus[i]; | ||||||||
| vpid_free(vcpu->state.vpid); | |||||||||
| free(vcpu->pir_desc, M_VMX); | |||||||||
| free(vcpu->apic_page, M_VMX); | |||||||||
| free(vcpu->vmcs, M_VMX); | |||||||||
| } | |||||||||
| free(vmx->msr_bitmap, M_VMX); | |||||||||
| free(vmx, M_VMX); | free(vmx, M_VMX); | ||||||||
| return; | return; | ||||||||
| } | } | ||||||||
| static register_t * | static register_t * | ||||||||
| vmxctx_regptr(struct vmxctx *vmxctx, int reg) | vmxctx_regptr(struct vmxctx *vmxctx, int reg) | ||||||||
| { | { | ||||||||
| ▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | |||||||||
| } | } | ||||||||
| static int | static int | ||||||||
| vmx_get_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t *retval) | vmx_get_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t *retval) | ||||||||
| { | { | ||||||||
| uint64_t gi; | uint64_t gi; | ||||||||
| int error; | int error; | ||||||||
| error = vmcs_getreg(&vmx->vmcs[vcpu], running, | error = vmcs_getreg(vmx->vcpus[vcpu].vmcs, running, | ||||||||
| VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi); | VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi); | ||||||||
| *retval = (gi & HWINTR_BLOCKING) ? 1 : 0; | *retval = (gi & HWINTR_BLOCKING) ? 1 : 0; | ||||||||
| return (error); | return (error); | ||||||||
| } | } | ||||||||
| static int | static int | ||||||||
| vmx_modify_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t val) | vmx_modify_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t val) | ||||||||
| { | { | ||||||||
| struct vmcs *vmcs; | struct vmcs *vmcs; | ||||||||
| uint64_t gi; | uint64_t gi; | ||||||||
| int error, ident; | int error, ident; | ||||||||
| /* | /* | ||||||||
| * Forcing the vcpu into an interrupt shadow is not supported. | * Forcing the vcpu into an interrupt shadow is not supported. | ||||||||
| */ | */ | ||||||||
| if (val) { | if (val) { | ||||||||
| error = EINVAL; | error = EINVAL; | ||||||||
| goto done; | goto done; | ||||||||
| } | } | ||||||||
| vmcs = &vmx->vmcs[vcpu]; | vmcs = vmx->vcpus[vcpu].vmcs; | ||||||||
| ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY); | ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY); | ||||||||
| error = vmcs_getreg(vmcs, running, ident, &gi); | error = vmcs_getreg(vmcs, running, ident, &gi); | ||||||||
| if (error == 0) { | if (error == 0) { | ||||||||
| gi &= ~HWINTR_BLOCKING; | gi &= ~HWINTR_BLOCKING; | ||||||||
| error = vmcs_setreg(vmcs, running, ident, gi); | error = vmcs_setreg(vmcs, running, ident, gi); | ||||||||
| } | } | ||||||||
| done: | done: | ||||||||
| VCPU_CTR2(vmx->vm, vcpu, "Setting intr_shadow to %#lx %s", val, | VCPU_CTR2(vmx->vm, vcpu, "Setting intr_shadow to %#lx %s", val, | ||||||||
| Show All 30 Lines | vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval) | ||||||||
| running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); | running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); | ||||||||
| if (running && hostcpu != curcpu) | if (running && hostcpu != curcpu) | ||||||||
| panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu); | panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu); | ||||||||
| if (reg == VM_REG_GUEST_INTR_SHADOW) | if (reg == VM_REG_GUEST_INTR_SHADOW) | ||||||||
| return (vmx_get_intr_shadow(vmx, vcpu, running, retval)); | return (vmx_get_intr_shadow(vmx, vcpu, running, retval)); | ||||||||
| if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0) | if (vmxctx_getreg(&vmx->vcpus[vcpu].ctx, reg, retval) == 0) | ||||||||
| return (0); | return (0); | ||||||||
| return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval)); | return (vmcs_getreg(vmx->vcpus[vcpu].vmcs, running, reg, retval)); | ||||||||
| } | } | ||||||||
| static int | static int | ||||||||
| vmx_setreg(void *arg, int vcpu, int reg, uint64_t val) | vmx_setreg(void *arg, int vcpu, int reg, uint64_t val) | ||||||||
| { | { | ||||||||
| int error, hostcpu, running, shadow; | int error, hostcpu, running, shadow; | ||||||||
| uint64_t ctls; | uint64_t ctls; | ||||||||
| pmap_t pmap; | pmap_t pmap; | ||||||||
| struct vmx *vmx = arg; | struct vmx *vmx = arg; | ||||||||
| struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; | |||||||||
| running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); | running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); | ||||||||
| if (running && hostcpu != curcpu) | if (running && hostcpu != curcpu) | ||||||||
| panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu); | panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu); | ||||||||
| if (reg == VM_REG_GUEST_INTR_SHADOW) | if (reg == VM_REG_GUEST_INTR_SHADOW) | ||||||||
| return (vmx_modify_intr_shadow(vmx, vcpu, running, val)); | return (vmx_modify_intr_shadow(vmx, vcpu, running, val)); | ||||||||
| if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0) | if (vmxctx_setreg(&vmx_vcpu->ctx, reg, val) == 0) | ||||||||
| return (0); | return (0); | ||||||||
| /* Do not permit user write access to VMCS fields by offset. */ | /* Do not permit user write access to VMCS fields by offset. */ | ||||||||
| if (reg < 0) | if (reg < 0) | ||||||||
| return (EINVAL); | return (EINVAL); | ||||||||
| error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val); | error = vmcs_setreg(vmx_vcpu->vmcs, running, reg, val); | ||||||||
| if (error == 0) { | if (error == 0) { | ||||||||
| /* | /* | ||||||||
| * If the "load EFER" VM-entry control is 1 then the | * If the "load EFER" VM-entry control is 1 then the | ||||||||
| * value of EFER.LMA must be identical to "IA-32e mode guest" | * value of EFER.LMA must be identical to "IA-32e mode guest" | ||||||||
| * bit in the VM-entry control. | * bit in the VM-entry control. | ||||||||
| */ | */ | ||||||||
| if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 && | if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 && | ||||||||
| (reg == VM_REG_GUEST_EFER)) { | (reg == VM_REG_GUEST_EFER)) { | ||||||||
| vmcs_getreg(&vmx->vmcs[vcpu], running, | vmcs_getreg(vmx_vcpu->vmcs, running, | ||||||||
| VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls); | VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls); | ||||||||
| if (val & EFER_LMA) | if (val & EFER_LMA) | ||||||||
| ctls |= VM_ENTRY_GUEST_LMA; | ctls |= VM_ENTRY_GUEST_LMA; | ||||||||
| else | else | ||||||||
| ctls &= ~VM_ENTRY_GUEST_LMA; | ctls &= ~VM_ENTRY_GUEST_LMA; | ||||||||
| vmcs_setreg(&vmx->vmcs[vcpu], running, | vmcs_setreg(vmx_vcpu->vmcs, running, | ||||||||
| VMCS_IDENT(VMCS_ENTRY_CTLS), ctls); | VMCS_IDENT(VMCS_ENTRY_CTLS), ctls); | ||||||||
| } | } | ||||||||
| shadow = vmx_shadow_reg(reg); | shadow = vmx_shadow_reg(reg); | ||||||||
| if (shadow > 0) { | if (shadow > 0) { | ||||||||
| /* | /* | ||||||||
| * Store the unmodified value in the shadow | * Store the unmodified value in the shadow | ||||||||
| */ | */ | ||||||||
| error = vmcs_setreg(&vmx->vmcs[vcpu], running, | error = vmcs_setreg(vmx_vcpu->vmcs, running, | ||||||||
| VMCS_IDENT(shadow), val); | VMCS_IDENT(shadow), val); | ||||||||
| } | } | ||||||||
| if (reg == VM_REG_GUEST_CR3) { | if (reg == VM_REG_GUEST_CR3) { | ||||||||
| /* | /* | ||||||||
| * Invalidate the guest vcpu's TLB mappings to emulate | * Invalidate the guest vcpu's TLB mappings to emulate | ||||||||
| * the behavior of updating %cr3. | * the behavior of updating %cr3. | ||||||||
| * | * | ||||||||
| * XXX the processor retains global mappings when %cr3 | * XXX the processor retains global mappings when %cr3 | ||||||||
| * is updated but vmx_invvpid() does not. | * is updated but vmx_invvpid() does not. | ||||||||
| */ | */ | ||||||||
| pmap = vmx->ctx[vcpu].pmap; | pmap = vmx_vcpu->ctx.pmap; | ||||||||
| vmx_invvpid(vmx, vcpu, pmap, running); | vmx_invvpid(vmx, vcpu, pmap, running); | ||||||||
| } | } | ||||||||
| } | } | ||||||||
| return (error); | return (error); | ||||||||
| } | } | ||||||||
| static int | static int | ||||||||
| vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) | vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) | ||||||||
| { | { | ||||||||
| int hostcpu, running; | int hostcpu, running; | ||||||||
| struct vmx *vmx = arg; | struct vmx *vmx = arg; | ||||||||
| running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); | running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); | ||||||||
| if (running && hostcpu != curcpu) | if (running && hostcpu != curcpu) | ||||||||
| panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu); | panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu); | ||||||||
| return (vmcs_getdesc(&vmx->vmcs[vcpu], running, reg, desc)); | return (vmcs_getdesc(vmx->vcpus[vcpu].vmcs, running, reg, desc)); | ||||||||
| } | } | ||||||||
| static int | static int | ||||||||
| vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) | vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) | ||||||||
| { | { | ||||||||
| int hostcpu, running; | int hostcpu, running; | ||||||||
| struct vmx *vmx = arg; | struct vmx *vmx = arg; | ||||||||
| running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); | running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); | ||||||||
| if (running && hostcpu != curcpu) | if (running && hostcpu != curcpu) | ||||||||
| panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu); | panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu); | ||||||||
| return (vmcs_setdesc(&vmx->vmcs[vcpu], running, reg, desc)); | return (vmcs_setdesc(vmx->vcpus[vcpu].vmcs, running, reg, desc)); | ||||||||
| } | } | ||||||||
| static int | static int | ||||||||
| vmx_getcap(void *arg, int vcpu, int type, int *retval) | vmx_getcap(void *arg, int vcpu, int type, int *retval) | ||||||||
| { | { | ||||||||
| struct vmx *vmx = arg; | struct vmx *vmx = arg; | ||||||||
| int vcap; | int vcap; | ||||||||
| int ret; | int ret; | ||||||||
| ret = ENOENT; | ret = ENOENT; | ||||||||
| vcap = vmx->cap[vcpu].set; | vcap = vmx->vcpus[vcpu].cap.set; | ||||||||
| switch (type) { | switch (type) { | ||||||||
| case VM_CAP_HALT_EXIT: | case VM_CAP_HALT_EXIT: | ||||||||
| if (cap_halt_exit) | if (cap_halt_exit) | ||||||||
| ret = 0; | ret = 0; | ||||||||
| break; | break; | ||||||||
| case VM_CAP_PAUSE_EXIT: | case VM_CAP_PAUSE_EXIT: | ||||||||
| if (cap_pause_exit) | if (cap_pause_exit) | ||||||||
| Show All 32 Lines | vmx_getcap(void *arg, int vcpu, int type, int *retval) | ||||||||
| return (ret); | return (ret); | ||||||||
| } | } | ||||||||
| static int | static int | ||||||||
| vmx_setcap(void *arg, int vcpu, int type, int val) | vmx_setcap(void *arg, int vcpu, int type, int val) | ||||||||
| { | { | ||||||||
| struct vmx *vmx = arg; | struct vmx *vmx = arg; | ||||||||
| struct vmcs *vmcs = &vmx->vmcs[vcpu]; | struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; | ||||||||
| struct vmcs *vmcs = vmx_vcpu->vmcs; | |||||||||
| struct vlapic *vlapic; | struct vlapic *vlapic; | ||||||||
| uint32_t baseval; | uint32_t baseval; | ||||||||
| uint32_t *pptr; | uint32_t *pptr; | ||||||||
| int error; | int error; | ||||||||
| int flag; | int flag; | ||||||||
| int reg; | int reg; | ||||||||
| int retval; | int retval; | ||||||||
| retval = ENOENT; | retval = ENOENT; | ||||||||
| pptr = NULL; | pptr = NULL; | ||||||||
| switch (type) { | switch (type) { | ||||||||
| case VM_CAP_HALT_EXIT: | case VM_CAP_HALT_EXIT: | ||||||||
| if (cap_halt_exit) { | if (cap_halt_exit) { | ||||||||
| retval = 0; | retval = 0; | ||||||||
| pptr = &vmx->cap[vcpu].proc_ctls; | pptr = &vmx_vcpu->cap.proc_ctls; | ||||||||
| baseval = *pptr; | baseval = *pptr; | ||||||||
| flag = PROCBASED_HLT_EXITING; | flag = PROCBASED_HLT_EXITING; | ||||||||
| reg = VMCS_PRI_PROC_BASED_CTLS; | reg = VMCS_PRI_PROC_BASED_CTLS; | ||||||||
| } | } | ||||||||
| break; | break; | ||||||||
| case VM_CAP_MTRAP_EXIT: | case VM_CAP_MTRAP_EXIT: | ||||||||
| if (cap_monitor_trap) { | if (cap_monitor_trap) { | ||||||||
| retval = 0; | retval = 0; | ||||||||
| pptr = &vmx->cap[vcpu].proc_ctls; | pptr = &vmx_vcpu->cap.proc_ctls; | ||||||||
| baseval = *pptr; | baseval = *pptr; | ||||||||
| flag = PROCBASED_MTF; | flag = PROCBASED_MTF; | ||||||||
| reg = VMCS_PRI_PROC_BASED_CTLS; | reg = VMCS_PRI_PROC_BASED_CTLS; | ||||||||
| } | } | ||||||||
| break; | break; | ||||||||
| case VM_CAP_PAUSE_EXIT: | case VM_CAP_PAUSE_EXIT: | ||||||||
| if (cap_pause_exit) { | if (cap_pause_exit) { | ||||||||
| retval = 0; | retval = 0; | ||||||||
| pptr = &vmx->cap[vcpu].proc_ctls; | pptr = &vmx_vcpu->cap.proc_ctls; | ||||||||
| baseval = *pptr; | baseval = *pptr; | ||||||||
| flag = PROCBASED_PAUSE_EXITING; | flag = PROCBASED_PAUSE_EXITING; | ||||||||
| reg = VMCS_PRI_PROC_BASED_CTLS; | reg = VMCS_PRI_PROC_BASED_CTLS; | ||||||||
| } | } | ||||||||
| break; | break; | ||||||||
| case VM_CAP_RDPID: | case VM_CAP_RDPID: | ||||||||
| case VM_CAP_RDTSCP: | case VM_CAP_RDTSCP: | ||||||||
| if (cap_rdpid || cap_rdtscp) | if (cap_rdpid || cap_rdtscp) | ||||||||
| /* | /* | ||||||||
| * Choose not to support enabling/disabling | * Choose not to support enabling/disabling | ||||||||
| * RDPID/RDTSCP via libvmmapi since, as per the | * RDPID/RDTSCP via libvmmapi since, as per the | ||||||||
| * discussion in vmx_modinit(), RDPID/RDTSCP are | * discussion in vmx_modinit(), RDPID/RDTSCP are | ||||||||
| * either always enabled or always disabled. | * either always enabled or always disabled. | ||||||||
| */ | */ | ||||||||
| error = EOPNOTSUPP; | error = EOPNOTSUPP; | ||||||||
| break; | break; | ||||||||
| case VM_CAP_UNRESTRICTED_GUEST: | case VM_CAP_UNRESTRICTED_GUEST: | ||||||||
| if (cap_unrestricted_guest) { | if (cap_unrestricted_guest) { | ||||||||
| retval = 0; | retval = 0; | ||||||||
| pptr = &vmx->cap[vcpu].proc_ctls2; | pptr = &vmx_vcpu->cap.proc_ctls2; | ||||||||
| baseval = *pptr; | baseval = *pptr; | ||||||||
| flag = PROCBASED2_UNRESTRICTED_GUEST; | flag = PROCBASED2_UNRESTRICTED_GUEST; | ||||||||
| reg = VMCS_SEC_PROC_BASED_CTLS; | reg = VMCS_SEC_PROC_BASED_CTLS; | ||||||||
| } | } | ||||||||
| break; | break; | ||||||||
| case VM_CAP_ENABLE_INVPCID: | case VM_CAP_ENABLE_INVPCID: | ||||||||
| if (cap_invpcid) { | if (cap_invpcid) { | ||||||||
| retval = 0; | retval = 0; | ||||||||
| pptr = &vmx->cap[vcpu].proc_ctls2; | pptr = &vmx_vcpu->cap.proc_ctls2; | ||||||||
| baseval = *pptr; | baseval = *pptr; | ||||||||
| flag = PROCBASED2_ENABLE_INVPCID; | flag = PROCBASED2_ENABLE_INVPCID; | ||||||||
| reg = VMCS_SEC_PROC_BASED_CTLS; | reg = VMCS_SEC_PROC_BASED_CTLS; | ||||||||
| } | } | ||||||||
| break; | break; | ||||||||
| case VM_CAP_BPT_EXIT: | case VM_CAP_BPT_EXIT: | ||||||||
| retval = 0; | retval = 0; | ||||||||
| /* Don't change the bitmap if we are tracing all exceptions. */ | /* Don't change the bitmap if we are tracing all exceptions. */ | ||||||||
| if (vmx->cap[vcpu].exc_bitmap != 0xffffffff) { | if (vmx_vcpu->cap.exc_bitmap != 0xffffffff) { | ||||||||
| pptr = &vmx->cap[vcpu].exc_bitmap; | pptr = &vmx_vcpu->cap.exc_bitmap; | ||||||||
| baseval = *pptr; | baseval = *pptr; | ||||||||
| flag = (1 << IDT_BP); | flag = (1 << IDT_BP); | ||||||||
| reg = VMCS_EXCEPTION_BITMAP; | reg = VMCS_EXCEPTION_BITMAP; | ||||||||
| } | } | ||||||||
| break; | break; | ||||||||
| case VM_CAP_IPI_EXIT: | case VM_CAP_IPI_EXIT: | ||||||||
| retval = 0; | retval = 0; | ||||||||
| Show All 23 Lines | if (pptr != NULL) { | ||||||||
| /* | /* | ||||||||
| * Update optional stored flags, and record | * Update optional stored flags, and record | ||||||||
| * setting | * setting | ||||||||
| */ | */ | ||||||||
| *pptr = baseval; | *pptr = baseval; | ||||||||
| } | } | ||||||||
| if (val) { | if (val) { | ||||||||
| vmx->cap[vcpu].set |= (1 << type); | vmx_vcpu->cap.set |= (1 << type); | ||||||||
| } else { | } else { | ||||||||
| vmx->cap[vcpu].set &= ~(1 << type); | vmx_vcpu->cap.set &= ~(1 << type); | ||||||||
| } | } | ||||||||
| return (0); | return (0); | ||||||||
| } | } | ||||||||
| static struct vmspace * | static struct vmspace * | ||||||||
| vmx_vmspace_alloc(vm_offset_t min, vm_offset_t max) | vmx_vmspace_alloc(vm_offset_t min, vm_offset_t max) | ||||||||
| { | { | ||||||||
| ▲ Show 20 Lines • Show All 188 Lines • ▼ Show 20 Lines | vmx_set_tmr(struct vlapic *vlapic, int vector, bool level) | ||||||||
| uint64_t mask, val; | uint64_t mask, val; | ||||||||
| KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector)); | KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector)); | ||||||||
| KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL), | KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL), | ||||||||
| ("vmx_set_tmr: vcpu cannot be running")); | ("vmx_set_tmr: vcpu cannot be running")); | ||||||||
| vlapic_vtx = (struct vlapic_vtx *)vlapic; | vlapic_vtx = (struct vlapic_vtx *)vlapic; | ||||||||
| vmx = vlapic_vtx->vmx; | vmx = vlapic_vtx->vmx; | ||||||||
| vmcs = &vmx->vmcs[vlapic->vcpuid]; | vmcs = vmx->vcpus[vlapic->vcpuid].vmcs; | ||||||||
| mask = 1UL << (vector % 64); | mask = 1UL << (vector % 64); | ||||||||
| VMPTRLD(vmcs); | VMPTRLD(vmcs); | ||||||||
| val = vmcs_read(VMCS_EOI_EXIT(vector)); | val = vmcs_read(VMCS_EOI_EXIT(vector)); | ||||||||
| if (level) | if (level) | ||||||||
| val |= mask; | val |= mask; | ||||||||
| else | else | ||||||||
| val &= ~mask; | val &= ~mask; | ||||||||
| vmcs_write(VMCS_EOI_EXIT(vector), val); | vmcs_write(VMCS_EOI_EXIT(vector), val); | ||||||||
| VMCLEAR(vmcs); | VMCLEAR(vmcs); | ||||||||
| } | } | ||||||||
| static void | static void | ||||||||
| vmx_enable_x2apic_mode_ts(struct vlapic *vlapic) | vmx_enable_x2apic_mode_ts(struct vlapic *vlapic) | ||||||||
| { | { | ||||||||
| struct vmx *vmx; | struct vmx *vmx; | ||||||||
| struct vmx_vcpu *vcpu; | |||||||||
Not Done Inline ActionsNit: most other functions call the variable vmx_vcpu. corvink: Nit: most other functions call the variable `vmx_vcpu`. | |||||||||
Done Inline ActionsBy the end of the series these are all named vcpu both here and in svm.c. At this stage of the series I used vmx_vcpu when there was already a local parameter int vcpu or the like that at a later stage became either void *vcpui or struct vmx_vcpu *vcpu. jhb: By the end of the series these are all named `vcpu` both here and in svm.c. At this stage of… | |||||||||
| struct vmcs *vmcs; | struct vmcs *vmcs; | ||||||||
| uint32_t proc_ctls; | uint32_t proc_ctls; | ||||||||
| int vcpuid; | int vcpuid; | ||||||||
| vcpuid = vlapic->vcpuid; | vcpuid = vlapic->vcpuid; | ||||||||
| vmx = ((struct vlapic_vtx *)vlapic)->vmx; | vmx = ((struct vlapic_vtx *)vlapic)->vmx; | ||||||||
| vmcs = &vmx->vmcs[vcpuid]; | vcpu = &vmx->vcpus[vcpuid]; | ||||||||
| vmcs = vcpu->vmcs; | |||||||||
| proc_ctls = vmx->cap[vcpuid].proc_ctls; | proc_ctls = vcpu->cap.proc_ctls; | ||||||||
| proc_ctls &= ~PROCBASED_USE_TPR_SHADOW; | proc_ctls &= ~PROCBASED_USE_TPR_SHADOW; | ||||||||
| proc_ctls |= PROCBASED_CR8_LOAD_EXITING; | proc_ctls |= PROCBASED_CR8_LOAD_EXITING; | ||||||||
| proc_ctls |= PROCBASED_CR8_STORE_EXITING; | proc_ctls |= PROCBASED_CR8_STORE_EXITING; | ||||||||
| vmx->cap[vcpuid].proc_ctls = proc_ctls; | vcpu->cap.proc_ctls = proc_ctls; | ||||||||
| VMPTRLD(vmcs); | VMPTRLD(vmcs); | ||||||||
| vmcs_write(VMCS_PRI_PROC_BASED_CTLS, proc_ctls); | vmcs_write(VMCS_PRI_PROC_BASED_CTLS, proc_ctls); | ||||||||
| VMCLEAR(vmcs); | VMCLEAR(vmcs); | ||||||||
| } | } | ||||||||
| static void | static void | ||||||||
| vmx_enable_x2apic_mode_vid(struct vlapic *vlapic) | vmx_enable_x2apic_mode_vid(struct vlapic *vlapic) | ||||||||
| { | { | ||||||||
| struct vmx *vmx; | struct vmx *vmx; | ||||||||
| struct vmx_vcpu *vcpu; | |||||||||
Not Done Inline ActionsNit: most other functions call it vmx_vcpu. corvink: Nit: most other functions call it `vmx_vcpu`. | |||||||||
| struct vmcs *vmcs; | struct vmcs *vmcs; | ||||||||
| uint32_t proc_ctls2; | uint32_t proc_ctls2; | ||||||||
| int vcpuid, error __diagused; | int vcpuid, error __diagused; | ||||||||
| vcpuid = vlapic->vcpuid; | vcpuid = vlapic->vcpuid; | ||||||||
| vmx = ((struct vlapic_vtx *)vlapic)->vmx; | vmx = ((struct vlapic_vtx *)vlapic)->vmx; | ||||||||
| vmcs = &vmx->vmcs[vcpuid]; | vcpu = &vmx->vcpus[vcpuid]; | ||||||||
| vmcs = vcpu->vmcs; | |||||||||
| proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; | proc_ctls2 = vcpu->cap.proc_ctls2; | ||||||||
| KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0, | KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0, | ||||||||
| ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2)); | ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2)); | ||||||||
| proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES; | proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES; | ||||||||
| proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE; | proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE; | ||||||||
| vmx->cap[vcpuid].proc_ctls2 = proc_ctls2; | vcpu->cap.proc_ctls2 = proc_ctls2; | ||||||||
| VMPTRLD(vmcs); | VMPTRLD(vmcs); | ||||||||
| vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2); | vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2); | ||||||||
| VMCLEAR(vmcs); | VMCLEAR(vmcs); | ||||||||
| if (vlapic->vcpuid == 0) { | if (vlapic->vcpuid == 0) { | ||||||||
| /* | /* | ||||||||
| * The nested page table mappings are shared by all vcpus | * The nested page table mappings are shared by all vcpus | ||||||||
| ▲ Show 20 Lines • Show All 121 Lines • ▼ Show 20 Lines | vmx_vlapic_init(void *arg, int vcpuid) | ||||||||
| struct vlapic *vlapic; | struct vlapic *vlapic; | ||||||||
| struct vlapic_vtx *vlapic_vtx; | struct vlapic_vtx *vlapic_vtx; | ||||||||
| vmx = arg; | vmx = arg; | ||||||||
| vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO); | vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO); | ||||||||
| vlapic->vm = vmx->vm; | vlapic->vm = vmx->vm; | ||||||||
| vlapic->vcpuid = vcpuid; | vlapic->vcpuid = vcpuid; | ||||||||
| vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid]; | vlapic->apic_page = (struct LAPIC *)vmx->vcpus[vcpuid].apic_page; | ||||||||
| vlapic_vtx = (struct vlapic_vtx *)vlapic; | vlapic_vtx = (struct vlapic_vtx *)vlapic; | ||||||||
| vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid]; | vlapic_vtx->pir_desc = vmx->vcpus[vcpuid].pir_desc; | ||||||||
| vlapic_vtx->vmx = vmx; | vlapic_vtx->vmx = vmx; | ||||||||
| if (tpr_shadowing) { | if (tpr_shadowing) { | ||||||||
| vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_ts; | vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_ts; | ||||||||
| } | } | ||||||||
| if (virtual_interrupt_delivery) { | if (virtual_interrupt_delivery) { | ||||||||
| vlapic->ops.set_intr_ready = vmx_set_intr_ready; | vlapic->ops.set_intr_ready = vmx_set_intr_ready; | ||||||||
| Show All 19 Lines | vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic) | ||||||||
| free(vlapic, M_VLAPIC); | free(vlapic, M_VLAPIC); | ||||||||
| } | } | ||||||||
| #ifdef BHYVE_SNAPSHOT | #ifdef BHYVE_SNAPSHOT | ||||||||
| static int | static int | ||||||||
| vmx_snapshot(void *arg, struct vm_snapshot_meta *meta) | vmx_snapshot(void *arg, struct vm_snapshot_meta *meta) | ||||||||
| { | { | ||||||||
| struct vmx *vmx; | struct vmx *vmx; | ||||||||
| struct vmx_vcpu *vcpu; | |||||||||
Not Done Inline ActionsNit: most other functions call it vmx_vcpu. corvink: Nit: most other functions call it `vmx_vcpu`. | |||||||||
| struct vmxctx *vmxctx; | struct vmxctx *vmxctx; | ||||||||
| int ret; | int ret; | ||||||||
| uint16_t i, maxcpus; | uint16_t i, maxcpus; | ||||||||
| vmx = arg; | vmx = arg; | ||||||||
| KASSERT(vmx != NULL, ("%s: arg was NULL", __func__)); | KASSERT(vmx != NULL, ("%s: arg was NULL", __func__)); | ||||||||
| maxcpus = vm_get_maxcpus(vmx->vm); | maxcpus = vm_get_maxcpus(vmx->vm); | ||||||||
| for (i = 0; i < maxcpus; i++) { | for (i = 0; i < maxcpus; i++) { | ||||||||
| SNAPSHOT_BUF_OR_LEAVE(vmx->guest_msrs[i], | vcpu = &vmx->vcpus[i]; | ||||||||
| sizeof(vmx->guest_msrs[i]), meta, ret, done); | |||||||||
| vmxctx = &vmx->ctx[i]; | SNAPSHOT_BUF_OR_LEAVE(vcpu->guest_msrs, | ||||||||
| sizeof(vcpu->guest_msrs), meta, ret, done); | |||||||||
| vmxctx = &vcpu->ctx; | |||||||||
| SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdi, meta, ret, done); | SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdi, meta, ret, done); | ||||||||
| SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rsi, meta, ret, done); | SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rsi, meta, ret, done); | ||||||||
| SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdx, meta, ret, done); | SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdx, meta, ret, done); | ||||||||
| SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rcx, meta, ret, done); | SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rcx, meta, ret, done); | ||||||||
| SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r8, meta, ret, done); | SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r8, meta, ret, done); | ||||||||
| SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r9, meta, ret, done); | SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r9, meta, ret, done); | ||||||||
| SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rax, meta, ret, done); | SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rax, meta, ret, done); | ||||||||
| SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbx, meta, ret, done); | SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbx, meta, ret, done); | ||||||||
| Show All 22 Lines | vmx_vmcx_snapshot(void *arg, struct vm_snapshot_meta *meta, int vcpu) | ||||||||
| struct vmcs *vmcs; | struct vmcs *vmcs; | ||||||||
| struct vmx *vmx; | struct vmx *vmx; | ||||||||
| int err, run, hostcpu; | int err, run, hostcpu; | ||||||||
| vmx = (struct vmx *)arg; | vmx = (struct vmx *)arg; | ||||||||
| err = 0; | err = 0; | ||||||||
| KASSERT(arg != NULL, ("%s: arg was NULL", __func__)); | KASSERT(arg != NULL, ("%s: arg was NULL", __func__)); | ||||||||
| vmcs = &vmx->vmcs[vcpu]; | vmcs = vmx->vcpus[vcpu].vmcs; | ||||||||
| run = vcpu_is_running(vmx->vm, vcpu, &hostcpu); | run = vcpu_is_running(vmx->vm, vcpu, &hostcpu); | ||||||||
| if (run && hostcpu != curcpu) { | if (run && hostcpu != curcpu) { | ||||||||
| printf("%s: %s%d is running", __func__, vm_name(vmx->vm), vcpu); | printf("%s: %s%d is running", __func__, vm_name(vmx->vm), vcpu); | ||||||||
| return (EINVAL); | return (EINVAL); | ||||||||
| } | } | ||||||||
| err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR0, meta); | err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR0, meta); | ||||||||
| ▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines | |||||||||
| static int | static int | ||||||||
| vmx_restore_tsc(void *arg, int vcpu, uint64_t offset) | vmx_restore_tsc(void *arg, int vcpu, uint64_t offset) | ||||||||
| { | { | ||||||||
| struct vmcs *vmcs; | struct vmcs *vmcs; | ||||||||
| struct vmx *vmx = (struct vmx *)arg; | struct vmx *vmx = (struct vmx *)arg; | ||||||||
| int error, running, hostcpu; | int error, running, hostcpu; | ||||||||
| KASSERT(arg != NULL, ("%s: arg was NULL", __func__)); | KASSERT(arg != NULL, ("%s: arg was NULL", __func__)); | ||||||||
| vmcs = &vmx->vmcs[vcpu]; | vmcs = vmx->vcpus[vcpu].vmcs; | ||||||||
| running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); | running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); | ||||||||
| if (running && hostcpu != curcpu) { | if (running && hostcpu != curcpu) { | ||||||||
| printf("%s: %s%d is running", __func__, vm_name(vmx->vm), vcpu); | printf("%s: %s%d is running", __func__, vm_name(vmx->vm), vcpu); | ||||||||
| return (EINVAL); | return (EINVAL); | ||||||||
| } | } | ||||||||
| if (!running) | if (!running) | ||||||||
| Show All 33 Lines | |||||||||
Shouldn't this use malloc_aligned() as well? If only to document the alignment requirement.