diff --git a/sys/amd64/include/vmm.h b/sys/amd64/include/vmm.h --- a/sys/amd64/include/vmm.h +++ b/sys/amd64/include/vmm.h @@ -167,27 +167,29 @@ typedef int (*vmm_cleanup_func_t)(void); typedef void (*vmm_resume_func_t)(void); typedef void * (*vmi_init_func_t)(struct vm *vm, struct pmap *pmap); -typedef int (*vmi_run_func_t)(void *vmi, int vcpu, register_t rip, +typedef int (*vmi_run_func_t)(void *vmi, void *vcpui, register_t rip, struct pmap *pmap, struct vm_eventinfo *info); typedef void (*vmi_cleanup_func_t)(void *vmi); -typedef int (*vmi_get_register_t)(void *vmi, int vcpu, int num, +typedef void * (*vmi_vcpu_init_func_t)(void *vmi, int vcpu_id); +typedef void (*vmi_vcpu_cleanup_func_t)(void *vmi, void *vcpui); +typedef int (*vmi_get_register_t)(void *vmi, void *vcpui, int num, uint64_t *retval); -typedef int (*vmi_set_register_t)(void *vmi, int vcpu, int num, +typedef int (*vmi_set_register_t)(void *vmi, void *vcpui, int num, uint64_t val); -typedef int (*vmi_get_desc_t)(void *vmi, int vcpu, int num, +typedef int (*vmi_get_desc_t)(void *vmi, void *vcpui, int num, struct seg_desc *desc); -typedef int (*vmi_set_desc_t)(void *vmi, int vcpu, int num, +typedef int (*vmi_set_desc_t)(void *vmi, void *vcpui, int num, struct seg_desc *desc); -typedef int (*vmi_get_cap_t)(void *vmi, int vcpu, int num, int *retval); -typedef int (*vmi_set_cap_t)(void *vmi, int vcpu, int num, int val); +typedef int (*vmi_get_cap_t)(void *vmi, void *vcpui, int num, int *retval); +typedef int (*vmi_set_cap_t)(void *vmi, void *vcpui, int num, int val); typedef struct vmspace * (*vmi_vmspace_alloc)(vm_offset_t min, vm_offset_t max); typedef void (*vmi_vmspace_free)(struct vmspace *vmspace); -typedef struct vlapic * (*vmi_vlapic_init)(void *vmi, int vcpu); +typedef struct vlapic * (*vmi_vlapic_init)(void *vmi, void *vcpui); typedef void (*vmi_vlapic_cleanup)(void *vmi, struct vlapic *vlapic); typedef int (*vmi_snapshot_t)(void *vmi, struct vm_snapshot_meta *meta); typedef int (*vmi_snapshot_vcpu_t)(void *vmi, struct vm_snapshot_meta *meta, - int vcpu); -typedef int (*vmi_restore_tsc_t)(void *vmi, int vcpuid, uint64_t now); + void *vcpui); +typedef int (*vmi_restore_tsc_t)(void *vmi, void *vcpui, uint64_t now); struct vmm_ops { vmm_init_func_t modinit; /* module wide initialization */ @@ -197,6 +199,8 @@ vmi_init_func_t init; /* vm-specific initialization */ vmi_run_func_t run; vmi_cleanup_func_t cleanup; + vmi_vcpu_init_func_t vcpu_init; + vmi_vcpu_cleanup_func_t vcpu_cleanup; vmi_get_register_t getreg; vmi_set_register_t setreg; vmi_get_desc_t getdesc; diff --git a/sys/amd64/vmm/amd/svm.h b/sys/amd64/vmm/amd/svm.h --- a/sys/amd64/vmm/amd/svm.h +++ b/sys/amd64/vmm/amd/svm.h @@ -33,6 +33,7 @@ struct pcpu; struct svm_softc; +struct svm_vcpu; /* * Guest register state that is saved outside the VMCB. @@ -68,7 +69,8 @@ void svm_launch(uint64_t pa, struct svm_regctx *gctx, struct pcpu *pcpu); #ifdef BHYVE_SNAPSHOT -int svm_set_tsc_offset(struct svm_softc *sc, int vcpu, uint64_t offset); +int svm_set_tsc_offset(struct svm_softc *sc, struct svm_vcpu *vcpu, + uint64_t offset); #endif #endif /* _SVM_H_ */ diff --git a/sys/amd64/vmm/amd/svm.c b/sys/amd64/vmm/amd/svm.c --- a/sys/amd64/vmm/amd/svm.c +++ b/sys/amd64/vmm/amd/svm.c @@ -132,8 +132,8 @@ static VMM_STAT_AMD(VCPU_INTINFO_INJECTED, "Events pending at VM entry"); static VMM_STAT_AMD(VMEXIT_VINTR, "VM exits due to interrupt window"); -static int svm_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc); -static int svm_setreg(void *arg, int vcpu, int ident, uint64_t val); +static int svm_getdesc(void *arg, void *vcpui, int reg, struct seg_desc *desc); +static int svm_setreg(void *arg, void *vcpui, int ident, uint64_t val); static __inline int flush_by_asid(void) @@ -283,18 +283,18 @@ #ifdef BHYVE_SNAPSHOT int -svm_set_tsc_offset(struct svm_softc *sc, int vcpu, uint64_t offset) +svm_set_tsc_offset(struct svm_softc *sc, struct svm_vcpu *vcpu, uint64_t offset) { int error; struct vmcb_ctrl *ctrl; - ctrl = svm_get_vmcb_ctrl(sc, vcpu); + ctrl = svm_get_vmcb_ctrl(vcpu); ctrl->tsc_offset = offset; - svm_set_dirty(sc, vcpu, VMCB_CACHE_I); + svm_set_dirty(vcpu, VMCB_CACHE_I); VCPU_CTR1(sc->vm, vcpu, "tsc offset changed to %#lx", offset); - error = vm_set_tsc_offset(sc->vm, vcpu, offset); + error = vm_set_tsc_offset(sc->vm, vcpu->vcpuid, offset); return (error); } @@ -382,26 +382,27 @@ } static __inline int -svm_get_intercept(struct svm_softc *sc, int vcpu, int idx, uint32_t bitmask) +svm_get_intercept(struct svm_softc *sc, struct svm_vcpu *vcpu, int idx, + uint32_t bitmask) { struct vmcb_ctrl *ctrl; KASSERT(idx >=0 && idx < 5, ("invalid intercept index %d", idx)); - ctrl = svm_get_vmcb_ctrl(sc, vcpu); + ctrl = svm_get_vmcb_ctrl(vcpu); return (ctrl->intercept[idx] & bitmask ? 1 : 0); } static __inline void -svm_set_intercept(struct svm_softc *sc, int vcpu, int idx, uint32_t bitmask, - int enabled) +svm_set_intercept(struct svm_softc *sc, struct svm_vcpu *vcpu, int idx, + uint32_t bitmask, int enabled) { struct vmcb_ctrl *ctrl; uint32_t oldval; KASSERT(idx >=0 && idx < 5, ("invalid intercept index %d", idx)); - ctrl = svm_get_vmcb_ctrl(sc, vcpu); + ctrl = svm_get_vmcb_ctrl(vcpu); oldval = ctrl->intercept[idx]; if (enabled) @@ -410,28 +411,30 @@ ctrl->intercept[idx] &= ~bitmask; if (ctrl->intercept[idx] != oldval) { - svm_set_dirty(sc, vcpu, VMCB_CACHE_I); - VCPU_CTR3(sc->vm, vcpu, "intercept[%d] modified " + svm_set_dirty(vcpu, VMCB_CACHE_I); + VCPU_CTR3(sc->vm, vcpu->vcpuid, "intercept[%d] modified " "from %#x to %#x", idx, oldval, ctrl->intercept[idx]); } } static __inline void -svm_disable_intercept(struct svm_softc *sc, int vcpu, int off, uint32_t bitmask) +svm_disable_intercept(struct svm_softc *sc, struct svm_vcpu *vcpu, int off, + uint32_t bitmask) { svm_set_intercept(sc, vcpu, off, bitmask, 0); } static __inline void -svm_enable_intercept(struct svm_softc *sc, int vcpu, int off, uint32_t bitmask) +svm_enable_intercept(struct svm_softc *sc, struct svm_vcpu *vcpu, int off, + uint32_t bitmask) { svm_set_intercept(sc, vcpu, off, bitmask, 1); } static void -vmcb_init(struct svm_softc *sc, int vcpu, uint64_t iopm_base_pa, +vmcb_init(struct svm_softc *sc, struct svm_vcpu *vcpu, uint64_t iopm_base_pa, uint64_t msrpm_base_pa, uint64_t np_pml4) { struct vmcb_ctrl *ctrl; @@ -439,8 +442,8 @@ uint32_t mask; int n; - ctrl = svm_get_vmcb_ctrl(sc, vcpu); - state = svm_get_vmcb_state(sc, vcpu); + ctrl = svm_get_vmcb_ctrl(vcpu); + state = svm_get_vmcb_state(vcpu); ctrl->iopm_base_pa = iopm_base_pa; ctrl->msrpm_base_pa = msrpm_base_pa; @@ -465,7 +468,7 @@ * Intercept everything when tracing guest exceptions otherwise * just intercept machine check exception. */ - if (vcpu_trace_exceptions(sc->vm, vcpu)) { + if (vcpu_trace_exceptions(sc->vm, vcpu->vcpuid)) { for (n = 0; n < 32; n++) { /* * Skip unimplemented vectors in the exception bitmap. @@ -506,7 +509,7 @@ svm_enable_intercept(sc, vcpu, VMCB_CTRL2_INTCPT, VMCB_INTCPT_CLGI); svm_enable_intercept(sc, vcpu, VMCB_CTRL2_INTCPT, VMCB_INTCPT_SKINIT); svm_enable_intercept(sc, vcpu, VMCB_CTRL2_INTCPT, VMCB_INTCPT_ICEBP); - if (vcpu_trap_wbinvd(sc->vm, vcpu)) { + if (vcpu_trap_wbinvd(sc->vm, vcpu->vcpuid)) { svm_enable_intercept(sc, vcpu, VMCB_CTRL2_INTCPT, VMCB_INTCPT_WBINVD); } @@ -559,10 +562,6 @@ svm_init(struct vm *vm, pmap_t pmap) { struct svm_softc *svm_sc; - struct svm_vcpu *vcpu; - vm_paddr_t msrpm_pa, iopm_pa, pml4_pa; - int i; - uint16_t maxcpus; svm_sc = malloc(sizeof (*svm_sc), M_SVM, M_WAITOK | M_ZERO); @@ -576,7 +575,7 @@ panic("contigmalloc of SVM IO bitmap failed"); svm_sc->vm = vm; - svm_sc->nptp = (vm_offset_t)vtophys(pmap->pm_pmltop); + svm_sc->nptp = vtophys(pmap->pm_pmltop); /* * Intercept read and write accesses to all MSRs. @@ -611,22 +610,27 @@ /* Intercept access to all I/O ports. */ memset(svm_sc->iopm_bitmap, 0xFF, SVM_IO_BITMAP_SIZE); - iopm_pa = vtophys(svm_sc->iopm_bitmap); - msrpm_pa = vtophys(svm_sc->msr_bitmap); - pml4_pa = svm_sc->nptp; - maxcpus = vm_get_maxcpus(svm_sc->vm); - for (i = 0; i < maxcpus; i++) { - vcpu = svm_get_vcpu(svm_sc, i); - vcpu->vmcb = malloc(sizeof(struct vmcb), M_SVM, M_WAITOK | M_ZERO); - vcpu->nextrip = ~0; - vcpu->lastcpu = NOCPU; - vcpu->vmcb_pa = vtophys(vcpu->vmcb); - vmcb_init(svm_sc, i, iopm_pa, msrpm_pa, pml4_pa); - svm_msr_guest_init(svm_sc, i); - } return (svm_sc); } +static void * +svm_vcpu_init(void *arg, int vcpuid) +{ + struct svm_softc *sc = arg; + struct svm_vcpu *vcpu; + + vcpu = malloc(sizeof(*vcpu), M_SVM, M_WAITOK | M_ZERO); + vcpu->vcpuid = vcpuid; + vcpu->vmcb = malloc(sizeof(struct vmcb), M_SVM, M_WAITOK | M_ZERO); + vcpu->nextrip = ~0; + vcpu->lastcpu = NOCPU; + vcpu->vmcb_pa = vtophys(vcpu->vmcb); + vmcb_init(sc, vcpu, vtophys(sc->iopm_bitmap), vtophys(sc->msr_bitmap), + sc->nptp); + svm_msr_guest_init(sc, vcpu); + return (vcpu); +} + /* * Collateral for a generic SVM VM-exit. */ @@ -719,8 +723,8 @@ } static void -svm_inout_str_seginfo(struct svm_softc *svm_sc, int vcpu, int64_t info1, - int in, struct vm_inout_str *vis) +svm_inout_str_seginfo(struct svm_softc *svm_sc, struct svm_vcpu *vcpu, + int64_t info1, int in, struct vm_inout_str *vis) { int error __diagused, s; @@ -773,7 +777,8 @@ * Handle guest I/O intercept. */ static int -svm_handle_io(struct svm_softc *svm_sc, int vcpu, struct vm_exit *vmexit) +svm_handle_io(struct svm_softc *svm_sc, struct svm_vcpu *vcpu, + struct vm_exit *vmexit) { struct vmcb_ctrl *ctrl; struct vmcb_state *state; @@ -782,9 +787,9 @@ uint64_t info1; int inout_string; - state = svm_get_vmcb_state(svm_sc, vcpu); - ctrl = svm_get_vmcb_ctrl(svm_sc, vcpu); - regs = svm_get_guest_regctx(svm_sc, vcpu); + state = svm_get_vmcb_state(vcpu); + ctrl = svm_get_vmcb_ctrl(vcpu); + regs = svm_get_guest_regctx(vcpu); info1 = ctrl->exitinfo1; inout_string = info1 & BIT(2) ? 1 : 0; @@ -810,7 +815,7 @@ if (inout_string) { vmexit->exitcode = VM_EXITCODE_INOUT_STR; vis = &vmexit->u.inout_str; - svm_paging_info(svm_get_vmcb(svm_sc, vcpu), &vis->paging); + svm_paging_info(svm_get_vmcb(vcpu), &vis->paging); vis->rflags = state->rflags; vis->cr0 = state->cr0; vis->index = svm_inout_str_index(regs, vmexit->u.inout.in); @@ -931,12 +936,12 @@ * Inject an event to vcpu as described in section 15.20, "Event injection". */ static void -svm_eventinject(struct svm_softc *sc, int vcpu, int intr_type, int vector, - uint32_t error, bool ec_valid) +svm_eventinject(struct svm_softc *sc, struct svm_vcpu *vcpu, int intr_type, + int vector, uint32_t error, bool ec_valid) { struct vmcb_ctrl *ctrl; - ctrl = svm_get_vmcb_ctrl(sc, vcpu); + ctrl = svm_get_vmcb_ctrl(vcpu); KASSERT((ctrl->eventinj & VMCB_EVENTINJ_VALID) == 0, ("%s: event already pending %#lx", __func__, ctrl->eventinj)); @@ -961,24 +966,25 @@ if (ec_valid) { ctrl->eventinj |= VMCB_EVENTINJ_EC_VALID; ctrl->eventinj |= (uint64_t)error << 32; - VCPU_CTR3(sc->vm, vcpu, "Injecting %s at vector %d errcode %#x", + VCPU_CTR3(sc->vm, vcpu->vcpuid, + "Injecting %s at vector %d errcode %#x", intrtype_to_str(intr_type), vector, error); } else { - VCPU_CTR2(sc->vm, vcpu, "Injecting %s at vector %d", + VCPU_CTR2(sc->vm, vcpu->vcpuid, "Injecting %s at vector %d", intrtype_to_str(intr_type), vector); } } static void -svm_update_virqinfo(struct svm_softc *sc, int vcpu) +svm_update_virqinfo(struct svm_softc *sc, struct svm_vcpu *vcpu) { struct vm *vm; struct vlapic *vlapic; struct vmcb_ctrl *ctrl; vm = sc->vm; - vlapic = vm_lapic(vm, vcpu); - ctrl = svm_get_vmcb_ctrl(sc, vcpu); + vlapic = vm_lapic(vm, vcpu->vcpuid); + ctrl = svm_get_vmcb_ctrl(vcpu); /* Update %cr8 in the emulated vlapic */ vlapic_set_cr8(vlapic, ctrl->v_tpr); @@ -989,12 +995,14 @@ } static void -svm_save_intinfo(struct svm_softc *svm_sc, int vcpu) +svm_save_intinfo(struct svm_softc *svm_sc, struct svm_vcpu *vcpu) { struct vmcb_ctrl *ctrl; uint64_t intinfo; + int vcpuid; - ctrl = svm_get_vmcb_ctrl(svm_sc, vcpu); + vcpuid = vcpu->vcpuid; + ctrl = svm_get_vmcb_ctrl(vcpu); intinfo = ctrl->exitintinfo; if (!VMCB_EXITINTINFO_VALID(intinfo)) return; @@ -1005,15 +1013,15 @@ * If a #VMEXIT happened during event delivery then record the event * that was being delivered. */ - VCPU_CTR2(svm_sc->vm, vcpu, "SVM:Pending INTINFO(0x%lx), vector=%d.\n", + VCPU_CTR2(svm_sc->vm, vcpuid, "SVM:Pending INTINFO(0x%lx), vector=%d.\n", intinfo, VMCB_EXITINTINFO_VECTOR(intinfo)); - vmm_stat_incr(svm_sc->vm, vcpu, VCPU_EXITINTINFO, 1); - vm_exit_intinfo(svm_sc->vm, vcpu, intinfo); + vmm_stat_incr(svm_sc->vm, vcpuid, VCPU_EXITINTINFO, 1); + vm_exit_intinfo(svm_sc->vm, vcpuid, intinfo); } #ifdef INVARIANTS static __inline int -vintr_intercept_enabled(struct svm_softc *sc, int vcpu) +vintr_intercept_enabled(struct svm_softc *sc, struct svm_vcpu *vcpu) { return (svm_get_intercept(sc, vcpu, VMCB_CTRL1_INTCPT, @@ -1022,11 +1030,11 @@ #endif static __inline void -enable_intr_window_exiting(struct svm_softc *sc, int vcpu) +enable_intr_window_exiting(struct svm_softc *sc, struct svm_vcpu *vcpu) { struct vmcb_ctrl *ctrl; - ctrl = svm_get_vmcb_ctrl(sc, vcpu); + ctrl = svm_get_vmcb_ctrl(vcpu); if (ctrl->v_irq && ctrl->v_intr_vector == 0) { KASSERT(ctrl->v_ign_tpr, ("%s: invalid v_ign_tpr", __func__)); @@ -1035,20 +1043,20 @@ return; } - VCPU_CTR0(sc->vm, vcpu, "Enable intr window exiting"); + VCPU_CTR0(sc->vm, vcpu->vcpuid, "Enable intr window exiting"); ctrl->v_irq = 1; ctrl->v_ign_tpr = 1; ctrl->v_intr_vector = 0; - svm_set_dirty(sc, vcpu, VMCB_CACHE_TPR); + svm_set_dirty(vcpu, VMCB_CACHE_TPR); svm_enable_intercept(sc, vcpu, VMCB_CTRL1_INTCPT, VMCB_INTCPT_VINTR); } static __inline void -disable_intr_window_exiting(struct svm_softc *sc, int vcpu) +disable_intr_window_exiting(struct svm_softc *sc, struct svm_vcpu *vcpu) { struct vmcb_ctrl *ctrl; - ctrl = svm_get_vmcb_ctrl(sc, vcpu); + ctrl = svm_get_vmcb_ctrl(vcpu); if (!ctrl->v_irq && ctrl->v_intr_vector == 0) { KASSERT(!vintr_intercept_enabled(sc, vcpu), @@ -1056,35 +1064,36 @@ return; } - VCPU_CTR0(sc->vm, vcpu, "Disable intr window exiting"); + VCPU_CTR0(sc->vm, vcpu->vcpuid, "Disable intr window exiting"); ctrl->v_irq = 0; ctrl->v_intr_vector = 0; - svm_set_dirty(sc, vcpu, VMCB_CACHE_TPR); + svm_set_dirty(vcpu, VMCB_CACHE_TPR); svm_disable_intercept(sc, vcpu, VMCB_CTRL1_INTCPT, VMCB_INTCPT_VINTR); } static int -svm_modify_intr_shadow(struct svm_softc *sc, int vcpu, uint64_t val) +svm_modify_intr_shadow(struct svm_softc *sc, struct svm_vcpu *vcpu, + uint64_t val) { struct vmcb_ctrl *ctrl; int oldval, newval; - ctrl = svm_get_vmcb_ctrl(sc, vcpu); + ctrl = svm_get_vmcb_ctrl(vcpu); oldval = ctrl->intr_shadow; newval = val ? 1 : 0; if (newval != oldval) { ctrl->intr_shadow = newval; - VCPU_CTR1(sc->vm, vcpu, "Setting intr_shadow to %d", newval); + VCPU_CTR1(sc->vm, vcpu->vcpuid, "Setting intr_shadow to %d", newval); } return (0); } static int -svm_get_intr_shadow(struct svm_softc *sc, int vcpu, uint64_t *val) +svm_get_intr_shadow(struct svm_softc *sc, struct svm_vcpu *vcpu, uint64_t *val) { struct vmcb_ctrl *ctrl; - ctrl = svm_get_vmcb_ctrl(sc, vcpu); + ctrl = svm_get_vmcb_ctrl(vcpu); *val = ctrl->intr_shadow; return (0); } @@ -1095,7 +1104,7 @@ * to track when the vcpu is done handling the NMI. */ static int -nmi_blocked(struct svm_softc *sc, int vcpu) +nmi_blocked(struct svm_softc *sc, struct svm_vcpu *vcpu) { int blocked; @@ -1105,21 +1114,21 @@ } static void -enable_nmi_blocking(struct svm_softc *sc, int vcpu) +enable_nmi_blocking(struct svm_softc *sc, struct svm_vcpu *vcpu) { KASSERT(!nmi_blocked(sc, vcpu), ("vNMI already blocked")); - VCPU_CTR0(sc->vm, vcpu, "vNMI blocking enabled"); + VCPU_CTR0(sc->vm, vcpu->vcpuid, "vNMI blocking enabled"); svm_enable_intercept(sc, vcpu, VMCB_CTRL1_INTCPT, VMCB_INTCPT_IRET); } static void -clear_nmi_blocking(struct svm_softc *sc, int vcpu) +clear_nmi_blocking(struct svm_softc *sc, struct svm_vcpu *vcpu) { int error __diagused; KASSERT(nmi_blocked(sc, vcpu), ("vNMI already unblocked")); - VCPU_CTR0(sc->vm, vcpu, "vNMI blocking cleared"); + VCPU_CTR0(sc->vm, vcpu->vcpuid, "vNMI blocking cleared"); /* * When the IRET intercept is cleared the vcpu will attempt to execute * the "iret" when it runs next. However, it is possible to inject @@ -1144,17 +1153,19 @@ #define EFER_MBZ_BITS 0xFFFFFFFFFFFF0200UL static int -svm_write_efer(struct svm_softc *sc, int vcpu, uint64_t newval, bool *retu) +svm_write_efer(struct svm_softc *sc, struct svm_vcpu *vcpu, uint64_t newval, + bool *retu) { struct vm_exit *vme; struct vmcb_state *state; uint64_t changed, lma, oldval; - int error __diagused; + int error __diagused, vcpuid; - state = svm_get_vmcb_state(sc, vcpu); + state = svm_get_vmcb_state(vcpu); + vcpuid = vcpu->vcpuid; oldval = state->efer; - VCPU_CTR2(sc->vm, vcpu, "wrmsr(efer) %#lx/%#lx", oldval, newval); + VCPU_CTR2(sc->vm, vcpuid, "wrmsr(efer) %#lx/%#lx", oldval, newval); newval &= ~0xFE; /* clear the Read-As-Zero (RAZ) bits */ changed = oldval ^ newval; @@ -1178,7 +1189,7 @@ goto gpf; if (newval & EFER_NXE) { - if (!vm_cpuid_capability(sc->vm, vcpu, VCC_NO_EXECUTE)) + if (!vm_cpuid_capability(sc->vm, vcpuid, VCC_NO_EXECUTE)) goto gpf; } @@ -1187,19 +1198,19 @@ * this is fixed flag guest attempt to set EFER_LMSLE as an error. */ if (newval & EFER_LMSLE) { - vme = vm_exitinfo(sc->vm, vcpu); + vme = vm_exitinfo(sc->vm, vcpuid); vm_exit_svm(vme, VMCB_EXIT_MSR, 1, 0); *retu = true; return (0); } if (newval & EFER_FFXSR) { - if (!vm_cpuid_capability(sc->vm, vcpu, VCC_FFXSR)) + if (!vm_cpuid_capability(sc->vm, vcpuid, VCC_FFXSR)) goto gpf; } if (newval & EFER_TCE) { - if (!vm_cpuid_capability(sc->vm, vcpu, VCC_TCE)) + if (!vm_cpuid_capability(sc->vm, vcpuid, VCC_TCE)) goto gpf; } @@ -1207,18 +1218,18 @@ KASSERT(error == 0, ("%s: error %d updating efer", __func__, error)); return (0); gpf: - vm_inject_gp(sc->vm, vcpu); + vm_inject_gp(sc->vm, vcpuid); return (0); } static int -emulate_wrmsr(struct svm_softc *sc, int vcpu, u_int num, uint64_t val, - bool *retu) +emulate_wrmsr(struct svm_softc *sc, struct svm_vcpu *vcpu, u_int num, + uint64_t val, bool *retu) { int error; if (lapic_msr(num)) - error = lapic_wrmsr(sc->vm, vcpu, num, val, retu); + error = lapic_wrmsr(sc->vm, vcpu->vcpuid, num, val, retu); else if (num == MSR_EFER) error = svm_write_efer(sc, vcpu, val, retu); else @@ -1228,7 +1239,8 @@ } static int -emulate_rdmsr(struct svm_softc *sc, int vcpu, u_int num, bool *retu) +emulate_rdmsr(struct svm_softc *sc, struct svm_vcpu *vcpu, u_int num, + bool *retu) { struct vmcb_state *state; struct svm_regctx *ctx; @@ -1236,13 +1248,13 @@ int error; if (lapic_msr(num)) - error = lapic_rdmsr(sc->vm, vcpu, num, &result, retu); + error = lapic_rdmsr(sc->vm, vcpu->vcpuid, num, &result, retu); else error = svm_rdmsr(sc, vcpu, num, &result, retu); if (error == 0) { - state = svm_get_vmcb_state(sc, vcpu); - ctx = svm_get_guest_regctx(sc, vcpu); + state = svm_get_vmcb_state(vcpu); + ctx = svm_get_guest_regctx(vcpu); state->rax = result & 0xffffffff; ctx->sctx_rdx = result >> 32; } @@ -1323,7 +1335,8 @@ } static int -svm_vmexit(struct svm_softc *svm_sc, int vcpu, struct vm_exit *vmexit) +svm_vmexit(struct svm_softc *svm_sc, struct svm_vcpu *vcpu, + struct vm_exit *vmexit) { struct vmcb *vmcb; struct vmcb_state *state; @@ -1332,12 +1345,14 @@ uint64_t code, info1, info2, val; uint32_t eax, ecx, edx; int error __diagused, errcode_valid, handled, idtvec, reflect; + int vcpuid; bool retu; - ctx = svm_get_guest_regctx(svm_sc, vcpu); - vmcb = svm_get_vmcb(svm_sc, vcpu); + ctx = svm_get_guest_regctx(vcpu); + vmcb = svm_get_vmcb(vcpu); state = &vmcb->state; ctrl = &vmcb->ctrl; + vcpuid = vcpu->vcpuid; handled = 0; code = ctrl->exitcode; @@ -1348,7 +1363,7 @@ vmexit->rip = state->rip; vmexit->inst_length = nrip_valid(code) ? ctrl->nrip - state->rip : 0; - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_COUNT, 1); + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_COUNT, 1); /* * #VMEXIT(INVALID) needs to be handled early because the VMCB is @@ -1380,18 +1395,18 @@ handled = 1; break; case VMCB_EXIT_VINTR: /* interrupt window exiting */ - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_VINTR, 1); + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_VINTR, 1); handled = 1; break; case VMCB_EXIT_INTR: /* external interrupt */ - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_EXTINT, 1); + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_EXTINT, 1); handled = 1; break; case VMCB_EXIT_NMI: /* external NMI */ handled = 1; break; case 0x40 ... 0x5F: - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_EXCEPTION, 1); + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_EXCEPTION, 1); reflect = 1; idtvec = code - 0x40; switch (idtvec) { @@ -1401,7 +1416,7 @@ * reflect the machine check back into the guest. */ reflect = 0; - VCPU_CTR0(svm_sc->vm, vcpu, "Vectoring to MCE handler"); + VCPU_CTR0(svm_sc->vm, vcpuid, "Vectoring to MCE handler"); __asm __volatile("int $18"); break; case IDT_PF: @@ -1435,7 +1450,7 @@ * event injection is identical to what it was when * the exception originally happened. */ - VCPU_CTR2(svm_sc->vm, vcpu, "Reset inst_length from %d " + VCPU_CTR2(svm_sc->vm, vcpuid, "Reset inst_length from %d " "to zero before injecting exception %d", vmexit->inst_length, idtvec); vmexit->inst_length = 0; @@ -1451,9 +1466,9 @@ if (reflect) { /* Reflect the exception back into the guest */ - VCPU_CTR2(svm_sc->vm, vcpu, "Reflecting exception " + VCPU_CTR2(svm_sc->vm, vcpuid, "Reflecting exception " "%d/%#x into the guest", idtvec, (int)info1); - error = vm_inject_exception(svm_sc->vm, vcpu, idtvec, + error = vm_inject_exception(svm_sc->vm, vcpuid, idtvec, errcode_valid, info1, 0); KASSERT(error == 0, ("%s: vm_inject_exception error %d", __func__, error)); @@ -1467,9 +1482,9 @@ retu = false; if (info1) { - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_WRMSR, 1); + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_WRMSR, 1); val = (uint64_t)edx << 32 | eax; - VCPU_CTR2(svm_sc->vm, vcpu, "wrmsr %#x val %#lx", + VCPU_CTR2(svm_sc->vm, vcpuid, "wrmsr %#x val %#lx", ecx, val); if (emulate_wrmsr(svm_sc, vcpu, ecx, val, &retu)) { vmexit->exitcode = VM_EXITCODE_WRMSR; @@ -1482,8 +1497,8 @@ ("emulate_wrmsr retu with bogus exitcode")); } } else { - VCPU_CTR1(svm_sc->vm, vcpu, "rdmsr %#x", ecx); - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_RDMSR, 1); + VCPU_CTR1(svm_sc->vm, vcpuid, "rdmsr %#x", ecx); + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_RDMSR, 1); if (emulate_rdmsr(svm_sc, vcpu, ecx, &retu)) { vmexit->exitcode = VM_EXITCODE_RDMSR; vmexit->u.msr.code = ecx; @@ -1497,40 +1512,40 @@ break; case VMCB_EXIT_IO: handled = svm_handle_io(svm_sc, vcpu, vmexit); - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_INOUT, 1); + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_INOUT, 1); break; case VMCB_EXIT_CPUID: - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_CPUID, 1); - handled = x86_emulate_cpuid(svm_sc->vm, vcpu, &state->rax, + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_CPUID, 1); + handled = x86_emulate_cpuid(svm_sc->vm, vcpuid, &state->rax, &ctx->sctx_rbx, &ctx->sctx_rcx, &ctx->sctx_rdx); break; case VMCB_EXIT_HLT: - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_HLT, 1); + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_HLT, 1); vmexit->exitcode = VM_EXITCODE_HLT; vmexit->u.hlt.rflags = state->rflags; break; case VMCB_EXIT_PAUSE: vmexit->exitcode = VM_EXITCODE_PAUSE; - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_PAUSE, 1); + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_PAUSE, 1); break; case VMCB_EXIT_NPF: /* EXITINFO2 contains the faulting guest physical address */ if (info1 & VMCB_NPF_INFO1_RSV) { - VCPU_CTR2(svm_sc->vm, vcpu, "nested page fault with " + VCPU_CTR2(svm_sc->vm, vcpuid, "nested page fault with " "reserved bits set: info1(%#lx) info2(%#lx)", info1, info2); - } else if (vm_mem_allocated(svm_sc->vm, vcpu, info2)) { + } else if (vm_mem_allocated(svm_sc->vm, vcpuid, info2)) { vmexit->exitcode = VM_EXITCODE_PAGING; vmexit->u.paging.gpa = info2; vmexit->u.paging.fault_type = npf_fault_type(info1); - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_NESTED_FAULT, 1); - VCPU_CTR3(svm_sc->vm, vcpu, "nested page fault " + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_NESTED_FAULT, 1); + VCPU_CTR3(svm_sc->vm, vcpuid, "nested page fault " "on gpa %#lx/%#lx at rip %#lx", info2, info1, state->rip); } else if (svm_npf_emul_fault(info1)) { svm_handle_inst_emul(vmcb, info2, vmexit); - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_INST_EMUL, 1); - VCPU_CTR3(svm_sc->vm, vcpu, "inst_emul fault " + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_INST_EMUL, 1); + VCPU_CTR3(svm_sc->vm, vcpuid, "inst_emul fault " "for gpa %#lx/%#lx at rip %#lx", info2, info1, state->rip); } @@ -1551,7 +1566,7 @@ case VMCB_EXIT_SKINIT: case VMCB_EXIT_ICEBP: case VMCB_EXIT_INVLPGA: - vm_inject_ud(svm_sc->vm, vcpu); + vm_inject_ud(svm_sc->vm, vcpuid); handled = 1; break; case VMCB_EXIT_INVD: @@ -1560,11 +1575,11 @@ handled = 1; break; default: - vmm_stat_incr(svm_sc->vm, vcpu, VMEXIT_UNKNOWN, 1); + vmm_stat_incr(svm_sc->vm, vcpuid, VMEXIT_UNKNOWN, 1); break; } - VCPU_CTR4(svm_sc->vm, vcpu, "%s %s vmexit at %#lx/%d", + VCPU_CTR4(svm_sc->vm, vcpuid, "%s %s vmexit at %#lx/%d", handled ? "handled" : "unhandled", exit_reason_to_str(code), vmexit->rip, vmexit->inst_length); @@ -1590,11 +1605,12 @@ } static void -svm_inj_intinfo(struct svm_softc *svm_sc, int vcpu) +svm_inj_intinfo(struct svm_softc *svm_sc, struct svm_vcpu *vcpu) { uint64_t intinfo; + int vcpuid = vcpu->vcpuid; - if (!vm_entry_intinfo(svm_sc->vm, vcpu, &intinfo)) + if (!vm_entry_intinfo(svm_sc->vm, vcpuid, &intinfo)) return; KASSERT(VMCB_EXITINTINFO_VALID(intinfo), ("%s: entry intinfo is not " @@ -1604,34 +1620,34 @@ VMCB_EXITINTINFO_VECTOR(intinfo), VMCB_EXITINTINFO_EC(intinfo), VMCB_EXITINTINFO_EC_VALID(intinfo)); - vmm_stat_incr(svm_sc->vm, vcpu, VCPU_INTINFO_INJECTED, 1); - VCPU_CTR1(svm_sc->vm, vcpu, "Injected entry intinfo: %#lx", intinfo); + vmm_stat_incr(svm_sc->vm, vcpuid, VCPU_INTINFO_INJECTED, 1); + VCPU_CTR1(svm_sc->vm, vcpuid, "Injected entry intinfo: %#lx", intinfo); } /* * Inject event to virtual cpu. */ static void -svm_inj_interrupts(struct svm_softc *sc, int vcpu, struct vlapic *vlapic) +svm_inj_interrupts(struct svm_softc *sc, struct svm_vcpu *vcpu, + struct vlapic *vlapic) { struct vmcb_ctrl *ctrl; struct vmcb_state *state; - struct svm_vcpu *vcpustate; uint8_t v_tpr; int vector, need_intr_window; int extint_pending; + int vcpuid = vcpu->vcpuid; - state = svm_get_vmcb_state(sc, vcpu); - ctrl = svm_get_vmcb_ctrl(sc, vcpu); - vcpustate = svm_get_vcpu(sc, vcpu); + state = svm_get_vmcb_state(vcpu); + ctrl = svm_get_vmcb_ctrl(vcpu); need_intr_window = 0; - if (vcpustate->nextrip != state->rip) { + if (vcpu->nextrip != state->rip) { ctrl->intr_shadow = 0; - VCPU_CTR2(sc->vm, vcpu, "Guest interrupt blocking " + VCPU_CTR2(sc->vm, vcpuid, "Guest interrupt blocking " "cleared due to rip change: %#lx/%#lx", - vcpustate->nextrip, state->rip); + vcpu->nextrip, state->rip); } /* @@ -1646,19 +1662,19 @@ svm_inj_intinfo(sc, vcpu); /* NMI event has priority over interrupts. */ - if (vm_nmi_pending(sc->vm, vcpu)) { + if (vm_nmi_pending(sc->vm, vcpuid)) { if (nmi_blocked(sc, vcpu)) { /* * Can't inject another NMI if the guest has not * yet executed an "iret" after the last NMI. */ - VCPU_CTR0(sc->vm, vcpu, "Cannot inject NMI due " + VCPU_CTR0(sc->vm, vcpuid, "Cannot inject NMI due " "to NMI-blocking"); } else if (ctrl->intr_shadow) { /* * Can't inject an NMI if the vcpu is in an intr_shadow. */ - VCPU_CTR0(sc->vm, vcpu, "Cannot inject NMI due to " + VCPU_CTR0(sc->vm, vcpuid, "Cannot inject NMI due to " "interrupt shadow"); need_intr_window = 1; goto done; @@ -1667,7 +1683,7 @@ * If there is already an exception/interrupt pending * then defer the NMI until after that. */ - VCPU_CTR1(sc->vm, vcpu, "Cannot inject NMI due to " + VCPU_CTR1(sc->vm, vcpuid, "Cannot inject NMI due to " "eventinj %#lx", ctrl->eventinj); /* @@ -1682,7 +1698,7 @@ */ ipi_cpu(curcpu, IPI_AST); /* XXX vmm_ipinum? */ } else { - vm_nmi_clear(sc->vm, vcpu); + vm_nmi_clear(sc->vm, vcpuid); /* Inject NMI, vector number is not used */ svm_eventinject(sc, vcpu, VMCB_EVENTINJ_TYPE_NMI, @@ -1691,11 +1707,11 @@ /* virtual NMI blocking is now in effect */ enable_nmi_blocking(sc, vcpu); - VCPU_CTR0(sc->vm, vcpu, "Injecting vNMI"); + VCPU_CTR0(sc->vm, vcpuid, "Injecting vNMI"); } } - extint_pending = vm_extint_pending(sc->vm, vcpu); + extint_pending = vm_extint_pending(sc->vm, vcpuid); if (!extint_pending) { if (!vlapic_pending_intr(vlapic, &vector)) goto done; @@ -1713,21 +1729,21 @@ * then we cannot inject the pending interrupt. */ if ((state->rflags & PSL_I) == 0) { - VCPU_CTR2(sc->vm, vcpu, "Cannot inject vector %d due to " + VCPU_CTR2(sc->vm, vcpuid, "Cannot inject vector %d due to " "rflags %#lx", vector, state->rflags); need_intr_window = 1; goto done; } if (ctrl->intr_shadow) { - VCPU_CTR1(sc->vm, vcpu, "Cannot inject vector %d due to " + VCPU_CTR1(sc->vm, vcpuid, "Cannot inject vector %d due to " "interrupt shadow", vector); need_intr_window = 1; goto done; } if (ctrl->eventinj & VMCB_EVENTINJ_VALID) { - VCPU_CTR2(sc->vm, vcpu, "Cannot inject vector %d due to " + VCPU_CTR2(sc->vm, vcpuid, "Cannot inject vector %d due to " "eventinj %#lx", vector, ctrl->eventinj); need_intr_window = 1; goto done; @@ -1738,7 +1754,7 @@ if (!extint_pending) { vlapic_intr_accepted(vlapic, vector); } else { - vm_extint_clear(sc->vm, vcpu); + vm_extint_clear(sc->vm, vcpuid); vatpic_intr_accepted(sc->vm, vector); } @@ -1764,10 +1780,10 @@ v_tpr = vlapic_get_cr8(vlapic); KASSERT(v_tpr <= 15, ("invalid v_tpr %#x", v_tpr)); if (ctrl->v_tpr != v_tpr) { - VCPU_CTR2(sc->vm, vcpu, "VMCB V_TPR changed from %#x to %#x", + VCPU_CTR2(sc->vm, vcpuid, "VMCB V_TPR changed from %#x to %#x", ctrl->v_tpr, v_tpr); ctrl->v_tpr = v_tpr; - svm_set_dirty(sc, vcpu, VMCB_CACHE_TPR); + svm_set_dirty(vcpu, VMCB_CACHE_TPR); } if (need_intr_window) { @@ -1809,9 +1825,8 @@ } static void -svm_pmap_activate(struct svm_softc *sc, int vcpuid, pmap_t pmap) +svm_pmap_activate(struct svm_softc *sc, struct svm_vcpu *vcpu, pmap_t pmap) { - struct svm_vcpu *vcpustate; struct vmcb_ctrl *ctrl; long eptgen; int cpu; @@ -1821,8 +1836,7 @@ CPU_SET_ATOMIC(cpu, &pmap->pm_active); smr_enter(pmap->pm_eptsmr); - vcpustate = svm_get_vcpu(sc, vcpuid); - ctrl = svm_get_vmcb_ctrl(sc, vcpuid); + ctrl = svm_get_vmcb_ctrl(vcpu); /* * The TLB entries associated with the vcpu's ASID are not valid @@ -1863,9 +1877,9 @@ eptgen = atomic_load_long(&pmap->pm_eptgen); ctrl->tlb_ctrl = VMCB_TLB_FLUSH_NOTHING; - if (vcpustate->asid.gen != asid[cpu].gen) { + if (vcpu->asid.gen != asid[cpu].gen) { alloc_asid = true; /* (c) and (d) */ - } else if (vcpustate->eptgen != eptgen) { + } else if (vcpu->eptgen != eptgen) { if (flush_by_asid()) ctrl->tlb_ctrl = VMCB_TLB_FLUSH_GUEST; /* (b1) */ else @@ -1893,11 +1907,11 @@ if (!flush_by_asid()) ctrl->tlb_ctrl = VMCB_TLB_FLUSH_ALL; } - vcpustate->asid.gen = asid[cpu].gen; - vcpustate->asid.num = asid[cpu].num; + vcpu->asid.gen = asid[cpu].gen; + vcpu->asid.num = asid[cpu].num; - ctrl->asid = vcpustate->asid.num; - svm_set_dirty(sc, vcpuid, VMCB_CACHE_ASID); + ctrl->asid = vcpu->asid.num; + svm_set_dirty(vcpu, VMCB_CACHE_ASID); /* * If this cpu supports "flush-by-asid" then the TLB * was not flushed after the generation bump. The TLB @@ -1906,11 +1920,11 @@ if (flush_by_asid()) ctrl->tlb_ctrl = VMCB_TLB_FLUSH_GUEST; } - vcpustate->eptgen = eptgen; + vcpu->eptgen = eptgen; KASSERT(ctrl->asid != 0, ("Guest ASID must be non-zero")); - KASSERT(ctrl->asid == vcpustate->asid.num, - ("ASID mismatch: %u/%u", ctrl->asid, vcpustate->asid.num)); + KASSERT(ctrl->asid == vcpu->asid.num, + ("ASID mismatch: %u/%u", ctrl->asid, vcpu->asid.num)); } static void @@ -1992,47 +2006,48 @@ * Start vcpu with specified RIP. */ static int -svm_run(void *arg, int vcpu, register_t rip, pmap_t pmap, +svm_run(void *arg, void *vcpui, register_t rip, pmap_t pmap, struct vm_eventinfo *evinfo) { struct svm_regctx *gctx; struct svm_softc *svm_sc; - struct svm_vcpu *vcpustate; + struct svm_vcpu *vcpu; struct vmcb_state *state; struct vmcb_ctrl *ctrl; struct vm_exit *vmexit; struct vlapic *vlapic; struct vm *vm; uint64_t vmcb_pa; - int handled; + int handled, vcpuid; uint16_t ldt_sel; svm_sc = arg; vm = svm_sc->vm; - vcpustate = svm_get_vcpu(svm_sc, vcpu); - state = svm_get_vmcb_state(svm_sc, vcpu); - ctrl = svm_get_vmcb_ctrl(svm_sc, vcpu); - vmexit = vm_exitinfo(vm, vcpu); - vlapic = vm_lapic(vm, vcpu); + vcpu = vcpui; + vcpuid = vcpu->vcpuid; + state = svm_get_vmcb_state(vcpu); + ctrl = svm_get_vmcb_ctrl(vcpu); + vmexit = vm_exitinfo(vm, vcpuid); + vlapic = vm_lapic(vm, vcpuid); - gctx = svm_get_guest_regctx(svm_sc, vcpu); - vmcb_pa = svm_sc->vcpu[vcpu].vmcb_pa; + gctx = svm_get_guest_regctx(vcpu); + vmcb_pa = vcpu->vmcb_pa; - if (vcpustate->lastcpu != curcpu) { + if (vcpu->lastcpu != curcpu) { /* * Force new ASID allocation by invalidating the generation. */ - vcpustate->asid.gen = 0; + vcpu->asid.gen = 0; /* * Invalidate the VMCB state cache by marking all fields dirty. */ - svm_set_dirty(svm_sc, vcpu, 0xffffffff); + svm_set_dirty(vcpu, 0xffffffff); /* * XXX - * Setting 'vcpustate->lastcpu' here is bit premature because + * Setting 'vcpu->lastcpu' here is bit premature because * we may return from this function without actually executing * the VMRUN instruction. This could happen if a rendezvous * or an AST is pending on the first time through the loop. @@ -2040,8 +2055,8 @@ * This works for now but any new side-effects of vcpu * migration should take this case into account. */ - vcpustate->lastcpu = curcpu; - vmm_stat_incr(vm, vcpu, VCPU_MIGRATIONS, 1); + vcpu->lastcpu = curcpu; + vmm_stat_incr(vm, vcpuid, VCPU_MIGRATIONS, 1); } svm_msr_guest_enter(svm_sc, vcpu); @@ -2061,32 +2076,32 @@ if (vcpu_suspended(evinfo)) { enable_gintr(); - vm_exit_suspended(vm, vcpu, state->rip); + vm_exit_suspended(vm, vcpuid, state->rip); break; } if (vcpu_rendezvous_pending(evinfo)) { enable_gintr(); - vm_exit_rendezvous(vm, vcpu, state->rip); + vm_exit_rendezvous(vm, vcpuid, state->rip); break; } if (vcpu_reqidle(evinfo)) { enable_gintr(); - vm_exit_reqidle(vm, vcpu, state->rip); + vm_exit_reqidle(vm, vcpuid, state->rip); break; } /* We are asked to give the cpu by scheduler. */ - if (vcpu_should_yield(vm, vcpu)) { + if (vcpu_should_yield(vm, vcpuid)) { enable_gintr(); - vm_exit_astpending(vm, vcpu, state->rip); + vm_exit_astpending(vm, vcpuid, state->rip); break; } - if (vcpu_debugged(vm, vcpu)) { + if (vcpu_debugged(vm, vcpuid)) { enable_gintr(); - vm_exit_debug(vm, vcpu, state->rip); + vm_exit_debug(vm, vcpuid, state->rip); break; } @@ -2107,12 +2122,12 @@ */ svm_pmap_activate(svm_sc, vcpu, pmap); - ctrl->vmcb_clean = vmcb_clean & ~vcpustate->dirty; - vcpustate->dirty = 0; - VCPU_CTR1(vm, vcpu, "vmcb clean %#x", ctrl->vmcb_clean); + ctrl->vmcb_clean = vmcb_clean & ~vcpu->dirty; + vcpu->dirty = 0; + VCPU_CTR1(vm, vcpuid, "vmcb clean %#x", ctrl->vmcb_clean); /* Launch Virtual Machine. */ - VCPU_CTR1(vm, vcpu, "Resume execution at %#lx", state->rip); + VCPU_CTR1(vm, vcpuid, "Resume execution at %#lx", state->rip); svm_dr_enter_guest(gctx); svm_launch(vmcb_pa, gctx, get_pcpu()); svm_dr_leave_guest(gctx); @@ -2133,7 +2148,7 @@ enable_gintr(); /* Update 'nextrip' */ - vcpustate->nextrip = state->rip; + vcpu->nextrip = state->rip; /* Handle #VMEXIT and if required return to user space. */ handled = svm_vmexit(svm_sc, vcpu, vmexit); @@ -2144,18 +2159,20 @@ return (0); } +static void +svm_vcpu_cleanup(void *arg, void *vcpui) +{ + struct svm_vcpu *vcpu = vcpui; + + free(vcpu->vmcb, M_SVM); + free(vcpu, M_SVM); +} + static void svm_cleanup(void *arg) { struct svm_softc *sc = arg; - struct svm_vcpu *vcpu; - uint16_t i, maxcpus; - maxcpus = vm_get_maxcpus(sc->vm); - for (i = 0; i < maxcpus; i++) { - vcpu = svm_get_vcpu(sc, i); - free(vcpu->vmcb, M_SVM); - } contigfree(sc->iopm_bitmap, SVM_IO_BITMAP_SIZE, M_SVM); contigfree(sc->msr_bitmap, SVM_MSR_BITMAP_SIZE, M_SVM); free(sc, M_SVM); @@ -2208,12 +2225,14 @@ } static int -svm_getreg(void *arg, int vcpu, int ident, uint64_t *val) +svm_getreg(void *arg, void *vcpui, int ident, uint64_t *val) { struct svm_softc *svm_sc; + struct svm_vcpu *vcpu; register_t *reg; svm_sc = arg; + vcpu = vcpui; if (ident == VM_REG_GUEST_INTR_SHADOW) { return (svm_get_intr_shadow(svm_sc, vcpu, val)); @@ -2223,24 +2242,27 @@ return (0); } - reg = swctx_regptr(svm_get_guest_regctx(svm_sc, vcpu), ident); + reg = swctx_regptr(svm_get_guest_regctx(vcpu), ident); if (reg != NULL) { *val = *reg; return (0); } - VCPU_CTR1(svm_sc->vm, vcpu, "svm_getreg: unknown register %#x", ident); + VCPU_CTR1(svm_sc->vm, vcpu->vcpuid, "svm_getreg: unknown register %#x", + ident); return (EINVAL); } static int -svm_setreg(void *arg, int vcpu, int ident, uint64_t val) +svm_setreg(void *arg, void *vcpui, int ident, uint64_t val) { struct svm_softc *svm_sc; + struct svm_vcpu *vcpu; register_t *reg; svm_sc = arg; + vcpu = vcpui; if (ident == VM_REG_GUEST_INTR_SHADOW) { return (svm_modify_intr_shadow(svm_sc, vcpu, val)); @@ -2253,7 +2275,7 @@ } } - reg = swctx_regptr(svm_get_guest_regctx(svm_sc, vcpu), ident); + reg = swctx_regptr(svm_get_guest_regctx(vcpu), ident); if (reg != NULL) { *reg = val; @@ -2271,32 +2293,33 @@ * whether 'running' is true/false. */ - VCPU_CTR1(svm_sc->vm, vcpu, "svm_setreg: unknown register %#x", ident); + VCPU_CTR1(svm_sc->vm, vcpu->vcpuid, "svm_setreg: unknown register %#x", + ident); return (EINVAL); } static int -svm_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) +svm_getdesc(void *arg, void *vcpui, int reg, struct seg_desc *desc) { - return (vmcb_getdesc(arg, vcpu, reg, desc)); + return (vmcb_getdesc(arg, vcpui, reg, desc)); } static int -svm_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) +svm_setdesc(void *arg, void *vcpui, int reg, struct seg_desc *desc) { - return (vmcb_setdesc(arg, vcpu, reg, desc)); + return (vmcb_setdesc(arg, vcpui, reg, desc)); } #ifdef BHYVE_SNAPSHOT static int -svm_snapshot_reg(void *arg, int vcpu, int ident, +svm_snapshot_reg(void *arg, void *vcpui, int ident, struct vm_snapshot_meta *meta) { int ret; uint64_t val; if (meta->op == VM_SNAPSHOT_SAVE) { - ret = svm_getreg(arg, vcpu, ident, &val); + ret = svm_getreg(arg, vcpui, ident, &val); if (ret != 0) goto done; @@ -2304,7 +2327,7 @@ } else if (meta->op == VM_SNAPSHOT_RESTORE) { SNAPSHOT_VAR_OR_LEAVE(val, meta, ret, done); - ret = svm_setreg(arg, vcpu, ident, val); + ret = svm_setreg(arg, vcpui, ident, val); if (ret != 0) goto done; } else { @@ -2318,13 +2341,15 @@ #endif static int -svm_setcap(void *arg, int vcpu, int type, int val) +svm_setcap(void *arg, void *vcpui, int type, int val) { struct svm_softc *sc; + struct svm_vcpu *vcpu; struct vlapic *vlapic; int error; sc = arg; + vcpu = vcpui; error = 0; switch (type) { case VM_CAP_HALT_EXIT: @@ -2341,7 +2366,7 @@ error = EINVAL; break; case VM_CAP_IPI_EXIT: - vlapic = vm_lapic(sc->vm, vcpu); + vlapic = vm_lapic(sc->vm, vcpu->vcpuid); vlapic->ipi_exit = val; break; default: @@ -2352,13 +2377,15 @@ } static int -svm_getcap(void *arg, int vcpu, int type, int *retval) +svm_getcap(void *arg, void *vcpui, int type, int *retval) { struct svm_softc *sc; + struct svm_vcpu *vcpu; struct vlapic *vlapic; int error; sc = arg; + vcpu = vcpui; error = 0; switch (type) { @@ -2374,7 +2401,7 @@ *retval = 1; /* unrestricted guest is always enabled */ break; case VM_CAP_IPI_EXIT: - vlapic = vm_lapic(sc->vm, vcpu); + vlapic = vm_lapic(sc->vm, vcpu->vcpuid); *retval = vlapic->ipi_exit; break; default: @@ -2397,15 +2424,17 @@ } static struct vlapic * -svm_vlapic_init(void *arg, int vcpuid) +svm_vlapic_init(void *arg, void *vcpui) { struct svm_softc *svm_sc; + struct svm_vcpu *vcpu; struct vlapic *vlapic; svm_sc = arg; + vcpu = vcpui; vlapic = malloc(sizeof(struct vlapic), M_SVM_VLAPIC, M_WAITOK | M_ZERO); vlapic->vm = svm_sc->vm; - vlapic->vcpuid = vcpuid; + vlapic->vcpuid = vcpu->vcpuid; vlapic->apic_page = malloc(PAGE_SIZE, M_SVM_VLAPIC, M_WAITOK | M_ZERO); vlapic_init(vlapic); @@ -2433,163 +2462,163 @@ } static int -svm_vcpu_snapshot(void *arg, struct vm_snapshot_meta *meta, int vcpuid) +svm_vcpu_snapshot(void *arg, struct vm_snapshot_meta *meta, void *vcpui) { struct svm_softc *sc; struct svm_vcpu *vcpu; int err, running, hostcpu; sc = (struct svm_softc *)arg; - vcpu = &sc->vcpu[vcpuid]; + vcpu = vcpui; err = 0; KASSERT(arg != NULL, ("%s: arg was NULL", __func__)); - running = vcpu_is_running(sc->vm, vcpuid, &hostcpu); + running = vcpu_is_running(sc->vm, vcpu->vcpuid, &hostcpu); if (running && hostcpu != curcpu) { printf("%s: %s%d is running", __func__, vm_name(sc->vm), - vcpuid); + vcpu->vcpuid); return (EINVAL); } - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_CR0, meta); - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_CR2, meta); - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_CR3, meta); - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_CR4, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_CR0, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_CR2, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_CR3, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_CR4, meta); - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_DR6, meta); - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_DR7, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_DR6, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_DR7, meta); - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_RAX, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_RAX, meta); - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_RSP, meta); - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_RIP, meta); - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_RFLAGS, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_RSP, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_RIP, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_RFLAGS, meta); /* Guest segments */ /* ES */ - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_ES, meta); - err += vmcb_snapshot_desc(sc, vcpuid, VM_REG_GUEST_ES, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_ES, meta); + err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_ES, meta); /* CS */ - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_CS, meta); - err += vmcb_snapshot_desc(sc, vcpuid, VM_REG_GUEST_CS, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_CS, meta); + err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_CS, meta); /* SS */ - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_SS, meta); - err += vmcb_snapshot_desc(sc, vcpuid, VM_REG_GUEST_SS, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_SS, meta); + err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_SS, meta); /* DS */ - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_DS, meta); - err += vmcb_snapshot_desc(sc, vcpuid, VM_REG_GUEST_DS, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_DS, meta); + err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_DS, meta); /* FS */ - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_FS, meta); - err += vmcb_snapshot_desc(sc, vcpuid, VM_REG_GUEST_FS, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_FS, meta); + err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_FS, meta); /* GS */ - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_GS, meta); - err += vmcb_snapshot_desc(sc, vcpuid, VM_REG_GUEST_GS, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_GS, meta); + err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_GS, meta); /* TR */ - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_TR, meta); - err += vmcb_snapshot_desc(sc, vcpuid, VM_REG_GUEST_TR, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_TR, meta); + err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_TR, meta); /* LDTR */ - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_LDTR, meta); - err += vmcb_snapshot_desc(sc, vcpuid, VM_REG_GUEST_LDTR, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_LDTR, meta); + err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_LDTR, meta); /* EFER */ - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_EFER, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_EFER, meta); /* IDTR and GDTR */ - err += vmcb_snapshot_desc(sc, vcpuid, VM_REG_GUEST_IDTR, meta); - err += vmcb_snapshot_desc(sc, vcpuid, VM_REG_GUEST_GDTR, meta); + err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_IDTR, meta); + err += vmcb_snapshot_desc(sc, vcpu, VM_REG_GUEST_GDTR, meta); /* Specific AMD registers */ - err += svm_snapshot_reg(sc, vcpuid, VM_REG_GUEST_INTR_SHADOW, meta); + err += svm_snapshot_reg(sc, vcpu, VM_REG_GUEST_INTR_SHADOW, meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_CR_INTERCEPT, 4), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_DR_INTERCEPT, 4), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_EXC_INTERCEPT, 4), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_INST1_INTERCEPT, 4), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_INST2_INTERCEPT, 4), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_PAUSE_FILTHRESH, 2), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_PAUSE_FILCNT, 2), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_ASID, 4), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_TLB_CTRL, 4), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_VIRQ, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_EXIT_REASON, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_EXITINFO1, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_EXITINFO2, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_EXITINTINFO, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_NP_ENABLE, 1), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_AVIC_BAR, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_AVIC_PAGE, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_AVIC_LT, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_AVIC_PT, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_CPL, 1), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_STAR, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_LSTAR, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_CSTAR, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_SFMASK, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_KERNELGBASE, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_SYSENTER_CS, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_SYSENTER_ESP, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_SYSENTER_EIP, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_GUEST_PAT, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_DBGCTL, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_BR_FROM, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_BR_TO, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_INT_FROM, 8), meta); - err += vmcb_snapshot_any(sc, vcpuid, + err += vmcb_snapshot_any(sc, vcpu, VMCB_ACCESS(VMCB_OFF_INT_TO, 8), meta); if (err != 0) goto done; @@ -2631,26 +2660,27 @@ /* Set all caches dirty */ if (meta->op == VM_SNAPSHOT_RESTORE) { - svm_set_dirty(sc, vcpuid, VMCB_CACHE_ASID); - svm_set_dirty(sc, vcpuid, VMCB_CACHE_IOPM); - svm_set_dirty(sc, vcpuid, VMCB_CACHE_I); - svm_set_dirty(sc, vcpuid, VMCB_CACHE_TPR); - svm_set_dirty(sc, vcpuid, VMCB_CACHE_CR2); - svm_set_dirty(sc, vcpuid, VMCB_CACHE_CR); - svm_set_dirty(sc, vcpuid, VMCB_CACHE_DT); - svm_set_dirty(sc, vcpuid, VMCB_CACHE_SEG); - svm_set_dirty(sc, vcpuid, VMCB_CACHE_NP); + svm_set_dirty(vcpu, VMCB_CACHE_ASID); + svm_set_dirty(vcpu, VMCB_CACHE_IOPM); + svm_set_dirty(vcpu, VMCB_CACHE_I); + svm_set_dirty(vcpu, VMCB_CACHE_TPR); + svm_set_dirty(vcpu, VMCB_CACHE_CR2); + svm_set_dirty(vcpu, VMCB_CACHE_CR); + svm_set_dirty(vcpu, VMCB_CACHE_DT); + svm_set_dirty(vcpu, VMCB_CACHE_SEG); + svm_set_dirty(vcpu, VMCB_CACHE_NP); } + done: return (err); } static int -svm_restore_tsc(void *arg, int vcpu, uint64_t offset) +svm_restore_tsc(void *arg, void *vcpui, uint64_t offset) { int err; - err = svm_set_tsc_offset(arg, vcpu, offset); + err = svm_set_tsc_offset(arg, vcpui, offset); return (err); } @@ -2663,6 +2693,8 @@ .init = svm_init, .run = svm_run, .cleanup = svm_cleanup, + .vcpu_init = svm_vcpu_init, + .vcpu_cleanup = svm_vcpu_cleanup, .getreg = svm_getreg, .setreg = svm_setreg, .getdesc = svm_getdesc, diff --git a/sys/amd64/vmm/amd/svm_msr.h b/sys/amd64/vmm/amd/svm_msr.h --- a/sys/amd64/vmm/amd/svm_msr.h +++ b/sys/amd64/vmm/amd/svm_msr.h @@ -32,15 +32,16 @@ #define _SVM_MSR_H_ struct svm_softc; +struct svm_vcpu; void svm_msr_init(void); -void svm_msr_guest_init(struct svm_softc *sc, int vcpu); -void svm_msr_guest_enter(struct svm_softc *sc, int vcpu); -void svm_msr_guest_exit(struct svm_softc *sc, int vcpu); +void svm_msr_guest_init(struct svm_softc *sc, struct svm_vcpu *vcpu); +void svm_msr_guest_enter(struct svm_softc *sc, struct svm_vcpu *vcpu); +void svm_msr_guest_exit(struct svm_softc *sc, struct svm_vcpu *vcpu); -int svm_wrmsr(struct svm_softc *sc, int vcpu, u_int num, uint64_t val, - bool *retu); -int svm_rdmsr(struct svm_softc *sc, int vcpu, u_int num, uint64_t *result, - bool *retu); +int svm_wrmsr(struct svm_softc *sc, struct svm_vcpu *vcpu, u_int num, + uint64_t val, bool *retu); +int svm_rdmsr(struct svm_softc *sc, struct svm_vcpu *vcpu, u_int num, + uint64_t *result, bool *retu); #endif /* _SVM_MSR_H_ */ diff --git a/sys/amd64/vmm/amd/svm_msr.c b/sys/amd64/vmm/amd/svm_msr.c --- a/sys/amd64/vmm/amd/svm_msr.c +++ b/sys/amd64/vmm/amd/svm_msr.c @@ -72,7 +72,7 @@ } void -svm_msr_guest_init(struct svm_softc *sc, int vcpu) +svm_msr_guest_init(struct svm_softc *sc, struct svm_vcpu *vcpu) { /* * All the MSRs accessible to the guest are either saved/restored by @@ -86,7 +86,7 @@ } void -svm_msr_guest_enter(struct svm_softc *sc, int vcpu) +svm_msr_guest_enter(struct svm_softc *sc, struct svm_vcpu *vcpu) { /* * Save host MSRs (if any) and restore guest MSRs (if any). @@ -94,7 +94,7 @@ } void -svm_msr_guest_exit(struct svm_softc *sc, int vcpu) +svm_msr_guest_exit(struct svm_softc *sc, struct svm_vcpu *vcpu) { /* * Save guest MSRs (if any) and restore host MSRs. @@ -108,44 +108,45 @@ } int -svm_rdmsr(struct svm_softc *sc, int vcpu, u_int num, uint64_t *result, +svm_rdmsr(struct svm_softc *sc, struct svm_vcpu *vcpu, u_int num, + uint64_t *result, bool *retu) +{ + int error = 0; + + switch (num) { + case MSR_MCG_CAP: + case MSR_MCG_STATUS: + *result = 0; + break; + case MSR_MTRRcap: + case MSR_MTRRdefType: + case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7: + case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1: + case MSR_MTRR64kBase: + case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1: + if (vm_rdmtrr(&vcpu->mtrr, num, result) != 0) { + vm_inject_gp(sc->vm, vcpu->vcpuid); + } + break; + case MSR_SYSCFG: + case MSR_AMDK8_IPM: + case MSR_EXTFEATURES: + *result = 0; + break; + default: + error = EINVAL; + break; + } + + return (error); +} + +int +svm_wrmsr(struct svm_softc *sc, struct svm_vcpu *vcpu, u_int num, uint64_t val, bool *retu) { int error = 0; - switch (num) { - case MSR_MCG_CAP: - case MSR_MCG_STATUS: - *result = 0; - break; - case MSR_MTRRcap: - case MSR_MTRRdefType: - case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7: - case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1: - case MSR_MTRR64kBase: - case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1: - if (vm_rdmtrr(&sc->vcpu[vcpu].mtrr, num, result) != 0) { - vm_inject_gp(sc->vm, vcpu); - } - break; - case MSR_SYSCFG: - case MSR_AMDK8_IPM: - case MSR_EXTFEATURES: - *result = 0; - break; - default: - error = EINVAL; - break; - } - - return (error); -} - -int -svm_wrmsr(struct svm_softc *sc, int vcpu, u_int num, uint64_t val, bool *retu) -{ - int error = 0; - switch (num) { case MSR_MCG_CAP: case MSR_MCG_STATUS: @@ -156,8 +157,8 @@ case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1: case MSR_MTRR64kBase: case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1: - if (vm_wrmtrr(&sc->vcpu[vcpu].mtrr, num, val) != 0) { - vm_inject_gp(sc->vm, vcpu); + if (vm_wrmtrr(&vcpu->mtrr, num, val) != 0) { + vm_inject_gp(sc->vm, vcpu->vcpuid); } break; case MSR_SYSCFG: diff --git a/sys/amd64/vmm/amd/svm_softc.h b/sys/amd64/vmm/amd/svm_softc.h --- a/sys/amd64/vmm/amd/svm_softc.h +++ b/sys/amd64/vmm/amd/svm_softc.h @@ -51,62 +51,52 @@ long eptgen; /* pmap->pm_eptgen when the vcpu last ran */ struct asid asid; struct vm_mtrr mtrr; + int vcpuid; }; /* * SVM softc, one per virtual machine. */ struct svm_softc { - struct svm_vcpu vcpu[VM_MAXCPU]; - vm_offset_t nptp; /* nested page table */ + vm_paddr_t nptp; /* nested page table */ uint8_t *iopm_bitmap; /* shared by all vcpus */ uint8_t *msr_bitmap; /* shared by all vcpus */ struct vm *vm; }; -static __inline struct svm_vcpu * -svm_get_vcpu(struct svm_softc *sc, int vcpu) -{ - - return (&(sc->vcpu[vcpu])); -} - static __inline struct vmcb * -svm_get_vmcb(struct svm_softc *sc, int vcpu) +svm_get_vmcb(struct svm_vcpu *vcpu) { - return ((sc->vcpu[vcpu].vmcb)); + return (vcpu->vmcb); } static __inline struct vmcb_state * -svm_get_vmcb_state(struct svm_softc *sc, int vcpu) +svm_get_vmcb_state(struct svm_vcpu *vcpu) { - return (&(sc->vcpu[vcpu].vmcb->state)); + return (&vcpu->vmcb->state); } static __inline struct vmcb_ctrl * -svm_get_vmcb_ctrl(struct svm_softc *sc, int vcpu) +svm_get_vmcb_ctrl(struct svm_vcpu *vcpu) { - return (&(sc->vcpu[vcpu].vmcb->ctrl)); + return (&vcpu->vmcb->ctrl); } static __inline struct svm_regctx * -svm_get_guest_regctx(struct svm_softc *sc, int vcpu) +svm_get_guest_regctx(struct svm_vcpu *vcpu) { - return (&(sc->vcpu[vcpu].swctx)); + return (&vcpu->swctx); } static __inline void -svm_set_dirty(struct svm_softc *sc, int vcpu, uint32_t dirtybits) +svm_set_dirty(struct svm_vcpu *vcpu, uint32_t dirtybits) { - struct svm_vcpu *vcpustate; - vcpustate = svm_get_vcpu(sc, vcpu); - - vcpustate->dirty |= dirtybits; + vcpu->dirty |= dirtybits; } #endif /* _SVM_SOFTC_H_ */ diff --git a/sys/amd64/vmm/amd/vmcb.h b/sys/amd64/vmm/amd/vmcb.h --- a/sys/amd64/vmm/amd/vmcb.h +++ b/sys/amd64/vmm/amd/vmcb.h @@ -234,6 +234,7 @@ #ifdef _KERNEL struct svm_softc; +struct svm_vcpu; struct vm_snapshot_meta; /* VMCB save state area segment format */ @@ -353,17 +354,23 @@ CTASSERT(sizeof(struct vmcb) == PAGE_SIZE); CTASSERT(offsetof(struct vmcb, state) == 0x400); -int vmcb_read(struct svm_softc *sc, int vcpu, int ident, uint64_t *retval); -int vmcb_write(struct svm_softc *sc, int vcpu, int ident, uint64_t val); -int vmcb_setdesc(void *arg, int vcpu, int ident, struct seg_desc *desc); -int vmcb_getdesc(void *arg, int vcpu, int ident, struct seg_desc *desc); +int vmcb_read(struct svm_softc *sc, struct svm_vcpu *vcpu, int ident, + uint64_t *retval); +int vmcb_write(struct svm_softc *sc, struct svm_vcpu *vcpu, int ident, + uint64_t val); +int vmcb_setdesc(struct svm_softc *sc, struct svm_vcpu *vcpu, int ident, + struct seg_desc *desc); +int vmcb_getdesc(struct svm_softc *sc, struct svm_vcpu *vcpu, int ident, + struct seg_desc *desc); int vmcb_seg(struct vmcb *vmcb, int ident, struct vmcb_segment *seg); #ifdef BHYVE_SNAPSHOT -int vmcb_getany(struct svm_softc *sc, int vcpu, int ident, uint64_t *val); -int vmcb_setany(struct svm_softc *sc, int vcpu, int ident, uint64_t val); -int vmcb_snapshot_desc(void *arg, int vcpu, int reg, +int vmcb_getany(struct svm_softc *sc, struct svm_vcpu *vcpu, int ident, + uint64_t *val); +int vmcb_setany(struct svm_softc *sc, struct svm_vcpu *vcpu, int ident, + uint64_t val); +int vmcb_snapshot_desc(struct svm_softc *sc, struct svm_vcpu *vcpu, int reg, struct vm_snapshot_meta *meta); -int vmcb_snapshot_any(struct svm_softc *sc, int vcpu, int ident, +int vmcb_snapshot_any(struct svm_softc *sc, struct svm_vcpu*vcpu, int ident, struct vm_snapshot_meta *meta); #endif diff --git a/sys/amd64/vmm/amd/vmcb.c b/sys/amd64/vmm/amd/vmcb.c --- a/sys/amd64/vmm/amd/vmcb.c +++ b/sys/amd64/vmm/amd/vmcb.c @@ -116,14 +116,14 @@ } static int -vmcb_access(struct svm_softc *softc, int vcpu, int write, int ident, - uint64_t *val) +vmcb_access(struct svm_softc *softc, struct svm_vcpu *vcpu, int write, + int ident, uint64_t *val) { struct vmcb *vmcb; int off, bytes; char *ptr; - vmcb = svm_get_vmcb(softc, vcpu); + vmcb = svm_get_vmcb(vcpu); off = VMCB_ACCESS_OFFSET(ident); bytes = VMCB_ACCESS_BYTES(ident); @@ -146,14 +146,14 @@ memcpy(val, ptr + off, bytes); break; default: - VCPU_CTR1(softc->vm, vcpu, + VCPU_CTR1(softc->vm, vcpu->vcpuid, "Invalid size %d for VMCB access: %d", bytes); return (EINVAL); } /* Invalidate all VMCB state cached by h/w. */ if (write) - svm_set_dirty(softc, vcpu, 0xffffffff); + svm_set_dirty(vcpu, 0xffffffff); return (0); } @@ -162,14 +162,15 @@ * Read from segment selector, control and general purpose register of VMCB. */ int -vmcb_read(struct svm_softc *sc, int vcpu, int ident, uint64_t *retval) +vmcb_read(struct svm_softc *sc, struct svm_vcpu *vcpu, int ident, + uint64_t *retval) { struct vmcb *vmcb; struct vmcb_state *state; struct vmcb_segment *seg; int err; - vmcb = svm_get_vmcb(sc, vcpu); + vmcb = svm_get_vmcb(vcpu); state = &vmcb->state; err = 0; @@ -252,14 +253,14 @@ * Write to segment selector, control and general purpose register of VMCB. */ int -vmcb_write(struct svm_softc *sc, int vcpu, int ident, uint64_t val) +vmcb_write(struct svm_softc *sc, struct svm_vcpu *vcpu, int ident, uint64_t val) { struct vmcb *vmcb; struct vmcb_state *state; struct vmcb_segment *seg; int err, dirtyseg; - vmcb = svm_get_vmcb(sc, vcpu); + vmcb = svm_get_vmcb(vcpu); state = &vmcb->state; dirtyseg = 0; err = 0; @@ -270,38 +271,38 @@ switch (ident) { case VM_REG_GUEST_CR0: state->cr0 = val; - svm_set_dirty(sc, vcpu, VMCB_CACHE_CR); + svm_set_dirty(vcpu, VMCB_CACHE_CR); break; case VM_REG_GUEST_CR2: state->cr2 = val; - svm_set_dirty(sc, vcpu, VMCB_CACHE_CR2); + svm_set_dirty(vcpu, VMCB_CACHE_CR2); break; case VM_REG_GUEST_CR3: state->cr3 = val; - svm_set_dirty(sc, vcpu, VMCB_CACHE_CR); + svm_set_dirty(vcpu, VMCB_CACHE_CR); break; case VM_REG_GUEST_CR4: state->cr4 = val; - svm_set_dirty(sc, vcpu, VMCB_CACHE_CR); + svm_set_dirty(vcpu, VMCB_CACHE_CR); break; case VM_REG_GUEST_DR6: state->dr6 = val; - svm_set_dirty(sc, vcpu, VMCB_CACHE_DR); + svm_set_dirty(vcpu, VMCB_CACHE_DR); break; case VM_REG_GUEST_DR7: state->dr7 = val; - svm_set_dirty(sc, vcpu, VMCB_CACHE_DR); + svm_set_dirty(vcpu, VMCB_CACHE_DR); break; case VM_REG_GUEST_EFER: /* EFER_SVM must always be set when the guest is executing */ state->efer = val | EFER_SVM; - svm_set_dirty(sc, vcpu, VMCB_CACHE_CR); + svm_set_dirty(vcpu, VMCB_CACHE_CR); break; case VM_REG_GUEST_RAX: @@ -334,7 +335,7 @@ __func__, ident)); seg->selector = val; if (dirtyseg) - svm_set_dirty(sc, vcpu, VMCB_CACHE_SEG); + svm_set_dirty(vcpu, VMCB_CACHE_SEG); break; case VM_REG_GUEST_GDTR: @@ -365,15 +366,14 @@ } int -vmcb_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) +vmcb_setdesc(struct svm_softc *sc, struct svm_vcpu *vcpu, int reg, + struct seg_desc *desc) { struct vmcb *vmcb; - struct svm_softc *sc; struct vmcb_segment *seg; uint16_t attrib; - sc = arg; - vmcb = svm_get_vmcb(sc, vcpu); + vmcb = svm_get_vmcb(vcpu); seg = vmcb_segptr(vmcb, reg); KASSERT(seg != NULL, ("%s: invalid segment descriptor %d", @@ -395,7 +395,7 @@ seg->attrib = attrib; } - VCPU_CTR4(sc->vm, vcpu, "Setting desc %d: base (%#lx), limit (%#x), " + VCPU_CTR4(sc->vm, vcpu->vcpuid, "Setting desc %d: base (%#lx), limit (%#x), " "attrib (%#x)", reg, seg->base, seg->limit, seg->attrib); switch (reg) { @@ -403,11 +403,11 @@ case VM_REG_GUEST_DS: case VM_REG_GUEST_ES: case VM_REG_GUEST_SS: - svm_set_dirty(sc, vcpu, VMCB_CACHE_SEG); + svm_set_dirty(vcpu, VMCB_CACHE_SEG); break; case VM_REG_GUEST_GDTR: case VM_REG_GUEST_IDTR: - svm_set_dirty(sc, vcpu, VMCB_CACHE_DT); + svm_set_dirty(vcpu, VMCB_CACHE_DT); break; default: break; @@ -417,14 +417,13 @@ } int -vmcb_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) +vmcb_getdesc(struct svm_softc *sc, struct svm_vcpu *vcpu, int reg, + struct seg_desc *desc) { struct vmcb *vmcb; - struct svm_softc *sc; struct vmcb_segment *seg; - sc = arg; - vmcb = svm_get_vmcb(sc, vcpu); + vmcb = svm_get_vmcb(vcpu); seg = vmcb_segptr(vmcb, reg); KASSERT(seg != NULL, ("%s: invalid segment descriptor %d", __func__, reg)); @@ -459,15 +458,11 @@ #ifdef BHYVE_SNAPSHOT int -vmcb_getany(struct svm_softc *sc, int vcpu, int ident, uint64_t *val) +vmcb_getany(struct svm_softc *sc, struct svm_vcpu *vcpu, int ident, + uint64_t *val) { int error = 0; - if (vcpu < 0 || vcpu >= vm_get_maxcpus(sc->vm)) { - error = EINVAL; - goto err; - } - if (ident >= VM_REG_LAST) { error = EINVAL; goto err; @@ -480,15 +475,11 @@ } int -vmcb_setany(struct svm_softc *sc, int vcpu, int ident, uint64_t val) +vmcb_setany(struct svm_softc *sc, struct svm_vcpu *vcpu, int ident, + uint64_t val) { int error = 0; - if (vcpu < 0 || vcpu >= vm_get_maxcpus(sc->vm)) { - error = EINVAL; - goto err; - } - if (ident >= VM_REG_LAST) { error = EINVAL; goto err; @@ -501,13 +492,14 @@ } int -vmcb_snapshot_desc(void *arg, int vcpu, int reg, struct vm_snapshot_meta *meta) +vmcb_snapshot_desc(struct svm_softc *sc, struct svm_vcpu *vcpu, int reg, + struct vm_snapshot_meta *meta) { int ret; struct seg_desc desc; if (meta->op == VM_SNAPSHOT_SAVE) { - ret = vmcb_getdesc(arg, vcpu, reg, &desc); + ret = vmcb_getdesc(sc, vcpu, reg, &desc); if (ret != 0) goto done; @@ -519,7 +511,7 @@ SNAPSHOT_VAR_OR_LEAVE(desc.limit, meta, ret, done); SNAPSHOT_VAR_OR_LEAVE(desc.access, meta, ret, done); - ret = vmcb_setdesc(arg, vcpu, reg, &desc); + ret = vmcb_setdesc(sc, vcpu, reg, &desc); if (ret != 0) goto done; } else { @@ -532,7 +524,7 @@ } int -vmcb_snapshot_any(struct svm_softc *sc, int vcpu, int ident, +vmcb_snapshot_any(struct svm_softc *sc, struct svm_vcpu *vcpu, int ident, struct vm_snapshot_meta *meta) { int ret; diff --git a/sys/amd64/vmm/intel/vmx.h b/sys/amd64/vmm/intel/vmx.h --- a/sys/amd64/vmm/intel/vmx.h +++ b/sys/amd64/vmm/intel/vmx.h @@ -31,6 +31,9 @@ #ifndef _VMX_H_ #define _VMX_H_ +#include +#include + #include "vmcs.h" #include "x86.h" @@ -131,15 +134,18 @@ struct vmxcap cap; struct vmxstate state; struct vm_mtrr mtrr; + int vcpuid; }; /* virtual machine softc */ struct vmx { - struct vmx_vcpu vcpus[VM_MAXCPU]; + struct vm *vm; char *msr_bitmap; + bool have_msr_tsc_aux; uint64_t eptp; - struct vm *vm; long eptgen[MAXCPU]; /* cached pmap->pm_eptgen */ + pmap_t pmap; + uint16_t vpids[VM_MAXCPU]; }; #define VMX_GUEST_VMEXIT 0 @@ -151,7 +157,8 @@ u_long vmx_fix_cr0(u_long cr0); u_long vmx_fix_cr4(u_long cr4); -int vmx_set_tsc_offset(struct vmx *vmx, int vcpu, uint64_t offset); +int vmx_set_tsc_offset(struct vmx *vmx, struct vmx_vcpu *vcpu, + uint64_t offset); extern char vmx_exit_guest[]; extern char vmx_exit_guest_flush_rsb[]; @@ -159,15 +166,8 @@ static inline bool vmx_have_msr_tsc_aux(struct vmx *vmx) { - int rdpid_rdtscp_bits = ((1 << VM_CAP_RDPID) | (1 << VM_CAP_RDTSCP)); - /* - * Since the values of these bits are uniform across all vCPUs - * (see discussion in vmx_modinit() and initialization of these bits - * in vmx_init()), just always use vCPU-zero's capability set and - * remove the need to require a vcpuid argument. - */ - return ((vmx->vcpus[0].cap.set & rdpid_rdtscp_bits) != 0); + return (vmx->have_msr_tsc_aux); } #endif diff --git a/sys/amd64/vmm/intel/vmx.c b/sys/amd64/vmm/intel/vmx.c --- a/sys/amd64/vmm/intel/vmx.c +++ b/sys/amd64/vmm/intel/vmx.c @@ -308,12 +308,12 @@ */ #define APIC_ACCESS_ADDRESS 0xFFFFF000 -static int vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc); -static int vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval); +static int vmx_getdesc(void *arg, void *vcpui, int reg, struct seg_desc *desc); +static int vmx_getreg(void *arg, void *vcpui, int reg, uint64_t *retval); static int vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val); static void vmx_inject_pir(struct vlapic *vlapic); #ifdef BHYVE_SNAPSHOT -static int vmx_restore_tsc(void *arg, int vcpu, uint64_t now); +static int vmx_restore_tsc(void *arg, void *vcpui, uint64_t now); #endif static inline bool @@ -1029,13 +1029,9 @@ static void * vmx_init(struct vm *vm, pmap_t pmap) { - int i, error; + int error; struct vmx *vmx; - struct vmcs *vmcs; - struct vmx_vcpu *vcpu; - uint32_t exc_bitmap; uint16_t maxcpus = vm_get_maxcpus(vm); - uint16_t vpid[maxcpus]; vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO); vmx->vm = vm; @@ -1095,7 +1091,7 @@ ((cap_rdpid || cap_rdtscp) && guest_msr_ro(vmx, MSR_TSC_AUX))) panic("vmx_init: error setting guest msr access"); - vpid_alloc(vpid, maxcpus); + vpid_alloc(vmx->vpids, maxcpus); if (virtual_interrupt_delivery) { error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE, @@ -1104,113 +1100,124 @@ KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error)); } - for (i = 0; i < maxcpus; i++) { - vcpu = &vmx->vcpus[i]; - - vcpu->vmcs = malloc(sizeof(*vmcs), M_VMX, M_WAITOK | M_ZERO); - vcpu->apic_page = malloc(PAGE_SIZE, M_VMX, M_WAITOK | M_ZERO); - vcpu->pir_desc = malloc_aligned(sizeof(*vcpu->pir_desc), 64, - M_VMX, M_WAITOK | M_ZERO); - - vmcs = vcpu->vmcs; - vmcs->identifier = vmx_revision(); - error = vmclear(vmcs); - if (error != 0) { - panic("vmx_init: vmclear error %d on vcpu %d\n", - error, i); - } - - vmx_msr_guest_init(vmx, i); - - error = vmcs_init(vmcs); - KASSERT(error == 0, ("vmcs_init error %d", error)); - - VMPTRLD(vmcs); - error = 0; - error += vmwrite(VMCS_HOST_RSP, (u_long)&vcpu->ctx); - error += vmwrite(VMCS_EPTP, vmx->eptp); - error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls); - error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls); - if (vcpu_trap_wbinvd(vm, i)) { - KASSERT(cap_wbinvd_exit, ("WBINVD trap not available")); - procbased_ctls2 |= PROCBASED2_WBINVD_EXITING; - } - error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2); - error += vmwrite(VMCS_EXIT_CTLS, exit_ctls); - error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls); - error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap)); - error += vmwrite(VMCS_VPID, vpid[i]); - - if (guest_l1d_flush && !guest_l1d_flush_sw) { - vmcs_write(VMCS_ENTRY_MSR_LOAD, pmap_kextract( - (vm_offset_t)&msr_load_list[0])); - vmcs_write(VMCS_ENTRY_MSR_LOAD_COUNT, - nitems(msr_load_list)); - vmcs_write(VMCS_EXIT_MSR_STORE, 0); - vmcs_write(VMCS_EXIT_MSR_STORE_COUNT, 0); - } - - /* exception bitmap */ - if (vcpu_trace_exceptions(vm, i)) - exc_bitmap = 0xffffffff; - else - exc_bitmap = 1 << IDT_MC; - error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap); - - vcpu->ctx.guest_dr6 = DBREG_DR6_RESERVED1; - error += vmwrite(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1); - - if (tpr_shadowing) { - error += vmwrite(VMCS_VIRTUAL_APIC, - vtophys(vcpu->apic_page)); - } - - if (virtual_interrupt_delivery) { - error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS); - error += vmwrite(VMCS_EOI_EXIT0, 0); - error += vmwrite(VMCS_EOI_EXIT1, 0); - error += vmwrite(VMCS_EOI_EXIT2, 0); - error += vmwrite(VMCS_EOI_EXIT3, 0); - } - if (posted_interrupts) { - error += vmwrite(VMCS_PIR_VECTOR, pirvec); - error += vmwrite(VMCS_PIR_DESC, - vtophys(vcpu->pir_desc)); - } - VMCLEAR(vmcs); - KASSERT(error == 0, ("vmx_init: error customizing the vmcs")); - - vcpu->cap.set = 0; - vcpu->cap.set |= cap_rdpid != 0 ? 1 << VM_CAP_RDPID : 0; - vcpu->cap.set |= cap_rdtscp != 0 ? 1 << VM_CAP_RDTSCP : 0; - vcpu->cap.proc_ctls = procbased_ctls; - vcpu->cap.proc_ctls2 = procbased_ctls2; - vcpu->cap.exc_bitmap = exc_bitmap; - - vcpu->state.nextrip = ~0; - vcpu->state.lastcpu = NOCPU; - vcpu->state.vpid = vpid[i]; - - /* - * Set up the CR0/4 shadows, and init the read shadow - * to the power-on register value from the Intel Sys Arch. - * CR0 - 0x60000010 - * CR4 - 0 - */ - error = vmx_setup_cr0_shadow(vmcs, 0x60000010); - if (error != 0) - panic("vmx_setup_cr0_shadow %d", error); - - error = vmx_setup_cr4_shadow(vmcs, 0); - if (error != 0) - panic("vmx_setup_cr4_shadow %d", error); - - vcpu->ctx.pmap = pmap; - } + vmx->have_msr_tsc_aux = (cap_rdpid != 0 || cap_rdtscp != 0); + vmx->pmap = pmap; return (vmx); } +static void * +vmx_vcpu_init(void *arg, int vcpuid) +{ + struct vmx *vmx = arg; + struct vmcs *vmcs; + struct vmx_vcpu *vcpu; + uint32_t exc_bitmap; + int error; + + vcpu = malloc(sizeof(*vcpu), M_VMX, M_WAITOK | M_ZERO); + vcpu->vcpuid = vcpuid; + vcpu->vmcs = malloc(sizeof(*vmcs), M_VMX, M_WAITOK | M_ZERO); + vcpu->apic_page = malloc(PAGE_SIZE, M_VMX, M_WAITOK | M_ZERO); + vcpu->pir_desc = malloc_aligned(sizeof(*vcpu->pir_desc), 64, M_VMX, + M_WAITOK | M_ZERO); + + vmcs = vcpu->vmcs; + vmcs->identifier = vmx_revision(); + error = vmclear(vmcs); + if (error != 0) { + panic("vmx_init: vmclear error %d on vcpu %d\n", + error, vcpuid); + } + + vmx_msr_guest_init(vmx, vcpu); + + error = vmcs_init(vmcs); + KASSERT(error == 0, ("vmcs_init error %d", error)); + + VMPTRLD(vmcs); + error = 0; + error += vmwrite(VMCS_HOST_RSP, (u_long)&vcpu->ctx); + error += vmwrite(VMCS_EPTP, vmx->eptp); + error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls); + error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls); + if (vcpu_trap_wbinvd(vmx->vm, vcpuid)) { + KASSERT(cap_wbinvd_exit, ("WBINVD trap not available")); + procbased_ctls2 |= PROCBASED2_WBINVD_EXITING; + } + error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2); + error += vmwrite(VMCS_EXIT_CTLS, exit_ctls); + error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls); + error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap)); + error += vmwrite(VMCS_VPID, vmx->vpids[vcpuid]); + + if (guest_l1d_flush && !guest_l1d_flush_sw) { + vmcs_write(VMCS_ENTRY_MSR_LOAD, pmap_kextract( + (vm_offset_t)&msr_load_list[0])); + vmcs_write(VMCS_ENTRY_MSR_LOAD_COUNT, + nitems(msr_load_list)); + vmcs_write(VMCS_EXIT_MSR_STORE, 0); + vmcs_write(VMCS_EXIT_MSR_STORE_COUNT, 0); + } + + /* exception bitmap */ + if (vcpu_trace_exceptions(vmx->vm, vcpuid)) + exc_bitmap = 0xffffffff; + else + exc_bitmap = 1 << IDT_MC; + error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap); + + vcpu->ctx.guest_dr6 = DBREG_DR6_RESERVED1; + error += vmwrite(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1); + + if (tpr_shadowing) { + error += vmwrite(VMCS_VIRTUAL_APIC, vtophys(vcpu->apic_page)); + } + + if (virtual_interrupt_delivery) { + error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS); + error += vmwrite(VMCS_EOI_EXIT0, 0); + error += vmwrite(VMCS_EOI_EXIT1, 0); + error += vmwrite(VMCS_EOI_EXIT2, 0); + error += vmwrite(VMCS_EOI_EXIT3, 0); + } + if (posted_interrupts) { + error += vmwrite(VMCS_PIR_VECTOR, pirvec); + error += vmwrite(VMCS_PIR_DESC, vtophys(vcpu->pir_desc)); + } + VMCLEAR(vmcs); + KASSERT(error == 0, ("vmx_init: error customizing the vmcs")); + + vcpu->cap.set = 0; + vcpu->cap.set |= cap_rdpid != 0 ? 1 << VM_CAP_RDPID : 0; + vcpu->cap.set |= cap_rdtscp != 0 ? 1 << VM_CAP_RDTSCP : 0; + vcpu->cap.proc_ctls = procbased_ctls; + vcpu->cap.proc_ctls2 = procbased_ctls2; + vcpu->cap.exc_bitmap = exc_bitmap; + + vcpu->state.nextrip = ~0; + vcpu->state.lastcpu = NOCPU; + vcpu->state.vpid = vmx->vpids[vcpuid]; + + /* + * Set up the CR0/4 shadows, and init the read shadow + * to the power-on register value from the Intel Sys Arch. + * CR0 - 0x60000010 + * CR4 - 0 + */ + error = vmx_setup_cr0_shadow(vmcs, 0x60000010); + if (error != 0) + panic("vmx_setup_cr0_shadow %d", error); + + error = vmx_setup_cr4_shadow(vmcs, 0); + if (error != 0) + panic("vmx_setup_cr4_shadow %d", error); + + vcpu->ctx.pmap = vmx->pmap; + + return (vcpu); +} + static int vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx) { @@ -1223,29 +1230,30 @@ } static __inline void -vmx_run_trace(struct vmx *vmx, int vcpu) +vmx_run_trace(struct vmx *vmx, struct vmx_vcpu *vcpu) { #ifdef KTR - VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip()); + VCPU_CTR1(vmx->vm, vcpu->vcpuid, "Resume execution at %#lx", + vmcs_guest_rip()); #endif } static __inline void -vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason, - int handled) +vmx_exit_trace(struct vmx *vmx, struct vmx_vcpu *vcpu, uint64_t rip, + uint32_t exit_reason, int handled) { #ifdef KTR - VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx", + VCPU_CTR3(vmx->vm, vcpu->vcpuid, "%s %s vmexit at 0x%0lx", handled ? "handled" : "unhandled", exit_reason_to_str(exit_reason), rip); #endif } static __inline void -vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip) +vmx_astpending_trace(struct vmx *vmx, struct vmx_vcpu *vcpu, uint64_t rip) { #ifdef KTR - VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip); + VCPU_CTR1(vmx->vm, vcpu->vcpuid, "astpending vmexit at 0x%0lx", rip); #endif } @@ -1256,12 +1264,12 @@ * Invalidate guest mappings identified by its vpid from the TLB. */ static __inline void -vmx_invvpid(struct vmx *vmx, int vcpu, pmap_t pmap, int running) +vmx_invvpid(struct vmx *vmx, struct vmx_vcpu *vcpu, pmap_t pmap, int running) { struct vmxstate *vmxstate; struct invvpid_desc invvpid_desc; - vmxstate = &vmx->vcpus[vcpu].state; + vmxstate = &vcpu->state; if (vmxstate->vpid == 0) return; @@ -1277,7 +1285,7 @@ } KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside " - "critical section", __func__, vcpu)); + "critical section", __func__, vcpu->vcpuid)); /* * Invalidate all mappings tagged with 'vpid' @@ -1300,7 +1308,7 @@ invvpid_desc.vpid = vmxstate->vpid; invvpid_desc.linear_addr = 0; invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc); - vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_DONE, 1); + vmm_stat_incr(vmx->vm, vcpu->vcpuid, VCPU_INVVPID_DONE, 1); } else { /* * The invvpid can be skipped if an invept is going to @@ -1308,22 +1316,22 @@ * will invalidate combined mappings tagged with * 'vmx->eptp' for all vpids. */ - vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1); + vmm_stat_incr(vmx->vm, vcpu->vcpuid, VCPU_INVVPID_SAVED, 1); } } static void -vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap) +vmx_set_pcpu_defaults(struct vmx *vmx, struct vmx_vcpu *vcpu, pmap_t pmap) { struct vmxstate *vmxstate; - vmxstate = &vmx->vcpus[vcpu].state; + vmxstate = &vcpu->state; if (vmxstate->lastcpu == curcpu) return; vmxstate->lastcpu = curcpu; - vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1); + vmm_stat_incr(vmx->vm, vcpu->vcpuid, VCPU_MIGRATIONS, 1); vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase()); vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase()); @@ -1337,69 +1345,65 @@ CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0); static void __inline -vmx_set_int_window_exiting(struct vmx *vmx, int vcpu) +vmx_set_int_window_exiting(struct vmx *vmx, struct vmx_vcpu *vcpu) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; - if ((vmx_vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) { - vmx_vcpu->cap.proc_ctls |= PROCBASED_INT_WINDOW_EXITING; - vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx_vcpu->cap.proc_ctls); - VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting"); + if ((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) { + vcpu->cap.proc_ctls |= PROCBASED_INT_WINDOW_EXITING; + vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls); + VCPU_CTR0(vmx->vm, vcpu->vcpuid, + "Enabling interrupt window exiting"); } } static void __inline -vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu) +vmx_clear_int_window_exiting(struct vmx *vmx, struct vmx_vcpu *vcpu) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; - KASSERT((vmx_vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0, - ("intr_window_exiting not set: %#x", vmx_vcpu->cap.proc_ctls)); - vmx_vcpu->cap.proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING; - vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx_vcpu->cap.proc_ctls); - VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting"); + KASSERT((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0, + ("intr_window_exiting not set: %#x", vcpu->cap.proc_ctls)); + vcpu->cap.proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING; + vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls); + VCPU_CTR0(vmx->vm, vcpu->vcpuid, "Disabling interrupt window exiting"); } static void __inline -vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu) +vmx_set_nmi_window_exiting(struct vmx *vmx, struct vmx_vcpu *vcpu) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; - if ((vmx_vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) { - vmx_vcpu->cap.proc_ctls |= PROCBASED_NMI_WINDOW_EXITING; - vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx_vcpu->cap.proc_ctls); - VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting"); + if ((vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) { + vcpu->cap.proc_ctls |= PROCBASED_NMI_WINDOW_EXITING; + vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls); + VCPU_CTR0(vmx->vm, vcpu->vcpuid, "Enabling NMI window exiting"); } } static void __inline -vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu) +vmx_clear_nmi_window_exiting(struct vmx *vmx, struct vmx_vcpu *vcpu) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; - KASSERT((vmx_vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0, - ("nmi_window_exiting not set %#x", vmx_vcpu->cap.proc_ctls)); - vmx_vcpu->cap.proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING; - vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx_vcpu->cap.proc_ctls); - VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting"); + KASSERT((vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0, + ("nmi_window_exiting not set %#x", vcpu->cap.proc_ctls)); + vcpu->cap.proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING; + vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls); + VCPU_CTR0(vmx->vm, vcpu->vcpuid, "Disabling NMI window exiting"); } int -vmx_set_tsc_offset(struct vmx *vmx, int vcpu, uint64_t offset) +vmx_set_tsc_offset(struct vmx *vmx, struct vmx_vcpu *vcpu, uint64_t offset) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; int error; - if ((vmx_vcpu->cap.proc_ctls & PROCBASED_TSC_OFFSET) == 0) { - vmx_vcpu->cap.proc_ctls |= PROCBASED_TSC_OFFSET; - vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx_vcpu->cap.proc_ctls); - VCPU_CTR0(vmx->vm, vcpu, "Enabling TSC offsetting"); + if ((vcpu->cap.proc_ctls & PROCBASED_TSC_OFFSET) == 0) { + vcpu->cap.proc_ctls |= PROCBASED_TSC_OFFSET; + vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls); + VCPU_CTR0(vmx->vm, vcpu->vcpuid, "Enabling TSC offsetting"); } error = vmwrite(VMCS_TSC_OFFSET, offset); #ifdef BHYVE_SNAPSHOT if (error == 0) - error = vm_set_tsc_offset(vmx->vm, vcpu, offset); + error = vm_set_tsc_offset(vmx->vm, vcpu->vcpuid, offset); #endif return (error); } @@ -1410,7 +1414,7 @@ VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) static void -vmx_inject_nmi(struct vmx *vmx, int vcpu) +vmx_inject_nmi(struct vmx *vmx, struct vmx_vcpu *vcpu) { uint32_t gi __diagused, info; @@ -1429,33 +1433,32 @@ info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID; vmcs_write(VMCS_ENTRY_INTR_INFO, info); - VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI"); + VCPU_CTR0(vmx->vm, vcpu->vcpuid, "Injecting vNMI"); /* Clear the request */ - vm_nmi_clear(vmx->vm, vcpu); + vm_nmi_clear(vmx->vm, vcpu->vcpuid); } static void -vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic, - uint64_t guestrip) +vmx_inject_interrupts(struct vmx *vmx, struct vmx_vcpu *vcpu, + struct vlapic *vlapic, uint64_t guestrip) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; int vector, need_nmi_exiting, extint_pending; uint64_t rflags, entryinfo; uint32_t gi, info; - if (vmx_vcpu->state.nextrip != guestrip) { + if (vcpu->state.nextrip != guestrip) { gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); if (gi & HWINTR_BLOCKING) { VCPU_CTR2(vmx->vm, vcpu, "Guest interrupt blocking " "cleared due to rip change: %#lx/%#lx", - vmx->vcpus[vcpu].state.nextrip, guestrip); + vcpu->state.nextrip, guestrip); gi &= ~HWINTR_BLOCKING; vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); } } - if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) { + if (vm_entry_intinfo(vmx->vm, vcpu->vcpuid, &entryinfo)) { KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry " "intinfo is not valid: %#lx", __func__, entryinfo)); @@ -1480,7 +1483,7 @@ vmcs_write(VMCS_ENTRY_INTR_INFO, info); } - if (vm_nmi_pending(vmx->vm, vcpu)) { + if (vm_nmi_pending(vmx->vm, vcpu->vcpuid)) { /* * If there are no conditions blocking NMI injection then * inject it directly here otherwise enable "NMI window @@ -1500,19 +1503,20 @@ vmx_inject_nmi(vmx, vcpu); need_nmi_exiting = 0; } else { - VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI " - "due to VM-entry intr info %#x", info); + VCPU_CTR1(vmx->vm, vcpu->vcpuid, "Cannot " + "inject NMI due to VM-entry intr info %#x", + info); } } else { - VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to " - "Guest Interruptibility-state %#x", gi); + VCPU_CTR1(vmx->vm, vcpu->vcpuid, "Cannot inject NMI " + "due to Guest Interruptibility-state %#x", gi); } if (need_nmi_exiting) vmx_set_nmi_window_exiting(vmx, vcpu); } - extint_pending = vm_extint_pending(vmx->vm, vcpu); + extint_pending = vm_extint_pending(vmx->vm, vcpu->vcpuid); if (!extint_pending && virtual_interrupt_delivery) { vmx_inject_pir(vlapic); @@ -1524,9 +1528,9 @@ * checking for pending interrupts. This is just an optimization and * not needed for correctness. */ - if ((vmx_vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) { - VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to " - "pending int_window_exiting"); + if ((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) { + VCPU_CTR0(vmx->vm, vcpu->vcpuid, "Skip interrupt injection " + "due to pending int_window_exiting"); return; } @@ -1560,15 +1564,15 @@ /* Check RFLAGS.IF and the interruptibility state of the guest */ rflags = vmcs_read(VMCS_GUEST_RFLAGS); if ((rflags & PSL_I) == 0) { - VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " - "rflags %#lx", vector, rflags); + VCPU_CTR2(vmx->vm, vcpu->vcpuid, "Cannot inject vector %d due " + "to rflags %#lx", vector, rflags); goto cantinject; } gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); if (gi & HWINTR_BLOCKING) { - VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " - "Guest Interruptibility-state %#x", vector, gi); + VCPU_CTR2(vmx->vm, vcpu->vcpuid, "Cannot inject vector %d due " + "to Guest Interruptibility-state %#x", vector, gi); goto cantinject; } @@ -1581,8 +1585,8 @@ * - An exception was injected above. * - An NMI was injected above or after "NMI window exiting" */ - VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " - "VM-entry intr info %#x", vector, info); + VCPU_CTR2(vmx->vm, vcpu->vcpuid, "Cannot inject vector %d due " + "to VM-entry intr info %#x", vector, info); goto cantinject; } @@ -1595,7 +1599,7 @@ /* Update the Local APIC ISR */ vlapic_intr_accepted(vlapic, vector); } else { - vm_extint_clear(vmx->vm, vcpu); + vm_extint_clear(vmx->vm, vcpu->vcpuid); vatpic_intr_accepted(vmx->vm, vector); /* @@ -1612,7 +1616,8 @@ vmx_set_int_window_exiting(vmx, vcpu); } - VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector); + VCPU_CTR1(vmx->vm, vcpu->vcpuid, "Injecting hwintr at vector %d", + vector); return; @@ -1634,29 +1639,29 @@ * hypervisor needs to restore virtual-NMI blocking before resuming the guest. */ static void -vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid) +vmx_restore_nmi_blocking(struct vmx *vmx, struct vmx_vcpu *vcpu) { uint32_t gi; - VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking"); + VCPU_CTR0(vmx->vm, vcpu->vcpuid, "Restore Virtual-NMI blocking"); gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING; vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); } static void -vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid) +vmx_clear_nmi_blocking(struct vmx *vmx, struct vmx_vcpu *vcpu) { uint32_t gi; - VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking"); + VCPU_CTR0(vmx->vm, vcpu->vcpuid, "Clear Virtual-NMI blocking"); gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING; vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); } static void -vmx_assert_nmi_blocking(struct vmx *vmx, int vcpuid) +vmx_assert_nmi_blocking(struct vmx *vmx, struct vmx_vcpu *vcpu) { uint32_t gi __diagused; @@ -1666,13 +1671,14 @@ } static int -vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) +vmx_emulate_xsetbv(struct vmx *vmx, struct vmx_vcpu *vcpu, + struct vm_exit *vmexit) { struct vmxctx *vmxctx; uint64_t xcrval; const struct xsave_limits *limits; - vmxctx = &vmx->vcpus[vcpu].ctx; + vmxctx = &vcpu->ctx; limits = vmm_get_xsave_limits(); /* @@ -1683,31 +1689,31 @@ /* Only xcr0 is supported. */ if (vmxctx->guest_rcx != 0) { - vm_inject_gp(vmx->vm, vcpu); + vm_inject_gp(vmx->vm, vcpu->vcpuid); return (HANDLED); } /* We only handle xcr0 if both the host and guest have XSAVE enabled. */ if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) { - vm_inject_ud(vmx->vm, vcpu); + vm_inject_ud(vmx->vm, vcpu->vcpuid); return (HANDLED); } xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff); if ((xcrval & ~limits->xcr0_allowed) != 0) { - vm_inject_gp(vmx->vm, vcpu); + vm_inject_gp(vmx->vm, vcpu->vcpuid); return (HANDLED); } if (!(xcrval & XFEATURE_ENABLED_X87)) { - vm_inject_gp(vmx->vm, vcpu); + vm_inject_gp(vmx->vm, vcpu->vcpuid); return (HANDLED); } /* AVX (YMM_Hi128) requires SSE. */ if (xcrval & XFEATURE_ENABLED_AVX && (xcrval & XFEATURE_AVX) != XFEATURE_AVX) { - vm_inject_gp(vmx->vm, vcpu); + vm_inject_gp(vmx->vm, vcpu->vcpuid); return (HANDLED); } @@ -1718,7 +1724,7 @@ if (xcrval & XFEATURE_AVX512 && (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) != (XFEATURE_AVX512 | XFEATURE_AVX)) { - vm_inject_gp(vmx->vm, vcpu); + vm_inject_gp(vmx->vm, vcpu->vcpuid); return (HANDLED); } @@ -1728,7 +1734,7 @@ */ if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) != ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) { - vm_inject_gp(vmx->vm, vcpu); + vm_inject_gp(vmx->vm, vcpu->vcpuid); return (HANDLED); } @@ -1742,11 +1748,11 @@ } static uint64_t -vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident) +vmx_get_guest_reg(struct vmx_vcpu *vcpu, int ident) { const struct vmxctx *vmxctx; - vmxctx = &vmx->vcpus[vcpu].ctx; + vmxctx = &vcpu->ctx; switch (ident) { case 0: @@ -1787,11 +1793,11 @@ } static void -vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval) +vmx_set_guest_reg(struct vmx_vcpu *vcpu, int ident, uint64_t regval) { struct vmxctx *vmxctx; - vmxctx = &vmx->vcpus[vcpu].ctx; + vmxctx = &vcpu->ctx; switch (ident) { case 0: @@ -1848,7 +1854,7 @@ } static int -vmx_emulate_cr0_access(struct vmx *vmx, int vcpu, uint64_t exitqual) +vmx_emulate_cr0_access(struct vmx_vcpu *vcpu, uint64_t exitqual) { uint64_t crval, regval; @@ -1856,7 +1862,7 @@ if ((exitqual & 0xf0) != 0x00) return (UNHANDLED); - regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf); + regval = vmx_get_guest_reg(vcpu, (exitqual >> 8) & 0xf); vmcs_write(VMCS_CR0_SHADOW, regval); @@ -1886,7 +1892,7 @@ } static int -vmx_emulate_cr4_access(struct vmx *vmx, int vcpu, uint64_t exitqual) +vmx_emulate_cr4_access(struct vmx_vcpu *vcpu, uint64_t exitqual) { uint64_t crval, regval; @@ -1894,7 +1900,7 @@ if ((exitqual & 0xf0) != 0x00) return (UNHANDLED); - regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf); + regval = vmx_get_guest_reg(vcpu, (exitqual >> 8) & 0xf); vmcs_write(VMCS_CR4_SHADOW, regval); @@ -1906,7 +1912,8 @@ } static int -vmx_emulate_cr8_access(struct vmx *vmx, int vcpu, uint64_t exitqual) +vmx_emulate_cr8_access(struct vmx *vmx, struct vmx_vcpu *vcpu, + uint64_t exitqual) { struct vlapic *vlapic; uint64_t cr8; @@ -1917,13 +1924,13 @@ return (UNHANDLED); } - vlapic = vm_lapic(vmx->vm, vcpu); + vlapic = vm_lapic(vmx->vm, vcpu->vcpuid); regnum = (exitqual >> 8) & 0xf; if (exitqual & 0x10) { cr8 = vlapic_get_cr8(vlapic); - vmx_set_guest_reg(vmx, vcpu, regnum, cr8); + vmx_set_guest_reg(vcpu, regnum, cr8); } else { - cr8 = vmx_get_guest_reg(vmx, vcpu, regnum); + cr8 = vmx_get_guest_reg(vcpu, regnum); vlapic_set_cr8(vlapic, cr8); } @@ -1979,26 +1986,26 @@ } static uint64_t -inout_str_index(struct vmx *vmx, int vcpuid, int in) +inout_str_index(struct vmx *vmx, struct vmx_vcpu *vcpu, int in) { uint64_t val; int error __diagused; enum vm_reg_name reg; reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI; - error = vmx_getreg(vmx, vcpuid, reg, &val); + error = vmx_getreg(vmx, vcpu, reg, &val); KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error)); return (val); } static uint64_t -inout_str_count(struct vmx *vmx, int vcpuid, int rep) +inout_str_count(struct vmx *vmx, struct vmx_vcpu *vcpu, int rep) { uint64_t val; int error __diagused; if (rep) { - error = vmx_getreg(vmx, vcpuid, VM_REG_GUEST_RCX, &val); + error = vmx_getreg(vmx, vcpu, VM_REG_GUEST_RCX, &val); KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error)); } else { val = 1; @@ -2025,8 +2032,8 @@ } static void -inout_str_seginfo(struct vmx *vmx, int vcpuid, uint32_t inst_info, int in, - struct vm_inout_str *vis) +inout_str_seginfo(struct vmx *vmx, struct vmx_vcpu *vcpu, uint32_t inst_info, + int in, struct vm_inout_str *vis) { int error __diagused, s; @@ -2037,7 +2044,7 @@ vis->seg_name = vm_segment_name(s); } - error = vmx_getdesc(vmx, vcpuid, vis->seg_name, &vis->seg_desc); + error = vmx_getdesc(vmx, vcpu, vis->seg_name, &vis->seg_desc); KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error)); } @@ -2126,25 +2133,25 @@ } static __inline int -apic_access_virtualization(struct vmx *vmx, int vcpuid) +apic_access_virtualization(struct vmx_vcpu *vcpu) { uint32_t proc_ctls2; - proc_ctls2 = vmx->vcpus[vcpuid].cap.proc_ctls2; + proc_ctls2 = vcpu->cap.proc_ctls2; return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0); } static __inline int -x2apic_virtualization(struct vmx *vmx, int vcpuid) +x2apic_virtualization(struct vmx_vcpu *vcpu) { uint32_t proc_ctls2; - proc_ctls2 = vmx->vcpus[vcpuid].cap.proc_ctls2; + proc_ctls2 = vcpu->cap.proc_ctls2; return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0); } static int -vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic, +vmx_handle_apic_write(struct vmx_vcpu *vcpu, struct vlapic *vlapic, uint64_t qual) { int error, handled, offset; @@ -2154,7 +2161,7 @@ handled = HANDLED; offset = APIC_WRITE_OFFSET(qual); - if (!apic_access_virtualization(vmx, vcpuid)) { + if (!apic_access_virtualization(vcpu)) { /* * In general there should not be any APIC write VM-exits * unless APIC-access virtualization is enabled. @@ -2162,7 +2169,7 @@ * However self-IPI virtualization can legitimately trigger * an APIC-write VM-exit so treat it specially. */ - if (x2apic_virtualization(vmx, vcpuid) && + if (x2apic_virtualization(vcpu) && offset == APIC_OFFSET_SELF_IPI) { apic_regs = (uint32_t *)(vlapic->apic_page); vector = apic_regs[APIC_OFFSET_SELF_IPI / 4]; @@ -2212,10 +2219,10 @@ } static bool -apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa) +apic_access_fault(struct vmx_vcpu *vcpu, uint64_t gpa) { - if (apic_access_virtualization(vmx, vcpuid) && + if (apic_access_virtualization(vcpu) && (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE)) return (true); else @@ -2223,12 +2230,12 @@ } static int -vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit) +vmx_handle_apic_access(struct vmx_vcpu *vcpu, struct vm_exit *vmexit) { uint64_t qual; int access_type, offset, allowed; - if (!apic_access_virtualization(vmx, vcpuid)) + if (!apic_access_virtualization(vcpu)) return (UNHANDLED); qual = vmexit->u.vmx.exit_qualification; @@ -2309,20 +2316,21 @@ } static int -emulate_wrmsr(struct vmx *vmx, int vcpuid, u_int num, uint64_t val, bool *retu) +emulate_wrmsr(struct vmx *vmx, struct vmx_vcpu *vcpu, u_int num, uint64_t val, + bool *retu) { int error; if (lapic_msr(num)) - error = lapic_wrmsr(vmx->vm, vcpuid, num, val, retu); + error = lapic_wrmsr(vmx->vm, vcpu->vcpuid, num, val, retu); else - error = vmx_wrmsr(vmx, vcpuid, num, val, retu); + error = vmx_wrmsr(vmx, vcpu, num, val, retu); return (error); } static int -emulate_rdmsr(struct vmx *vmx, int vcpuid, u_int num, bool *retu) +emulate_rdmsr(struct vmx *vmx, struct vmx_vcpu *vcpu, u_int num, bool *retu) { struct vmxctx *vmxctx; uint64_t result; @@ -2330,13 +2338,13 @@ int error; if (lapic_msr(num)) - error = lapic_rdmsr(vmx->vm, vcpuid, num, &result, retu); + error = lapic_rdmsr(vmx->vm, vcpu->vcpuid, num, &result, retu); else - error = vmx_rdmsr(vmx, vcpuid, num, &result, retu); + error = vmx_rdmsr(vmx, vcpu, num, &result, retu); if (error == 0) { eax = result; - vmxctx = &vmx->vcpus[vcpuid].ctx; + vmxctx = &vcpu->ctx; error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax); KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error)); @@ -2349,10 +2357,9 @@ } static int -vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) +vmx_exit_process(struct vmx *vmx, struct vmx_vcpu *vcpu, struct vm_exit *vmexit) { int error, errcode, errcode_valid, handled, in; - struct vmx_vcpu *vmx_vcpu; struct vmxctx *vmxctx; struct vlapic *vlapic; struct vm_inout_str *vis; @@ -2360,21 +2367,22 @@ uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info; uint32_t intr_type, intr_vec, reason; uint64_t exitintinfo, qual, gpa; + int vcpuid; bool retu; CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0); CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0); handled = UNHANDLED; - vmx_vcpu = &vmx->vcpus[vcpu]; - vmxctx = &vmx_vcpu->ctx; + vmxctx = &vcpu->ctx; + vcpuid = vcpu->vcpuid; qual = vmexit->u.vmx.exit_qualification; reason = vmexit->u.vmx.exit_reason; vmexit->exitcode = VM_EXITCODE_BOGUS; - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1); - SDT_PROBE3(vmm, vmx, exit, entry, vmx, vcpu, vmexit); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_COUNT, 1); + SDT_PROBE3(vmm, vmx, exit, entry, vmx, vcpuid, vmexit); /* * VM-entry failures during or after loading guest state. @@ -2383,7 +2391,7 @@ * as most VM-exit fields are not populated as usual. */ if (__predict_false(reason == EXIT_REASON_MCE_DURING_ENTRY)) { - VCPU_CTR0(vmx->vm, vcpu, "Handling MCE during VM-entry"); + VCPU_CTR0(vmx->vm, vcpuid, "Handling MCE during VM-entry"); __asm __volatile("int $18"); return (1); } @@ -2404,7 +2412,7 @@ idtvec_err = vmcs_idt_vectoring_err(); exitintinfo |= (uint64_t)idtvec_err << 32; } - error = vm_exit_intinfo(vmx->vm, vcpu, exitintinfo); + error = vm_exit_intinfo(vmx->vm, vcpuid, exitintinfo); KASSERT(error == 0, ("%s: vm_set_intinfo error %d", __func__, error)); @@ -2477,21 +2485,21 @@ } } vmexit->exitcode = VM_EXITCODE_TASK_SWITCH; - SDT_PROBE4(vmm, vmx, exit, taskswitch, vmx, vcpu, vmexit, ts); - VCPU_CTR4(vmx->vm, vcpu, "task switch reason %d, tss 0x%04x, " + SDT_PROBE4(vmm, vmx, exit, taskswitch, vmx, vcpuid, vmexit, ts); + VCPU_CTR4(vmx->vm, vcpuid, "task switch reason %d, tss 0x%04x, " "%s errcode 0x%016lx", ts->reason, ts->tsssel, ts->ext ? "external" : "internal", ((uint64_t)ts->errcode << 32) | ts->errcode_valid); break; case EXIT_REASON_CR_ACCESS: - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1); - SDT_PROBE4(vmm, vmx, exit, craccess, vmx, vcpu, vmexit, qual); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_CR_ACCESS, 1); + SDT_PROBE4(vmm, vmx, exit, craccess, vmx, vcpuid, vmexit, qual); switch (qual & 0xf) { case 0: - handled = vmx_emulate_cr0_access(vmx, vcpu, qual); + handled = vmx_emulate_cr0_access(vcpu, qual); break; case 4: - handled = vmx_emulate_cr4_access(vmx, vcpu, qual); + handled = vmx_emulate_cr4_access(vcpu, qual); break; case 8: handled = vmx_emulate_cr8_access(vmx, vcpu, qual); @@ -2499,11 +2507,11 @@ } break; case EXIT_REASON_RDMSR: - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_RDMSR, 1); retu = false; ecx = vmxctx->guest_rcx; - VCPU_CTR1(vmx->vm, vcpu, "rdmsr 0x%08x", ecx); - SDT_PROBE4(vmm, vmx, exit, rdmsr, vmx, vcpu, vmexit, ecx); + VCPU_CTR1(vmx->vm, vcpuid, "rdmsr 0x%08x", ecx); + SDT_PROBE4(vmm, vmx, exit, rdmsr, vmx, vcpuid, vmexit, ecx); error = emulate_rdmsr(vmx, vcpu, ecx, &retu); if (error) { vmexit->exitcode = VM_EXITCODE_RDMSR; @@ -2517,14 +2525,14 @@ } break; case EXIT_REASON_WRMSR: - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_WRMSR, 1); retu = false; eax = vmxctx->guest_rax; ecx = vmxctx->guest_rcx; edx = vmxctx->guest_rdx; - VCPU_CTR2(vmx->vm, vcpu, "wrmsr 0x%08x value 0x%016lx", + VCPU_CTR2(vmx->vm, vcpuid, "wrmsr 0x%08x value 0x%016lx", ecx, (uint64_t)edx << 32 | eax); - SDT_PROBE5(vmm, vmx, exit, wrmsr, vmx, vmexit, vcpu, ecx, + SDT_PROBE5(vmm, vmx, exit, wrmsr, vmx, vmexit, vcpuid, ecx, (uint64_t)edx << 32 | eax); error = emulate_wrmsr(vmx, vcpu, ecx, (uint64_t)edx << 32 | eax, &retu); @@ -2541,8 +2549,8 @@ } break; case EXIT_REASON_HLT: - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1); - SDT_PROBE3(vmm, vmx, exit, halt, vmx, vcpu, vmexit); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_HLT, 1); + SDT_PROBE3(vmm, vmx, exit, halt, vmx, vcpuid, vmexit); vmexit->exitcode = VM_EXITCODE_HLT; vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS); if (virtual_interrupt_delivery) @@ -2552,19 +2560,19 @@ vmexit->u.hlt.intr_status = 0; break; case EXIT_REASON_MTF: - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1); - SDT_PROBE3(vmm, vmx, exit, mtrap, vmx, vcpu, vmexit); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_MTRAP, 1); + SDT_PROBE3(vmm, vmx, exit, mtrap, vmx, vcpuid, vmexit); vmexit->exitcode = VM_EXITCODE_MTRAP; vmexit->inst_length = 0; break; case EXIT_REASON_PAUSE: - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1); - SDT_PROBE3(vmm, vmx, exit, pause, vmx, vcpu, vmexit); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_PAUSE, 1); + SDT_PROBE3(vmm, vmx, exit, pause, vmx, vcpuid, vmexit); vmexit->exitcode = VM_EXITCODE_PAUSE; break; case EXIT_REASON_INTR_WINDOW: - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1); - SDT_PROBE3(vmm, vmx, exit, intrwindow, vmx, vcpu, vmexit); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_INTR_WINDOW, 1); + SDT_PROBE3(vmm, vmx, exit, intrwindow, vmx, vcpuid, vmexit); vmx_clear_int_window_exiting(vmx, vcpu); return (1); case EXIT_REASON_EXT_INTR: @@ -2579,7 +2587,7 @@ */ intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); SDT_PROBE4(vmm, vmx, exit, interrupt, - vmx, vcpu, vmexit, intr_info); + vmx, vcpuid, vmexit, intr_info); /* * XXX: Ignore this exit if VMCS_INTR_VALID is not set. @@ -2596,18 +2604,18 @@ * This is special. We want to treat this as an 'handled' * VM-exit but not increment the instruction pointer. */ - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_EXTINT, 1); return (1); case EXIT_REASON_NMI_WINDOW: - SDT_PROBE3(vmm, vmx, exit, nmiwindow, vmx, vcpu, vmexit); + SDT_PROBE3(vmm, vmx, exit, nmiwindow, vmx, vcpuid, vmexit); /* Exit to allow the pending virtual NMI to be injected */ - if (vm_nmi_pending(vmx->vm, vcpu)) + if (vm_nmi_pending(vmx->vm, vcpuid)) vmx_inject_nmi(vmx, vcpu); vmx_clear_nmi_window_exiting(vmx, vcpu); - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_NMI_WINDOW, 1); return (1); case EXIT_REASON_INOUT: - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_INOUT, 1); vmexit->exitcode = VM_EXITCODE_INOUT; vmexit->u.inout.bytes = (qual & 0x7) + 1; vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0; @@ -2627,15 +2635,15 @@ vis->addrsize = inout_str_addrsize(inst_info); inout_str_seginfo(vmx, vcpu, inst_info, in, vis); } - SDT_PROBE3(vmm, vmx, exit, inout, vmx, vcpu, vmexit); + SDT_PROBE3(vmm, vmx, exit, inout, vmx, vcpuid, vmexit); break; case EXIT_REASON_CPUID: - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1); - SDT_PROBE3(vmm, vmx, exit, cpuid, vmx, vcpu, vmexit); - handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_CPUID, 1); + SDT_PROBE3(vmm, vmx, exit, cpuid, vmx, vcpuid, vmexit); + handled = vmx_handle_cpuid(vmx->vm, vcpuid, vmxctx); break; case EXIT_REASON_EXCEPTION: - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_EXCEPTION, 1); intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); KASSERT((intr_info & VMCS_INTR_VALID) != 0, ("VM exit interruption info invalid: %#x", intr_info)); @@ -2668,7 +2676,7 @@ * the machine check back into the guest. */ if (intr_vec == IDT_MC) { - VCPU_CTR0(vmx->vm, vcpu, "Vectoring to MCE handler"); + VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to MCE handler"); __asm __volatile("int $18"); return (1); } @@ -2678,7 +2686,7 @@ * debug exceptions, bounce them out to userland. */ if (intr_type == VMCS_INTR_T_SWEXCEPTION && intr_vec == IDT_BP && - (vmx_vcpu->cap.set & (1 << VM_CAP_BPT_EXIT))) { + (vcpu->cap.set & (1 << VM_CAP_BPT_EXIT))) { vmexit->exitcode = VM_EXITCODE_BPT; vmexit->u.bpt.inst_length = vmexit->inst_length; vmexit->inst_length = 0; @@ -2706,11 +2714,11 @@ errcode_valid = 1; errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE); } - VCPU_CTR2(vmx->vm, vcpu, "Reflecting exception %d/%#x into " + VCPU_CTR2(vmx->vm, vcpuid, "Reflecting exception %d/%#x into " "the guest", intr_vec, errcode); SDT_PROBE5(vmm, vmx, exit, exception, - vmx, vcpu, vmexit, intr_vec, errcode); - error = vm_inject_exception(vmx->vm, vcpu, intr_vec, + vmx, vcpuid, vmexit, intr_vec, errcode); + error = vm_inject_exception(vmx->vm, vcpuid, intr_vec, errcode_valid, errcode, 0); KASSERT(error == 0, ("%s: vm_inject_exception error %d", __func__, error)); @@ -2723,20 +2731,20 @@ * this must be an instruction that accesses MMIO space. */ gpa = vmcs_gpa(); - if (vm_mem_allocated(vmx->vm, vcpu, gpa) || - apic_access_fault(vmx, vcpu, gpa)) { + if (vm_mem_allocated(vmx->vm, vcpuid, gpa) || + apic_access_fault(vcpu, gpa)) { vmexit->exitcode = VM_EXITCODE_PAGING; vmexit->inst_length = 0; vmexit->u.paging.gpa = gpa; vmexit->u.paging.fault_type = ept_fault_type(qual); - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_NESTED_FAULT, 1); SDT_PROBE5(vmm, vmx, exit, nestedfault, - vmx, vcpu, vmexit, gpa, qual); + vmx, vcpuid, vmexit, gpa, qual); } else if (ept_emulation_fault(qual)) { vmexit_inst_emul(vmexit, gpa, vmcs_gla()); - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_INST_EMUL, 1); SDT_PROBE4(vmm, vmx, exit, mmiofault, - vmx, vcpu, vmexit, gpa); + vmx, vcpuid, vmexit, gpa); } /* * If Virtual NMIs control is 1 and the VM-exit is due to an @@ -2753,12 +2761,12 @@ case EXIT_REASON_VIRTUALIZED_EOI: vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI; vmexit->u.ioapic_eoi.vector = qual & 0xFF; - SDT_PROBE3(vmm, vmx, exit, eoi, vmx, vcpu, vmexit); + SDT_PROBE3(vmm, vmx, exit, eoi, vmx, vcpuid, vmexit); vmexit->inst_length = 0; /* trap-like */ break; case EXIT_REASON_APIC_ACCESS: - SDT_PROBE3(vmm, vmx, exit, apicaccess, vmx, vcpu, vmexit); - handled = vmx_handle_apic_access(vmx, vcpu, vmexit); + SDT_PROBE3(vmm, vmx, exit, apicaccess, vmx, vcpuid, vmexit); + handled = vmx_handle_apic_access(vcpu, vmexit); break; case EXIT_REASON_APIC_WRITE: /* @@ -2766,25 +2774,25 @@ * pointing to the next instruction. */ vmexit->inst_length = 0; - vlapic = vm_lapic(vmx->vm, vcpu); + vlapic = vm_lapic(vmx->vm, vcpuid); SDT_PROBE4(vmm, vmx, exit, apicwrite, - vmx, vcpu, vmexit, vlapic); - handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual); + vmx, vcpuid, vmexit, vlapic); + handled = vmx_handle_apic_write(vcpu, vlapic, qual); break; case EXIT_REASON_XSETBV: - SDT_PROBE3(vmm, vmx, exit, xsetbv, vmx, vcpu, vmexit); + SDT_PROBE3(vmm, vmx, exit, xsetbv, vmx, vcpuid, vmexit); handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit); break; case EXIT_REASON_MONITOR: - SDT_PROBE3(vmm, vmx, exit, monitor, vmx, vcpu, vmexit); + SDT_PROBE3(vmm, vmx, exit, monitor, vmx, vcpuid, vmexit); vmexit->exitcode = VM_EXITCODE_MONITOR; break; case EXIT_REASON_MWAIT: - SDT_PROBE3(vmm, vmx, exit, mwait, vmx, vcpu, vmexit); + SDT_PROBE3(vmm, vmx, exit, mwait, vmx, vcpuid, vmexit); vmexit->exitcode = VM_EXITCODE_MWAIT; break; case EXIT_REASON_TPR: - vlapic = vm_lapic(vmx->vm, vcpu); + vlapic = vm_lapic(vmx->vm, vcpuid); vlapic_sync_tpr(vlapic); vmexit->inst_length = 0; handled = HANDLED; @@ -2799,7 +2807,7 @@ case EXIT_REASON_VMWRITE: case EXIT_REASON_VMXOFF: case EXIT_REASON_VMXON: - SDT_PROBE3(vmm, vmx, exit, vminsn, vmx, vcpu, vmexit); + SDT_PROBE3(vmm, vmx, exit, vminsn, vmx, vcpuid, vmexit); vmexit->exitcode = VM_EXITCODE_VMINSN; break; case EXIT_REASON_INVD: @@ -2809,8 +2817,8 @@ break; default: SDT_PROBE4(vmm, vmx, exit, unknown, - vmx, vcpu, vmexit, reason); - vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1); + vmx, vcpuid, vmexit, reason); + vmm_stat_incr(vmx->vm, vcpuid, VMEXIT_UNKNOWN, 1); break; } @@ -2847,7 +2855,7 @@ } SDT_PROBE4(vmm, vmx, exit, return, - vmx, vcpu, vmexit, handled); + vmx, vcpuid, vmexit, handled); return (handled); } @@ -2885,7 +2893,8 @@ * clear NMI blocking. */ static __inline void -vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit) +vmx_exit_handle_nmi(struct vmx *vmx, struct vmx_vcpu *vcpu, + struct vm_exit *vmexit) { uint32_t intr_info; @@ -2901,7 +2910,7 @@ if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) { KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due " "to NMI has invalid vector: %#x", intr_info)); - VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler"); + VCPU_CTR0(vmx->vm, vcpu->vcpuid, "Vectoring to NMI handler"); __asm __volatile("int $2"); } } @@ -2999,12 +3008,12 @@ } static int -vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap, +vmx_run(void *arg, void *vcpui, register_t rip, pmap_t pmap, struct vm_eventinfo *evinfo) { - int rc, handled, launched; + int rc, handled, launched, vcpuid; struct vmx *vmx; - struct vmx_vcpu *vmx_vcpu; + struct vmx_vcpu *vcpu; struct vm *vm; struct vmxctx *vmxctx; struct vmcs *vmcs; @@ -3016,11 +3025,12 @@ vmx = arg; vm = vmx->vm; - vmx_vcpu = &vmx->vcpus[vcpu]; - vmcs = vmx_vcpu->vmcs; - vmxctx = &vmx_vcpu->ctx; - vlapic = vm_lapic(vm, vcpu); - vmexit = vm_exitinfo(vm, vcpu); + vcpu = vcpui; + vcpuid = vcpu->vcpuid; + vmcs = vcpu->vmcs; + vmxctx = &vcpu->ctx; + vlapic = vm_lapic(vm, vcpuid); + vmexit = vm_exitinfo(vm, vcpuid); launched = 0; KASSERT(vmxctx->pmap == pmap, @@ -3075,33 +3085,33 @@ */ if (vcpu_suspended(evinfo)) { enable_intr(); - vm_exit_suspended(vmx->vm, vcpu, rip); + vm_exit_suspended(vmx->vm, vcpuid, rip); break; } if (vcpu_rendezvous_pending(evinfo)) { enable_intr(); - vm_exit_rendezvous(vmx->vm, vcpu, rip); + vm_exit_rendezvous(vmx->vm, vcpuid, rip); break; } if (vcpu_reqidle(evinfo)) { enable_intr(); - vm_exit_reqidle(vmx->vm, vcpu, rip); + vm_exit_reqidle(vmx->vm, vcpuid, rip); break; } - if (vcpu_should_yield(vm, vcpu)) { + if (vcpu_should_yield(vm, vcpuid)) { enable_intr(); - vm_exit_astpending(vmx->vm, vcpu, rip); + vm_exit_astpending(vmx->vm, vcpuid, rip); vmx_astpending_trace(vmx, vcpu, rip); handled = HANDLED; break; } - if (vcpu_debugged(vm, vcpu)) { + if (vcpu_debugged(vm, vcpuid)) { enable_intr(); - vm_exit_debug(vmx->vm, vcpu, rip); + vm_exit_debug(vmx->vm, vcpuid, rip); break; } @@ -3110,7 +3120,7 @@ * must be updated right before entering the guest. */ if (tpr_shadowing && !virtual_interrupt_delivery) { - if ((vmx_vcpu->cap.proc_ctls & PROCBASED_USE_TPR_SHADOW) != 0) { + if ((vcpu->cap.proc_ctls & PROCBASED_USE_TPR_SHADOW) != 0) { vmcs_write(VMCS_TPR_THRESHOLD, vlapic_get_cr8(vlapic)); } } @@ -3174,7 +3184,7 @@ vmexit->u.vmx.exit_qualification = vmcs_exit_qualification(); /* Update 'nextrip' */ - vmx_vcpu->state.nextrip = rip; + vcpu->state.nextrip = rip; if (rc == VMX_GUEST_VMEXIT) { vmx_exit_handle_nmi(vmx, vcpu, vmexit); @@ -3199,7 +3209,7 @@ handled, vmexit->exitcode); } - VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d", + VCPU_CTR1(vm, vcpuid, "returning from vmx_run: exitcode %d", vmexit->exitcode); VMCLEAR(vmcs); @@ -3208,26 +3218,26 @@ return (0); } +static void +vmx_vcpu_cleanup(void *arg, void *vcpui) +{ + struct vmx_vcpu *vcpu = vcpui; + + vpid_free(vcpu->state.vpid); + free(vcpu->pir_desc, M_VMX); + free(vcpu->apic_page, M_VMX); + free(vcpu->vmcs, M_VMX); + free(vcpu, M_VMX); +} + static void vmx_cleanup(void *arg) { - int i; - struct vmx_vcpu *vcpu; struct vmx *vmx = arg; - uint16_t maxcpus; - if (apic_access_virtualization(vmx, 0)) + if (virtual_interrupt_delivery) vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); - maxcpus = vm_get_maxcpus(vmx->vm); - for (i = 0; i < maxcpus; i++) { - vcpu = &vmx->vcpus[i]; - vpid_free(vcpu->state.vpid); - free(vcpu->pir_desc, M_VMX); - free(vcpu->apic_page, M_VMX); - free(vcpu->vmcs, M_VMX); - } - free(vmx->msr_bitmap, M_VMX); free(vmx, M_VMX); @@ -3312,19 +3322,20 @@ } static int -vmx_get_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t *retval) +vmx_get_intr_shadow(struct vmx_vcpu *vcpu, int running, uint64_t *retval) { uint64_t gi; int error; - error = vmcs_getreg(vmx->vcpus[vcpu].vmcs, running, + error = vmcs_getreg(vcpu->vmcs, running, VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi); *retval = (gi & HWINTR_BLOCKING) ? 1 : 0; return (error); } static int -vmx_modify_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t val) +vmx_modify_intr_shadow(struct vmx *vmx, struct vmx_vcpu *vcpu, int running, + uint64_t val) { struct vmcs *vmcs; uint64_t gi; @@ -3338,7 +3349,7 @@ goto done; } - vmcs = vmx->vcpus[vcpu].vmcs; + vmcs = vcpu->vmcs; ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY); error = vmcs_getreg(vmcs, running, ident, &gi); if (error == 0) { @@ -3346,7 +3357,7 @@ error = vmcs_setreg(vmcs, running, ident, gi); } done: - VCPU_CTR2(vmx->vm, vcpu, "Setting intr_shadow to %#lx %s", val, + VCPU_CTR2(vmx->vm, vcpu->vcpuid, "Setting intr_shadow to %#lx %s", val, error ? "failed" : "succeeded"); return (error); } @@ -3373,48 +3384,51 @@ } static int -vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval) +vmx_getreg(void *arg, void *vcpui, int reg, uint64_t *retval) { int running, hostcpu; struct vmx *vmx = arg; + struct vmx_vcpu *vcpu = vcpui; - running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); + running = vcpu_is_running(vmx->vm, vcpu->vcpuid, &hostcpu); if (running && hostcpu != curcpu) - panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu); + panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), + vcpu->vcpuid); if (reg == VM_REG_GUEST_INTR_SHADOW) - return (vmx_get_intr_shadow(vmx, vcpu, running, retval)); + return (vmx_get_intr_shadow(vcpu, running, retval)); - if (vmxctx_getreg(&vmx->vcpus[vcpu].ctx, reg, retval) == 0) + if (vmxctx_getreg(&vcpu->ctx, reg, retval) == 0) return (0); - return (vmcs_getreg(vmx->vcpus[vcpu].vmcs, running, reg, retval)); + return (vmcs_getreg(vcpu->vmcs, running, reg, retval)); } static int -vmx_setreg(void *arg, int vcpu, int reg, uint64_t val) +vmx_setreg(void *arg, void *vcpui, int reg, uint64_t val) { int error, hostcpu, running, shadow; uint64_t ctls; pmap_t pmap; struct vmx *vmx = arg; - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; + struct vmx_vcpu *vcpu = vcpui; - running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); + running = vcpu_is_running(vmx->vm, vcpu->vcpuid, &hostcpu); if (running && hostcpu != curcpu) - panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu); + panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), + vcpu->vcpuid); if (reg == VM_REG_GUEST_INTR_SHADOW) return (vmx_modify_intr_shadow(vmx, vcpu, running, val)); - if (vmxctx_setreg(&vmx_vcpu->ctx, reg, val) == 0) + if (vmxctx_setreg(&vcpu->ctx, reg, val) == 0) return (0); /* Do not permit user write access to VMCS fields by offset. */ if (reg < 0) return (EINVAL); - error = vmcs_setreg(vmx_vcpu->vmcs, running, reg, val); + error = vmcs_setreg(vcpu->vmcs, running, reg, val); if (error == 0) { /* @@ -3424,13 +3438,13 @@ */ if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 && (reg == VM_REG_GUEST_EFER)) { - vmcs_getreg(vmx_vcpu->vmcs, running, + vmcs_getreg(vcpu->vmcs, running, VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls); if (val & EFER_LMA) ctls |= VM_ENTRY_GUEST_LMA; else ctls &= ~VM_ENTRY_GUEST_LMA; - vmcs_setreg(vmx_vcpu->vmcs, running, + vmcs_setreg(vcpu->vmcs, running, VMCS_IDENT(VMCS_ENTRY_CTLS), ctls); } @@ -3439,7 +3453,7 @@ /* * Store the unmodified value in the shadow */ - error = vmcs_setreg(vmx_vcpu->vmcs, running, + error = vmcs_setreg(vcpu->vmcs, running, VMCS_IDENT(shadow), val); } @@ -3451,7 +3465,7 @@ * XXX the processor retains global mappings when %cr3 * is updated but vmx_invvpid() does not. */ - pmap = vmx_vcpu->ctx.pmap; + pmap = vcpu->ctx.pmap; vmx_invvpid(vmx, vcpu, pmap, running); } } @@ -3460,41 +3474,45 @@ } static int -vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) +vmx_getdesc(void *arg, void *vcpui, int reg, struct seg_desc *desc) { int hostcpu, running; struct vmx *vmx = arg; + struct vmx_vcpu *vcpu = vcpui; - running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); + running = vcpu_is_running(vmx->vm, vcpu->vcpuid, &hostcpu); if (running && hostcpu != curcpu) - panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu); + panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), + vcpu->vcpuid); - return (vmcs_getdesc(vmx->vcpus[vcpu].vmcs, running, reg, desc)); + return (vmcs_getdesc(vcpu->vmcs, running, reg, desc)); } static int -vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) +vmx_setdesc(void *arg, void *vcpui, int reg, struct seg_desc *desc) { int hostcpu, running; struct vmx *vmx = arg; + struct vmx_vcpu *vcpu = vcpui; - running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); + running = vcpu_is_running(vmx->vm, vcpu->vcpuid, &hostcpu); if (running && hostcpu != curcpu) - panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu); + panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), + vcpu->vcpuid); - return (vmcs_setdesc(vmx->vcpus[vcpu].vmcs, running, reg, desc)); + return (vmcs_setdesc(vcpu->vmcs, running, reg, desc)); } static int -vmx_getcap(void *arg, int vcpu, int type, int *retval) +vmx_getcap(void *arg, void *vcpui, int type, int *retval) { - struct vmx *vmx = arg; + struct vmx_vcpu *vcpu = vcpui; int vcap; int ret; ret = ENOENT; - vcap = vmx->vcpus[vcpu].cap.set; + vcap = vcpu->cap.set; switch (type) { case VM_CAP_HALT_EXIT: @@ -3540,11 +3558,11 @@ } static int -vmx_setcap(void *arg, int vcpu, int type, int val) +vmx_setcap(void *arg, void *vcpui, int type, int val) { struct vmx *vmx = arg; - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpu]; - struct vmcs *vmcs = vmx_vcpu->vmcs; + struct vmx_vcpu *vcpu = vcpui; + struct vmcs *vmcs = vcpu->vmcs; struct vlapic *vlapic; uint32_t baseval; uint32_t *pptr; @@ -3560,7 +3578,7 @@ case VM_CAP_HALT_EXIT: if (cap_halt_exit) { retval = 0; - pptr = &vmx_vcpu->cap.proc_ctls; + pptr = &vcpu->cap.proc_ctls; baseval = *pptr; flag = PROCBASED_HLT_EXITING; reg = VMCS_PRI_PROC_BASED_CTLS; @@ -3569,7 +3587,7 @@ case VM_CAP_MTRAP_EXIT: if (cap_monitor_trap) { retval = 0; - pptr = &vmx_vcpu->cap.proc_ctls; + pptr = &vcpu->cap.proc_ctls; baseval = *pptr; flag = PROCBASED_MTF; reg = VMCS_PRI_PROC_BASED_CTLS; @@ -3578,7 +3596,7 @@ case VM_CAP_PAUSE_EXIT: if (cap_pause_exit) { retval = 0; - pptr = &vmx_vcpu->cap.proc_ctls; + pptr = &vcpu->cap.proc_ctls; baseval = *pptr; flag = PROCBASED_PAUSE_EXITING; reg = VMCS_PRI_PROC_BASED_CTLS; @@ -3598,7 +3616,7 @@ case VM_CAP_UNRESTRICTED_GUEST: if (cap_unrestricted_guest) { retval = 0; - pptr = &vmx_vcpu->cap.proc_ctls2; + pptr = &vcpu->cap.proc_ctls2; baseval = *pptr; flag = PROCBASED2_UNRESTRICTED_GUEST; reg = VMCS_SEC_PROC_BASED_CTLS; @@ -3607,7 +3625,7 @@ case VM_CAP_ENABLE_INVPCID: if (cap_invpcid) { retval = 0; - pptr = &vmx_vcpu->cap.proc_ctls2; + pptr = &vcpu->cap.proc_ctls2; baseval = *pptr; flag = PROCBASED2_ENABLE_INVPCID; reg = VMCS_SEC_PROC_BASED_CTLS; @@ -3617,8 +3635,8 @@ retval = 0; /* Don't change the bitmap if we are tracing all exceptions. */ - if (vmx_vcpu->cap.exc_bitmap != 0xffffffff) { - pptr = &vmx_vcpu->cap.exc_bitmap; + if (vcpu->cap.exc_bitmap != 0xffffffff) { + pptr = &vcpu->cap.exc_bitmap; baseval = *pptr; flag = (1 << IDT_BP); reg = VMCS_EXCEPTION_BITMAP; @@ -3627,7 +3645,7 @@ case VM_CAP_IPI_EXIT: retval = 0; - vlapic = vm_lapic(vmx->vm, vcpu); + vlapic = vm_lapic(vmx->vm, vcpu->vcpuid); vlapic->ipi_exit = val; break; default: @@ -3658,9 +3676,9 @@ } if (val) { - vmx_vcpu->cap.set |= (1 << type); + vcpu->cap.set |= (1 << type); } else { - vmx_vcpu->cap.set &= ~(1 << type); + vcpu->cap.set &= ~(1 << type); } return (0); @@ -3682,6 +3700,7 @@ struct vlapic vlapic; struct pir_desc *pir_desc; struct vmx *vmx; + struct vmx_vcpu *vcpu; u_int pending_prio; }; @@ -3855,7 +3874,6 @@ vmx_set_tmr(struct vlapic *vlapic, int vector, bool level) { struct vlapic_vtx *vlapic_vtx; - struct vmx *vmx; struct vmcs *vmcs; uint64_t mask, val; @@ -3864,8 +3882,7 @@ ("vmx_set_tmr: vcpu cannot be running")); vlapic_vtx = (struct vlapic_vtx *)vlapic; - vmx = vlapic_vtx->vmx; - vmcs = vmx->vcpus[vlapic->vcpuid].vmcs; + vmcs = vlapic_vtx->vcpu->vmcs; mask = 1UL << (vector % 64); VMPTRLD(vmcs); @@ -3881,15 +3898,13 @@ static void vmx_enable_x2apic_mode_ts(struct vlapic *vlapic) { - struct vmx *vmx; + struct vlapic_vtx *vlapic_vtx; struct vmx_vcpu *vcpu; struct vmcs *vmcs; uint32_t proc_ctls; - int vcpuid; - vcpuid = vlapic->vcpuid; - vmx = ((struct vlapic_vtx *)vlapic)->vmx; - vcpu = &vmx->vcpus[vcpuid]; + vlapic_vtx = (struct vlapic_vtx *)vlapic; + vcpu = vlapic_vtx->vcpu; vmcs = vcpu->vmcs; proc_ctls = vcpu->cap.proc_ctls; @@ -3906,15 +3921,16 @@ static void vmx_enable_x2apic_mode_vid(struct vlapic *vlapic) { + struct vlapic_vtx *vlapic_vtx; struct vmx *vmx; struct vmx_vcpu *vcpu; struct vmcs *vmcs; uint32_t proc_ctls2; - int vcpuid, error __diagused; + int error __diagused; - vcpuid = vlapic->vcpuid; - vmx = ((struct vlapic_vtx *)vlapic)->vmx; - vcpu = &vmx->vcpus[vcpuid]; + vlapic_vtx = (struct vlapic_vtx *)vlapic; + vmx = vlapic_vtx->vmx; + vcpu = vlapic_vtx->vcpu; vmcs = vcpu->vmcs; proc_ctls2 = vcpu->cap.proc_ctls2; @@ -4050,22 +4066,25 @@ } static struct vlapic * -vmx_vlapic_init(void *arg, int vcpuid) +vmx_vlapic_init(void *arg, void *vcpui) { struct vmx *vmx; + struct vmx_vcpu *vcpu; struct vlapic *vlapic; struct vlapic_vtx *vlapic_vtx; vmx = arg; + vcpu = vcpui; vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO); vlapic->vm = vmx->vm; - vlapic->vcpuid = vcpuid; - vlapic->apic_page = (struct LAPIC *)vmx->vcpus[vcpuid].apic_page; + vlapic->vcpuid = vcpu->vcpuid; + vlapic->apic_page = (struct LAPIC *)vcpu->apic_page; vlapic_vtx = (struct vlapic_vtx *)vlapic; - vlapic_vtx->pir_desc = vmx->vcpus[vcpuid].pir_desc; + vlapic_vtx->pir_desc = vcpu->pir_desc; vlapic_vtx->vmx = vmx; + vlapic_vtx->vcpu = vcpu; if (tpr_shadowing) { vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_ts; @@ -4103,7 +4122,7 @@ } static int -vmx_vcpu_snapshot(void *arg, struct vm_snapshot_meta *meta, int vcpuid) +vmx_vcpu_snapshot(void *arg, struct vm_snapshot_meta *meta, void *vcpui) { struct vmcs *vmcs; struct vmx *vmx; @@ -4112,16 +4131,16 @@ int err, run, hostcpu; vmx = (struct vmx *)arg; + vcpu = vcpui; err = 0; KASSERT(arg != NULL, ("%s: arg was NULL", __func__)); - vcpu = &vmx->vcpus[vcpuid]; vmcs = vcpu->vmcs; - run = vcpu_is_running(vmx->vm, vcpuid, &hostcpu); + run = vcpu_is_running(vmx->vm, vcpu->vcpuid, &hostcpu); if (run && hostcpu != curcpu) { printf("%s: %s%d is running", __func__, vm_name(vmx->vm), - vcpuid); + vcpu->vcpuid); return (EINVAL); } @@ -4211,18 +4230,20 @@ } static int -vmx_restore_tsc(void *arg, int vcpu, uint64_t offset) +vmx_restore_tsc(void *arg, void *vcpui, uint64_t offset) { struct vmcs *vmcs; struct vmx *vmx = (struct vmx *)arg; + struct vmx_vcpu *vcpu = vcpui; int error, running, hostcpu; KASSERT(arg != NULL, ("%s: arg was NULL", __func__)); - vmcs = vmx->vcpus[vcpu].vmcs; + vmcs = vcpu->vmcs; - running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); + running = vcpu_is_running(vmx->vm, vcpu->vcpuid, &hostcpu); if (running && hostcpu != curcpu) { - printf("%s: %s%d is running", __func__, vm_name(vmx->vm), vcpu); + printf("%s: %s%d is running", __func__, vm_name(vmx->vm), + vcpu->vcpuid); return (EINVAL); } @@ -4244,6 +4265,8 @@ .init = vmx_init, .run = vmx_run, .cleanup = vmx_cleanup, + .vcpu_init = vmx_vcpu_init, + .vcpu_cleanup = vmx_vcpu_cleanup, .getreg = vmx_getreg, .setreg = vmx_setreg, .getdesc = vmx_getdesc, diff --git a/sys/amd64/vmm/intel/vmx_msr.h b/sys/amd64/vmm/intel/vmx_msr.h --- a/sys/amd64/vmm/intel/vmx_msr.h +++ b/sys/amd64/vmm/intel/vmx_msr.h @@ -34,13 +34,15 @@ struct vmx; void vmx_msr_init(void); -void vmx_msr_guest_init(struct vmx *vmx, int vcpuid); -void vmx_msr_guest_enter_tsc_aux(struct vmx *vmx, int vcpuid); -void vmx_msr_guest_enter(struct vmx *vmx, int vcpuid); -void vmx_msr_guest_exit(struct vmx *vmx, int vcpuid); -void vmx_msr_guest_exit_tsc_aux(struct vmx *vmx, int vcpuid); -int vmx_rdmsr(struct vmx *, int vcpuid, u_int num, uint64_t *val, bool *retu); -int vmx_wrmsr(struct vmx *, int vcpuid, u_int num, uint64_t val, bool *retu); +void vmx_msr_guest_init(struct vmx *vmx, struct vmx_vcpu *vcpu); +void vmx_msr_guest_enter_tsc_aux(struct vmx *vmx, struct vmx_vcpu *vcpu); +void vmx_msr_guest_enter(struct vmx *vmx, struct vmx_vcpu *vcpu); +void vmx_msr_guest_exit(struct vmx *vmx, struct vmx_vcpu *vcpu); +void vmx_msr_guest_exit_tsc_aux(struct vmx *vmx, struct vmx_vcpu *vcpu); +int vmx_rdmsr(struct vmx *, struct vmx_vcpu *vcpu, u_int num, uint64_t *val, + bool *retu); +int vmx_wrmsr(struct vmx *, struct vmx_vcpu *vcpu, u_int num, uint64_t val, + bool *retu); uint32_t vmx_revision(void); diff --git a/sys/amd64/vmm/intel/vmx_msr.c b/sys/amd64/vmm/intel/vmx_msr.c --- a/sys/amd64/vmm/intel/vmx_msr.c +++ b/sys/amd64/vmm/intel/vmx_msr.c @@ -314,15 +314,13 @@ } void -vmx_msr_guest_init(struct vmx *vmx, int vcpuid) +vmx_msr_guest_init(struct vmx *vmx, struct vmx_vcpu *vcpu) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpuid]; - /* * The permissions bitmap is shared between all vcpus so initialize it * once when initializing the vBSP. */ - if (vcpuid == 0) { + if (vcpu->vcpuid == 0) { guest_msr_rw(vmx, MSR_LSTAR); guest_msr_rw(vmx, MSR_CSTAR); guest_msr_rw(vmx, MSR_STAR); @@ -333,7 +331,7 @@ /* * Initialize guest IA32_PAT MSR with default value after reset. */ - vmx_vcpu->guest_msrs[IDX_MSR_PAT] = PAT_VALUE(0, PAT_WRITE_BACK) | + vcpu->guest_msrs[IDX_MSR_PAT] = PAT_VALUE(0, PAT_WRITE_BACK) | PAT_VALUE(1, PAT_WRITE_THROUGH) | PAT_VALUE(2, PAT_UNCACHED) | PAT_VALUE(3, PAT_UNCACHEABLE) | @@ -346,24 +344,22 @@ } void -vmx_msr_guest_enter(struct vmx *vmx, int vcpuid) +vmx_msr_guest_enter(struct vmx *vmx, struct vmx_vcpu *vcpu) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpuid]; /* Save host MSRs (in particular, KGSBASE) and restore guest MSRs */ update_pcb_bases(curpcb); - wrmsr(MSR_LSTAR, vmx_vcpu->guest_msrs[IDX_MSR_LSTAR]); - wrmsr(MSR_CSTAR, vmx_vcpu->guest_msrs[IDX_MSR_CSTAR]); - wrmsr(MSR_STAR, vmx_vcpu->guest_msrs[IDX_MSR_STAR]); - wrmsr(MSR_SF_MASK, vmx_vcpu->guest_msrs[IDX_MSR_SF_MASK]); - wrmsr(MSR_KGSBASE, vmx_vcpu->guest_msrs[IDX_MSR_KGSBASE]); + wrmsr(MSR_LSTAR, vcpu->guest_msrs[IDX_MSR_LSTAR]); + wrmsr(MSR_CSTAR, vcpu->guest_msrs[IDX_MSR_CSTAR]); + wrmsr(MSR_STAR, vcpu->guest_msrs[IDX_MSR_STAR]); + wrmsr(MSR_SF_MASK, vcpu->guest_msrs[IDX_MSR_SF_MASK]); + wrmsr(MSR_KGSBASE, vcpu->guest_msrs[IDX_MSR_KGSBASE]); } void -vmx_msr_guest_enter_tsc_aux(struct vmx *vmx, int vcpuid) +vmx_msr_guest_enter_tsc_aux(struct vmx *vmx, struct vmx_vcpu *vcpu) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpuid]; - uint64_t guest_tsc_aux = vmx_vcpu->guest_msrs[IDX_MSR_TSC_AUX]; + uint64_t guest_tsc_aux = vcpu->guest_msrs[IDX_MSR_TSC_AUX]; uint32_t host_aux = cpu_auxmsr(); if (vmx_have_msr_tsc_aux(vmx) && guest_tsc_aux != host_aux) @@ -371,16 +367,15 @@ } void -vmx_msr_guest_exit(struct vmx *vmx, int vcpuid) +vmx_msr_guest_exit(struct vmx *vmx, struct vmx_vcpu *vcpu) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpuid]; /* Save guest MSRs */ - vmx_vcpu->guest_msrs[IDX_MSR_LSTAR] = rdmsr(MSR_LSTAR); - vmx_vcpu->guest_msrs[IDX_MSR_CSTAR] = rdmsr(MSR_CSTAR); - vmx_vcpu->guest_msrs[IDX_MSR_STAR] = rdmsr(MSR_STAR); - vmx_vcpu->guest_msrs[IDX_MSR_SF_MASK] = rdmsr(MSR_SF_MASK); - vmx_vcpu->guest_msrs[IDX_MSR_KGSBASE] = rdmsr(MSR_KGSBASE); + vcpu->guest_msrs[IDX_MSR_LSTAR] = rdmsr(MSR_LSTAR); + vcpu->guest_msrs[IDX_MSR_CSTAR] = rdmsr(MSR_CSTAR); + vcpu->guest_msrs[IDX_MSR_STAR] = rdmsr(MSR_STAR); + vcpu->guest_msrs[IDX_MSR_SF_MASK] = rdmsr(MSR_SF_MASK); + vcpu->guest_msrs[IDX_MSR_KGSBASE] = rdmsr(MSR_KGSBASE); /* Restore host MSRs */ wrmsr(MSR_LSTAR, host_msrs[IDX_MSR_LSTAR]); @@ -392,16 +387,15 @@ } void -vmx_msr_guest_exit_tsc_aux(struct vmx *vmx, int vcpuid) +vmx_msr_guest_exit_tsc_aux(struct vmx *vmx, struct vmx_vcpu *vcpu) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpuid]; - uint64_t guest_tsc_aux = vmx_vcpu->guest_msrs[IDX_MSR_TSC_AUX]; + uint64_t guest_tsc_aux = vcpu->guest_msrs[IDX_MSR_TSC_AUX]; uint32_t host_aux = cpu_auxmsr(); if (vmx_have_msr_tsc_aux(vmx) && guest_tsc_aux != host_aux) /* * Note that it is not necessary to save the guest value - * here; vmx->guest_msrs[vcpuid][IDX_MSR_TSC_AUX] always + * here; vcpu->guest_msrs[IDX_MSR_TSC_AUX] always * contains the current value since it is updated whenever * the guest writes to it (which is expected to be very * rare). @@ -410,9 +404,9 @@ } int -vmx_rdmsr(struct vmx *vmx, int vcpuid, u_int num, uint64_t *val, bool *retu) +vmx_rdmsr(struct vmx *vmx, struct vmx_vcpu *vcpu, u_int num, uint64_t *val, + bool *retu) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpuid]; int error; error = 0; @@ -428,8 +422,8 @@ case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1: case MSR_MTRR64kBase: case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1: - if (vm_rdmtrr(&vmx_vcpu->mtrr, num, val) != 0) { - vm_inject_gp(vmx->vm, vcpuid); + if (vm_rdmtrr(&vcpu->mtrr, num, val) != 0) { + vm_inject_gp(vmx->vm, vcpu->vcpuid); } break; case MSR_IA32_MISC_ENABLE: @@ -443,7 +437,7 @@ *val = turbo_ratio_limit; break; case MSR_PAT: - *val = vmx_vcpu->guest_msrs[IDX_MSR_PAT]; + *val = vcpu->guest_msrs[IDX_MSR_PAT]; break; default: error = EINVAL; @@ -453,9 +447,9 @@ } int -vmx_wrmsr(struct vmx *vmx, int vcpuid, u_int num, uint64_t val, bool *retu) +vmx_wrmsr(struct vmx *vmx, struct vmx_vcpu *vcpu, u_int num, uint64_t val, + bool *retu) { - struct vmx_vcpu *vmx_vcpu = &vmx->vcpus[vcpuid]; uint64_t changed; int error; @@ -471,8 +465,8 @@ case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1: case MSR_MTRR64kBase: case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1: - if (vm_wrmtrr(&vmx_vcpu->mtrr, num, val) != 0) { - vm_inject_gp(vmx->vm, vcpuid); + if (vm_wrmtrr(&vcpu->mtrr, num, val) != 0) { + vm_inject_gp(vmx->vm, vcpu->vcpuid); } break; case MSR_IA32_MISC_ENABLE: @@ -497,12 +491,12 @@ break; case MSR_PAT: if (pat_valid(val)) - vmx_vcpu->guest_msrs[IDX_MSR_PAT] = val; + vcpu->guest_msrs[IDX_MSR_PAT] = val; else - vm_inject_gp(vmx->vm, vcpuid); + vm_inject_gp(vmx->vm, vcpu->vcpuid); break; case MSR_TSC: - error = vmx_set_tsc_offset(vmx, vcpuid, val - rdtsc()); + error = vmx_set_tsc_offset(vmx, vcpu, val - rdtsc()); break; case MSR_TSC_AUX: if (vmx_have_msr_tsc_aux(vmx)) @@ -511,9 +505,9 @@ * value when it is called immediately before guest * entry. */ - vmx_vcpu->guest_msrs[IDX_MSR_TSC_AUX] = val; + vcpu->guest_msrs[IDX_MSR_TSC_AUX] = val; else - vm_inject_gp(vmx->vm, vcpuid); + vm_inject_gp(vmx->vm, vcpu->vcpuid); break; default: error = EINVAL; diff --git a/sys/amd64/vmm/vmm.c b/sys/amd64/vmm/vmm.c --- a/sys/amd64/vmm/vmm.c +++ b/sys/amd64/vmm/vmm.c @@ -106,6 +106,7 @@ enum vcpu_state state; /* (o) vcpu state */ int hostcpu; /* (o) vcpu's host cpu */ int reqidle; /* (i) request vcpu to idle */ + void *cookie; /* (i) cpu-specific data */ struct vlapic *vlapic; /* (i) APIC device model */ enum x2apic_state x2apic_state; /* (i) APIC mode */ uint64_t exitintinfo; /* (i) events pending at VM exit */ @@ -208,30 +209,32 @@ DEFINE_VMMOPS_IFUNC(int, modcleanup, (void)) DEFINE_VMMOPS_IFUNC(void, modresume, (void)) DEFINE_VMMOPS_IFUNC(void *, init, (struct vm *vm, struct pmap *pmap)) -DEFINE_VMMOPS_IFUNC(int, run, (void *vmi, int vcpu, register_t rip, +DEFINE_VMMOPS_IFUNC(int, run, (void *vmi, void *vcpui, register_t rip, struct pmap *pmap, struct vm_eventinfo *info)) DEFINE_VMMOPS_IFUNC(void, cleanup, (void *vmi)) -DEFINE_VMMOPS_IFUNC(int, getreg, (void *vmi, int vcpu, int num, +DEFINE_VMMOPS_IFUNC(void *, vcpu_init, (void *vmi, int vcpu_id)) +DEFINE_VMMOPS_IFUNC(void, vcpu_cleanup, (void *vmi, void *vcpui)) +DEFINE_VMMOPS_IFUNC(int, getreg, (void *vmi, void *vcpui, int num, uint64_t *retval)) -DEFINE_VMMOPS_IFUNC(int, setreg, (void *vmi, int vcpu, int num, +DEFINE_VMMOPS_IFUNC(int, setreg, (void *vmi, void *vcpui, int num, uint64_t val)) -DEFINE_VMMOPS_IFUNC(int, getdesc, (void *vmi, int vcpu, int num, +DEFINE_VMMOPS_IFUNC(int, getdesc, (void *vmi, void *vcpui, int num, struct seg_desc *desc)) -DEFINE_VMMOPS_IFUNC(int, setdesc, (void *vmi, int vcpu, int num, +DEFINE_VMMOPS_IFUNC(int, setdesc, (void *vmi, void *vcpui, int num, struct seg_desc *desc)) -DEFINE_VMMOPS_IFUNC(int, getcap, (void *vmi, int vcpu, int num, int *retval)) -DEFINE_VMMOPS_IFUNC(int, setcap, (void *vmi, int vcpu, int num, int val)) +DEFINE_VMMOPS_IFUNC(int, getcap, (void *vmi, void *vcpui, int num, int *retval)) +DEFINE_VMMOPS_IFUNC(int, setcap, (void *vmi, void *vcpui, int num, int val)) DEFINE_VMMOPS_IFUNC(struct vmspace *, vmspace_alloc, (vm_offset_t min, vm_offset_t max)) DEFINE_VMMOPS_IFUNC(void, vmspace_free, (struct vmspace *vmspace)) -DEFINE_VMMOPS_IFUNC(struct vlapic *, vlapic_init, (void *vmi, int vcpu)) +DEFINE_VMMOPS_IFUNC(struct vlapic *, vlapic_init, (void *vmi, void *vcpui)) DEFINE_VMMOPS_IFUNC(void, vlapic_cleanup, (void *vmi, struct vlapic *vlapic)) #ifdef BHYVE_SNAPSHOT DEFINE_VMMOPS_IFUNC(int, snapshot, (void *vmi, struct vm_snapshot_meta *meta)) DEFINE_VMMOPS_IFUNC(int, vcpu_snapshot, (void *vmi, struct vm_snapshot_meta - *meta, int vcpu)) -DEFINE_VMMOPS_IFUNC(int, restore_tsc, (void *vmi, int vcpuid, uint64_t now)) + *meta, void *vcpui)) +DEFINE_VMMOPS_IFUNC(int, restore_tsc, (void *vmi, void *vcpui, uint64_t now)) #endif #define fpu_start_emulating() load_cr0(rcr0() | CR0_TS) @@ -293,12 +296,20 @@ } #endif +static __inline void * +vcpu_cookie(struct vm *vm, int i) +{ + return (vm->vcpu[i].cookie); +} + static void vcpu_cleanup(struct vm *vm, int i, bool destroy) { struct vcpu *vcpu = &vm->vcpu[i]; vmmops_vlapic_cleanup(vm->cookie, vcpu->vlapic); + vmmops_vcpu_cleanup(vm->cookie, vcpu->cookie); + vcpu->cookie = NULL; if (destroy) { vmm_stat_free(vcpu->stats); fpu_save_area_free(vcpu->guestfpu); @@ -326,7 +337,8 @@ vcpu->tsc_offset = 0; } - vcpu->vlapic = vmmops_vlapic_init(vm->cookie, vcpu_id); + vcpu->cookie = vmmops_vcpu_init(vm->cookie, vcpu_id); + vcpu->vlapic = vmmops_vlapic_init(vm->cookie, vcpu->cookie); vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED); vcpu->reqidle = 0; vcpu->exitintinfo = 0; @@ -1070,7 +1082,8 @@ if (reg >= VM_REG_LAST) return (EINVAL); - return (vmmops_getreg(vm->cookie, vcpu, reg, retval)); + return (vmmops_getreg(vm->cookie, vcpu_cookie(vm, vcpu), reg, + retval)); } int @@ -1085,13 +1098,13 @@ if (reg >= VM_REG_LAST) return (EINVAL); - error = vmmops_setreg(vm->cookie, vcpuid, reg, val); + vcpu = &vm->vcpu[vcpuid]; + error = vmmops_setreg(vm->cookie, vcpu->cookie, reg, val); if (error || reg != VM_REG_GUEST_RIP) return (error); /* Set 'nextrip' to match the value of %rip */ VCPU_CTR1(vm, vcpuid, "Setting nextrip to %#lx", val); - vcpu = &vm->vcpu[vcpuid]; vcpu->nextrip = val; return (0); } @@ -1139,7 +1152,7 @@ if (!is_segment_register(reg) && !is_descriptor_table(reg)) return (EINVAL); - return (vmmops_getdesc(vm->cookie, vcpu, reg, desc)); + return (vmmops_getdesc(vm->cookie, vcpu_cookie(vm, vcpu), reg, desc)); } int @@ -1152,7 +1165,7 @@ if (!is_segment_register(reg) && !is_descriptor_table(reg)) return (EINVAL); - return (vmmops_setdesc(vm->cookie, vcpu, reg, desc)); + return (vmmops_setdesc(vm->cookie, vcpu_cookie(vm, vcpu), reg, desc)); } static void @@ -1767,7 +1780,8 @@ restore_guest_fpustate(vcpu); vcpu_require_state(vm, vcpuid, VCPU_RUNNING); - error = vmmops_run(vm->cookie, vcpuid, vcpu->nextrip, pmap, &evinfo); + error = vmmops_run(vm->cookie, vcpu->cookie, vcpu->nextrip, pmap, + &evinfo); vcpu_require_state(vm, vcpuid, VCPU_FROZEN); save_guest_fpustate(vcpu); @@ -2273,7 +2287,7 @@ if (type < 0 || type >= VM_CAP_MAX) return (EINVAL); - return (vmmops_getcap(vm->cookie, vcpu, type, retval)); + return (vmmops_getcap(vm->cookie, vcpu_cookie(vm, vcpu), type, retval)); } int @@ -2285,7 +2299,7 @@ if (type < 0 || type >= VM_CAP_MAX) return (EINVAL); - return (vmmops_setcap(vm->cookie, vcpu, type, val)); + return (vmmops_setcap(vm->cookie, vcpu_cookie(vm, vcpu), type, val)); } struct vlapic * @@ -2850,16 +2864,19 @@ } static int -vm_snapshot_vmcx(struct vm *vm, struct vm_snapshot_meta *meta) +vm_snapshot_vcpu(struct vm *vm, struct vm_snapshot_meta *meta) { int error; + struct vcpu *vcpu; uint16_t i, maxcpus; error = 0; maxcpus = vm_get_maxcpus(vm); for (i = 0; i < maxcpus; i++) { - error = vmmops_vcpu_snapshot(vm->cookie, meta, i); + vcpu = &vm->vcpu[i]; + + error = vmmops_vcpu_snapshot(vm->cookie, meta, vcpu->cookie); if (error != 0) { printf("%s: failed to snapshot vmcs/vmcb data for " "vCPU: %d; error: %d\n", __func__, i, error); @@ -2884,7 +2901,7 @@ ret = vmmops_snapshot(vm->cookie, meta); break; case STRUCT_VMCX: - ret = vm_snapshot_vmcx(vm, meta); + ret = vm_snapshot_vcpu(vm, meta); break; case STRUCT_VM: ret = vm_snapshot_vm(vm, meta); @@ -2950,8 +2967,8 @@ for (i = 0; i < maxcpus; i++) { vcpu = &vm->vcpu[i]; - error = vmmops_restore_tsc(vm->cookie, i, vcpu->tsc_offset - - now); + error = vmmops_restore_tsc(vm->cookie, vcpu->cookie, + vcpu->tsc_offset - now); if (error) return (error); }