Differential D19495 Diff 69728 sys/amd64/vmm/vmm.c

Changeset View

Standalone View

sys/amd64/vmm/vmm.c

Show All 25 Lines
* SUCH DAMAGE.		* SUCH DAMAGE.
*		*
* $FreeBSD$		* $FreeBSD$
*/		*/

#include <sys/cdefs.h>		#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");		__FBSDID("$FreeBSD$");

		#include "opt_bhyve_snapshot.h"

#include <sys/param.h>		#include <sys/param.h>
#include <sys/systm.h>		#include <sys/systm.h>
#include <sys/kernel.h>		#include <sys/kernel.h>
#include <sys/module.h>		#include <sys/module.h>
#include <sys/sysctl.h>		#include <sys/sysctl.h>
#include <sys/malloc.h>		#include <sys/malloc.h>
#include <sys/pcpu.h>		#include <sys/pcpu.h>
#include <sys/lock.h>		#include <sys/lock.h>
#include <sys/mutex.h>		#include <sys/mutex.h>
#include <sys/proc.h>		#include <sys/proc.h>
#include <sys/rwlock.h>		#include <sys/rwlock.h>
#include <sys/sched.h>		#include <sys/sched.h>
#include <sys/smp.h>		#include <sys/smp.h>
#include <sys/systm.h>		#include <sys/vnode.h>

#include <vm/vm.h>		#include <vm/vm.h>
#include <vm/vm_object.h>		#include <vm/vm_object.h>
#include <vm/vm_page.h>		#include <vm/vm_page.h>
#include <vm/pmap.h>		#include <vm/pmap.h>
#include <vm/vm_map.h>		#include <vm/vm_map.h>
#include <vm/vm_extern.h>		#include <vm/vm_extern.h>
#include <vm/vm_param.h>		#include <vm/vm_param.h>
		#include <vm/vm_pager.h>
		#include <vm/vm_kern.h>
		#include <vm/vnode_pager.h>
		#include <vm/swap_pager.h>
		#include <vm/uma.h>

#include <machine/cpu.h>		#include <machine/cpu.h>
#include <machine/pcb.h>		#include <machine/pcb.h>
#include <machine/smp.h>		#include <machine/smp.h>
#include <machine/md_var.h>		#include <machine/md_var.h>
#include <x86/psl.h>		#include <x86/psl.h>
#include <x86/apicreg.h>		#include <x86/apicreg.h>

#include <machine/vmm.h>		#include <machine/vmm.h>
#include <machine/vmm_dev.h>		#include <machine/vmm_dev.h>
#include <machine/vmm_instruction_emul.h>		#include <machine/vmm_instruction_emul.h>
		#include <machine/vmm_snapshot.h>

#include "vmm_ioport.h"		#include "vmm_ioport.h"
#include "vmm_ktr.h"		#include "vmm_ktr.h"
#include "vmm_host.h"		#include "vmm_host.h"
#include "vmm_mem.h"		#include "vmm_mem.h"
#include "vmm_util.h"		#include "vmm_util.h"
#include "vatpic.h"		#include "vatpic.h"
#include "vatpit.h"		#include "vatpit.h"
Show All 31 Lines	struct vcpu {
int exc_vector; /* (x) exception collateral */		int exc_vector; /* (x) exception collateral */
int exc_errcode_valid;		int exc_errcode_valid;
uint32_t exc_errcode;		uint32_t exc_errcode;
struct savefpu guestfpu; / (a,i) guest fpu state */		struct savefpu guestfpu; / (a,i) guest fpu state */
uint64_t guest_xcr0; /* (i) guest %xcr0 register */		uint64_t guest_xcr0; /* (i) guest %xcr0 register */
void stats; / (a,i) statistics */		void stats; / (a,i) statistics */
struct vm_exit exitinfo; /* (x) exit reason and collateral */		struct vm_exit exitinfo; /* (x) exit reason and collateral */
uint64_t nextrip; /* (x) next instruction to execute */		uint64_t nextrip; /* (x) next instruction to execute */
		uint64_t tsc_offset; /* (o) TSC offsetting */
};		};

#define vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx))		#define vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx))
#define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)		#define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
#define vcpu_lock(v) mtx_lock_spin(&((v)->mtx))		#define vcpu_lock(v) mtx_lock_spin(&((v)->mtx))
#define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx))		#define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx))
#define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED)		#define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED)

▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines
#define VMGETCAP(vmi, vcpu, num, retval) \		#define VMGETCAP(vmi, vcpu, num, retval) \
(ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO)		(ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO)
#define VMSETCAP(vmi, vcpu, num, val) \		#define VMSETCAP(vmi, vcpu, num, val) \
(ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO)		(ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO)
#define VLAPIC_INIT(vmi, vcpu) \		#define VLAPIC_INIT(vmi, vcpu) \
(ops != NULL ? (*ops->vlapic_init)(vmi, vcpu) : NULL)		(ops != NULL ? (*ops->vlapic_init)(vmi, vcpu) : NULL)
#define VLAPIC_CLEANUP(vmi, vlapic) \		#define VLAPIC_CLEANUP(vmi, vlapic) \
(ops != NULL ? (*ops->vlapic_cleanup)(vmi, vlapic) : NULL)		(ops != NULL ? (*ops->vlapic_cleanup)(vmi, vlapic) : NULL)
		#ifdef BHYVE_SNAPSHOT
		#define VM_SNAPSHOT_VMI(vmi, meta) \
		(ops != NULL ? (*ops->vmsnapshot)(vmi, meta) : ENXIO)
		#define VM_SNAPSHOT_VMCX(vmi, meta, vcpuid) \
		(ops != NULL ? (*ops->vmcx_snapshot)(vmi, meta, vcpuid) : ENXIO)
		#define VM_RESTORE_TSC(vmi, vcpuid, offset) \
		(ops != NULL ? (*ops->vm_restore_tsc)(vmi, vcpuid, offset) : ENXIO)
		#endif

#define fpu_start_emulating() load_cr0(rcr0() \| CR0_TS)		#define fpu_start_emulating() load_cr0(rcr0() \| CR0_TS)
#define fpu_stop_emulating() clts()		#define fpu_stop_emulating() clts()

SDT_PROVIDER_DEFINE(vmm);		SDT_PROVIDER_DEFINE(vmm);

static MALLOC_DEFINE(M_VM, "vm", "vm");		static MALLOC_DEFINE(M_VM, "vm", "vm");

▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines	vcpu_init(struct vm *vm, int vcpu_id, bool create)
if (create) {		if (create) {
KASSERT(!vcpu_lock_initialized(vcpu), ("vcpu %d already "		KASSERT(!vcpu_lock_initialized(vcpu), ("vcpu %d already "
"initialized", vcpu_id));		"initialized", vcpu_id));
vcpu_lock_init(vcpu);		vcpu_lock_init(vcpu);
vcpu->state = VCPU_IDLE;		vcpu->state = VCPU_IDLE;
vcpu->hostcpu = NOCPU;		vcpu->hostcpu = NOCPU;
vcpu->guestfpu = fpu_save_area_alloc();		vcpu->guestfpu = fpu_save_area_alloc();
vcpu->stats = vmm_stat_alloc();		vcpu->stats = vmm_stat_alloc();
		vcpu->tsc_offset = 0;
}		}

vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id);		vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id);
vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED);		vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED);
vcpu->reqidle = 0;		vcpu->reqidle = 0;
vcpu->exitintinfo = 0;		vcpu->exitintinfo = 0;
vcpu->nmi_pending = 0;		vcpu->nmi_pending = 0;
vcpu->extint_pending = 0;		vcpu->extint_pending = 0;
▲ Show 20 Lines • Show All 497 Lines • ▼ Show 20 Lines	if (mmnext != NULL) {
if (flags)		if (flags)
*flags = mmnext->flags;		*flags = mmnext->flags;
return (0);		return (0);
} else {		} else {
return (ENOENT);		return (ENOENT);
}		}
}		}

static void		static void
vm_free_memmap(struct vm *vm, int ident)		vm_free_memmap(struct vm *vm, int ident)
{		{
struct mem_map *mm;		struct mem_map *mm;
int error;		int error;
		pmooney_pfmooney.comUnsubmitted Not Done Inline Actions Why is this here with no consumer? pmooney_pfmooney.com: Why is this here with no consumer?
		darius.mihaim_gmail.comUnsubmitted Done Inline Actions Removed it; was a leftover from an older implementation. darius.mihaim_gmail.com: Removed it; was a leftover from an older implementation.

mm = &vm->mem_maps[ident];		mm = &vm->mem_maps[ident];
if (mm->len) {		if (mm->len) {
error = vm_map_remove(&vm->vmspace->vm_map, mm->gpa,		error = vm_map_remove(&vm->vmspace->vm_map, mm->gpa,
mm->gpa + mm->len);		mm->gpa + mm->len);
KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d",		KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d",
__func__, error));		__func__, error));
bzero(mm, sizeof(struct mem_map));		bzero(mm, sizeof(struct mem_map));
}		}
}		}

		xistence_0x58.comUnsubmitted Done Inline Actions Single statement if/else does not require braces. xistence_0x58.com: Single statement if/else does not require braces.
static __inline bool		static __inline bool
sysmem_mapping(struct vm vm, struct mem_map mm)		sysmem_mapping(struct vm vm, struct mem_map mm)
{		{

if (mm->len != 0 && vm->mem_segs[mm->segid].sysmem)		if (mm->len != 0 && vm->mem_segs[mm->segid].sysmem)
return (true);		return (true);
else		else
return (false);		return (false);
▲ Show 20 Lines • Show All 885 Lines • ▼ Show 20 Lines	restart:
vcpu_require_state(vm, vcpuid, VCPU_RUNNING);		vcpu_require_state(vm, vcpuid, VCPU_RUNNING);
error = VMRUN(vm->cookie, vcpuid, vcpu->nextrip, pmap, &evinfo);		error = VMRUN(vm->cookie, vcpuid, vcpu->nextrip, pmap, &evinfo);
vcpu_require_state(vm, vcpuid, VCPU_FROZEN);		vcpu_require_state(vm, vcpuid, VCPU_FROZEN);

save_guest_fpustate(vcpu);		save_guest_fpustate(vcpu);

vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval);		vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval);

critical_exit();		critical_exit();
		jhbAuthorUnsubmitted Not Done Inline Actions Why is this check here? jhb: Why is this check here?
		darius.mihaim_gmail.comUnsubmitted Done Inline Actions Removed it; was forgotten from a previous implementation. darius.mihaim_gmail.com: Removed it; was forgotten from a previous implementation.

if (error == 0) {		if (error == 0) {
retu = false;		retu = false;
vcpu->nextrip = vme->rip + vme->inst_length;		vcpu->nextrip = vme->rip + vme->inst_length;
switch (vme->exitcode) {		switch (vme->exitcode) {
case VM_EXITCODE_REQIDLE:		case VM_EXITCODE_REQIDLE:
error = vm_handle_reqidle(vm, vcpuid, &retu);		error = vm_handle_reqidle(vm, vcpuid, &retu);
break;		break;
▲ Show 20 Lines • Show All 993 Lines • ▼ Show 20 Lines	vm_get_wiredcnt(struct vm vm, int vcpu, struct vmm_stat_type stat)
if (vcpu == 0) {		if (vcpu == 0) {
vmm_stat_set(vm, vcpu, VMM_MEM_WIRED,		vmm_stat_set(vm, vcpu, VMM_MEM_WIRED,
PAGE_SIZE * pmap_wired_count(vmspace_pmap(vm->vmspace)));		PAGE_SIZE * pmap_wired_count(vmspace_pmap(vm->vmspace)));
}		}
}		}

VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt);		VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt);
VMM_STAT_FUNC(VMM_MEM_WIRED, "Wired memory", vm_get_wiredcnt);		VMM_STAT_FUNC(VMM_MEM_WIRED, "Wired memory", vm_get_wiredcnt);

		#ifdef BHYVE_SNAPSHOT
		static int
		vm_snapshot_vcpus(struct vm vm, struct vm_snapshot_meta meta)
		{
		int ret;
		int i;
		struct vcpu *vcpu;

		for (i = 0; i < VM_MAXCPU; i++) {
		vcpu = &vm->vcpu[i];

		SNAPSHOT_VAR_OR_LEAVE(vcpu->x2apic_state, meta, ret, done);
		SNAPSHOT_VAR_OR_LEAVE(vcpu->exitintinfo, meta, ret, done);
		SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_vector, meta, ret, done);
		SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_errcode_valid, meta, ret, done);
		SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_errcode, meta, ret, done);
		SNAPSHOT_VAR_OR_LEAVE(vcpu->guest_xcr0, meta, ret, done);
		SNAPSHOT_VAR_OR_LEAVE(vcpu->exitinfo, meta, ret, done);
		SNAPSHOT_VAR_OR_LEAVE(vcpu->nextrip, meta, ret, done);
		/* XXX we're cheating here, since the value of tsc_offset as
		* saved here is actually the value of the guest's TSC value.
		*
		* It will be turned turned back into an actual offset when the
		* TSC restore function is called
		*/
		SNAPSHOT_VAR_OR_LEAVE(vcpu->tsc_offset, meta, ret, done);
		}

		done:
		return (ret);
		}

		static int
		vm_snapshot_vm(struct vm vm, struct vm_snapshot_meta meta)
		{
		int ret;
		int i;
		uint64_t now;

		ret = 0;
		now = rdtsc();

		if (meta->op == VM_SNAPSHOT_SAVE) {
		/* XXX make tsc_offset take the value TSC proper as seen by the
		* guest
		*/
		for (i = 0; i < VM_MAXCPU; i++)
		vm->vcpu[i].tsc_offset += now;
		}

		ret = vm_snapshot_vcpus(vm, meta);
		if (ret != 0) {
		printf("%s: failed to copy vm data to user buffer", __func__);
		goto done;
		}

		if (meta->op == VM_SNAPSHOT_SAVE) {
		/* XXX turn tsc_offset back into an offset; actual value is only
		* required for restore; using it otherwise would be wrong
		*/
		for (i = 0; i < VM_MAXCPU; i++)
		vm->vcpu[i].tsc_offset -= now;
		}

		done:
		return (ret);
		}

		static int
		vm_snapshot_vmcx(struct vm vm, struct vm_snapshot_meta meta)
		{
		int i, error;

		error = 0;

		for (i = 0; i < VM_MAXCPU; i++) {
		error = VM_SNAPSHOT_VMCX(vm->cookie, meta, i);
		if (error != 0) {
		printf("%s: failed to snapshot vmcs/vmcb data for "
		"vCPU: %d; error: %d\n", __func__, i, error);
		goto done;
		}
		}

		done:
		return (error);
		}

		/*
		* Save kernel-side structures to user-space for snapshotting.
		*/
		int
		vm_snapshot_req(struct vm vm, struct vm_snapshot_meta meta)
		{
		int ret = 0;

		switch (meta->dev_req) {
		case STRUCT_VMX:
		ret = VM_SNAPSHOT_VMI(vm->cookie, meta);
		break;
		case STRUCT_VMCX:
		ret = vm_snapshot_vmcx(vm, meta);
		break;
		case STRUCT_VM:
		ret = vm_snapshot_vm(vm, meta);
		break;
		case STRUCT_VIOAPIC:
		ret = vioapic_snapshot(vm_ioapic(vm), meta);
		break;
		case STRUCT_VLAPIC:
		ret = vlapic_snapshot(vm, meta);
		break;
		case STRUCT_VHPET:
		ret = vhpet_snapshot(vm_hpet(vm), meta);
		break;
		case STRUCT_VATPIC:
		ret = vatpic_snapshot(vm_atpic(vm), meta);
		break;
		case STRUCT_VATPIT:
		ret = vatpit_snapshot(vm_atpit(vm), meta);
		break;
		case STRUCT_VPMTMR:
		ret = vpmtmr_snapshot(vm_pmtmr(vm), meta);
		break;
		case STRUCT_VRTC:
		ret = vrtc_snapshot(vm_rtc(vm), meta);
		break;
		default:
		printf("%s: failed to find the requested type %#x\n",
		__func__, meta->dev_req);
		ret = (EINVAL);
		}
		return (ret);
		}

		int
		vm_set_tsc_offset(struct vm *vm, int vcpuid, uint64_t offset)
		{
		struct vcpu *vcpu;

		if (vcpuid < 0 \|\| vcpuid >= VM_MAXCPU)
		return (EINVAL);

		vcpu = &vm->vcpu[vcpuid];
		vcpu->tsc_offset = offset;

		return (0);
		}

		int
		vm_restore_time(struct vm *vm)
		{
		jhbAuthorUnsubmitted Not Done Inline Actions Not to change right now, but for the one-liner wrapper functions that are only used once like this (vm_snapshot_lapic) I would probably just inline the call to vlapic_snapshot() directly in this switch instead. jhb: Not to change right now, but for the one-liner wrapper functions that are only used once like…
		darius.mihaim_gmail.comUnsubmitted Done Inline Actions Removed the one-line wrapper functions. darius.mihaim_gmail.com: Removed the one-line wrapper functions.
		int error, i;
		uint64_t now;
		struct vcpu *vcpu;

		now = rdtsc();

		error = vhpet_restore_time(vm_hpet(vm));
		if (error)
		return (error);

		for (i = 0; i < nitems(vm->vcpu); i++) {
		vcpu = &vm->vcpu[i];

		error = VM_RESTORE_TSC(vm->cookie, i, vcpu->tsc_offset - now);
		if (error)
		return (error);
		}

		return (0);
		}
		#endif