Page MenuHomeFreeBSD

D61708.1795085006.diff
No OneTemporary

Authored By
Unknown
Size
64 KB
Referenced Files
None
Subscribers
None

D61708.1795085006.diff

diff --git a/share/man/man4/vmm.4 b/share/man/man4/vmm.4
--- a/share/man/man4/vmm.4
+++ b/share/man/man4/vmm.4
@@ -25,7 +25,7 @@
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
.\" SUCH DAMAGE.
.\"
-.Dd August 29, 2026
+.Dd November 18, 2026
.Dt VMM 4
.Os
.Sh NAME
@@ -134,6 +134,13 @@
.It Va hw.vmm.maxcpu
Maximum number of virtual CPUs.
The default is the number of physical CPUs in the system.
+.It Va hw.vmm.vmx.nested
+Offer VMX to guests on Intel CPUs, so that a guest can run virtual machines
+of its own.
+The guest's hypervisor has to be a 64-bit one and has to do without EPT,
+VPID, the TPR shadow and I/O and MSR bitmaps.
+Setting this tunable disables APIC virtualization for all guests.
+The default is 0.
.El
.Sh FILES
.Bl -tag -width /dev/vmm.io/* -compact
diff --git a/sys/amd64/include/vmm.h b/sys/amd64/include/vmm.h
--- a/sys/amd64/include/vmm.h
+++ b/sys/amd64/include/vmm.h
@@ -194,6 +194,7 @@
DECLARE_VMMOPS_FUNC(void, vmspace_free, (struct vmspace *vmspace));
DECLARE_VMMOPS_FUNC(struct vlapic *, vlapic_init, (void *vcpui));
DECLARE_VMMOPS_FUNC(void, vlapic_cleanup, (struct vlapic *vlapic));
+DECLARE_VMMOPS_FUNC(int, vminsn, (void *vcpui));
DECLARE_VMMOPS_FUNC(int, vcpu_snapshot, (void *vcpui,
struct vm_snapshot_meta *meta));
DECLARE_VMMOPS_FUNC(int, restore_tsc, (void *vcpui, uint64_t now));
@@ -219,6 +220,7 @@
vmmops_vmspace_free_t vmspace_free;
vmmops_vlapic_init_t vlapic_init;
vmmops_vlapic_cleanup_t vlapic_cleanup;
+ vmmops_vminsn_t vminsn; /* VM_EXITCODE_VMINSN */
/* checkpoint operations */
vmmops_vcpu_snapshot_t vcpu_snapshot;
diff --git a/sys/amd64/vmm/amd/svm.c b/sys/amd64/vmm/amd/svm.c
--- a/sys/amd64/vmm/amd/svm.c
+++ b/sys/amd64/vmm/amd/svm.c
@@ -2618,6 +2618,15 @@
free(vlapic, M_SVM_VLAPIC);
}
+static int
+svm_vminsn(void *vcpui)
+{
+ struct svm_vcpu *vcpu = vcpui;
+
+ vm_inject_ud(vcpu->vcpu);
+ return (0);
+}
+
#ifdef BHYVE_SNAPSHOT
static int
svm_vcpu_snapshot(void *vcpui, struct vm_snapshot_meta *meta)
@@ -2853,6 +2862,7 @@
.vmspace_free = svm_vmspace_free,
.vlapic_init = svm_vlapic_init,
.vlapic_cleanup = svm_vlapic_cleanup,
+ .vminsn = svm_vminsn,
#ifdef BHYVE_SNAPSHOT
.vcpu_snapshot = svm_vcpu_snapshot,
.restore_tsc = svm_restore_tsc,
diff --git a/sys/amd64/vmm/intel/vmx.h b/sys/amd64/vmm/intel/vmx.h
--- a/sys/amd64/vmm/intel/vmx.h
+++ b/sys/amd64/vmm/intel/vmx.h
@@ -112,6 +112,8 @@
} __aligned(64);
CTASSERT(sizeof(struct pir_desc) == 64);
+struct vmx_nested;
+
/* Index into the 'guest_msrs[]' array */
enum {
IDX_MSR_LSTAR,
@@ -128,6 +130,7 @@
struct vmx *vmx;
struct vcpu *vcpu;
struct vmcs *vmcs;
+ struct vmx_nested *nested;
struct apic_page *apic_page;
struct pir_desc *pir_desc;
uint64_t guest_msrs[GUEST_MSR_NUM];
diff --git a/sys/amd64/vmm/intel/vmx.c b/sys/amd64/vmm/intel/vmx.c
--- a/sys/amd64/vmm/intel/vmx.c
+++ b/sys/amd64/vmm/intel/vmx.c
@@ -73,6 +73,7 @@
#include "vmx_cpufunc.h"
#include "vmx.h"
#include "vmx_msr.h"
+#include "vmx_nested.h"
#include "x86.h"
#include "vmx_controls.h"
#include "io/ppt.h"
@@ -209,6 +210,10 @@
CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
&posted_interrupts, 0, "APICv posted interrupt support");
+static int vmx_nested;
+SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, nested, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
+ &vmx_nested, 0, "Nested virtualization: guests are offered VMX");
+
static int pirvec = -1;
SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
&pirvec, 0, "APICv posted interrupt vector");
@@ -868,9 +873,16 @@
PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
+ /*
+ * Nested virtualization does without APIC virtualization: an
+ * interrupt for a guest has to be found pending in its vlapic and
+ * acknowledged there while a guest of its own runs.
+ */
+ TUNABLE_INT_FETCH("hw.vmm.vmx.nested", &vmx_nested);
+
error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
procbased2_vid_bits, 0, &tmp);
- if (error == 0 && tpr_shadowing) {
+ if (error == 0 && tpr_shadowing && !vmx_nested) {
virtual_interrupt_delivery = 1;
#ifndef BURN_BRIDGES
TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
@@ -984,6 +996,8 @@
vpid_init();
vmx_msr_init();
+ if (vmx_nested)
+ vmx_nested_init();
/* enable VMX operation */
vmxon_region = kmem_malloc((mp_maxid + 1) * PAGE_SIZE,
@@ -1128,6 +1142,48 @@
return (vmx);
}
+/*
+ * Nested virtualization: the VMCS that a guest of the guest runs on. The
+ * host state and the host's controls are set here, the rest comes from the
+ * guest's own VMCS at each of its VM entries.
+ */
+static void
+vmx_nested_vcpu_init(struct vmx_vcpu *vcpu)
+{
+ struct vmcs *vmcs;
+ int error;
+
+ vmcs = malloc_aligned(sizeof(*vmcs), PAGE_SIZE, M_VMX,
+ M_WAITOK | M_ZERO);
+ vmcs->identifier = vmx_revision();
+ error = vmclear(vmcs);
+ if (error != 0) {
+ panic("%s: vmclear error %d on vcpu %d", __func__, error,
+ vcpu->vcpuid);
+ }
+ error = vmcs_init(vmcs);
+ KASSERT(error == 0, ("vmcs_init error %d", error));
+
+ VMPTRLD(vmcs);
+ error = 0;
+ error += vmwrite(VMCS_HOST_RSP, (u_long)&vcpu->ctx);
+ error += vmwrite(VMCS_EPTP, vcpu->vmx->eptp);
+ error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, PROCBASED2_ENABLE_EPT);
+ error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
+ error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
+ if (guest_l1d_flush && !guest_l1d_flush_sw) {
+ vmcs_write(VMCS_ENTRY_MSR_LOAD, pmap_kextract(
+ (vm_offset_t)&msr_load_list[0]));
+ vmcs_write(VMCS_ENTRY_MSR_LOAD_COUNT,
+ nitems(msr_load_list));
+ }
+ VMCLEAR(vmcs);
+ KASSERT(error == 0, ("%s: error customizing the vmcs", __func__));
+
+ vcpu->nested = vmx_nested_alloc();
+ vcpu->nested->vmcs02 = vmcs;
+}
+
static void *
vmx_vcpu_init(void *vmi, struct vcpu *vcpu1, int vcpuid)
{
@@ -1244,17 +1300,24 @@
vcpu->ctx.pmap = vmx->pmap;
+ if (vmx_nested)
+ vmx_nested_vcpu_init(vcpu);
+
return (vcpu);
}
static int
vmx_handle_cpuid(struct vmx_vcpu *vcpu, struct vmxctx *vmxctx)
{
+ uint32_t func;
int handled;
+ func = vmxctx->guest_rax;
handled = x86_emulate_cpuid(vcpu->vcpu, (uint64_t *)&vmxctx->guest_rax,
(uint64_t *)&vmxctx->guest_rbx, (uint64_t *)&vmxctx->guest_rcx,
(uint64_t *)&vmxctx->guest_rdx);
+ if (vcpu->nested != NULL && func == CPUID_0000_0001)
+ vmxctx->guest_rcx |= CPUID2_VMX;
return (handled);
}
@@ -2376,6 +2439,34 @@
return (error);
}
+/*
+ * Nested virtualization: record what the emulation of a VMX instruction
+ * needs from the VMCS. vmx_nested_exit() emulates it with the vCPU frozen.
+ */
+static void
+vmx_nested_insn_exit(struct vmx_vcpu *vcpu, uint32_t reason, uint64_t qual)
+{
+ struct vmx_nested_insn *insn;
+ uint64_t mask;
+
+ insn = &vcpu->nested->insn;
+ insn->reason = reason;
+ insn->qual = qual;
+ insn->inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO);
+ mask = vmcs_read(VMCS_CR0_MASK);
+ insn->cr0 = (vmcs_read(VMCS_GUEST_CR0) & ~mask) |
+ (vmcs_read(VMCS_CR0_SHADOW) & mask);
+ mask = vmcs_read(VMCS_CR4_MASK);
+ insn->cr4 = (vmcs_read(VMCS_GUEST_CR4) & ~mask) |
+ (vmcs_read(VMCS_CR4_SHADOW) & mask);
+ insn->rflags = vmcs_read(VMCS_GUEST_RFLAGS);
+ insn->interruptibility =
+ vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
+ insn->seg_base = vmcs_read(VMCS_GUEST_ES_BASE +
+ 2 * ((insn->inst_info >> 15) & 0x7));
+ vmx_paging_info(&insn->paging);
+}
+
static int
vmx_exit_process(struct vmx *vmx, struct vmx_vcpu *vcpu, struct vm_exit *vmexit)
{
@@ -2839,6 +2930,8 @@
case EXIT_REASON_VMXON:
SDT_PROBE3(vmm, vmx, exit, vminsn, vmx, vcpuid, vmexit);
vmexit->exitcode = VM_EXITCODE_VMINSN;
+ if (vcpu->nested != NULL)
+ vmx_nested_insn_exit(vcpu, reason, qual);
break;
case EXIT_REASON_INVD:
case EXIT_REASON_WBINVD:
@@ -3039,6 +3132,237 @@
CPU_CLR_ATOMIC(curcpu, &pmap->pm_active);
}
+/*
+ * Nested virtualization: run the guest of a guest (L2) on its VMCS until it
+ * has a VM exit that the guest (L1) is to handle, which vmx_nested_exit()
+ * does with the vCPU frozen, or until the vCPU has to leave vmx_run() for
+ * another reason. The VM exits that are the host's own are handled here.
+ */
+static int
+vmx_run_nested(struct vmx_vcpu *vcpu, register_t rip, pmap_t pmap,
+ struct vm_eventinfo *evinfo)
+{
+ struct vmx *vmx;
+ struct vmxctx *vmxctx;
+ struct vmx_nested_l2exit *exit;
+ struct vm_exit *vmexit;
+ struct vlapic *vlapic;
+ struct region_descriptor gdtr, idtr;
+ uint64_t gpa, qual;
+ uint32_t idtvec_info, intr_info, reason;
+ uint16_t ldt_sel;
+ int launched;
+ bool host, nmi;
+
+ vmx = vcpu->vmx;
+ vmxctx = &vcpu->ctx;
+ exit = &vcpu->nested->exit;
+ vlapic = vm_lapic(vcpu->vcpu);
+ vmexit = vm_exitinfo(vcpu->vcpu);
+ launched = 0;
+
+ /* L1 continues at 'rip' once L2 has had a VM exit to it. */
+ vmexit->rip = rip;
+ vmexit->inst_length = 0;
+ vmexit->exitcode = VM_EXITCODE_BOGUS;
+ memset(exit, 0, sizeof(*exit));
+
+ vmx_msr_guest_enter(vcpu);
+
+ VMPTRLD(vcpu->nested->vmcs02);
+ vmcs_write(VMCS_HOST_CR3, rcr3());
+ vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
+ vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
+ vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
+
+ for (;;) {
+ /* See vmx_run() for why interrupts are disabled from here. */
+ disable_intr();
+
+ if (vcpu_suspended(evinfo)) {
+ enable_intr();
+ vm_exit_suspended(vcpu->vcpu, rip);
+ break;
+ }
+
+ if (vcpu_rendezvous_pending(vcpu->vcpu, evinfo)) {
+ enable_intr();
+ vm_exit_rendezvous(vcpu->vcpu, rip);
+ break;
+ }
+
+ if (vcpu_reqidle(evinfo)) {
+ enable_intr();
+ vm_exit_reqidle(vcpu->vcpu, rip);
+ break;
+ }
+
+ if (vcpu_should_yield(vcpu->vcpu)) {
+ enable_intr();
+ vm_exit_astpending(vcpu->vcpu, rip);
+ break;
+ }
+
+ if (vcpu_debugged(vcpu->vcpu)) {
+ enable_intr();
+ vm_exit_debug(vcpu->vcpu, rip);
+ break;
+ }
+
+ /*
+ * An NMI or an interrupt for L1 is a VM exit of L2. An event
+ * that is to be injected into L2 is delivered first: L2 is
+ * entered with an interrupt for the host pending, so that it
+ * has a VM exit right after the delivery.
+ */
+ nmi = vm_nmi_pending(vcpu->vcpu);
+ if (nmi || vlapic_pending_intr(vlapic, NULL)) {
+ if ((vmcs_read(VMCS_ENTRY_INTR_INFO) &
+ VMCS_INTR_VALID) != 0) {
+ ipi_cpu(curcpu, IPI_AST);
+ } else {
+ enable_intr();
+ if (nmi) {
+ exit->reason = EXIT_REASON_EXCEPTION;
+ exit->intr_info = VMCS_INTR_VALID |
+ VMCS_INTR_T_NMI | IDT_NMI;
+ } else {
+ exit->reason = EXIT_REASON_EXT_INTR;
+ }
+ vmexit->exitcode = VM_EXITCODE_VMINSN;
+ break;
+ }
+ }
+
+ /* See vmx_run() for all of this. */
+ sgdt(&gdtr);
+ sidt(&idtr);
+ ldt_sel = sldt();
+ vmx_msr_guest_enter_tsc_aux(vmx, vcpu);
+ vmx_dr_enter_guest(vmxctx);
+ vmx_pmap_activate(vmx, pmap);
+
+ exit->rc = vmx_enter_guest(vmxctx, vmx, launched);
+
+ vmx_pmap_deactivate(vmx, pmap);
+ vmx_dr_leave_guest(vmxctx);
+ vmx_msr_guest_exit_tsc_aux(vmx, vcpu);
+ bare_lgdt(&gdtr);
+ lidt(&idtr);
+ lldt(ldt_sel);
+
+ if (exit->rc != VMX_GUEST_VMEXIT) {
+ /* L1's VMLAUNCH or VMRESUME fails. */
+ enable_intr();
+ exit->inst_error = vmcs_instruction_error();
+ vmexit->exitcode = VM_EXITCODE_VMINSN;
+ break;
+ }
+ launched = 1;
+
+ reason = vmcs_read(VMCS_EXIT_REASON);
+ qual = vmcs_exit_qualification();
+ intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
+ vmexit->u.vmx.exit_reason = reason & 0xffff;
+ vmx_exit_handle_nmi(vcpu, vmexit);
+ enable_intr();
+
+ host = false;
+ switch (reason) {
+ case EXIT_REASON_EXT_INTR:
+ if ((intr_info & VMCS_INTR_VALID) != 0)
+ vmx_trigger_hostintr(intr_info & 0xff);
+ host = true;
+ break;
+ case EXIT_REASON_EXCEPTION:
+ /* The NMI handler has been called already. */
+ if ((intr_info & VMCS_INTR_T_MASK) ==
+ VMCS_INTR_T_NMI) {
+ host = true;
+ } else if ((intr_info & 0xff) == IDT_MC) {
+ __asm __volatile("int $18");
+ host = true;
+ }
+ break;
+ case EXIT_REASON_EPT_FAULT:
+ /*
+ * L2 runs on L1's shadow page tables: this is a fault
+ * in L1's memory, or L1 has made a mistake.
+ */
+ gpa = vmcs_gpa();
+ if (vm_mem_allocated(vcpu->vcpu, gpa)) {
+ vmexit->exitcode = VM_EXITCODE_PAGING;
+ vmexit->u.paging.gpa = gpa;
+ vmexit->u.paging.fault_type =
+ ept_fault_type(qual);
+ vmm_stat_incr(vcpu->vcpu,
+ VMEXIT_NESTED_FAULT, 1);
+ } else {
+ vm_suspend(vmx->vm, VM_SUSPEND_TRIPLEFAULT);
+ }
+ if ((vmcs_idt_vectoring_info() &
+ VMCS_IDT_VEC_VALID) == 0 &&
+ (qual & EXIT_QUAL_NMIUDTI) != 0 &&
+ (vmcs_read(VMCS_PIN_BASED_CTLS) &
+ PINBASED_VIRTUAL_NMI) != 0)
+ vmx_restore_nmi_blocking(vcpu);
+ host = true;
+ break;
+ case EXIT_REASON_EPT_MISCONFIG:
+ vm_suspend(vmx->vm, VM_SUSPEND_TRIPLEFAULT);
+ host = true;
+ break;
+ case EXIT_REASON_MCE_DURING_ENTRY | (1U << 31):
+ __asm __volatile("int $18");
+ break;
+ }
+
+ if (host) {
+ /*
+ * An event that was being delivered to L2 is
+ * delivered again at the next VM entry. Bit 12 of
+ * the IDT-vectoring information is not defined for
+ * the injection.
+ */
+ idtvec_info = vmcs_idt_vectoring_info();
+ if ((idtvec_info & VMCS_IDT_VEC_VALID) != 0) {
+ if ((idtvec_info & VMCS_INTR_T_MASK) ==
+ VMCS_INTR_T_NMI)
+ vmx_clear_nmi_blocking(vcpu);
+ idtvec_info &= ~(1 << 12);
+ vmcs_write(VMCS_ENTRY_INTR_INFO, idtvec_info);
+ if ((idtvec_info & VMCS_INTR_DEL_ERRCODE) != 0)
+ vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR,
+ vmcs_idt_vectoring_err());
+ vmcs_write(VMCS_ENTRY_INST_LENGTH,
+ vmexit_instruction_length());
+ }
+ if (vmexit->exitcode != VM_EXITCODE_BOGUS)
+ break;
+ continue;
+ }
+
+ /* The VM exit is L1's to handle. */
+ exit->reason = reason;
+ exit->qual = qual;
+ exit->intr_info = intr_info;
+ exit->intr_errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE);
+ exit->idt_info = vmcs_idt_vectoring_info();
+ exit->idt_errcode = vmcs_idt_vectoring_err();
+ exit->inst_length = vmexit_instruction_length();
+ exit->inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO);
+ exit->gla = vmcs_gla();
+ exit->gpa = vmcs_gpa();
+ vmexit->exitcode = VM_EXITCODE_VMINSN;
+ break;
+ }
+
+ VMCLEAR(vcpu->nested->vmcs02);
+ vmx_msr_guest_exit(vcpu);
+
+ return (0);
+}
+
static int
vmx_run(void *vcpui, register_t rip, pmap_t pmap, struct vm_eventinfo *evinfo)
{
@@ -3064,6 +3388,9 @@
KASSERT(vmxctx->pmap == pmap,
("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
+ if (vcpu->nested != NULL && vcpu->nested->l2)
+ return (vmx_run_nested(vcpu, rip, pmap, evinfo));
+
vmx_msr_guest_enter(vcpu);
VMPTRLD(vmcs);
@@ -3246,12 +3573,22 @@
return (0);
}
+static int
+vmx_vminsn(void *vcpui)
+{
+ return (vmx_nested_exit(vcpui));
+}
+
static void
vmx_vcpu_cleanup(void *vcpui)
{
struct vmx_vcpu *vcpu = vcpui;
vpid_free(vcpu->state.vpid);
+ if (vcpu->nested != NULL) {
+ free(vcpu->nested->vmcs02, M_VMX);
+ vmx_nested_free(vcpu->nested);
+ }
free(vcpu->pir_desc, M_VMX);
free(vcpu->apic_page, M_VMX);
free(vcpu->vmcs, M_VMX);
@@ -4179,6 +4516,10 @@
vmx = vcpu->vmx;
vmcs = vcpu->vmcs;
+ /* The state of a guest in VMX operation is not in the snapshot. */
+ if (vcpu->nested != NULL && vcpu->nested->vmxon)
+ return (EBUSY);
+
run = vcpu_is_running(vcpu->vcpu, &hostcpu);
if (run && hostcpu != curcpu) {
printf("%s: %s%d is running", __func__, vm_name(vmx->vm),
@@ -4326,6 +4667,7 @@
.vmspace_free = vmx_vmspace_free,
.vlapic_init = vmx_vlapic_init,
.vlapic_cleanup = vmx_vlapic_cleanup,
+ .vminsn = vmx_vminsn,
#ifdef BHYVE_SNAPSHOT
.vcpu_snapshot = vmx_vcpu_snapshot,
.restore_tsc = vmx_restore_tsc,
diff --git a/sys/amd64/vmm/intel/vmx_msr.c b/sys/amd64/vmm/intel/vmx_msr.c
--- a/sys/amd64/vmm/intel/vmx_msr.c
+++ b/sys/amd64/vmm/intel/vmx_msr.c
@@ -39,6 +39,7 @@
#include "vmx.h"
#include "vmx_msr.h"
+#include "vmx_nested.h"
#include "x86.h"
static bool
@@ -433,6 +434,13 @@
case MSR_PAT:
*val = vcpu->guest_msrs[IDX_MSR_PAT];
break;
+ case MSR_IA32_FEATURE_CONTROL:
+ case MSR_VMX_BASIC ... MSR_VMX_TRUE_ENTRY_CTLS:
+ if (vcpu->nested != NULL)
+ error = vmx_nested_rdmsr(num, val);
+ else
+ error = EINVAL;
+ break;
default:
error = EINVAL;
break;
diff --git a/sys/amd64/vmm/intel/vmx_nested.h b/sys/amd64/vmm/intel/vmx_nested.h
new file mode 100644
--- /dev/null
+++ b/sys/amd64/vmm/intel/vmx_nested.h
@@ -0,0 +1,89 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026 Wanpeng Qian <wanpengqian@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _VMX_NESTED_H_
+#define _VMX_NESTED_H_
+
+/*
+ * Nested virtualization: a guest (L1) that uses VMX itself to run a guest
+ * of its own (L2). See the comment at the top of vmx_nested.c.
+ */
+
+struct vmcs;
+struct vmcs12;
+struct vmx_vcpu;
+
+/* A VMX instruction of L1, recorded at its VM exit for vmx_nested_exit(). */
+struct vmx_nested_insn {
+ uint32_t reason;
+ uint32_t inst_info;
+ uint64_t qual; /* displacement of the memory operand */
+ uint64_t cr0; /* CR0 and CR4 as L1 reads them */
+ uint64_t cr4;
+ uint64_t rflags;
+ uint32_t interruptibility;
+ uint64_t seg_base; /* base of the operand's segment */
+ struct vm_guest_paging paging;
+};
+
+/* A VM exit of L2, read from vmcs02 for vmx_nested_exit(). */
+struct vmx_nested_l2exit {
+ int rc; /* from vmx_enter_guest() */
+ uint32_t inst_error;
+ uint32_t reason;
+ uint32_t intr_info;
+ uint32_t intr_errcode;
+ uint32_t idt_info;
+ uint32_t idt_errcode;
+ uint32_t inst_length;
+ uint32_t inst_info;
+ uint64_t qual;
+ uint64_t gla;
+ uint64_t gpa;
+};
+
+struct vmx_nested {
+ bool vmxon; /* L1 is in VMX operation */
+ bool l2; /* L2 runs: vmx_run() enters vmcs02 */
+ bool launching; /* the pending entry is a VMLAUNCH */
+ uint64_t vmxon_ptr;
+ uint64_t vmptr; /* L1's current-VMCS pointer */
+ struct vmcs12 *vmcs12; /* copy of L1's current VMCS */
+ struct vmcs *vmcs02; /* the VMCS that L2 runs on */
+ uint64_t cr0_host; /* CR0/CR4 bits that vmcs02 owns but */
+ uint64_t cr4_host; /* L1's guest/host masks do not */
+ struct vmx_nested_insn insn;
+ struct vmx_nested_l2exit exit;
+};
+
+void vmx_nested_init(void);
+struct vmx_nested *vmx_nested_alloc(void);
+void vmx_nested_free(struct vmx_nested *nested);
+int vmx_nested_rdmsr(u_int num, uint64_t *val);
+int vmx_nested_exit(struct vmx_vcpu *vcpu);
+
+#endif
diff --git a/sys/amd64/vmm/intel/vmx_nested.c b/sys/amd64/vmm/intel/vmx_nested.c
new file mode 100644
--- /dev/null
+++ b/sys/amd64/vmm/intel/vmx_nested.c
@@ -0,0 +1,1334 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2026 Wanpeng Qian <wanpengqian@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Nested virtualization for Intel CPUs: a guest (L1) is offered VMX and runs
+ * a guest of its own (L2). It is off unless the hw.vmm.vmx.nested tunable is
+ * set. This is the smallest set of features that runs an L2:
+ *
+ * - CPUID.1:ECX.VMX, IA32_FEATURE_CONTROL and the VMX capability MSRs. L1 is
+ * not offered EPT, VPID, unrestricted guest, APIC virtualization, the TPR
+ * shadow, VMCS shadowing, I/O or MSR bitmaps, MSR load/store areas or the
+ * preemption timer. It has to run L2 on shadow page tables and it has to
+ * be a 64-bit hypervisor.
+ *
+ * - VMXON, VMXOFF, VMCLEAR, VMPTRLD, VMPTRST, VMREAD and VMWRITE are emulated
+ * on L1's VMCS ("vmcs12"), a page of its memory that only this file knows
+ * the layout of. The current one is cached in the vCPU and written back at
+ * VMCLEAR, at a VMPTRLD of another one and at VMXOFF.
+ *
+ * - VMLAUNCH and VMRESUME check vmcs12 against the capability MSRs and merge
+ * it into "vmcs02", a second VMCS of the vCPU which vmx_run() then enters
+ * instead of L1's ("vmcs01"). L2's page tables are L1's shadow tables and
+ * hold L1-physical addresses, so that vmcs02 uses the VM's EPT unchanged.
+ *
+ * - A VM exit of L2 that is not the host's own (a host interrupt or NMI, an
+ * EPT violation in L1's memory) is reflected to L1: the exit information
+ * and L2's state go to vmcs12 and L1 continues with the state of vmcs12's
+ * host-state area. An interrupt or NMI for L1 that arrives while L2 runs is
+ * an external-interrupt or NMI exit of L2.
+ *
+ * Nothing in vmcs12 is trusted. vmcs02 takes L2's state, the event to inject
+ * and the controls that L1 was offered from it. The EPT pointer, the host
+ * state and everything else stay the host's.
+ *
+ * Not implemented: the segment limit check of the instructions' memory
+ * operand, INIT blocking and A20M in VMX operation, PIC (ExtINT) interrupts
+ * for L1 while L2 runs, snapshots of a vCPU in VMX operation.
+ *
+ * "SDM" is the Intel 64 and IA-32 Architectures Software Developer's Manual,
+ * Volume 3, order number 325384-081US, which the section numbers refer to.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/proc.h>
+#include <sys/reg.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/md_var.h>
+#include <machine/psl.h>
+#include <machine/segments.h>
+#include <machine/specialreg.h>
+#include <machine/vmm.h>
+
+#include <dev/vmm/vmm_mem.h>
+#include <dev/vmm/vmm_vm.h>
+
+#include "vlapic.h"
+#include "vmx_cpufunc.h"
+#include "vmcs.h"
+#include "vmx.h"
+#include "vmx_controls.h"
+#include "vmx_nested.h"
+
+static MALLOC_DEFINE(M_VMXNESTED, "vmx_nested", "nested VMX");
+
+#define MSR_VMX_MISC 0x485
+#define MSR_VMX_VMCS_ENUM 0x48a
+#define MSR_VMX_FIRST MSR_VMX_BASIC
+#define MSR_VMX_LAST MSR_VMX_TRUE_ENTRY_CTLS
+
+#define VMX_BASIC_INOUT_INFO (1UL << 54)
+#define VMX_BASIC_TRUE_CTLS (1UL << 55)
+#define VMX_MISC_SAVE_LMA (1UL << 5)
+
+/*
+ * The default1 controls, which the MSRs that are not the TRUE ones report as
+ * 1 (SDM A.2, A.3.1, A.3.2, A.4.1 and A.5).
+ */
+#define PINBASED_DEFAULT1 0x00000016
+#define PROCBASED_DEFAULT1 0x0401e172
+#define VM_EXIT_DEFAULT1 0x00036dff
+#define VM_ENTRY_DEFAULT1 0x000011ff
+
+/*
+ * The controls that L1 has to set, besides those that the CPU requires, and
+ * the ones that it may set. L1 has to take the exits that the host needs L2
+ * to have in any case, so that every reflected exit is one that it asked for.
+ */
+#define PINBASED_L1_MUST (PINBASED_EXTINT_EXITING | PINBASED_NMI_EXITING)
+#define PINBASED_L1_MAY PINBASED_VIRTUAL_NMI
+#define PROCBASED_L1_MUST \
+ (PROCBASED_HLT_EXITING | PROCBASED_MWAIT_EXITING | \
+ PROCBASED_MONITOR_EXITING | PROCBASED_IO_EXITING | \
+ PROCBASED_CR8_LOAD_EXITING | PROCBASED_CR8_STORE_EXITING)
+#define PROCBASED_L1_MAY \
+ (PROCBASED_INT_WINDOW_EXITING | PROCBASED_TSC_OFFSET | \
+ PROCBASED_INVLPG_EXITING | PROCBASED_RDPMC_EXITING | \
+ PROCBASED_RDTSC_EXITING | PROCBASED_CR3_LOAD_EXITING | \
+ PROCBASED_CR3_STORE_EXITING | PROCBASED_NMI_WINDOW_EXITING | \
+ PROCBASED_MOV_DR_EXITING | PROCBASED_PAUSE_EXITING)
+#define VM_EXIT_L1_MUST VM_EXIT_HOST_LMA
+#define VM_EXIT_L1_MAY \
+ (VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACKNOWLEDGE_INTERRUPT | \
+ VM_EXIT_SAVE_EFER | VM_EXIT_LOAD_EFER)
+#define VM_ENTRY_L1_MUST 0
+#define VM_ENTRY_L1_MAY \
+ (VM_ENTRY_LOAD_DEBUG_CONTROLS | VM_ENTRY_GUEST_LMA | \
+ VM_ENTRY_LOAD_EFER)
+
+/* VM-instruction error numbers (SDM 31.4). */
+#define VMXERR_VMCLEAR_ADDR 2
+#define VMXERR_VMCLEAR_VMXON 3
+#define VMXERR_VMLAUNCH_NOT_CLEAR 4
+#define VMXERR_VMRESUME_NOT_LAUNCHED 5
+#define VMXERR_ENTRY_CTLS 7
+#define VMXERR_ENTRY_HOST_STATE 8
+#define VMXERR_VMPTRLD_ADDR 9
+#define VMXERR_VMPTRLD_VMXON 10
+#define VMXERR_VMPTRLD_REVISION 11
+#define VMXERR_FIELD 12
+#define VMXERR_READONLY_FIELD 13
+#define VMXERR_VMXON_IN_ROOT 15
+#define VMXERR_ENTRY_MOVSS 26
+
+#define EXIT_REASON_ENTRY_FAIL (1U << 31)
+
+/* Exit qualifications of a VM-entry failure with the guest state (SDM 27.8). */
+#define EXIT_QUAL_ENTRY_NONE 0
+#define EXIT_QUAL_ENTRY_PDPTE 2
+
+/* IA32_EFER bits that are not reserved on a CPU that has VMX. */
+#define EFER_VMX_VALID (EFER_SCE | EFER_LME | EFER_LMA | EFER_NXE)
+
+/*
+ * Bits of CR0 that a VM exit leaves as they are (SDM 28.5.1): ET, NW, CD and
+ * the reserved bits 63:32, 28:19, 17 and 15:6.
+ */
+#define CR0_VMEXIT_KEEP \
+ (CR0_NW | CR0_CD | 0xffffffff1ffaffd0)
+
+/* The revision identifier of vmcs12: "bhyv". */
+#define VMCS12_REVISION 0x62687976
+#define VMCS12_MAGIC 0x32317376 /* VMCLEAR has been done */
+#define VMPTR_INVALID (~0UL)
+
+/*
+ * The fields of vmcs12 by their encoding (SDM Appendix B), the 64-bit ones
+ * by the "full" one, sorted for vmcs12_slot(). A field exists whether its
+ * feature is offered to L1 or not; the controls are checked before any of
+ * them is used.
+ */
+static const uint16_t vmcs12_fields[] = {
+ /* 16-bit control, guest-state and host-state fields */
+ 0x0000, 0x0002, 0x0004, 0x0006, 0x0008,
+ 0x0800, 0x0802, 0x0804, 0x0806, 0x0808, 0x080a, 0x080c, 0x080e,
+ 0x0810, 0x0812, 0x0814,
+ 0x0c00, 0x0c02, 0x0c04, 0x0c06, 0x0c08, 0x0c0a, 0x0c0c,
+ /* 64-bit control, read-only, guest-state and host-state fields */
+ 0x2000, 0x2002, 0x2004, 0x2006, 0x2008, 0x200a, 0x200c, 0x200e,
+ 0x2010, 0x2012, 0x2014, 0x2016, 0x2018, 0x201a, 0x201c, 0x201e,
+ 0x2020, 0x2022, 0x2024, 0x2026, 0x2028, 0x202a, 0x202c, 0x202e,
+ 0x2030, 0x2032, 0x2034, 0x2036, 0x2038, 0x203a, 0x203c, 0x203e,
+ 0x2040, 0x2042, 0x2044, 0x204a, 0x204c,
+ 0x2400,
+ 0x2800, 0x2802, 0x2804, 0x2806, 0x2808, 0x280a, 0x280c, 0x280e,
+ 0x2810, 0x2812, 0x2814, 0x2816, 0x2818,
+ 0x2c00, 0x2c02, 0x2c04, 0x2c06,
+ /* 32-bit control, read-only, guest-state and host-state fields */
+ 0x4000, 0x4002, 0x4004, 0x4006, 0x4008, 0x400a, 0x400c, 0x400e,
+ 0x4010, 0x4012, 0x4014, 0x4016, 0x4018, 0x401a, 0x401c, 0x401e,
+ 0x4020, 0x4022, 0x4024,
+ 0x4400, 0x4402, 0x4404, 0x4406, 0x4408, 0x440a, 0x440c, 0x440e,
+ 0x4800, 0x4802, 0x4804, 0x4806, 0x4808, 0x480a, 0x480c, 0x480e,
+ 0x4810, 0x4812, 0x4814, 0x4816, 0x4818, 0x481a, 0x481c, 0x481e,
+ 0x4820, 0x4822, 0x4824, 0x4826, 0x4828, 0x482a, 0x482e,
+ 0x4c00,
+ /* natural-width control, read-only, guest-state, host-state fields */
+ 0x6000, 0x6002, 0x6004, 0x6006, 0x6008, 0x600a, 0x600c, 0x600e,
+ 0x6400, 0x6402, 0x6404, 0x6406, 0x6408, 0x640a,
+ 0x6800, 0x6802, 0x6804, 0x6806, 0x6808, 0x680a, 0x680c, 0x680e,
+ 0x6810, 0x6812, 0x6814, 0x6816, 0x6818, 0x681a, 0x681c, 0x681e,
+ 0x6820, 0x6822, 0x6824, 0x6826, 0x6828, 0x682a, 0x682c,
+ 0x6c00, 0x6c02, 0x6c04, 0x6c06, 0x6c08, 0x6c0a, 0x6c0c, 0x6c0e,
+ 0x6c10, 0x6c12, 0x6c14, 0x6c16, 0x6c18, 0x6c1a, 0x6c1c,
+};
+
+struct vmcs12 {
+ uint32_t revision;
+ uint32_t abort;
+ uint32_t magic;
+ uint32_t launched;
+ uint64_t field[nitems(vmcs12_fields)];
+};
+CTASSERT(sizeof(struct vmcs12) <= PAGE_SIZE);
+
+/* The structure of a field's encoding (SDM 25.11.2). */
+#define VMCS_WIDTH(enc) (((enc) >> 13) & 0x3)
+#define VMCS_WIDTH_16 0
+#define VMCS_WIDTH_64 1
+#define VMCS_WIDTH_32 2
+#define VMCS_TYPE(enc) (((enc) >> 10) & 0x3)
+#define VMCS_TYPE_READONLY 1
+#define VMCS_INDEX(enc) (((enc) >> 1) & 0x1ff)
+
+/* L2's state that vmcs12 and vmcs02 hold alike. */
+static const uint16_t vmcs12_guest_state[] = {
+ VMCS_GUEST_ES_SELECTOR, VMCS_GUEST_CS_SELECTOR, VMCS_GUEST_SS_SELECTOR,
+ VMCS_GUEST_DS_SELECTOR, VMCS_GUEST_FS_SELECTOR, VMCS_GUEST_GS_SELECTOR,
+ VMCS_GUEST_LDTR_SELECTOR, VMCS_GUEST_TR_SELECTOR,
+ VMCS_GUEST_ES_LIMIT, VMCS_GUEST_CS_LIMIT, VMCS_GUEST_SS_LIMIT,
+ VMCS_GUEST_DS_LIMIT, VMCS_GUEST_FS_LIMIT, VMCS_GUEST_GS_LIMIT,
+ VMCS_GUEST_LDTR_LIMIT, VMCS_GUEST_TR_LIMIT, VMCS_GUEST_GDTR_LIMIT,
+ VMCS_GUEST_IDTR_LIMIT,
+ VMCS_GUEST_ES_ACCESS_RIGHTS, VMCS_GUEST_CS_ACCESS_RIGHTS,
+ VMCS_GUEST_SS_ACCESS_RIGHTS, VMCS_GUEST_DS_ACCESS_RIGHTS,
+ VMCS_GUEST_FS_ACCESS_RIGHTS, VMCS_GUEST_GS_ACCESS_RIGHTS,
+ VMCS_GUEST_LDTR_ACCESS_RIGHTS, VMCS_GUEST_TR_ACCESS_RIGHTS,
+ VMCS_GUEST_ES_BASE, VMCS_GUEST_CS_BASE, VMCS_GUEST_SS_BASE,
+ VMCS_GUEST_DS_BASE, VMCS_GUEST_FS_BASE, VMCS_GUEST_GS_BASE,
+ VMCS_GUEST_LDTR_BASE, VMCS_GUEST_TR_BASE, VMCS_GUEST_GDTR_BASE,
+ VMCS_GUEST_IDTR_BASE,
+ VMCS_GUEST_CR3, VMCS_GUEST_RSP, VMCS_GUEST_RIP, VMCS_GUEST_RFLAGS,
+ VMCS_GUEST_PENDING_DBG_EXCEPTIONS, VMCS_GUEST_IA32_SYSENTER_CS,
+ VMCS_GUEST_IA32_SYSENTER_ESP, VMCS_GUEST_IA32_SYSENTER_EIP,
+ VMCS_GUEST_INTERRUPTIBILITY, VMCS_GUEST_ACTIVITY,
+};
+
+/* The VMX capability MSRs as L1 reads them. */
+static uint64_t nested_msrs[MSR_VMX_LAST - MSR_VMX_FIRST + 1];
+static uint32_t nested_msrs_present;
+
+#define NESTED_MSR(num) (nested_msrs[(num) - MSR_VMX_FIRST])
+
+static void
+nested_msr_set(u_int num, uint64_t val)
+{
+ NESTED_MSR(num) = val;
+ nested_msrs_present |= 1U << (num - MSR_VMX_FIRST);
+}
+
+/*
+ * A control may be set by L1 if the CPU has it and it is offered. It has to
+ * be set if the CPU or the host requires it. Bits 31:0 of the MSR are the
+ * controls that must be 1, bits 63:32 the ones that may be 1 (SDM A.3.1).
+ */
+static void
+nested_msr_set_ctls(u_int msr, u_int true_msr, uint32_t default1,
+ uint32_t must, uint32_t may)
+{
+ uint64_t host;
+
+ host = rdmsr((rdmsr(MSR_VMX_BASIC) & VMX_BASIC_TRUE_CTLS) != 0 ?
+ true_msr : msr);
+ must |= (uint32_t)host;
+ may = ((host >> 32) & may) | must;
+ nested_msr_set(true_msr, (uint64_t)may << 32 | must);
+ nested_msr_set(msr, (uint64_t)(may | default1) << 32 |
+ (must | default1));
+}
+
+void
+vmx_nested_init(void)
+{
+ u_int i, maxidx;
+
+ /*
+ * SDM A.1: the revision identifier, 4096 bytes for a VMCS, the
+ * write-back memory type, the instruction information of INS and OUTS
+ * if the CPU has it, the TRUE capability MSRs.
+ */
+ nested_msr_set(MSR_VMX_BASIC, VMCS12_REVISION |
+ (uint64_t)PAGE_SIZE << 32 | (uint64_t)PAT_WRITE_BACK << 50 |
+ (rdmsr(MSR_VMX_BASIC) & VMX_BASIC_INOUT_INFO) |
+ VMX_BASIC_TRUE_CTLS);
+ nested_msr_set_ctls(MSR_VMX_PINBASED_CTLS, MSR_VMX_TRUE_PINBASED_CTLS,
+ PINBASED_DEFAULT1, PINBASED_L1_MUST, PINBASED_L1_MAY);
+ nested_msr_set_ctls(MSR_VMX_PROCBASED_CTLS,
+ MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_DEFAULT1,
+ PROCBASED_L1_MUST, PROCBASED_L1_MAY);
+ nested_msr_set_ctls(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
+ VM_EXIT_DEFAULT1, VM_EXIT_L1_MUST, VM_EXIT_L1_MAY);
+ nested_msr_set_ctls(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS,
+ VM_ENTRY_DEFAULT1, VM_ENTRY_L1_MUST, VM_ENTRY_L1_MAY);
+ /*
+ * SDM A.6: VM exits store EFER.LMA into "IA-32e mode guest". The
+ * rest is zero: no preemption timer, no activity state but active,
+ * no CR3 targets, no VMWRITE to the VM-exit information fields. The
+ * MSR does not have a way to say that the MSR areas are not there.
+ */
+ nested_msr_set(MSR_VMX_MISC, VMX_MISC_SAVE_LMA);
+ nested_msr_set(MSR_VMX_CR0_FIXED0, rdmsr(MSR_VMX_CR0_FIXED0));
+ nested_msr_set(MSR_VMX_CR0_FIXED1, rdmsr(MSR_VMX_CR0_FIXED1));
+ nested_msr_set(MSR_VMX_CR4_FIXED0, rdmsr(MSR_VMX_CR4_FIXED0));
+ nested_msr_set(MSR_VMX_CR4_FIXED1, rdmsr(MSR_VMX_CR4_FIXED1));
+ /* SDM A.9: the highest index of a field's encoding. */
+ maxidx = 0;
+ for (i = 0; i < nitems(vmcs12_fields); i++)
+ maxidx = MAX(maxidx, VMCS_INDEX(vmcs12_fields[i]));
+ nested_msr_set(MSR_VMX_VMCS_ENUM, maxidx << 1);
+}
+
+/* Returns EINVAL for an MSR that L1 does not have. */
+int
+vmx_nested_rdmsr(u_int num, uint64_t *val)
+{
+ if (num == MSR_IA32_FEATURE_CONTROL) {
+ *val = IA32_FEATURE_CONTROL_LOCK | IA32_FEATURE_CONTROL_VMX_EN;
+ return (0);
+ }
+ if (num < MSR_VMX_FIRST || num > MSR_VMX_LAST ||
+ (nested_msrs_present & (1U << (num - MSR_VMX_FIRST))) == 0)
+ return (EINVAL);
+ *val = NESTED_MSR(num);
+ return (0);
+}
+
+struct vmx_nested *
+vmx_nested_alloc(void)
+{
+ struct vmx_nested *nested;
+
+ nested = malloc(sizeof(*nested), M_VMXNESTED, M_WAITOK | M_ZERO);
+ nested->vmcs12 = malloc(PAGE_SIZE, M_VMXNESTED, M_WAITOK | M_ZERO);
+ nested->vmptr = VMPTR_INVALID;
+ return (nested);
+}
+
+void
+vmx_nested_free(struct vmx_nested *nested)
+{
+ if (nested == NULL)
+ return;
+ free(nested->vmcs12, M_VMXNESTED);
+ free(nested, M_VMXNESTED);
+}
+
+/* The index of a field in vmcs12, or -1 if there is no such field. */
+static int
+vmcs12_slot(uint64_t enc)
+{
+ int lo, hi, mid;
+
+ if (enc > 0x7fff || (enc & (1 << 12)) != 0)
+ return (-1);
+ /* Only 64-bit fields have a "high" encoding. */
+ if ((enc & 1) != 0 && VMCS_WIDTH(enc) != VMCS_WIDTH_64)
+ return (-1);
+ enc &= ~1UL;
+ lo = 0;
+ hi = nitems(vmcs12_fields) - 1;
+ while (lo <= hi) {
+ mid = (lo + hi) / 2;
+ if (vmcs12_fields[mid] == enc)
+ return (mid);
+ if (vmcs12_fields[mid] < enc)
+ lo = mid + 1;
+ else
+ hi = mid - 1;
+ }
+ return (-1);
+}
+
+#define V12(nested, enc) ((nested)->vmcs12->field[vmcs12_slot(enc)])
+
+/* VMsucceed, VMfailInvalid and VMfailValid (SDM 31.2). */
+static void
+nested_set_rflags(struct vmx_vcpu *vcpu, uint64_t flags)
+{
+ uint64_t rflags;
+ int error __diagused;
+
+ rflags = vcpu->nested->insn.rflags;
+ rflags &= ~(PSL_C | PSL_PF | PSL_AF | PSL_Z | PSL_N | PSL_V);
+ error = vm_set_register(vcpu->vcpu, VM_REG_GUEST_RFLAGS,
+ rflags | flags);
+ KASSERT(error == 0, ("%s: error %d setting rflags", __func__, error));
+}
+
+static void
+nested_succeed(struct vmx_vcpu *vcpu)
+{
+ nested_set_rflags(vcpu, 0);
+}
+
+static void
+nested_fail(struct vmx_vcpu *vcpu, uint32_t error)
+{
+ struct vmx_nested *nested = vcpu->nested;
+
+ if (nested->vmptr == VMPTR_INVALID) {
+ nested_set_rflags(vcpu, PSL_C);
+ } else {
+ V12(nested, VMCS_INSTRUCTION_ERROR) = error;
+ nested_set_rflags(vcpu, PSL_Z);
+ }
+}
+
+static void
+nested_fail_invalid(struct vmx_vcpu *vcpu)
+{
+ nested_set_rflags(vcpu, PSL_C);
+}
+
+/* Register numbers of the instruction information and exit qualification. */
+static const int nested_gpr[16] = {
+ VM_REG_GUEST_RAX, VM_REG_GUEST_RCX, VM_REG_GUEST_RDX, VM_REG_GUEST_RBX,
+ VM_REG_GUEST_RSP, VM_REG_GUEST_RBP, VM_REG_GUEST_RSI, VM_REG_GUEST_RDI,
+ VM_REG_GUEST_R8, VM_REG_GUEST_R9, VM_REG_GUEST_R10, VM_REG_GUEST_R11,
+ VM_REG_GUEST_R12, VM_REG_GUEST_R13, VM_REG_GUEST_R14, VM_REG_GUEST_R15,
+};
+
+static uint64_t
+nested_get_gpr(struct vmx_vcpu *vcpu, int reg)
+{
+ uint64_t val;
+ int error __diagused;
+
+ error = vm_get_register(vcpu->vcpu, nested_gpr[reg & 0xf], &val);
+ KASSERT(error == 0, ("%s: error %d getting reg", __func__, error));
+ return (val);
+}
+
+/*
+ * The VM-exit instruction information of the VMX instructions (SDM 28.2.5,
+ * tables 28-13 and 28-14).
+ */
+#define INST_INFO_SCALE(info) ((info) & 0x3)
+#define INST_INFO_REG1(info) (((info) >> 3) & 0xf)
+#define INST_INFO_ADDR_SIZE(info) (((info) >> 7) & 0x7)
+#define INST_INFO_IS_REG (1 << 10)
+#define INST_INFO_SEGMENT(info) (((info) >> 15) & 0x7)
+#define INST_INFO_INDEX(info) (((info) >> 18) & 0xf)
+#define INST_INFO_NO_INDEX (1 << 22)
+#define INST_INFO_BASE(info) (((info) >> 23) & 0xf)
+#define INST_INFO_NO_BASE (1 << 27)
+#define INST_INFO_REG2(info) (((info) >> 28) & 0xf)
+
+static bool
+nested_insn_64bit(const struct vmx_nested_insn *insn)
+{
+ return (insn->paging.cpu_mode == CPU_MODE_64BIT);
+}
+
+/*
+ * The linear address of the instruction's memory operand. The exit
+ * qualification is the displacement, or for RIP-relative addressing the sum
+ * of the displacement and RIP, in which case there is no base register
+ * (SDM 28.2.1).
+ */
+static uint64_t
+nested_insn_gla(struct vmx_vcpu *vcpu)
+{
+ const struct vmx_nested_insn *insn = &vcpu->nested->insn;
+ uint64_t gla;
+ uint32_t info;
+ int seg;
+
+ info = insn->inst_info;
+ gla = insn->qual;
+ if ((info & INST_INFO_NO_BASE) == 0)
+ gla += nested_get_gpr(vcpu, INST_INFO_BASE(info));
+ if ((info & INST_INFO_NO_INDEX) == 0)
+ gla += nested_get_gpr(vcpu, INST_INFO_INDEX(info)) <<
+ INST_INFO_SCALE(info);
+ if (INST_INFO_ADDR_SIZE(info) == 0)
+ gla &= 0xffff;
+ else if (INST_INFO_ADDR_SIZE(info) == 1)
+ gla &= 0xffffffff;
+ /* Only FS and GS have a base in 64-bit mode. */
+ seg = INST_INFO_SEGMENT(info);
+ if (!nested_insn_64bit(insn) || seg == 4 || seg == 5)
+ gla += insn->seg_base;
+ if (!nested_insn_64bit(insn))
+ gla &= 0xffffffff;
+ return (gla);
+}
+
+/*
+ * Read or write the instruction's memory operand. Returns false after
+ * injecting an exception: the instruction is not completed.
+ */
+static bool
+nested_insn_mem(struct vmx_vcpu *vcpu, void *buf, size_t len, bool write)
+{
+ struct vm_copyinfo copyinfo[2];
+ struct vm_guest_paging paging;
+ int error, fault;
+
+ paging = vcpu->nested->insn.paging;
+ error = vm_copy_setup(vcpu->vcpu, &paging, nested_insn_gla(vcpu), len,
+ write ? VM_PROT_WRITE : VM_PROT_READ, copyinfo, nitems(copyinfo),
+ &fault);
+ if (error != 0) {
+ vm_inject_gp(vcpu->vcpu);
+ return (false);
+ }
+ if (fault != 0)
+ return (false);
+ if (write)
+ vm_copyout(buf, copyinfo, len);
+ else
+ vm_copyin(copyinfo, buf, len);
+ vm_copy_teardown(copyinfo, nitems(copyinfo));
+ return (true);
+}
+
+/* A VMXON or VMCS pointer has to be page-aligned and within MAXPHYADDR. */
+static bool
+nested_ptr_valid(uint64_t ptr)
+{
+ return ((ptr & PAGE_MASK) == 0 && (ptr >> cpu_maxphyaddr) == 0);
+}
+
+/* Copy from or to L1's memory. Returns false if it is not all memory. */
+static bool
+nested_copy(struct vmx_vcpu *vcpu, uint64_t gpa, void *buf, size_t len,
+ bool write)
+{
+ void *cookie, *p;
+
+ KASSERT((gpa & PAGE_MASK) + len <= PAGE_SIZE,
+ ("%s: copy crosses a page boundary", __func__));
+ p = vm_gpa_hold(vcpu->vcpu, gpa, len,
+ write ? VM_PROT_WRITE : VM_PROT_READ, &cookie);
+ if (p == NULL)
+ return (false);
+ if (write)
+ memcpy(p, buf, len);
+ else
+ memcpy(buf, p, len);
+ vm_gpa_release(cookie);
+ return (true);
+}
+
+/* Write the cached vmcs12 back to L1's memory. */
+static void
+nested_vmcs12_flush(struct vmx_vcpu *vcpu)
+{
+ struct vmx_nested *nested = vcpu->nested;
+
+ if (nested->vmptr == VMPTR_INVALID)
+ return;
+ nested->vmcs12->magic = VMCS12_MAGIC;
+ (void)nested_copy(vcpu, nested->vmptr, nested->vmcs12,
+ sizeof(struct vmcs12), true);
+}
+
+static uint64_t
+nested_cr_fixed(u_int msr)
+{
+ return (NESTED_MSR(msr));
+}
+
+static bool
+nested_cr0_valid(uint64_t cr0)
+{
+ uint64_t fixed0, fixed1;
+
+ fixed0 = nested_cr_fixed(MSR_VMX_CR0_FIXED0);
+ fixed1 = nested_cr_fixed(MSR_VMX_CR0_FIXED1);
+ return ((cr0 & fixed0) == fixed0 && (cr0 & ~fixed1) == 0);
+}
+
+static bool
+nested_cr4_valid(uint64_t cr4)
+{
+ uint64_t fixed0, fixed1;
+
+ fixed0 = nested_cr_fixed(MSR_VMX_CR4_FIXED0);
+ fixed1 = nested_cr_fixed(MSR_VMX_CR4_FIXED1);
+ return ((cr4 & fixed0) == fixed0 && (cr4 & ~fixed1) == 0);
+}
+
+static bool
+nested_ctls_valid(uint32_t ctls, u_int msr)
+{
+ uint32_t must, may;
+
+ must = (uint32_t)NESTED_MSR(msr);
+ may = NESTED_MSR(msr) >> 32;
+ return ((ctls & must) == must && (ctls & ~may) == 0);
+}
+
+static bool
+nested_canonical(uint64_t addr)
+{
+ return ((int64_t)(addr << 16) >> 16 == (int64_t)addr);
+}
+
+/*
+ * The checks on the VMX controls (SDM 27.2.1), as far as the controls are
+ * offered to L1. The fields for the event to inject are checked by the CPU
+ * when it enters vmcs02. Returns 0 or a VM-instruction error.
+ */
+static int
+nested_check_ctls(struct vmx_nested *nested)
+{
+ if (!nested_ctls_valid(V12(nested, VMCS_PIN_BASED_CTLS),
+ MSR_VMX_TRUE_PINBASED_CTLS) ||
+ !nested_ctls_valid(V12(nested, VMCS_PRI_PROC_BASED_CTLS),
+ MSR_VMX_TRUE_PROCBASED_CTLS) ||
+ !nested_ctls_valid(V12(nested, VMCS_EXIT_CTLS),
+ MSR_VMX_TRUE_EXIT_CTLS) ||
+ !nested_ctls_valid(V12(nested, VMCS_ENTRY_CTLS),
+ MSR_VMX_TRUE_ENTRY_CTLS))
+ return (VMXERR_ENTRY_CTLS);
+ /* "NMI-window exiting" needs "virtual NMIs" (SDM 27.2.1.1). */
+ if ((V12(nested, VMCS_PRI_PROC_BASED_CTLS) &
+ PROCBASED_NMI_WINDOW_EXITING) != 0 &&
+ (V12(nested, VMCS_PIN_BASED_CTLS) & PINBASED_VIRTUAL_NMI) == 0)
+ return (VMXERR_ENTRY_CTLS);
+ /*
+ * No CR3 targets, as IA32_VMX_MISC says. The MSR areas are not
+ * implemented: this is where the emulation differs from a CPU, which
+ * always has them.
+ */
+ if (V12(nested, VMCS_CR3_TARGET_COUNT) != 0 ||
+ V12(nested, VMCS_EXIT_MSR_STORE_COUNT) != 0 ||
+ V12(nested, VMCS_EXIT_MSR_LOAD_COUNT) != 0 ||
+ V12(nested, VMCS_ENTRY_MSR_LOAD_COUNT) != 0)
+ return (VMXERR_ENTRY_CTLS);
+ return (0);
+}
+
+/*
+ * The checks on the host-state area (SDM 27.2.2 to 27.2.4), which is what L1
+ * continues with after a VM exit of L2. L1 is in IA-32e mode, since "host
+ * address-space size" is a control that it must set. Returns 0 or a
+ * VM-instruction error.
+ */
+static int
+nested_check_host_state(struct vmx_nested *nested)
+{
+ static const uint16_t selectors[] = {
+ VMCS_HOST_ES_SELECTOR, VMCS_HOST_CS_SELECTOR,
+ VMCS_HOST_SS_SELECTOR, VMCS_HOST_DS_SELECTOR,
+ VMCS_HOST_FS_SELECTOR, VMCS_HOST_GS_SELECTOR,
+ VMCS_HOST_TR_SELECTOR,
+ };
+ static const uint16_t addresses[] = {
+ VMCS_HOST_FS_BASE, VMCS_HOST_GS_BASE, VMCS_HOST_TR_BASE,
+ VMCS_HOST_GDTR_BASE, VMCS_HOST_IDTR_BASE,
+ VMCS_HOST_IA32_SYSENTER_ESP, VMCS_HOST_IA32_SYSENTER_EIP,
+ VMCS_HOST_RIP,
+ };
+ uint64_t efer;
+ u_int i;
+
+ if (!nested_cr0_valid(V12(nested, VMCS_HOST_CR0)) ||
+ !nested_cr4_valid(V12(nested, VMCS_HOST_CR4)) ||
+ (V12(nested, VMCS_HOST_CR4) & CR4_PAE) == 0 ||
+ ((V12(nested, VMCS_HOST_CR4) & CR4_CET) != 0 &&
+ (V12(nested, VMCS_HOST_CR0) & CR0_WP) == 0) ||
+ (V12(nested, VMCS_HOST_CR3) >> cpu_maxphyaddr) != 0)
+ return (VMXERR_ENTRY_HOST_STATE);
+ for (i = 0; i < nitems(selectors); i++) {
+ if ((V12(nested, selectors[i]) & (SEL_RPL_MASK | SEL_LDT)) != 0)
+ return (VMXERR_ENTRY_HOST_STATE);
+ }
+ if (V12(nested, VMCS_HOST_CS_SELECTOR) == 0 ||
+ V12(nested, VMCS_HOST_TR_SELECTOR) == 0)
+ return (VMXERR_ENTRY_HOST_STATE);
+ for (i = 0; i < nitems(addresses); i++) {
+ if (!nested_canonical(V12(nested, addresses[i])))
+ return (VMXERR_ENTRY_HOST_STATE);
+ }
+ if ((V12(nested, VMCS_EXIT_CTLS) & VM_EXIT_LOAD_EFER) != 0) {
+ efer = V12(nested, VMCS_HOST_IA32_EFER);
+ if ((efer & ~(uint64_t)EFER_VMX_VALID) != 0 ||
+ (efer & (EFER_LMA | EFER_LME)) != (EFER_LMA | EFER_LME))
+ return (VMXERR_ENTRY_HOST_STATE);
+ }
+ return (0);
+}
+
+/*
+ * Merge vmcs12 into vmcs02 for a VM entry. The CPU does the checks on L2's
+ * state (SDM 27.3.1) when it enters vmcs02. Fails if the PDPTEs of an L2
+ * with PAE paging cannot be read.
+ */
+static int
+nested_vmcs02_load(struct vmx_vcpu *vcpu)
+{
+ struct vmx_nested *nested = vcpu->nested;
+ uint64_t cr, debugctl, dr7, efer, mask, mask12, pdpte[4], tsc_offset;
+ uint32_t entry12, proc, proc01;
+ u_int i;
+
+ entry12 = V12(nested, VMCS_ENTRY_CTLS);
+
+ /*
+ * With EPT the CPU takes the PDPTEs of a guest that uses PAE paging
+ * from the VMCS. L1 was not offered EPT and does not set them, so
+ * they are read from L2's CR3 as a CPU without EPT does (SDM
+ * 27.3.2.4).
+ */
+ memset(pdpte, 0, sizeof(pdpte));
+ if ((V12(nested, VMCS_GUEST_CR0) & CR0_PG) != 0 &&
+ (V12(nested, VMCS_GUEST_CR4) & CR4_PAE) != 0 &&
+ (entry12 & VM_ENTRY_GUEST_LMA) == 0 &&
+ !nested_copy(vcpu, V12(nested, VMCS_GUEST_CR3) & 0xffffffe0, pdpte,
+ sizeof(pdpte), false))
+ return (EFAULT);
+
+ critical_enter();
+ VMPTRLD(vcpu->vmcs);
+ proc01 = vmcs_read(VMCS_PRI_PROC_BASED_CTLS);
+ tsc_offset = (proc01 & PROCBASED_TSC_OFFSET) != 0 ?
+ vmcs_read(VMCS_TSC_OFFSET) : 0;
+ dr7 = vmcs_read(VMCS_GUEST_DR7);
+ debugctl = vmcs_read(VMCS_GUEST_IA32_DEBUGCTL);
+ efer = vmcs_read(VMCS_GUEST_IA32_EFER);
+ VMCLEAR(vcpu->vmcs);
+
+ VMPTRLD(nested->vmcs02);
+ /*
+ * The controls. L1's are valid and include all that the host needs.
+ * The TSC offset of L1 applies to L2 as well.
+ */
+ proc = V12(nested, VMCS_PRI_PROC_BASED_CTLS);
+ if ((proc & PROCBASED_TSC_OFFSET) != 0)
+ tsc_offset += V12(nested, VMCS_TSC_OFFSET);
+ proc |= PROCBASED_SECONDARY_CONTROLS | (proc01 & PROCBASED_TSC_OFFSET);
+ vmcs_write(VMCS_PIN_BASED_CTLS, V12(nested, VMCS_PIN_BASED_CTLS));
+ vmcs_write(VMCS_PRI_PROC_BASED_CTLS, proc);
+ vmcs_write(VMCS_TSC_OFFSET, tsc_offset);
+ vmcs_write(VMCS_ENTRY_CTLS,
+ (vmcs_read(VMCS_ENTRY_CTLS) & ~VM_ENTRY_GUEST_LMA) |
+ (entry12 & VM_ENTRY_GUEST_LMA));
+ vmcs_write(VMCS_EXCEPTION_BITMAP,
+ V12(nested, VMCS_EXCEPTION_BITMAP) | (1 << IDT_MC));
+ vmcs_write(VMCS_PF_ERROR_MASK, V12(nested, VMCS_PF_ERROR_MASK));
+ vmcs_write(VMCS_PF_ERROR_MATCH, V12(nested, VMCS_PF_ERROR_MATCH));
+ vmcs_write(VMCS_ENTRY_INTR_INFO, V12(nested, VMCS_ENTRY_INTR_INFO));
+ vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR,
+ V12(nested, VMCS_ENTRY_EXCEPTION_ERROR));
+ vmcs_write(VMCS_ENTRY_INST_LENGTH,
+ V12(nested, VMCS_ENTRY_INST_LENGTH));
+
+ /* L2's state. */
+ for (i = 0; i < nitems(vmcs12_guest_state); i++)
+ vmcs_write(vmcs12_guest_state[i],
+ V12(nested, vmcs12_guest_state[i]));
+ for (i = 0; i < nitems(pdpte); i++)
+ vmcs_write(VMCS_GUEST_PDPTE0 + 2 * i, pdpte[i]);
+
+ /*
+ * CR0 and CR4 with their guest/host masks and read shadows (SDM
+ * 25.6.6). The host owns the bits that it fixes in addition to the
+ * ones that L1 owns. L2 reads the former as L1 has set them.
+ */
+ cr = V12(nested, VMCS_GUEST_CR0);
+ mask = vmx_fix_cr0(0) | ~vmx_fix_cr0(~0UL);
+ mask12 = V12(nested, VMCS_CR0_MASK);
+ vmcs_write(VMCS_GUEST_CR0, vmx_fix_cr0(cr));
+ vmcs_write(VMCS_CR0_MASK, mask12 | mask);
+ vmcs_write(VMCS_CR0_SHADOW,
+ (V12(nested, VMCS_CR0_SHADOW) & mask12) | (cr & ~mask12));
+ nested->cr0_host = mask & ~mask12;
+ cr = V12(nested, VMCS_GUEST_CR4);
+ mask = vmx_fix_cr4(0) | ~vmx_fix_cr4(~0UL);
+ mask12 = V12(nested, VMCS_CR4_MASK);
+ vmcs_write(VMCS_GUEST_CR4, vmx_fix_cr4(cr));
+ vmcs_write(VMCS_CR4_MASK, mask12 | mask);
+ vmcs_write(VMCS_CR4_SHADOW,
+ (V12(nested, VMCS_CR4_SHADOW) & mask12) | (cr & ~mask12));
+ nested->cr4_host = mask & ~mask12;
+
+ /*
+ * DR7, IA32_DEBUGCTL and IA32_EFER are always loaded from vmcs02.
+ * They are L1's unless it has asked for them to be loaded, except
+ * for EFER.LMA and EFER.LME, which follow "IA-32e mode guest" (SDM
+ * 27.3.2.1).
+ */
+ if ((entry12 & VM_ENTRY_LOAD_DEBUG_CONTROLS) != 0) {
+ dr7 = V12(nested, VMCS_GUEST_DR7);
+ debugctl = V12(nested, VMCS_GUEST_IA32_DEBUGCTL);
+ }
+ if ((entry12 & VM_ENTRY_LOAD_EFER) != 0)
+ efer = V12(nested, VMCS_GUEST_IA32_EFER);
+ else if ((entry12 & VM_ENTRY_GUEST_LMA) != 0)
+ efer |= EFER_LMA | EFER_LME;
+ else
+ efer &= ~(EFER_LMA | EFER_LME);
+ vmcs_write(VMCS_GUEST_DR7, dr7);
+ vmcs_write(VMCS_GUEST_IA32_DEBUGCTL, debugctl);
+ vmcs_write(VMCS_GUEST_IA32_EFER, efer);
+ VMCLEAR(nested->vmcs02);
+ critical_exit();
+ return (0);
+}
+
+static void nested_reflect(struct vmx_vcpu *vcpu);
+
+/*
+ * A VM entry that fails with L2's state: a VM exit with bit 31 set in the
+ * exit reason, which leaves L2's state in vmcs12 as it is (SDM 27.8).
+ */
+static void
+nested_entry_failure(struct vmx_vcpu *vcpu, uint64_t qual)
+{
+ struct vmx_nested_l2exit *exit = &vcpu->nested->exit;
+
+ memset(exit, 0, sizeof(*exit));
+ exit->reason = EXIT_REASON_INVAL_VMCS | EXIT_REASON_ENTRY_FAIL;
+ exit->qual = qual;
+ nested_reflect(vcpu);
+}
+
+/*
+ * VMLAUNCH and VMRESUME (SDM 31.3). The instruction fails if the controls
+ * or the host-state area of vmcs12 are not valid (SDM 27.2).
+ */
+static void
+nested_enter(struct vmx_vcpu *vcpu, bool launch)
+{
+ struct vmx_nested *nested = vcpu->nested;
+ int error;
+
+ if (nested->vmptr == VMPTR_INVALID) {
+ nested_fail_invalid(vcpu);
+ return;
+ }
+ if ((nested->insn.interruptibility &
+ VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) != 0) {
+ nested_fail(vcpu, VMXERR_ENTRY_MOVSS);
+ return;
+ }
+ if (launch && nested->vmcs12->launched != 0) {
+ nested_fail(vcpu, VMXERR_VMLAUNCH_NOT_CLEAR);
+ return;
+ }
+ if (!launch && nested->vmcs12->launched == 0) {
+ nested_fail(vcpu, VMXERR_VMRESUME_NOT_LAUNCHED);
+ return;
+ }
+ error = nested_check_ctls(nested);
+ if (error == 0)
+ error = nested_check_host_state(nested);
+ if (error != 0) {
+ nested_fail(vcpu, error);
+ return;
+ }
+
+ /*
+ * Only the active state is offered (SDM 27.3.1.5). The CPU would
+ * find this in vmcs02 as well, but the other states must not get that
+ * far: in the wait-for-SIPI state nothing gets it out of L2 again.
+ */
+ if (V12(nested, VMCS_GUEST_ACTIVITY) != 0) {
+ nested_entry_failure(vcpu, EXIT_QUAL_ENTRY_NONE);
+ return;
+ }
+ if (nested_vmcs02_load(vcpu) != 0) {
+ nested_entry_failure(vcpu, EXIT_QUAL_ENTRY_PDPTE);
+ return;
+ }
+ nested->launching = launch;
+ nested->l2 = true;
+}
+
+/*
+ * Emulate a VMX instruction of L1 as its operation section says (SDM 31.3).
+ * The CPU has done the checks that come before the VM exit, except for the
+ * ones on VMX operation and CR4.VMXE, which it does with the host's state.
+ */
+static void
+nested_insn(struct vmx_vcpu *vcpu)
+{
+ struct vmx_nested *nested = vcpu->nested;
+ const struct vmx_nested_insn *insn = &nested->insn;
+ uint64_t enc, ptr, val;
+ uint32_t header[2], info, revision;
+ int size, slot;
+
+ info = insn->inst_info;
+ size = nested_insn_64bit(insn) ? 8 : 4;
+
+ /* The checks that all of the instructions have in common. */
+ if ((insn->cr0 & CR0_PE) == 0 || (insn->rflags & PSL_VM) != 0 ||
+ insn->paging.cpu_mode == CPU_MODE_COMPATIBILITY ||
+ (insn->reason == EXIT_REASON_VMXON ?
+ (insn->cr4 & CR4_VMXE) == 0 : !nested->vmxon)) {
+ vm_inject_ud(vcpu->vcpu);
+ return;
+ }
+ if (insn->paging.cpl != 0) {
+ vm_inject_gp(vcpu->vcpu);
+ return;
+ }
+
+ switch (insn->reason) {
+ case EXIT_REASON_VMXON:
+ if (nested->vmxon) {
+ nested_fail(vcpu, VMXERR_VMXON_IN_ROOT);
+ break;
+ }
+ if (!nested_cr0_valid(insn->cr0) ||
+ !nested_cr4_valid(insn->cr4)) {
+ vm_inject_gp(vcpu->vcpu);
+ break;
+ }
+ if (!nested_insn_mem(vcpu, &ptr, sizeof(ptr), false))
+ break;
+ if (!nested_ptr_valid(ptr) || !nested_copy(vcpu, ptr,
+ &revision, sizeof(revision), false) ||
+ revision != VMCS12_REVISION) {
+ nested_fail_invalid(vcpu);
+ break;
+ }
+ nested->vmxon = true;
+ nested->vmxon_ptr = ptr;
+ nested->vmptr = VMPTR_INVALID;
+ nested_succeed(vcpu);
+ break;
+ case EXIT_REASON_VMXOFF:
+ nested_vmcs12_flush(vcpu);
+ nested->vmxon = false;
+ nested->vmptr = VMPTR_INVALID;
+ nested_succeed(vcpu);
+ break;
+ case EXIT_REASON_VMCLEAR:
+ if (!nested_insn_mem(vcpu, &ptr, sizeof(ptr), false))
+ break;
+ if (!nested_ptr_valid(ptr)) {
+ nested_fail(vcpu, VMXERR_VMCLEAR_ADDR);
+ break;
+ }
+ if (ptr == nested->vmxon_ptr) {
+ nested_fail(vcpu, VMXERR_VMCLEAR_VMXON);
+ break;
+ }
+ if (ptr == nested->vmptr) {
+ nested->vmcs12->launched = 0;
+ nested_vmcs12_flush(vcpu);
+ nested->vmptr = VMPTR_INVALID;
+ } else {
+ /* Only the launch state changes. */
+ header[0] = VMCS12_MAGIC;
+ header[1] = 0;
+ (void)nested_copy(vcpu,
+ ptr + offsetof(struct vmcs12, magic), header,
+ sizeof(header), true);
+ }
+ nested_succeed(vcpu);
+ break;
+ case EXIT_REASON_VMPTRLD:
+ if (!nested_insn_mem(vcpu, &ptr, sizeof(ptr), false))
+ break;
+ if (!nested_ptr_valid(ptr)) {
+ nested_fail(vcpu, VMXERR_VMPTRLD_ADDR);
+ break;
+ }
+ if (ptr == nested->vmxon_ptr) {
+ nested_fail(vcpu, VMXERR_VMPTRLD_VMXON);
+ break;
+ }
+ if (!nested_copy(vcpu, ptr, &revision, sizeof(revision),
+ false) || revision != VMCS12_REVISION) {
+ nested_fail(vcpu, VMXERR_VMPTRLD_REVISION);
+ break;
+ }
+ if (ptr != nested->vmptr) {
+ nested_vmcs12_flush(vcpu);
+ nested->vmptr = ptr;
+ (void)nested_copy(vcpu, ptr, nested->vmcs12,
+ sizeof(struct vmcs12), false);
+ /* A VMCS that was never cleared starts out empty. */
+ if (nested->vmcs12->magic != VMCS12_MAGIC) {
+ memset(nested->vmcs12, 0,
+ sizeof(struct vmcs12));
+ nested->vmcs12->revision = VMCS12_REVISION;
+ }
+ }
+ nested_succeed(vcpu);
+ break;
+ case EXIT_REASON_VMPTRST:
+ ptr = nested->vmptr;
+ if (nested_insn_mem(vcpu, &ptr, sizeof(ptr), true))
+ nested_succeed(vcpu);
+ break;
+ case EXIT_REASON_VMREAD:
+ if (nested->vmptr == VMPTR_INVALID) {
+ nested_fail_invalid(vcpu);
+ break;
+ }
+ enc = nested_get_gpr(vcpu, INST_INFO_REG2(info));
+ if (size == 4)
+ enc &= 0xffffffff;
+ slot = vmcs12_slot(enc);
+ if (slot < 0) {
+ nested_fail(vcpu, VMXERR_FIELD);
+ break;
+ }
+ val = nested->vmcs12->field[slot];
+ if ((enc & 1) != 0)
+ val >>= 32;
+ if (size == 4)
+ val &= 0xffffffff;
+ if ((info & INST_INFO_IS_REG) != 0)
+ vm_set_register(vcpu->vcpu,
+ nested_gpr[INST_INFO_REG1(info)], val);
+ else if (!nested_insn_mem(vcpu, &val, size, true))
+ break;
+ nested_succeed(vcpu);
+ break;
+ case EXIT_REASON_VMWRITE:
+ if (nested->vmptr == VMPTR_INVALID) {
+ nested_fail_invalid(vcpu);
+ break;
+ }
+ val = 0;
+ if ((info & INST_INFO_IS_REG) != 0)
+ val = nested_get_gpr(vcpu, INST_INFO_REG1(info));
+ else if (!nested_insn_mem(vcpu, &val, size, false))
+ break;
+ enc = nested_get_gpr(vcpu, INST_INFO_REG2(info));
+ if (size == 4) {
+ val &= 0xffffffff;
+ enc &= 0xffffffff;
+ }
+ slot = vmcs12_slot(enc);
+ if (slot < 0) {
+ nested_fail(vcpu, VMXERR_FIELD);
+ break;
+ }
+ if (VMCS_TYPE(enc) == VMCS_TYPE_READONLY) {
+ nested_fail(vcpu, VMXERR_READONLY_FIELD);
+ break;
+ }
+ if ((enc & 1) != 0)
+ val = (nested->vmcs12->field[slot] & 0xffffffff) |
+ val << 32;
+ else if (VMCS_WIDTH(enc) == VMCS_WIDTH_16)
+ val &= 0xffff;
+ else if (VMCS_WIDTH(enc) == VMCS_WIDTH_32)
+ val &= 0xffffffff;
+ nested->vmcs12->field[slot] = val;
+ nested_succeed(vcpu);
+ break;
+ case EXIT_REASON_VMLAUNCH:
+ case EXIT_REASON_VMRESUME:
+ nested_enter(vcpu, insn->reason == EXIT_REASON_VMLAUNCH);
+ break;
+ default:
+ /* VMCALL, INVEPT, INVVPID and VMFUNC. */
+ vm_inject_ud(vcpu->vcpu);
+ break;
+ }
+}
+
+/* The segment registers in the order of their VMCS fields. */
+enum {
+ SEG_ES,
+ SEG_CS,
+ SEG_SS,
+ SEG_DS,
+ SEG_FS,
+ SEG_GS,
+ SEG_LDTR,
+ SEG_TR,
+};
+
+/* Segment access rights of the state that a VM exit loads (SDM 28.5.2). */
+#define SEG_AR_CODE64 0xa09b /* accessed, present, L, G */
+#define SEG_AR_DATA 0xc093 /* accessed, present, D/B, G */
+#define SEG_AR_TSS 0x008b /* busy 64-bit TSS, present */
+#define SEG_AR_UNUSABLE 0x10000
+
+static void
+nested_set_seg(int seg, uint16_t selector, uint64_t base, uint32_t limit,
+ uint32_t access)
+{
+ vmcs_write(VMCS_GUEST_ES_SELECTOR + 2 * seg, selector);
+ vmcs_write(VMCS_GUEST_ES_BASE + 2 * seg, base);
+ vmcs_write(VMCS_GUEST_ES_LIMIT + 2 * seg, limit);
+ vmcs_write(VMCS_GUEST_ES_ACCESS_RIGHTS + 2 * seg, access);
+}
+
+/*
+ * Load L1's state from the host-state area of vmcs12 as a VM exit does (SDM
+ * 28.5). The checks at the VM entry have made sure that it is a 64-bit
+ * state.
+ */
+static void
+nested_load_host_state(struct vmx_vcpu *vcpu)
+{
+ static const struct {
+ int seg;
+ uint16_t selector;
+ uint16_t base;
+ } segs[] = {
+ { SEG_ES, VMCS_HOST_ES_SELECTOR, 0 },
+ { SEG_SS, VMCS_HOST_SS_SELECTOR, 0 },
+ { SEG_DS, VMCS_HOST_DS_SELECTOR, 0 },
+ { SEG_FS, VMCS_HOST_FS_SELECTOR, VMCS_HOST_FS_BASE },
+ { SEG_GS, VMCS_HOST_GS_SELECTOR, VMCS_HOST_GS_BASE },
+ };
+ struct vmx_nested *nested = vcpu->nested;
+ uint64_t cr0, efer, mask;
+ uint16_t selector;
+ u_int i;
+ int error __diagused;
+
+ critical_enter();
+ VMPTRLD(vcpu->vmcs);
+ /* Control registers, DR7 and MSRs (SDM 28.5.1). */
+ mask = vmcs_read(VMCS_CR0_MASK);
+ cr0 = (vmcs_read(VMCS_GUEST_CR0) & ~mask) |
+ (vmcs_read(VMCS_CR0_SHADOW) & mask);
+ cr0 = (cr0 & CR0_VMEXIT_KEEP) |
+ (V12(nested, VMCS_HOST_CR0) & ~CR0_VMEXIT_KEEP);
+ vmcs_write(VMCS_GUEST_CR0, vmx_fix_cr0(cr0));
+ vmcs_write(VMCS_CR0_SHADOW, cr0);
+ vmcs_write(VMCS_GUEST_CR4, vmx_fix_cr4(V12(nested, VMCS_HOST_CR4)));
+ vmcs_write(VMCS_CR4_SHADOW, V12(nested, VMCS_HOST_CR4));
+ vmcs_write(VMCS_GUEST_CR3, V12(nested, VMCS_HOST_CR3));
+ if ((V12(nested, VMCS_EXIT_CTLS) & VM_EXIT_LOAD_EFER) != 0)
+ efer = V12(nested, VMCS_HOST_IA32_EFER);
+ else
+ efer = vmcs_read(VMCS_GUEST_IA32_EFER) | EFER_LMA | EFER_LME;
+ vmcs_write(VMCS_GUEST_IA32_EFER, efer);
+ vmcs_write(VMCS_ENTRY_CTLS,
+ vmcs_read(VMCS_ENTRY_CTLS) | VM_ENTRY_GUEST_LMA);
+ vmcs_write(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1);
+
+ /* Segment and descriptor-table registers (SDM 28.5.2). */
+ nested_set_seg(SEG_CS, V12(nested, VMCS_HOST_CS_SELECTOR), 0,
+ 0xffffffff, SEG_AR_CODE64);
+ for (i = 0; i < nitems(segs); i++) {
+ selector = V12(nested, segs[i].selector);
+ nested_set_seg(segs[i].seg, selector,
+ segs[i].base != 0 ? V12(nested, segs[i].base) : 0,
+ 0xffffffff, selector != 0 ? SEG_AR_DATA :
+ SEG_AR_DATA | SEG_AR_UNUSABLE);
+ }
+ nested_set_seg(SEG_TR, V12(nested, VMCS_HOST_TR_SELECTOR),
+ V12(nested, VMCS_HOST_TR_BASE), 0x67, SEG_AR_TSS);
+ nested_set_seg(SEG_LDTR, 0, 0, 0, SEG_AR_UNUSABLE);
+ vmcs_write(VMCS_GUEST_GDTR_BASE, V12(nested, VMCS_HOST_GDTR_BASE));
+ vmcs_write(VMCS_GUEST_GDTR_LIMIT, 0xffff);
+ vmcs_write(VMCS_GUEST_IDTR_BASE, V12(nested, VMCS_HOST_IDTR_BASE));
+ vmcs_write(VMCS_GUEST_IDTR_LIMIT, 0xffff);
+
+ /* RSP, RFLAGS and the non-register state (SDM 28.5.3, 28.5.5). */
+ vmcs_write(VMCS_GUEST_RSP, V12(nested, VMCS_HOST_RSP));
+ vmcs_write(VMCS_GUEST_RFLAGS, PSL_RESERVED_DEFAULT);
+ vmcs_write(VMCS_GUEST_INTERRUPTIBILITY,
+ vmcs_read(VMCS_GUEST_INTERRUPTIBILITY) &
+ ~(VMCS_INTERRUPTIBILITY_STI_BLOCKING |
+ VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING));
+ vmcs_write(VMCS_GUEST_IA32_SYSENTER_CS,
+ V12(nested, VMCS_HOST_IA32_SYSENTER_CS));
+ vmcs_write(VMCS_GUEST_IA32_SYSENTER_ESP,
+ V12(nested, VMCS_HOST_IA32_SYSENTER_ESP));
+ vmcs_write(VMCS_GUEST_IA32_SYSENTER_EIP,
+ V12(nested, VMCS_HOST_IA32_SYSENTER_EIP));
+ vmcs_write(VMCS_GUEST_IA32_DEBUGCTL, 0);
+ vmcs_write(VMCS_GUEST_PENDING_DBG_EXCEPTIONS, 0);
+ VMCLEAR(vcpu->vmcs);
+ critical_exit();
+
+ /* CR3 has changed: flush the VPID's mappings, as vmx_setreg() does. */
+ vcpu->state.lastcpu = NOCPU;
+
+ /* This also tells vm_run() where the vCPU continues. */
+ error = vm_set_register(vcpu->vcpu, VM_REG_GUEST_RIP,
+ V12(nested, VMCS_HOST_RIP));
+ KASSERT(error == 0, ("%s: error %d setting rip", __func__, error));
+}
+
+/*
+ * Reflect a VM exit of L2 to L1: the exit information (SDM 28.2) and L2's
+ * state (SDM 28.3) go to vmcs12, L1's state comes from its host-state area.
+ */
+static void
+nested_reflect(struct vmx_vcpu *vcpu)
+{
+ struct vmx_nested *nested = vcpu->nested;
+ const struct vmx_nested_l2exit *exit = &nested->exit;
+ uint64_t cr0, cr4, efer;
+ uint32_t exit12;
+ u_int i;
+
+ /* A VM-entry failure leaves L2's state in vmcs12 as it is. */
+ if ((exit->reason & EXIT_REASON_ENTRY_FAIL) == 0) {
+ exit12 = V12(nested, VMCS_EXIT_CTLS);
+
+ critical_enter();
+ VMPTRLD(nested->vmcs02);
+ for (i = 0; i < nitems(vmcs12_guest_state); i++)
+ V12(nested, vmcs12_guest_state[i]) =
+ vmcs_read(vmcs12_guest_state[i]);
+ cr0 = (vmcs_read(VMCS_GUEST_CR0) & ~nested->cr0_host) |
+ (vmcs_read(VMCS_CR0_SHADOW) & nested->cr0_host);
+ cr4 = (vmcs_read(VMCS_GUEST_CR4) & ~nested->cr4_host) |
+ (vmcs_read(VMCS_CR4_SHADOW) & nested->cr4_host);
+ if ((exit12 & VM_EXIT_SAVE_DEBUG_CONTROLS) != 0) {
+ V12(nested, VMCS_GUEST_DR7) =
+ vmcs_read(VMCS_GUEST_DR7);
+ V12(nested, VMCS_GUEST_IA32_DEBUGCTL) =
+ vmcs_read(VMCS_GUEST_IA32_DEBUGCTL);
+ }
+ efer = vmcs_read(VMCS_GUEST_IA32_EFER);
+ VMCLEAR(nested->vmcs02);
+ critical_exit();
+
+ V12(nested, VMCS_GUEST_CR0) = cr0;
+ V12(nested, VMCS_GUEST_CR4) = cr4;
+ if ((exit12 & VM_EXIT_SAVE_EFER) != 0)
+ V12(nested, VMCS_GUEST_IA32_EFER) = efer;
+ V12(nested, VMCS_ENTRY_CTLS) &= ~(uint64_t)VM_ENTRY_GUEST_LMA;
+ if ((efer & EFER_LMA) != 0)
+ V12(nested, VMCS_ENTRY_CTLS) |= VM_ENTRY_GUEST_LMA;
+ /* The event was injected or is in the IDT-vectoring info. */
+ V12(nested, VMCS_ENTRY_INTR_INFO) &= ~(uint64_t)VMCS_INTR_VALID;
+ if (nested->launching)
+ nested->vmcs12->launched = 1;
+ }
+ V12(nested, VMCS_EXIT_REASON) = exit->reason;
+ V12(nested, VMCS_EXIT_QUALIFICATION) = exit->qual;
+ V12(nested, VMCS_EXIT_INTR_INFO) = exit->intr_info;
+ V12(nested, VMCS_EXIT_INTR_ERRCODE) = exit->intr_errcode;
+ V12(nested, VMCS_IDT_VECTORING_INFO) = exit->idt_info;
+ V12(nested, VMCS_IDT_VECTORING_ERROR) = exit->idt_errcode;
+ V12(nested, VMCS_EXIT_INSTRUCTION_LENGTH) = exit->inst_length;
+ V12(nested, VMCS_EXIT_INSTRUCTION_INFO) = exit->inst_info;
+ V12(nested, VMCS_GUEST_LINEAR_ADDRESS) = exit->gla;
+ V12(nested, VMCS_GUEST_PHYSICAL_ADDRESS) = exit->gpa;
+
+ nested_load_host_state(vcpu);
+ nested->l2 = false;
+}
+
+/* vmx_run() has left L2. */
+static void
+nested_l2_exit(struct vmx_vcpu *vcpu)
+{
+ struct vmx_nested *nested = vcpu->nested;
+ struct vmx_nested_l2exit *exit = &nested->exit;
+ struct vlapic *vlapic;
+ int vector;
+ bool nmi;
+
+ nmi = false;
+ if (exit->rc != VMX_GUEST_VMEXIT) {
+ /* The VM entry failed: so does L1's instruction. */
+ nested->l2 = false;
+ nested_fail(vcpu, exit->inst_error != 0 ? exit->inst_error :
+ VMXERR_ENTRY_CTLS);
+ return;
+ }
+
+ switch (exit->reason) {
+ case EXIT_REASON_EXCEPTION:
+ /* An NMI for L1. The host's own do not get here. */
+ if ((exit->intr_info & VMCS_INTR_T_MASK) != VMCS_INTR_T_NMI)
+ break;
+ if (!vm_nmi_pending(vcpu->vcpu))
+ return;
+ vm_nmi_clear(vcpu->vcpu);
+ nmi = true;
+ break;
+ case EXIT_REASON_EXT_INTR:
+ /* An interrupt for L1. The host's own do not get here. */
+ vlapic = vm_lapic(vcpu->vcpu);
+ if (!vlapic_pending_intr(vlapic, &vector))
+ return;
+ if ((V12(nested, VMCS_EXIT_CTLS) &
+ VM_EXIT_ACKNOWLEDGE_INTERRUPT) != 0) {
+ vlapic_intr_accepted(vlapic, vector);
+ exit->intr_info = VMCS_INTR_VALID |
+ VMCS_INTR_T_HWINTR | vector;
+ }
+ break;
+ }
+ nested_reflect(vcpu);
+
+ /* A VM exit that an NMI causes blocks NMIs (SDM 28.5.5). */
+ if (nmi) {
+ critical_enter();
+ VMPTRLD(vcpu->vmcs);
+ vmcs_write(VMCS_GUEST_INTERRUPTIBILITY,
+ vmcs_read(VMCS_GUEST_INTERRUPTIBILITY) |
+ VMCS_INTERRUPTIBILITY_NMI_BLOCKING);
+ VMCLEAR(vcpu->vmcs);
+ critical_exit();
+ }
+}
+
+/*
+ * A VMX instruction of L1 or a VM exit of L2, both VM_EXITCODE_VMINSN, with
+ * the vCPU frozen: L1's memory can be accessed and its registers set.
+ */
+int
+vmx_nested_exit(struct vmx_vcpu *vcpu)
+{
+ if (vcpu->nested == NULL)
+ vm_inject_ud(vcpu->vcpu);
+ else if (vcpu->nested->l2)
+ nested_l2_exit(vcpu);
+ else
+ nested_insn(vcpu);
+ return (0);
+}
diff --git a/sys/amd64/vmm/vmm.c b/sys/amd64/vmm/vmm.c
--- a/sys/amd64/vmm/vmm.c
+++ b/sys/amd64/vmm/vmm.c
@@ -148,6 +148,7 @@
DEFINE_VMMOPS_IFUNC(void, vmspace_free, (struct vmspace *vmspace))
DEFINE_VMMOPS_IFUNC(struct vlapic *, vlapic_init, (void *vcpui))
DEFINE_VMMOPS_IFUNC(void, vlapic_cleanup, (struct vlapic *vlapic))
+DEFINE_VMMOPS_IFUNC(int, vminsn, (void *vcpui))
#ifdef BHYVE_SNAPSHOT
DEFINE_VMMOPS_IFUNC(int, vcpu_snapshot, (void *vcpui,
struct vm_snapshot_meta *meta))
@@ -1257,9 +1258,11 @@
break;
case VM_EXITCODE_MONITOR:
case VM_EXITCODE_MWAIT:
- case VM_EXITCODE_VMINSN:
vm_inject_ud(vcpu);
break;
+ case VM_EXITCODE_VMINSN:
+ error = vmmops_vminsn(vcpu->cookie);
+ break;
default:
retu = true; /* handled in userland */
break;
diff --git a/sys/modules/vmm/Makefile b/sys/modules/vmm/Makefile
--- a/sys/modules/vmm/Makefile
+++ b/sys/modules/vmm/Makefile
@@ -110,6 +110,7 @@
SRCS+= ept.c \
vmcs.c \
vmx_msr.c \
+ vmx_nested.c \
vmx_support.S \
vmx.c \
vtd.c

File Metadata

Mime Type
text/plain
Expires
Thu, Nov 19, 10:43 AM (16 h, 16 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
41843903
Default Alt Text
D61708.1795085006.diff (64 KB)

Event Timeline