Page MenuHomeFreeBSD
Files F157281462

D56553.id175976.diff
No OneTemporary
Actions

D56553.id175976.diff
View Options

diff --git a/sys/arm64/vmm/arm64.h b/sys/arm64/vmm/arm64.h
--- a/sys/arm64/vmm/arm64.h
+++ b/sys/arm64/vmm/arm64.h
@@ -224,6 +224,12 @@
uint64_t hfgrtr2_el2;
uint64_t hfgwtr2_el2;
+ /* Storage for *host* timer registers */
+ struct {
+ uint64_t cnthctl_el2;
+ uint64_t cntvoff_el2;
+ } host_timer_regs;
+
uint64_t el2_addr; /* The address of this in el2 space */
struct hyp *hyp;
struct vcpu *vcpu;
diff --git a/sys/arm64/vmm/vmm_hyp.c b/sys/arm64/vmm/vmm_hyp.c
--- a/sys/arm64/vmm/vmm_hyp.c
+++ b/sys/arm64/vmm/vmm_hyp.c
@@ -41,85 +41,9 @@
uint64_t VMM_HYP_FUNC(do_call_guest)(struct hypctx *);
static void
-vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest,
- bool ecv_poff)
+vmm_hyp_reg_store_pmu_debug(struct hypctx *hypctx, bool guest)
{
uint64_t dfr0;
-
- if (guest) {
- /* Store the timer registers */
- hypctx->vtimer_cpu.cntkctl_el1 =
- READ_SPECIALREG(EL1_REG(CNTKCTL));
- hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 =
- READ_SPECIALREG(EL0_REG(CNTV_CVAL));
- hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0 =
- READ_SPECIALREG(EL0_REG(CNTV_CTL));
- }
- if (guest_or_nonvhe(guest) && ecv_poff) {
- /*
- * If we have ECV then the guest could modify these registers.
- * If VHE is enabled then the kernel will see a different view
- * of the registers, so doesn't need to handle them.
- */
- hypctx->vtimer_cpu.phys_timer.cntx_cval_el0 =
- READ_SPECIALREG(EL0_REG(CNTP_CVAL));
- hypctx->vtimer_cpu.phys_timer.cntx_ctl_el0 =
- READ_SPECIALREG(EL0_REG(CNTP_CTL));
- }
-
- if (guest) {
- /* Store the GICv3 registers */
- hypctx->vgic_v3_regs.ich_eisr_el2 =
- READ_SPECIALREG(ich_eisr_el2);
- hypctx->vgic_v3_regs.ich_elrsr_el2 =
- READ_SPECIALREG(ich_elrsr_el2);
- hypctx->vgic_v3_regs.ich_hcr_el2 =
- READ_SPECIALREG(ich_hcr_el2);
- hypctx->vgic_v3_regs.ich_misr_el2 =
- READ_SPECIALREG(ich_misr_el2);
- hypctx->vgic_v3_regs.ich_vmcr_el2 =
- READ_SPECIALREG(ich_vmcr_el2);
- switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
-#define STORE_LR(x) \
- case x: \
- hypctx->vgic_v3_regs.ich_lr_el2[x] = \
- READ_SPECIALREG(ich_lr ## x ##_el2)
- STORE_LR(15);
- STORE_LR(14);
- STORE_LR(13);
- STORE_LR(12);
- STORE_LR(11);
- STORE_LR(10);
- STORE_LR(9);
- STORE_LR(8);
- STORE_LR(7);
- STORE_LR(6);
- STORE_LR(5);
- STORE_LR(4);
- STORE_LR(3);
- STORE_LR(2);
- STORE_LR(1);
- default:
- STORE_LR(0);
-#undef STORE_LR
- }
-
- switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
-#define STORE_APR(x) \
- case x: \
- hypctx->vgic_v3_regs.ich_ap0r_el2[x] = \
- READ_SPECIALREG(ich_ap0r ## x ##_el2); \
- hypctx->vgic_v3_regs.ich_ap1r_el2[x] = \
- READ_SPECIALREG(ich_ap1r ## x ##_el2)
- STORE_APR(3);
- STORE_APR(2);
- STORE_APR(1);
- default:
- STORE_APR(0);
-#undef STORE_APR
- }
- }
-
hypctx_write_sys_reg(hypctx, DBGCLAIMSET_EL1, READ_SPECIALREG(dbgclaimset_el1));
dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
@@ -231,12 +155,112 @@
#undef STORE_PMU
}
+ if (!guest)
+ hypctx->mdcr_el2 = READ_SPECIALREG(mdcr_el2);
+}
+
+static void
+vmm_hyp_reg_store_vgic(struct hypctx *hypctx, bool guest)
+{
+ hypctx->vgic_v3_regs.ich_hcr_el2 = READ_SPECIALREG(ich_hcr_el2);
+ hypctx->vgic_v3_regs.ich_vmcr_el2 = READ_SPECIALREG(ich_vmcr_el2);
+
+ if (!guest)
+ return;
+
+ /* Store the GICv3 registers */
+ hypctx->vgic_v3_regs.ich_eisr_el2 = READ_SPECIALREG(ich_eisr_el2);
+ hypctx->vgic_v3_regs.ich_elrsr_el2 = READ_SPECIALREG(ich_elrsr_el2);
+ hypctx->vgic_v3_regs.ich_misr_el2 = READ_SPECIALREG(ich_misr_el2);
+
+ switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
+#define STORE_LR(x) \
+case x: \
+ hypctx->vgic_v3_regs.ich_lr_el2[x] = \
+ READ_SPECIALREG(ich_lr ## x ##_el2)
+ STORE_LR(15);
+ STORE_LR(14);
+ STORE_LR(13);
+ STORE_LR(12);
+ STORE_LR(11);
+ STORE_LR(10);
+ STORE_LR(9);
+ STORE_LR(8);
+ STORE_LR(7);
+ STORE_LR(6);
+ STORE_LR(5);
+ STORE_LR(4);
+ STORE_LR(3);
+ STORE_LR(2);
+ STORE_LR(1);
+ default:
+ STORE_LR(0);
+#undef STORE_LR
+ }
+
+ switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
+#define STORE_APR(x) \
+case x: \
+ hypctx->vgic_v3_regs.ich_ap0r_el2[x] = \
+ READ_SPECIALREG(ich_ap0r ## x ##_el2); \
+ hypctx->vgic_v3_regs.ich_ap1r_el2[x] = \
+ READ_SPECIALREG(ich_ap1r ## x ##_el2)
+ STORE_APR(3);
+ STORE_APR(2);
+ STORE_APR(1);
+ default:
+ STORE_APR(0);
+#undef STORE_APR
+ }
+}
+
+static void
+vmm_hyp_reg_store_timer(struct hypctx *hypctx, struct hyp *hyp, bool guest)
+{
+ bool ecv_poff;
+ ecv_poff = (hyp->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0;
+
+ if (guest) {
+ /* Store the timer registers */
+ hypctx->vtimer_cpu.cntkctl_el1 =
+ READ_SPECIALREG(EL1_REG(CNTKCTL));
+ hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 =
+ READ_SPECIALREG(EL0_REG(CNTV_CVAL));
+ hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0 =
+ READ_SPECIALREG(EL0_REG(CNTV_CTL));
+ } else {
+ hypctx->vtimer_cpu.cntkctl_el1 =
+ READ_SPECIALREG(cntkctl_el1);
+ hypctx->host_timer_regs.cnthctl_el2 =
+ READ_SPECIALREG(cnthctl_el2);
+ hypctx->host_timer_regs.cntvoff_el2 =
+ READ_SPECIALREG(cntvoff_el2);
+ }
+ if (guest_or_nonvhe(guest) && ecv_poff) {
+ /*
+ * If we have ECV then the guest could modify these registers.
+ * If VHE is enabled then the kernel will see a different view
+ * of the registers, so doesn't need to handle them.
+ */
+ hypctx->vtimer_cpu.phys_timer.cntx_cval_el0 =
+ READ_SPECIALREG(EL0_REG(CNTP_CVAL));
+ hypctx->vtimer_cpu.phys_timer.cntx_ctl_el0 =
+ READ_SPECIALREG(EL0_REG(CNTP_CTL));
+ }
+
+}
+
+static void
+vmm_hyp_reg_store_special(struct hypctx *hypctx, struct hyp *hyp, bool guest)
+{
/* Store the special to from the trapframe */
hypctx->tf.tf_sp = READ_SPECIALREG(sp_el1);
hypctx->tf.tf_elr = READ_SPECIALREG(elr_el2);
hypctx->tf.tf_spsr = READ_SPECIALREG(spsr_el2);
+
if (guest) {
hypctx->tf.tf_esr = READ_SPECIALREG(esr_el2);
+ hypctx->exit_info.far_el2 = READ_SPECIALREG(far_el2);
hypctx_write_sys_reg(hypctx, PAR_EL1, READ_SPECIALREG(par_el1));
}
@@ -276,21 +300,30 @@
hypctx->hcr_el2 = READ_SPECIALREG(hcr_el2);
hypctx->vpidr_el2 = READ_SPECIALREG(vpidr_el2);
hypctx->vmpidr_el2 = READ_SPECIALREG(vmpidr_el2);
+
+ if (!guest) {
+ #ifndef VMM_VHE
+ if ((hyp->feats & HYP_FEAT_HCX) != 0)
+ hypctx->hcrx_el2 = READ_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2));
+ #endif
+ }
}
static void
-vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest,
- bool ecv_poff)
+vmm_hyp_reg_restore_special(struct hypctx *hypctx, struct hyp *hyp, bool guest)
{
- uint64_t dfr0;
-
- /* Restore the special registers */
WRITE_SPECIALREG(hcr_el2, hypctx->hcr_el2);
+ /* Always restore the register in non-VHE mode, only write it for the guest in VHE mode */
+#ifdef VMM_VHE
if (guest) {
- if ((hyp->feats & HYP_FEAT_HCX) != 0)
- WRITE_SPECIALREG(HCRX_EL2_REG, hypctx->hcrx_el2);
+#endif
+ if ((hyp->feats & HYP_FEAT_HCX) != 0)
+ WRITE_SPECIALREG(HCRX_EL2_REG, hypctx->hcrx_el2);
+#ifdef VMM_VHE
}
+#endif
+
isb();
#ifdef VMM_VHE
@@ -358,6 +391,12 @@
WRITE_SPECIALREG(sp_el1, hypctx->tf.tf_sp);
WRITE_SPECIALREG(elr_el2, hypctx->tf.tf_elr);
WRITE_SPECIALREG(spsr_el2, hypctx->tf.tf_spsr);
+}
+
+static void
+vmm_hyp_reg_restore_pmu_debug(struct hypctx *hypctx)
+{
+ uint64_t dfr0;
/* Restore the PMU registers */
WRITE_SPECIALREG(pmcr_el0, hypctx_read_sys_reg(hypctx, PMCR_EL0));
@@ -474,17 +513,23 @@
LOAD_DBG_WRP(0);
#undef LOAD_DBG_WRP
}
+}
+
+static void
+vmm_hyp_reg_restore_timer(struct hypctx *hypctx, struct hyp *hyp, bool guest)
+{
+ bool ecv_poff;
+ ecv_poff = (hyp->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0;
+
+ WRITE_SPECIALREG(cntkctl_el1, hypctx->vtimer_cpu.cntkctl_el1);
if (guest) {
- /* Load the timer registers */
- WRITE_SPECIALREG(EL1_REG(CNTKCTL),
- hypctx->vtimer_cpu.cntkctl_el1);
+ WRITE_SPECIALREG(cnthctl_el2, hyp->vtimer.cnthctl_el2);
+ WRITE_SPECIALREG(cntvoff_el2, hyp->vtimer.cntvoff_el2);
WRITE_SPECIALREG(EL0_REG(CNTV_CVAL),
hypctx->vtimer_cpu.virt_timer.cntx_cval_el0);
WRITE_SPECIALREG(EL0_REG(CNTV_CTL),
hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0);
- WRITE_SPECIALREG(cnthctl_el2, hyp->vtimer.cnthctl_el2);
- WRITE_SPECIALREG(cntvoff_el2, hyp->vtimer.cntvoff_el2);
if (ecv_poff) {
/*
@@ -495,6 +540,9 @@
hyp->vtimer.cntvoff_el2);
isb();
}
+ } else {
+ WRITE_SPECIALREG(cnthctl_el2, hypctx->host_timer_regs.cnthctl_el2);
+ WRITE_SPECIALREG(cntvoff_el2, hypctx->host_timer_regs.cntvoff_el2);
}
if (guest_or_nonvhe(guest) && ecv_poff) {
/*
@@ -507,105 +555,87 @@
WRITE_SPECIALREG(EL0_REG(CNTP_CTL),
hypctx->vtimer_cpu.phys_timer.cntx_ctl_el0);
}
+}
- if (guest) {
- /* Load the GICv3 registers */
- WRITE_SPECIALREG(ich_hcr_el2, hypctx->vgic_v3_regs.ich_hcr_el2);
- WRITE_SPECIALREG(ich_vmcr_el2,
- hypctx->vgic_v3_regs.ich_vmcr_el2);
- switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
+static void
+vmm_hyp_reg_restore_vgic(struct hypctx *hypctx, bool guest)
+{
+ /* Load the GICv3 registers */
+ WRITE_SPECIALREG(ich_hcr_el2, hypctx->vgic_v3_regs.ich_hcr_el2);
+ WRITE_SPECIALREG(ich_vmcr_el2,
+ hypctx->vgic_v3_regs.ich_vmcr_el2);
+
+ if (!guest)
+ return;
+
+ switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
#define LOAD_LR(x) \
- case x: \
- WRITE_SPECIALREG(ich_lr ## x ##_el2, \
- hypctx->vgic_v3_regs.ich_lr_el2[x])
- LOAD_LR(15);
- LOAD_LR(14);
- LOAD_LR(13);
- LOAD_LR(12);
- LOAD_LR(11);
- LOAD_LR(10);
- LOAD_LR(9);
- LOAD_LR(8);
- LOAD_LR(7);
- LOAD_LR(6);
- LOAD_LR(5);
- LOAD_LR(4);
- LOAD_LR(3);
- LOAD_LR(2);
- LOAD_LR(1);
- default:
- LOAD_LR(0);
+case x: \
+ WRITE_SPECIALREG(ich_lr ## x ##_el2, \
+ hypctx->vgic_v3_regs.ich_lr_el2[x])
+ LOAD_LR(15);
+ LOAD_LR(14);
+ LOAD_LR(13);
+ LOAD_LR(12);
+ LOAD_LR(11);
+ LOAD_LR(10);
+ LOAD_LR(9);
+ LOAD_LR(8);
+ LOAD_LR(7);
+ LOAD_LR(6);
+ LOAD_LR(5);
+ LOAD_LR(4);
+ LOAD_LR(3);
+ LOAD_LR(2);
+ LOAD_LR(1);
+ default:
+ LOAD_LR(0);
#undef LOAD_LR
- }
+ }
- switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
+ switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
#define LOAD_APR(x) \
- case x: \
- WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \
- hypctx->vgic_v3_regs.ich_ap0r_el2[x]); \
- WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \
- hypctx->vgic_v3_regs.ich_ap1r_el2[x])
- LOAD_APR(3);
- LOAD_APR(2);
- LOAD_APR(1);
- default:
- LOAD_APR(0);
+case x: \
+ WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \
+ hypctx->vgic_v3_regs.ich_ap0r_el2[x]); \
+ WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \
+ hypctx->vgic_v3_regs.ich_ap1r_el2[x])
+ LOAD_APR(3);
+ LOAD_APR(2);
+ LOAD_APR(1);
+ default:
+ LOAD_APR(0);
#undef LOAD_APR
- }
}
}
-static uint64_t
-vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx)
+static void
+vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest)
{
- struct hypctx host_hypctx;
- uint64_t cntvoff_el2;
- uint64_t ich_hcr_el2, ich_vmcr_el2, cnthctl_el2, cntkctl_el1;
-#ifndef VMM_VHE
- uint64_t hcrx_el2;
-#endif
- uint64_t ret;
- uint64_t s1e1r, hpfar_el2;
- bool ecv_poff, hpfar_valid;
-
- /* Allows uniform implementation of guest and host register reloads */
- host_hypctx.vncr_el2 = hypctx->host_ctx_addr;
-
- ecv_poff = (hyp->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0;
- vmm_hyp_reg_store(&host_hypctx, NULL, false, ecv_poff);
-#ifndef VMM_VHE
- if ((hyp->feats & HYP_FEAT_HCX) != 0)
- hcrx_el2 = READ_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2));
-#endif
-
- /* Save the host special registers */
- cnthctl_el2 = READ_SPECIALREG(cnthctl_el2);
- cntkctl_el1 = READ_SPECIALREG(cntkctl_el1);
- cntvoff_el2 = READ_SPECIALREG(cntvoff_el2);
-
- ich_hcr_el2 = READ_SPECIALREG(ich_hcr_el2);
- ich_vmcr_el2 = READ_SPECIALREG(ich_vmcr_el2);
-
- vmm_hyp_reg_restore(hypctx, hyp, true, ecv_poff);
-
- /* Load the common hypervisor registers */
- WRITE_SPECIALREG(vttbr_el2, hyp->vttbr_el2);
-
- host_hypctx.mdcr_el2 = READ_SPECIALREG(mdcr_el2);
- WRITE_SPECIALREG(mdcr_el2, hypctx->mdcr_el2);
-
- /* Call into the guest */
- ret = VMM_HYP_FUNC(do_call_guest)(hypctx);
+ vmm_hyp_reg_store_vgic(hypctx, guest);
+ vmm_hyp_reg_store_timer(hypctx, hyp, guest);
+ vmm_hyp_reg_store_pmu_debug(hypctx, guest);
+ vmm_hyp_reg_store_special(hypctx, hyp, guest);
+}
- WRITE_SPECIALREG(mdcr_el2, host_hypctx.mdcr_el2);
- isb();
+static void
+vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest)
+{
+ vmm_hyp_reg_restore_special(hypctx, hyp, guest);
+ vmm_hyp_reg_restore_pmu_debug(hypctx);
+ vmm_hyp_reg_restore_timer(hypctx, hyp, guest);
+ vmm_hyp_reg_restore_vgic(hypctx, guest);
+}
- /* Store the exit info */
- hypctx->exit_info.far_el2 = READ_SPECIALREG(far_el2);
- vmm_hyp_reg_store(hypctx, hyp, true, ecv_poff);
+static void
+vmm_hyp_handle_guest_exit(struct hypctx *hypctx, struct hypctx *host_hypctx,
+ uint64_t *ret)
+{
+ bool hpfar_valid;
+ uint64_t s1e1r, hpfar_el2;
hpfar_valid = true;
- if (ret == EXCP_TYPE_EL1_SYNC) {
+ if (*ret == EXCP_TYPE_EL1_SYNC) {
switch (ESR_ELx_EXCEPTION(hypctx->tf.tf_esr)) {
case EXCP_INSN_ABORT_L:
case EXCP_DATA_ABORT_L:
@@ -648,24 +678,44 @@
hpfar_el2 <<= HPFAR_EL2_FIPA_SHIFT;
hypctx->exit_info.hpfar_el2 = hpfar_el2;
} else {
- ret = EXCP_TYPE_REENTER;
+ *ret = EXCP_TYPE_REENTER;
}
}
+}
- vmm_hyp_reg_restore(&host_hypctx, NULL, false, ecv_poff);
+/* Minimum guest entry logic that should not be reordered */
+static uint64_t
+__vmm_hyp_call_guest(struct hypctx *hypctx, struct hypctx *host_hypctx)
+{
+ uint64_t ret;
+ WRITE_SPECIALREG(vttbr_el2, hypctx->hyp->vttbr_el2);
+ WRITE_SPECIALREG(mdcr_el2, hypctx->mdcr_el2);
+ /* Call into the guest */
+ ret = VMM_HYP_FUNC(do_call_guest)(hypctx);
+ WRITE_SPECIALREG(mdcr_el2, host_hypctx->mdcr_el2);
-#ifndef VMM_VHE
- if ((hyp->feats & HYP_FEAT_HCX) != 0)
- WRITE_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2), hcrx_el2);
-#endif
+ isb();
+
+ return ret;
+}
+
+static uint64_t
+vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx)
+{
+ struct hypctx host_hypctx;
+ uint64_t ret;
+
+ /* Allows uniform implementation of guest and host register reloads */
+ host_hypctx.vncr_el2 = hypctx->host_ctx_addr;
+
+ vmm_hyp_reg_store(&host_hypctx, NULL, false);
+ vmm_hyp_reg_restore(hypctx, hyp, true);
- /* Restore the host special registers */
- WRITE_SPECIALREG(ich_hcr_el2, ich_hcr_el2);
- WRITE_SPECIALREG(ich_vmcr_el2, ich_vmcr_el2);
+ ret = __vmm_hyp_call_guest(hypctx, &host_hypctx);
- WRITE_SPECIALREG(cnthctl_el2, cnthctl_el2);
- WRITE_SPECIALREG(cntkctl_el1, cntkctl_el1);
- WRITE_SPECIALREG(cntvoff_el2, cntvoff_el2);
+ vmm_hyp_reg_store(hypctx, hyp, true);
+ vmm_hyp_handle_guest_exit(hypctx, &host_hypctx, &ret);
+ vmm_hyp_reg_restore(&host_hypctx, NULL, false);
return (ret);
}

File Metadata

Mime Type
text/plain
Expires
Wed, May 20, 11:57 PM (7 h, 1 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
33364127
Default Alt Text
D56553.id175976.diff (14 KB)

Event Timeline