D56556.id179463.diff
No OneTemporary
Actions

Size

37 KB

Referenced Files

None

Subscribers

None

D56556.id179463.diff
View Options

	diff --git a/sys/arm64/vmm/arm64.h b/sys/arm64/vmm/arm64.h
	--- a/sys/arm64/vmm/arm64.h
	+++ b/sys/arm64/vmm/arm64.h
	@@ -56,7 +56,22 @@
	#define DBGWCR_EL1(n) (DBGWCR0_EL1 + MIN(n, 15))
	#define DBGWVR_EL1(n) (DBGWVR0_EL1 + MIN(n, 15))

	+#define HOST_ICH_LR_EL2(n) (HOST_ICH_LR0_EL2 + MIN(n, VGIC_ICH_LR_NUM_MAX-1))
	+#define HOST_ICH_AP0R_EL2(n) (HOST_ICH_AP0R0_EL2 + MIN(n, VGIC_ICH_APR_NUM_MAX-1))
	+#define HOST_ICH_AP1R_EL2(n) (HOST_ICH_AP1R0_EL2 + MIN(n, VGIC_ICH_APR_NUM_MAX-1))
	+#define GPR_X(n) (GPR_X0 + MIN(n, 30))
	+
	enum hypctx_sysreg {
	+ GPR_LR,
	+ GPR_X0,
	+ GPR_X30 = GPR_X0 + 30,
	+
	+ /* These can be reordered freely */
	+ HOST_SP_EL1,
	+ HOST_ELR_EL2,
	+ HOST_SPSR_EL2,
	+ HOST_ESR_EL2,
	+
	/*
	* EL1 & EL0 registers.
	*/
	@@ -69,7 +84,7 @@
	PAR_EL1, /* Physical Address Register */

	/* PMU Registers */
	- PMCR_EL0, /* Performance Monitors Control Register */
	+ PMCR_EL0, /* Performance Monitors Control Register */
	PMCCNTR_EL0,
	PMCCFILTR_EL0,
	PMUSERENR_EL0,
	@@ -87,13 +102,13 @@
	/* DBG Registers */
	DBGCLAIMSET_EL1,
	/* Access these through macros defined above, e.g. DBGBCR_EL1(5) */
	- DBGBCR0_EL1, /* Debug Breakpoint Control Registers */
	+ DBGBCR0_EL1, /* Debug Breakpoint Control Registers */
	DBGBCR15_EL1 = DBGBCR0_EL1 + 15,
	- DBGBVR0_EL1, /* Debug Breakpoint Value Registers */
	+ DBGBVR0_EL1, /* Debug Breakpoint Value Registers */
	DBGBVR15_EL1 = DBGBVR0_EL1 + 15,
	- DBGWCR0_EL1, /* Debug Watchpoint Control Registers */
	+ DBGWCR0_EL1, /* Debug Watchpoint Control Registers */
	DBGWCR15_EL1 = DBGWCR0_EL1 + 15,
	- DBGWVR0_EL1, /* Debug Watchpoint Value Registers */
	+ DBGWVR0_EL1, /* Debug Watchpoint Value Registers */
	DBGWVR15_EL1 = DBGWVR0_EL1 + 15,

	/* EL2 registers used to control the guest, but not exposed to it */
	@@ -125,7 +140,29 @@

	/* Exit info registers */
	HOST_FAR_EL2, /* Fault Address Register */
	- HOST_HPFAR_EL2, /* Hypervisor IPA Fault Address Register */
	+ HOST_HPFAR_EL2, /* Hypervisor IPA Fault Address Register */
	+
	+ HOST_ICH_EISR_EL2, /* End of Interrupt Status Register */
	+ HOST_ICH_ELRSR_EL2, /* Empty List Register Status Register */
	+ HOST_ICH_HCR_EL2, /* Hyp Control Register */
	+ HOST_ICH_MISR_EL2, /* Maintenance Interrupt State Register */
	+ HOST_ICH_VMCR_EL2, /* Virtual Machine Control Register */
	+
	+ /*
	+ * The List Registers are part of the VM context and are modified on a
	+ * world switch. They need to be allocated statically so they are
	+ * mapped in the EL2 translation tables when struct hypctx is mapped.
	+ */
	+ HOST_ICH_LR0_EL2,
	+ HOST_ICH_LR_MAX_EL2 = HOST_ICH_LR0_EL2 + (VGIC_ICH_LR_NUM_MAX - 1),
	+ /* Active Priorities Registers for Group 0 and 1 interrupts */
	+ HOST_ICH_AP0R0_EL2,
	+ HOST_ICH_AP0R_MAX_EL2 = HOST_ICH_AP0R0_EL2 + (VGIC_ICH_APR_NUM_MAX - 1),
	+ HOST_ICH_AP1R0_EL2,
	+ HOST_ICH_AP1R_MAX_EL2 = HOST_ICH_AP1R0_EL2 + (VGIC_ICH_APR_NUM_MAX - 1),
	+
	+ HOST_CNTHCTL_EL2,
	+ HOST_CNTVOFF_EL2,

	NR_NON_VNCR_REGS,

	@@ -226,14 +263,12 @@
	* Per-vCPU hypervisor state.
	*/
	struct hypctx {
	- struct trapframe tf;
	/* Non-VNCR register state */
	uint64_t sys_regs[NR_NON_VNCR_REGS];

	struct hyp *hyp;
	struct vcpu *vcpu;

	- struct vtimer vtimer;
	struct vtimer_cpu vtimer_cpu;

	uint64_t setcaps; /* Currently enabled capabilities. */
	@@ -242,7 +277,10 @@
	uint64_t debug_spsr; /* Saved guest SPSR */
	uint64_t debug_mdscr; /* Saved guest MDSCR */

	- struct vgic_v3_regs vgic_v3_regs;
	+ struct {
	+ uint16_t ich_lr_num;
	+ uint16_t ich_apr_num;
	+ } vgic_v3;
	struct vgic_v3_cpu *vgic_cpu;
	bool has_exception;
	bool dbg_oslock;
	@@ -267,12 +305,13 @@
	((uint64_t )((char )hypctx->vncr_regs + REG_VNCR_OFFSET(reg)))
	#endif

	+/* Calling this with markers like reg=VNCR_START or reg=NR_NON_VNCR_REGS
	+ is a bad idea */
	static inline uint64_t *
	hypctx_sys_reg(struct hypctx hypctx, int reg / enum hypctx_sysreg */)
	{
	if (reg > VNCR_START)
	return (__hypctx_vncr_sysreg(hypctx, reg));
	- /* Calling this with reg=VNCR_START or reg=NR_NON_VNCR_REGS is a bad idea */
	return (&hypctx->sys_regs[reg]);
	}

	diff --git a/sys/arm64/vmm/hyp_genassym.c b/sys/arm64/vmm/hyp_genassym.c
	new file mode 100644
	--- /dev/null
	+++ b/sys/arm64/vmm/hyp_genassym.c
	@@ -0,0 +1,35 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause
	+ *
	+ * Copyright (c) 2026 Kajetan Puchalski <kajetan.puchalski@arm.com>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ */
	+
	+#include <sys/param.h>
	+#include <sys/assym.h>
	+#include <sys/proc.h>
	+
	+#include "arm64.h"
	+
	+ASSYM(HYPCTX_GPR_LR, offsetof(struct hypctx, sys_regs[GPR_LR]) / sizeof(uint64_t));
	+ASSYM(HYPCTX_GPR_X0, offsetof(struct hypctx, sys_regs[GPR_X0]) / sizeof(uint64_t));
	diff --git a/sys/arm64/vmm/io/vgic_v3.h b/sys/arm64/vmm/io/vgic_v3.h
	--- a/sys/arm64/vmm/io/vgic_v3.h
	+++ b/sys/arm64/vmm/io/vgic_v3.h
	@@ -32,26 +32,4 @@
	#define VGIC_ICH_LR_NUM_MAX 16
	#define VGIC_ICH_APR_NUM_MAX 4

	-/* Registers accessed by EL2 */
	-struct vgic_v3_regs {
	- uint32_t ich_eisr_el2; /* End of Interrupt Status Register */
	- uint32_t ich_elrsr_el2; /* Empty List register Status Register (ICH_ELRSR_EL2) */
	- uint32_t ich_hcr_el2; /* Hyp Control Register */
	- uint32_t ich_misr_el2; /* Maintenance Interrupt State Register */
	- uint32_t ich_vmcr_el2; /* Virtual Machine Control Register */
	-
	- /*
	- * The List Registers are part of the VM context and are modified on a
	- * world switch. They need to be allocated statically so they are
	- * mapped in the EL2 translation tables when struct hypctx is mapped.
	- */
	- uint64_t ich_lr_el2[VGIC_ICH_LR_NUM_MAX];
	- uint16_t ich_lr_num;
	-
	- /* Active Priorities Registers for Group 0 and 1 interrupts */
	- uint16_t ich_apr_num;
	- uint32_t ich_ap0r_el2[VGIC_ICH_APR_NUM_MAX];
	- uint32_t ich_ap1r_el2[VGIC_ICH_APR_NUM_MAX];
	-};
	-
	#endif /* !_VMM_VGIC_V3_H_ */
	diff --git a/sys/arm64/vmm/io/vgic_v3.c b/sys/arm64/vmm/io/vgic_v3.c
	--- a/sys/arm64/vmm/io/vgic_v3.c
	+++ b/sys/arm64/vmm/io/vgic_v3.c
	@@ -503,7 +503,7 @@
	*
	* Maintenance interrupts are disabled.
	*/
	- hypctx->vgic_v3_regs.ich_hcr_el2 = ICH_HCR_EL2_En;
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_HCR_EL2, ICH_HCR_EL2_En);

	/*
	* Configure the Interrupt Controller Virtual Machine Control Register.
	@@ -518,20 +518,21 @@
	* ICH_VMCR_EL2_VENG0: virtual Group 0 interrupts enabled.
	* ICH_VMCR_EL2_VENG1: virtual Group 1 interrupts enabled.
	*/
	- hypctx->vgic_v3_regs.ich_vmcr_el2 =
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_VMCR_EL2,
	(virt_features.min_prio << ICH_VMCR_EL2_VPMR_SHIFT) \|
	- ICH_VMCR_EL2_VBPR1_NO_PREEMPTION \| ICH_VMCR_EL2_VBPR0_NO_PREEMPTION;
	- hypctx->vgic_v3_regs.ich_vmcr_el2 &= ~ICH_VMCR_EL2_VEOIM;
	- hypctx->vgic_v3_regs.ich_vmcr_el2 \|= ICH_VMCR_EL2_VENG0 \|
	+ ICH_VMCR_EL2_VBPR1_NO_PREEMPTION \|
	+ ICH_VMCR_EL2_VBPR0_NO_PREEMPTION);
	+ *hypctx_sys_reg(hypctx, HOST_ICH_VMCR_EL2) &= ~ICH_VMCR_EL2_VEOIM;
	+ *hypctx_sys_reg(hypctx, HOST_ICH_VMCR_EL2) \|= ICH_VMCR_EL2_VENG0 \|
	ICH_VMCR_EL2_VENG1;

	- hypctx->vgic_v3_regs.ich_lr_num = virt_features.ich_lr_num;
	- for (i = 0; i < hypctx->vgic_v3_regs.ich_lr_num; i++)
	- hypctx->vgic_v3_regs.ich_lr_el2[i] = 0UL;
	+ hypctx->vgic_v3.ich_lr_num = virt_features.ich_lr_num;
	+ for (i = 0; i < hypctx->vgic_v3.ich_lr_num; i++)
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_LR_EL2(i), 0UL);
	vgic_cpu->ich_lr_used = 0;
	TAILQ_INIT(&vgic_cpu->irq_act_pend);

	- hypctx->vgic_v3_regs.ich_apr_num = virt_features.ich_apr_num;
	+ hypctx->vgic_v3.ich_apr_num = virt_features.ich_apr_num;
	}

	static void
	@@ -2118,7 +2119,7 @@
	*/
	mtx_lock_spin(&vgic_cpu->lr_mtx);

	- hypctx->vgic_v3_regs.ich_hcr_el2 &= ~ICH_HCR_EL2_UIE;
	+ *hypctx_sys_reg(hypctx, HOST_ICH_HCR_EL2) &= ~ICH_HCR_EL2_UIE;

	/* Exit early if there are no buffered interrupts */
	if (TAILQ_EMPTY(&vgic_cpu->irq_act_pend))
	@@ -2128,33 +2129,34 @@
	__func__, vgic_cpu->ich_lr_used));

	i = 0;
	- hypctx->vgic_v3_regs.ich_elrsr_el2 =
	- (1u << hypctx->vgic_v3_regs.ich_lr_num) - 1;
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_ELRSR_EL2,
	+ (1u << hypctx->vgic_v3.ich_lr_num) - 1);
	TAILQ_FOREACH(irq, &vgic_cpu->irq_act_pend, act_pend_list) {
	/* No free list register, stop searching for IRQs */
	- if (i == hypctx->vgic_v3_regs.ich_lr_num)
	+ if (i == hypctx->vgic_v3.ich_lr_num)
	break;

	if (!irq->enabled)
	continue;

	- hypctx->vgic_v3_regs.ich_lr_el2[i] = ICH_LR_EL2_GROUP1 \|
	- ((uint64_t)irq->priority << ICH_LR_EL2_PRIO_SHIFT) \|
	- irq->irq;
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_LR_EL2(i),
	+ ICH_LR_EL2_GROUP1 \|
	+ ((uint64_t)irq->priority << ICH_LR_EL2_PRIO_SHIFT) \|
	+ irq->irq);

	if (irq->active) {
	- hypctx->vgic_v3_regs.ich_lr_el2[i] \|=
	+ *hypctx_sys_reg(hypctx, HOST_ICH_LR_EL2(i)) \|=
	ICH_LR_EL2_STATE_ACTIVE;
	}

	#ifdef notyet
	/* TODO: Check why this is needed */
	if ((irq->config & _MASK) == LEVEL)
	- hypctx->vgic_v3_regs.ich_lr_el2[i] \|= ICH_LR_EL2_EOI;
	+ *hypctx_sys_reg(hypctx, HOST_ICH_LR_EL2(i)) \|= ICH_LR_EL2_EOI;
	#endif

	if (!irq->active && vgic_v3_irq_pending(irq)) {
	- hypctx->vgic_v3_regs.ich_lr_el2[i] \|=
	+ *hypctx_sys_reg(hypctx, HOST_ICH_LR_EL2(i)) \|=
	ICH_LR_EL2_STATE_PENDING;

	/*
	@@ -2196,8 +2198,8 @@
	* access unlocked.
	*/
	for (i = 0; i < vgic_cpu->ich_lr_used; i++) {
	- lr = hypctx->vgic_v3_regs.ich_lr_el2[i];
	- hypctx->vgic_v3_regs.ich_lr_el2[i] = 0;
	+ lr = hypctx_read_sys_reg(hypctx, HOST_ICH_LR_EL2(i));
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_LR_EL2(i), 0);

	irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
	ICH_LR_EL2_VINTID(lr));
	@@ -2244,7 +2246,7 @@
	vgic_v3_release_irq(irq);
	}

	- hypctx->vgic_v3_regs.ich_hcr_el2 &= ~ICH_HCR_EL2_EOICOUNT_MASK;
	+ *hypctx_sys_reg(hypctx, HOST_ICH_HCR_EL2) &= ~ICH_HCR_EL2_EOICOUNT_MASK;
	vgic_cpu->ich_lr_used = 0;
	}

	diff --git a/sys/arm64/vmm/io/vtimer.h b/sys/arm64/vmm/io/vtimer.h
	--- a/sys/arm64/vmm/io/vtimer.h
	+++ b/sys/arm64/vmm/io/vtimer.h
	@@ -37,11 +37,6 @@
	struct hyp;
	struct hypctx;

	-struct vtimer {
	- uint64_t cnthctl_el2;
	- uint64_t cntvoff_el2;
	-};
	-
	struct vtimer_timer {
	struct callout callout;
	struct mtx mtx;
	diff --git a/sys/arm64/vmm/io/vtimer.c b/sys/arm64/vmm/io/vtimer.c
	--- a/sys/arm64/vmm/io/vtimer.c
	+++ b/sys/arm64/vmm/io/vtimer.c
	@@ -97,7 +97,7 @@
	}

	cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
	- hypctx->vtimer.cntvoff_el2;
	+ hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
	if (hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 < cntpct_el0)
	vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
	GT_VIRT_IRQ, true);
	@@ -186,17 +186,17 @@
	*
	* TODO: Don't trap when FEAT_ECV is present
	*/
	- hypctx->vtimer.cnthctl_el2 =
	+ hypctx_write_sys_reg(hypctx, HOST_CNTHCTL_EL2,
	CNTHCTL_E2H_EL0VCTEN_NOTRAP \|
	- CNTHCTL_E2H_EL0VTEN_NOTRAP;
	+ CNTHCTL_E2H_EL0VTEN_NOTRAP);
	if (ecv_poff) {
	- hypctx->vtimer.cnthctl_el2 \|=
	+ *hypctx_sys_reg(hypctx, HOST_CNTHCTL_EL2) \|=
	CNTHCTL_E2H_EL0PCTEN_NOTRAP \|
	CNTHCTL_E2H_EL0PTEN_NOTRAP \|
	CNTHCTL_E2H_EL1PCTEN_NOTRAP \|
	CNTHCTL_E2H_EL1PTEN_NOTRAP;
	} else {
	- hypctx->vtimer.cnthctl_el2 \|=
	+ *hypctx_sys_reg(hypctx, HOST_CNTHCTL_EL2) \|=
	CNTHCTL_E2H_EL0PCTEN_TRAP \|
	CNTHCTL_E2H_EL0PTEN_TRAP \|
	CNTHCTL_E2H_EL1PCTEN_TRAP \|
	@@ -209,20 +209,20 @@
	* CNTHCTL_EL1PCTEN: trap access to CNTPCT_EL0
	*/
	if (ecv_poff) {
	- hypctx->vtimer.cnthctl_el2 =
	+ hypctx_write_sys_reg(hypctx, HOST_CNTHCTL_EL2,
	CNTHCTL_EL1PCTEN_NOTRAP \|
	- CNTHCTL_EL1PCEN_NOTRAP;
	+ CNTHCTL_EL1PCEN_NOTRAP);
	} else {
	- hypctx->vtimer.cnthctl_el2 =
	+ hypctx_write_sys_reg(hypctx, HOST_CNTHCTL_EL2,
	CNTHCTL_EL1PCTEN_TRAP \|
	- CNTHCTL_EL1PCEN_TRAP;
	+ CNTHCTL_EL1PCEN_TRAP);
	}
	}

	if (ecv_poff)
	- hypctx->vtimer.cnthctl_el2 \|= CNTHCTL_ECV_EN;
	+ *hypctx_sys_reg(hypctx, HOST_CNTHCTL_EL2) \|= CNTHCTL_ECV_EN;

	- hypctx->vtimer.cntvoff_el2 = hypctx->hyp->cntvoff_el2;
	+ hypctx_write_sys_reg(hypctx, HOST_CNTVOFF_EL2, hypctx->hyp->cntvoff_el2);
	}

	void
	@@ -280,10 +280,10 @@
	uint64_t cntpct_el0;

	cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
	- hypctx->vtimer.cntvoff_el2;
	+ hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
	vtime_sync_timer(hypctx, &hypctx->vtimer_cpu.virt_timer, cntpct_el0);
	/* If FEAT_ECV_POFF is in use then we need to sync the physical timer */
	- if ((hypctx->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0) {
	+ if ((hypctx_read_sys_reg(hypctx, HOST_CNTHCTL_EL2) & CNTHCTL_ECV_EN) != 0) {
	vtime_sync_timer(hypctx, &hypctx->vtimer_cpu.phys_timer,
	cntpct_el0);
	}
	@@ -322,7 +322,7 @@
	else
	timer = &hypctx->vtimer_cpu.virt_timer;
	cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
	- hypctx->vtimer.cntvoff_el2;
	+ hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
	if (timer->cntx_cval_el0 < cntpct_el0) {
	/* Timer set in the past, trigger interrupt */
	vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
	@@ -379,7 +379,7 @@
	hypctx = vcpu_get_cookie(vcpu);
	vtimer_cpu = &hypctx->vtimer_cpu;

	- cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hypctx->vtimer.cntvoff_el2;
	+ cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
	if (vtimer_cpu->phys_timer.cntx_cval_el0 < cntpct_el0)
	/* Timer condition met */
	*rval = vtimer_cpu->phys_timer.cntx_ctl_el0 \| CNTP_CTL_ISTATUS;
	@@ -422,7 +422,7 @@
	struct hypctx *hypctx;

	hypctx = vcpu_get_cookie(vcpu);
	- *rval = READ_SPECIALREG(cntpct_el0) - hypctx->vtimer.cntvoff_el2;
	+ *rval = READ_SPECIALREG(cntpct_el0) - hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
	return (0);
	}

	@@ -485,7 +485,7 @@
	*rval = (uint32_t)RES1;
	} else {
	cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
	- hypctx->vtimer.cntvoff_el2;
	+ hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
	*rval = vtimer_cpu->phys_timer.cntx_cval_el0 - cntpct_el0;
	}

	@@ -502,7 +502,7 @@
	hypctx = vcpu_get_cookie(vcpu);
	vtimer_cpu = &hypctx->vtimer_cpu;

	- cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hypctx->vtimer.cntvoff_el2;
	+ cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
	vtimer_cpu->phys_timer.cntx_cval_el0 = (int32_t)wval + cntpct_el0;

	vtimer_remove_irq(hypctx, vcpu);
	diff --git a/sys/arm64/vmm/vmm.c b/sys/arm64/vmm/vmm.c
	--- a/sys/arm64/vmm/vmm.c
	+++ b/sys/arm64/vmm/vmm.c
	@@ -862,13 +862,13 @@

	hypctx = vcpu_get_cookie(vcpu);

	- if ((hypctx->tf.tf_esr & ESR_ELx_ISS_MASK) != 0)
	+ if ((hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ESR_ELx_ISS_MASK) != 0)
	return (1);

	vme->exitcode = VM_EXITCODE_SMCCC;
	- vme->u.smccc_call.func_id = hypctx->tf.tf_x[0];
	+ vme->u.smccc_call.func_id = hypctx_read_sys_reg(hypctx, GPR_X(0));
	for (i = 0; i < nitems(vme->u.smccc_call.args); i++)
	- vme->u.smccc_call.args[i] = hypctx->tf.tf_x[i + 1];
	+ vme->u.smccc_call.args[i] = hypctx_read_sys_reg(hypctx, GPR_X(i + 1));

	*retu = true;
	return (0);
	diff --git a/sys/arm64/vmm/vmm_arm64.c b/sys/arm64/vmm/vmm_arm64.c
	--- a/sys/arm64/vmm/vmm_arm64.c
	+++ b/sys/arm64/vmm/vmm_arm64.c
	@@ -657,7 +657,7 @@
	~(TTBR_ASID_MASK \| TTBR_CnP);
	paging->tcr_el1 = hypctx_read_sys_reg(hypctx, TCR_EL1);
	paging->tcr2_el1 = hypctx_read_sys_reg(hypctx, TCR2_EL1);
	- paging->flags = hypctx->tf.tf_spsr & (PSR_M_MASK \| PSR_M_32);
	+ paging->flags = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2) & (PSR_M_MASK \| PSR_M_32);
	if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) & SCTLR_M) != 0)
	paging->flags \|= VM_GP_MMU_ENABLED;
	}
	@@ -683,7 +683,7 @@
	{
	uint64_t esr;

	- if ((hypctx->tf.tf_spsr & PSR_M_MASK) == PSR_M_EL0t)
	+ if ((hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2) & PSR_M_MASK) == PSR_M_EL0t)
	esr = EXCP_INSN_ABORT_L << ESR_ELx_EC_SHIFT;
	else
	esr = EXCP_INSN_ABORT << ESR_ELx_EC_SHIFT;
	@@ -691,7 +691,7 @@
	if (dabort)
	esr \|= EXCP_DATA_ABORT_L << ESR_ELx_EC_SHIFT;
	/* Set the IL bit if set by hardware */
	- esr \|= hypctx->tf.tf_esr & ESR_ELx_IL;
	+ esr \|= hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ESR_ELx_IL;

	vmmops_exception(hypctx, esr \| fsc, far);
	}
	@@ -703,8 +703,8 @@
	uint64_t gpa;
	uint32_t esr_ec, esr_iss;

	- esr_ec = ESR_ELx_EXCEPTION(hypctx->tf.tf_esr);
	- esr_iss = hypctx->tf.tf_esr & ESR_ELx_ISS_MASK;
	+ esr_ec = ESR_ELx_EXCEPTION(hypctx_read_sys_reg(hypctx, HOST_ESR_EL2));
	+ esr_iss = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ESR_ELx_ISS_MASK;

	switch (esr_ec) {
	case EXCP_UNKNOWN:
	@@ -713,7 +713,7 @@
	vme_ret->exitcode = VM_EXITCODE_HYP;
	break;
	case EXCP_TRAP_WFI_WFE:
	- if ((hypctx->tf.tf_esr & 0x3) == 0) { /* WFI */
	+ if ((hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & 0x3) == 0) { /* WFI */
	vmm_stat_incr(hypctx->vcpu, VMEXIT_WFI, 1);
	vme_ret->exitcode = VM_EXITCODE_WFI;
	} else {
	@@ -742,7 +742,7 @@
	case EXCP_DATA_ABORT_L:
	vmm_stat_incr(hypctx->vcpu, esr_ec == EXCP_DATA_ABORT_L ?
	VMEXIT_DATA_ABORT : VMEXIT_INSN_ABORT, 1);
	- switch (hypctx->tf.tf_esr & ISS_DATA_DFSC_MASK) {
	+ switch (hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ISS_DATA_DFSC_MASK) {
	case ISS_DATA_DFSC_TF_L0:
	case ISS_DATA_DFSC_TF_L1:
	case ISS_DATA_DFSC_TF_L2:
	@@ -767,7 +767,7 @@
	if (vm_mem_allocated(hypctx->vcpu, gpa)) {
	vme_ret->exitcode = VM_EXITCODE_PAGING;
	vme_ret->inst_length = 0;
	- vme_ret->u.paging.esr = hypctx->tf.tf_esr;
	+ vme_ret->u.paging.esr = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2);
	vme_ret->u.paging.gpa = gpa;
	} else if (esr_ec == EXCP_INSN_ABORT_L) {
	/*
	@@ -1092,7 +1092,7 @@
	int
	vmmops_run(void vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo evinfo)
	{
	- uint64_t excp_type, vbar_el1;
	+ uint64_t excp_type, vbar_el1, new_spsr;
	int handled;
	register_t daif;
	struct hyp *hyp;
	@@ -1106,36 +1106,43 @@
	vcpu = hypctx->vcpu;
	vme = vm_exitinfo(vcpu);

	- hypctx->tf.tf_elr = (uint64_t)pc;
	+ hypctx_write_sys_reg(hypctx, HOST_ELR_EL2, (uint64_t)pc);

	for (;;) {
	if (hypctx->has_exception) {
	hypctx->has_exception = false;
	- hypctx_write_sys_reg(hypctx, ELR_EL1, hypctx->tf.tf_elr);
	+ hypctx_write_sys_reg(hypctx, ELR_EL1,
	+ hypctx_read_sys_reg(hypctx, HOST_ELR_EL2));

	- mode = hypctx->tf.tf_spsr & (PSR_M_MASK \| PSR_M_32);
	+ mode = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2) &
	+ (PSR_M_MASK \| PSR_M_32);

	vbar_el1 = hypctx_read_sys_reg(hypctx, VBAR_EL1);
	if (mode == PSR_M_EL1t) {
	- hypctx->tf.tf_elr = vbar_el1 + 0x0;
	+ hypctx_write_sys_reg(hypctx, HOST_ELR_EL2,
	+ vbar_el1 + 0x0);
	} else if (mode == PSR_M_EL1h) {
	- hypctx->tf.tf_elr = vbar_el1 + 0x200;
	+ hypctx_write_sys_reg(hypctx, HOST_ELR_EL2,
	+ vbar_el1 + 0x200);
	} else if ((mode & PSR_M_32) == PSR_M_64) {
	/* 64-bit EL0 */
	- hypctx->tf.tf_elr = vbar_el1 + 0x400;
	+ hypctx_write_sys_reg(hypctx, HOST_ELR_EL2,
	+ vbar_el1 + 0x400);
	} else {
	/* 32-bit EL0 */
	- hypctx->tf.tf_elr = vbar_el1 + 0x600;
	+ hypctx_write_sys_reg(hypctx, HOST_ELR_EL2,
	+ vbar_el1 + 0x600);
	}

	/* Set the new spsr */
	- hypctx_write_sys_reg(hypctx, SPSR_EL1, hypctx->tf.tf_spsr);
	+ new_spsr = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2);
	+ hypctx_write_sys_reg(hypctx, SPSR_EL1, new_spsr);

	/* Set the new cpsr */
	- hypctx->tf.tf_spsr = hypctx_read_sys_reg(hypctx,
	- SPSR_EL1) & PSR_FLAGS;
	-
	- hypctx->tf.tf_spsr \|= PSR_DAIF \| PSR_M_EL1h;
	+ hypctx_write_sys_reg(hypctx, HOST_SPSR_EL2,
	+ new_spsr & PSR_FLAGS);
	+ *hypctx_sys_reg(hypctx, HOST_SPSR_EL2) \|= PSR_DAIF \|
	+ PSR_M_EL1h;

	/*
	* Update fields that may change on exeption entry
	@@ -1143,12 +1150,15 @@
	*/
	if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) &
	SCTLR_SPAN) == 0)
	- hypctx->tf.tf_spsr \|= PSR_PAN;
	+ *hypctx_sys_reg(hypctx,
	+ HOST_SPSR_EL2) \|= PSR_PAN;
	if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) &
	SCTLR_DSSBS) == 0)
	- hypctx->tf.tf_spsr &= ~PSR_SSBS;
	+ *hypctx_sys_reg(hypctx,
	+ HOST_SPSR_EL2) &= ~PSR_SSBS;
	else
	- hypctx->tf.tf_spsr \|= PSR_SSBS;
	+ *hypctx_sys_reg(hypctx,
	+ HOST_SPSR_EL2) \|= PSR_SSBS;
	}

	daif = intr_disable();
	@@ -1168,7 +1178,8 @@

	/* Activate the stage2 pmap so the vmid is valid */
	pmap_activate_vm(pmap);
	- hypctx_write_sys_reg(hypctx, HOST_VTTBR_EL2, pmap_to_ttbr0(pmap));
	+ hypctx_write_sys_reg(hypctx, HOST_VTTBR_EL2,
	+ pmap_to_ttbr0(pmap));

	/*
	* TODO: What happens if a timer interrupt is asserted exactly
	@@ -1193,10 +1204,10 @@
	if (excp_type == EXCP_TYPE_MAINT_IRQ)
	continue;

	- vme->pc = hypctx->tf.tf_elr;
	+ vme->pc = hypctx_read_sys_reg(hypctx, HOST_ELR_EL2);
	vme->inst_length = INSN_SIZE;
	vme->u.hyp.exception_nr = excp_type;
	- vme->u.hyp.esr_el2 = hypctx->tf.tf_esr;
	+ vme->u.hyp.esr_el2 = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2);
	vme->u.hyp.far_el2 = hypctx_read_sys_reg(hypctx, HOST_FAR_EL2);
	vme->u.hyp.hpfar_el2 = hypctx_read_sys_reg(hypctx, HOST_HPFAR_EL2);

	@@ -1207,7 +1218,7 @@
	break;
	else
	/* Resume guest execution from the next instruction. */
	- hypctx->tf.tf_elr += vme->inst_length;
	+ *hypctx_sys_reg(hypctx, HOST_ELR_EL2) += vme->inst_length;
	}

	return (0);
	@@ -1268,15 +1279,15 @@
	{
	switch (reg) {
	case VM_REG_GUEST_X0 ... VM_REG_GUEST_X29:
	- return (&hypctx->tf.tf_x[reg]);
	+ return hypctx_sys_reg(hypctx, GPR_X(reg));
	case VM_REG_GUEST_LR:
	- return (&hypctx->tf.tf_lr);
	+ return hypctx_sys_reg(hypctx, GPR_LR);
	case VM_REG_GUEST_SP:
	- return (&hypctx->tf.tf_sp);
	+ return hypctx_sys_reg(hypctx, HOST_SP_EL1);
	case VM_REG_GUEST_CPSR:
	- return (&hypctx->tf.tf_spsr);
	+ return hypctx_sys_reg(hypctx, HOST_SPSR_EL2);
	case VM_REG_GUEST_PC:
	- return (&hypctx->tf.tf_elr);
	+ return hypctx_sys_reg(hypctx, HOST_ELR_EL2);
	case VM_REG_GUEST_SCTLR_EL1:
	return hypctx_sys_reg(hypctx, SCTLR_EL1);
	case VM_REG_GUEST_TTBR0_EL1:
	@@ -1383,6 +1394,7 @@
	{
	struct hypctx *hypctx = vcpui;
	int ret;
	+ uint64_t host_spsr_el2;

	ret = 0;

	@@ -1400,38 +1412,42 @@
	break;

	if (val != 0) {
	- hypctx->debug_spsr \|= (hypctx->tf.tf_spsr & PSR_SS);
	- hypctx->debug_mdscr \|=
	- (hypctx_read_sys_reg(hypctx, MDSCR_EL1) & MDSCR_SS);
	+ hypctx->debug_spsr \|= (hypctx_read_sys_reg(hypctx,
	+ HOST_SPSR_EL2) & PSR_SS);
	+ hypctx->debug_mdscr \|= (hypctx_read_sys_reg(hypctx,
	+ MDSCR_EL1) & MDSCR_SS);

	- hypctx->tf.tf_spsr \|= PSR_SS;
	+ *hypctx_sys_reg(hypctx, HOST_SPSR_EL2) \|= PSR_SS;
	*hypctx_sys_reg(hypctx, MDSCR_EL1) \|= MDSCR_SS;
	*hypctx_sys_reg(hypctx, HOST_MDCR_EL2) \|= MDCR_EL2_TDE;
	} else {
	- hypctx->tf.tf_spsr &= ~PSR_SS;
	- hypctx->tf.tf_spsr \|= hypctx->debug_spsr;
	+ *hypctx_sys_reg(hypctx, HOST_SPSR_EL2) &= ~PSR_SS;
	+ *hypctx_sys_reg(hypctx,
	+ HOST_SPSR_EL2) \|= hypctx->debug_spsr;
	hypctx->debug_spsr &= ~PSR_SS;
	*hypctx_sys_reg(hypctx, MDSCR_EL1) &= ~MDSCR_SS;
	- *hypctx_sys_reg(hypctx, MDSCR_EL1) \|= hypctx->debug_mdscr;
	+ *hypctx_sys_reg(hypctx,
	+ MDSCR_EL1) \|= hypctx->debug_mdscr;
	hypctx->debug_mdscr &= ~MDSCR_SS;
	if ((hypctx->setcaps & (1ul << VM_CAP_BRK_EXIT)) == 0)
	- *hypctx_sys_reg(hypctx, HOST_MDCR_EL2) &= ~MDCR_EL2_TDE;
	+ *hypctx_sys_reg(hypctx,
	+ HOST_MDCR_EL2) &= ~MDCR_EL2_TDE;
	}
	break;
	case VM_CAP_MASK_HWINTR:
	if ((val != 0) == ((hypctx->setcaps & (1ul << num)) != 0))
	break;

	+ host_spsr_el2 = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2);
	if (val != 0) {
	- hypctx->debug_spsr \|= (hypctx->tf.tf_spsr &
	- (PSR_I \| PSR_F));
	- hypctx->tf.tf_spsr \|= PSR_I \| PSR_F;
	+ hypctx->debug_spsr \|= (host_spsr_el2 & (PSR_I \| PSR_F));
	+ host_spsr_el2 \|= PSR_I \| PSR_F;
	} else {
	- hypctx->tf.tf_spsr &= ~(PSR_I \| PSR_F);
	- hypctx->tf.tf_spsr \|= (hypctx->debug_spsr &
	- (PSR_I \| PSR_F));
	+ host_spsr_el2 &= ~(PSR_I \| PSR_F);
	+ host_spsr_el2 \|= (hypctx->debug_spsr & (PSR_I \| PSR_F));
	hypctx->debug_spsr &= ~(PSR_I \| PSR_F);
	}
	+ hypctx_write_sys_reg(hypctx, HOST_SPSR_EL2, host_spsr_el2);
	break;
	default:
	ret = ENOENT;
	diff --git a/sys/arm64/vmm/vmm_hyp.c b/sys/arm64/vmm/vmm_hyp.c
	--- a/sys/arm64/vmm/vmm_hyp.c
	+++ b/sys/arm64/vmm/vmm_hyp.c
	@@ -174,22 +174,27 @@
	static void
	vmm_hyp_reg_store_vgic(struct hypctx *hypctx, bool guest)
	{
	- hypctx->vgic_v3_regs.ich_hcr_el2 = READ_SPECIALREG(ich_hcr_el2);
	- hypctx->vgic_v3_regs.ich_vmcr_el2 = READ_SPECIALREG(ich_vmcr_el2);
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_HCR_EL2,
	+ READ_SPECIALREG(ich_hcr_el2));
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_VMCR_EL2,
	+ READ_SPECIALREG(ich_vmcr_el2));

	if (!guest)
	return;

	/* Store the GICv3 registers */
	- hypctx->vgic_v3_regs.ich_eisr_el2 = READ_SPECIALREG(ich_eisr_el2);
	- hypctx->vgic_v3_regs.ich_elrsr_el2 = READ_SPECIALREG(ich_elrsr_el2);
	- hypctx->vgic_v3_regs.ich_misr_el2 = READ_SPECIALREG(ich_misr_el2);
	-
	- switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
	-#define STORE_LR(x) \
	-case x: \
	- hypctx->vgic_v3_regs.ich_lr_el2[x] = \
	- READ_SPECIALREG(ich_lr ## x ##_el2)
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_EISR_EL2,
	+ READ_SPECIALREG(ich_eisr_el2));
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_ELRSR_EL2,
	+ READ_SPECIALREG(ich_elrsr_el2));
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_MISR_EL2,
	+ READ_SPECIALREG(ich_misr_el2));
	+
	+ switch (hypctx->vgic_v3.ich_lr_num - 1) {
	+#define STORE_LR(x) \
	+case x: \
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_LR_EL2(x), \
	+ READ_SPECIALREG(ich_lr ## x ##_el2))
	STORE_LR(15);
	STORE_LR(14);
	STORE_LR(13);
	@@ -210,13 +215,13 @@
	#undef STORE_LR
	}

	- switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
	-#define STORE_APR(x) \
	-case x: \
	- hypctx->vgic_v3_regs.ich_ap0r_el2[x] = \
	- READ_SPECIALREG(ich_ap0r ## x ##_el2); \
	- hypctx->vgic_v3_regs.ich_ap1r_el2[x] = \
	- READ_SPECIALREG(ich_ap1r ## x ##_el2)
	+ switch (hypctx->vgic_v3.ich_apr_num - 1) {
	+#define STORE_APR(x) \
	+case x: \
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_AP0R_EL2(x), \
	+ READ_SPECIALREG(ich_ap0r ## x ##_el2)); \
	+ hypctx_write_sys_reg(hypctx, HOST_ICH_AP1R_EL2(x), \
	+ READ_SPECIALREG(ich_ap1r ## x ##_el2))
	STORE_APR(3);
	STORE_APR(2);
	STORE_APR(1);
	@@ -242,13 +247,14 @@
	} else {
	hypctx->vtimer_cpu.cntkctl_el1 =
	READ_SPECIALREG(cntkctl_el1);
	- hypctx->vtimer.cnthctl_el2 =
	- READ_SPECIALREG(cnthctl_el2);
	- hypctx->vtimer.cntvoff_el2 =
	- READ_SPECIALREG(cntvoff_el2);
	+ hypctx_write_sys_reg(hypctx, HOST_CNTHCTL_EL2,
	+ READ_SPECIALREG(cnthctl_el2));
	+ hypctx_write_sys_reg(hypctx, HOST_CNTVOFF_EL2,
	+ READ_SPECIALREG(cntvoff_el2));
	}

	- ecv_poff = (hypctx->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0;
	+ ecv_poff = (hypctx_read_sys_reg(hypctx, HOST_CNTHCTL_EL2) &
	+ CNTHCTL_ECV_EN) != 0;

	if (guest_or_nonvhe(guest) && ecv_poff) {
	/*
	@@ -267,16 +273,17 @@
	vmm_hyp_reg_store_special(struct hypctx hypctx, struct hyp hyp, bool guest)
	{
	/* Store the special to from the trapframe */
	- hypctx->tf.tf_sp = READ_SPECIALREG(sp_el1);
	- hypctx->tf.tf_elr = READ_SPECIALREG(elr_el2);
	- hypctx->tf.tf_spsr = READ_SPECIALREG(spsr_el2);
	+ hypctx_write_sys_reg(hypctx, HOST_SP_EL1, READ_SPECIALREG(sp_el1));
	+ hypctx_write_sys_reg(hypctx, HOST_ELR_EL2, READ_SPECIALREG(elr_el2));
	+ hypctx_write_sys_reg(hypctx, HOST_SPSR_EL2, READ_SPECIALREG(spsr_el2));

	if (guest) {
	- hypctx->tf.tf_esr = READ_SPECIALREG(esr_el2);
	+ hypctx_write_sys_reg(hypctx, HOST_ESR_EL2,
	+ READ_SPECIALREG(esr_el2));
	hypctx_write_sys_reg(hypctx, HOST_FAR_EL2,
	READ_SPECIALREG(far_el2));
	- hypctx_write_sys_reg(hypctx,
	- PAR_EL1, READ_SPECIALREG(par_el1));
	+ hypctx_write_sys_reg(hypctx, PAR_EL1,
	+ READ_SPECIALREG(par_el1));
	}

	/* Store the guest special registers */
	@@ -439,9 +446,9 @@
	hypctx_read_sys_reg(hypctx, HOST_VMPIDR_EL2));

	/* Load the special regs from the trapframe */
	- WRITE_SPECIALREG(sp_el1, hypctx->tf.tf_sp);
	- WRITE_SPECIALREG(elr_el2, hypctx->tf.tf_elr);
	- WRITE_SPECIALREG(spsr_el2, hypctx->tf.tf_spsr);
	+ WRITE_SPECIALREG(sp_el1, hypctx_read_sys_reg(hypctx, HOST_SP_EL1));
	+ WRITE_SPECIALREG(elr_el2, hypctx_read_sys_reg(hypctx, HOST_ELR_EL2));
	+ WRITE_SPECIALREG(spsr_el2, hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2));
	}

	static void
	@@ -581,10 +588,13 @@
	vmm_hyp_reg_restore_timer(struct hypctx hypctx, struct hyp hyp, bool guest)
	{
	bool ecv_poff;
	- ecv_poff = (hypctx->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0;
	+ ecv_poff = (hypctx_read_sys_reg(hypctx, HOST_CNTHCTL_EL2) &
	+ CNTHCTL_ECV_EN) != 0;

	- WRITE_SPECIALREG(cnthctl_el2, hypctx->vtimer.cnthctl_el2);
	- WRITE_SPECIALREG(cntvoff_el2, hypctx->vtimer.cntvoff_el2);
	+ WRITE_SPECIALREG(cnthctl_el2,
	+ hypctx_read_sys_reg(hypctx, HOST_CNTHCTL_EL2));
	+ WRITE_SPECIALREG(cntvoff_el2,
	+ hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2));

	if (guest) {
	WRITE_SPECIALREG(EL1_REG(CNTKCTL),
	@@ -600,7 +610,7 @@
	* to keep in sync.
	*/
	WRITE_SPECIALREG(CNTPOFF_EL2_REG,
	- hypctx->vtimer.cntvoff_el2);
	+ hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2));
	isb();
	}
	} else {
	@@ -623,18 +633,19 @@
	vmm_hyp_reg_restore_vgic(struct hypctx *hypctx, bool guest)
	{
	/* Load the GICv3 registers */
	- WRITE_SPECIALREG(ich_hcr_el2, hypctx->vgic_v3_regs.ich_hcr_el2);
	+ WRITE_SPECIALREG(ich_hcr_el2,
	+ hypctx_read_sys_reg(hypctx, HOST_ICH_HCR_EL2));
	WRITE_SPECIALREG(ich_vmcr_el2,
	- hypctx->vgic_v3_regs.ich_vmcr_el2);
	+ hypctx_read_sys_reg(hypctx, HOST_ICH_VMCR_EL2));

	if (!guest)
	return;

	- switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
	-#define LOAD_LR(x) \
	-case x: \
	- WRITE_SPECIALREG(ich_lr ## x ##_el2, \
	- hypctx->vgic_v3_regs.ich_lr_el2[x])
	+ switch (hypctx->vgic_v3.ich_lr_num - 1) {
	+#define LOAD_LR(x) \
	+case x: \
	+ WRITE_SPECIALREG(ich_lr ## x ##_el2, \
	+ hypctx_read_sys_reg(hypctx, HOST_ICH_LR_EL2(x)))
	LOAD_LR(15);
	LOAD_LR(14);
	LOAD_LR(13);
	@@ -655,18 +666,18 @@
	#undef LOAD_LR
	}

	- switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
	-#define LOAD_APR(x) \
	- case x: \
	- WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \
	- hypctx->vgic_v3_regs.ich_ap0r_el2[x]); \
	- WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \
	- hypctx->vgic_v3_regs.ich_ap1r_el2[x])
	- LOAD_APR(3);
	- LOAD_APR(2);
	- LOAD_APR(1);
	- default:
	- LOAD_APR(0);
	+ switch (hypctx->vgic_v3.ich_apr_num - 1) {
	+#define LOAD_APR(x) \
	+case x: \
	+ WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \
	+ hypctx_read_sys_reg(hypctx, HOST_ICH_AP0R_EL2(x))); \
	+ WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \
	+ hypctx_read_sys_reg(hypctx, HOST_ICH_AP1R_EL2(x)))
	+ LOAD_APR(3);
	+ LOAD_APR(2);
	+ LOAD_APR(1);
	+ default:
	+ LOAD_APR(0);
	#undef LOAD_APR
	}
	}
	@@ -693,11 +704,12 @@
	vmm_hyp_handle_guest_exit(struct hypctx hypctx, uint64_t ret)
	{
	bool hpfar_valid;
	- uint64_t s1e1r, hpfar_el2;
	+ uint64_t s1e1r, hpfar_el2, host_esr_el2, host_far_el2;

	hpfar_valid = true;
	+ host_esr_el2 = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2);
	if (*ret == EXCP_TYPE_EL1_SYNC) {
	- switch (ESR_ELx_EXCEPTION(hypctx->tf.tf_esr)) {
	+ switch (ESR_ELx_EXCEPTION(host_esr_el2)) {
	case EXCP_INSN_ABORT_L:
	case EXCP_DATA_ABORT_L:
	/*
	@@ -713,9 +725,9 @@
	*
	* TODO: Add a case for Arm erratum 834220.
	*/
	- if ((hypctx->tf.tf_esr & ISS_DATA_S1PTW) != 0)
	+ if ((host_esr_el2 & ISS_DATA_S1PTW) != 0)
	break;
	- switch (hypctx->tf.tf_esr & ISS_DATA_DFSC_MASK) {
	+ switch (host_esr_el2 & ISS_DATA_DFSC_MASK) {
	case ISS_DATA_DFSC_PF_L1:
	case ISS_DATA_DFSC_PF_L2:
	case ISS_DATA_DFSC_PF_L3:
	@@ -726,14 +738,15 @@
	}
	}
	if (hpfar_valid) {
	- hypctx_write_sys_reg(hypctx, HOST_HPFAR_EL2, READ_SPECIALREG(hpfar_el2));
	+ hypctx_write_sys_reg(hypctx, HOST_HPFAR_EL2,
	+ READ_SPECIALREG(hpfar_el2));
	} else {
	/*
	* TODO: There is a risk the at instruction could cause an
	* exception here. We should handle it & return a failure.
	*/
	- s1e1r =
	- arm64_address_translate_s1e1r(hypctx_read_sys_reg(hypctx, HOST_FAR_EL2));
	+ host_far_el2 = hypctx_read_sys_reg(hypctx, HOST_FAR_EL2);
	+ s1e1r = arm64_address_translate_s1e1r(host_far_el2);
	if (PAR_SUCCESS(s1e1r)) {
	hpfar_el2 = (s1e1r & PAR_PA_MASK) >> PAR_PA_SHIFT;
	hpfar_el2 <<= HPFAR_EL2_FIPA_SHIFT;
	diff --git a/sys/arm64/vmm/vmm_hyp_exception.S b/sys/arm64/vmm/vmm_hyp_exception.S
	--- a/sys/arm64/vmm/vmm_hyp_exception.S
	+++ b/sys/arm64/vmm/vmm_hyp_exception.S
	@@ -35,6 +35,7 @@
	#include <machine/hypervisor.h>

	#include "assym.inc"
	+#include "hyp_assym.h"
	#include "hyp.h"

	.macro save_host_registers
	@@ -86,26 +87,26 @@
	/* Restore the hypctx pointer */
	mrs x0, tpidr_el2

	- stp x2, x3, [x0, #(TF_X + 2 * 8)]
	- stp x4, x5, [x0, #(TF_X + 4 * 8)]
	- stp x6, x7, [x0, #(TF_X + 6 * 8)]
	- stp x8, x9, [x0, #(TF_X + 8 * 8)]
	- stp x10, x11, [x0, #(TF_X + 10 * 8)]
	- stp x12, x13, [x0, #(TF_X + 12 * 8)]
	- stp x14, x15, [x0, #(TF_X + 14 * 8)]
	- stp x16, x17, [x0, #(TF_X + 16 * 8)]
	- stp x18, x19, [x0, #(TF_X + 18 * 8)]
	- stp x20, x21, [x0, #(TF_X + 20 * 8)]
	- stp x22, x23, [x0, #(TF_X + 22 * 8)]
	- stp x24, x25, [x0, #(TF_X + 24 * 8)]
	- stp x26, x27, [x0, #(TF_X + 26 * 8)]
	- stp x28, x29, [x0, #(TF_X + 28 * 8)]
	-
	- str lr, [x0, #(TF_LR)]
	+ stp x2, x3, [x0, #((HYPCTX_GPR_X0 + 2) * 8)]
	+ stp x4, x5, [x0, #((HYPCTX_GPR_X0 + 4) * 8)]
	+ stp x6, x7, [x0, #((HYPCTX_GPR_X0 + 6) * 8)]
	+ stp x8, x9, [x0, #((HYPCTX_GPR_X0 + 8) * 8)]
	+ stp x10, x11, [x0, #((HYPCTX_GPR_X0 + 10) * 8)]
	+ stp x12, x13, [x0, #((HYPCTX_GPR_X0 + 12) * 8)]
	+ stp x14, x15, [x0, #((HYPCTX_GPR_X0 + 14) * 8)]
	+ stp x16, x17, [x0, #((HYPCTX_GPR_X0 + 16) * 8)]
	+ stp x18, x19, [x0, #((HYPCTX_GPR_X0 + 18) * 8)]
	+ stp x20, x21, [x0, #((HYPCTX_GPR_X0 + 20) * 8)]
	+ stp x22, x23, [x0, #((HYPCTX_GPR_X0 + 22) * 8)]
	+ stp x24, x25, [x0, #((HYPCTX_GPR_X0 + 24) * 8)]
	+ stp x26, x27, [x0, #((HYPCTX_GPR_X0 + 26) * 8)]
	+ stp x28, x29, [x0, #((HYPCTX_GPR_X0 + 28) * 8)]
	+
	+ str lr, [x0, #(HYPCTX_GPR_LR * 8)]

	/* Restore the saved x0 & x1 and save them */
	ldp x2, x3, [sp], #(2 * 8)
	- stp x2, x3, [x0, #(TF_X + 0 * 8)]
	+ stp x2, x3, [x0, #((HYPCTX_GPR_X0 + 0) * 8)]
	.endm

	.macro restore_guest_registers
	@@ -113,25 +114,25 @@
	* Copy the guest x0 and x1 to the stack so we can restore them
	* after loading the other registers.
	*/
	- ldp x2, x3, [x0, #(TF_X + 0 * 8)]
	+ ldp x2, x3, [x0, #((HYPCTX_GPR_X0 + 0) * 8)]
	stp x2, x3, [sp, #-(2 * 8)]!

	- ldr lr, [x0, #(TF_LR)]
	-
	- ldp x28, x29, [x0, #(TF_X + 28 * 8)]
	- ldp x26, x27, [x0, #(TF_X + 26 * 8)]
	- ldp x24, x25, [x0, #(TF_X + 24 * 8)]
	- ldp x22, x23, [x0, #(TF_X + 22 * 8)]
	- ldp x20, x21, [x0, #(TF_X + 20 * 8)]
	- ldp x18, x19, [x0, #(TF_X + 18 * 8)]
	- ldp x16, x17, [x0, #(TF_X + 16 * 8)]
	- ldp x14, x15, [x0, #(TF_X + 14 * 8)]
	- ldp x12, x13, [x0, #(TF_X + 12 * 8)]
	- ldp x10, x11, [x0, #(TF_X + 10 * 8)]
	- ldp x8, x9, [x0, #(TF_X + 8 * 8)]
	- ldp x6, x7, [x0, #(TF_X + 6 * 8)]
	- ldp x4, x5, [x0, #(TF_X + 4 * 8)]
	- ldp x2, x3, [x0, #(TF_X + 2 * 8)]
	+ ldr lr, [x0, #(HYPCTX_GPR_LR * 8)]
	+
	+ ldp x28, x29, [x0, #((HYPCTX_GPR_X0 + 28) * 8)]
	+ ldp x26, x27, [x0, #((HYPCTX_GPR_X0 + 26) * 8)]
	+ ldp x24, x25, [x0, #((HYPCTX_GPR_X0 + 24) * 8)]
	+ ldp x22, x23, [x0, #((HYPCTX_GPR_X0 + 22) * 8)]
	+ ldp x20, x21, [x0, #((HYPCTX_GPR_X0 + 20) * 8)]
	+ ldp x18, x19, [x0, #((HYPCTX_GPR_X0 + 18) * 8)]
	+ ldp x16, x17, [x0, #((HYPCTX_GPR_X0 + 16) * 8)]
	+ ldp x14, x15, [x0, #((HYPCTX_GPR_X0 + 14) * 8)]
	+ ldp x12, x13, [x0, #((HYPCTX_GPR_X0 + 12) * 8)]
	+ ldp x10, x11, [x0, #((HYPCTX_GPR_X0 + 10) * 8)]
	+ ldp x8, x9, [x0, #((HYPCTX_GPR_X0 + 8) * 8)]
	+ ldp x6, x7, [x0, #((HYPCTX_GPR_X0 + 6) * 8)]
	+ ldp x4, x5, [x0, #((HYPCTX_GPR_X0 + 4) * 8)]
	+ ldp x2, x3, [x0, #((HYPCTX_GPR_X0 + 2) * 8)]

	ldp x0, x1, [sp], #(2 * 8)
	.endm
	diff --git a/sys/arm64/vmm/vmm_reset.c b/sys/arm64/vmm/vmm_reset.c
	--- a/sys/arm64/vmm/vmm_reset.c
	+++ b/sys/arm64/vmm/vmm_reset.c
	@@ -53,8 +53,6 @@

	el2ctx = vcpu;

	- set_arch_unknown(el2ctx->tf);
	-
	/*
	* Guest starts with:
	* ~SCTLR_M: MMU off
	@@ -150,9 +148,9 @@
	* Disable interrupts in the guest. The guest OS will re-enable
	* them.
	*/
	- el2ctx->tf.tf_spsr = PSR_D \| PSR_A \| PSR_I \| PSR_F;
	+ hypctx_write_sys_reg(el2ctx, HOST_SPSR_EL2, PSR_D \| PSR_A \| PSR_I \| PSR_F);
	/* Use the EL1 stack when taking exceptions to EL1 */
	- el2ctx->tf.tf_spsr \|= PSR_M_EL1h;
	+ *hypctx_sys_reg(el2ctx, HOST_SPSR_EL2) \|= PSR_M_EL1h;

	/* FEAT_FGT traps */
	if ((el2ctx->hyp->feats & HYP_FEAT_FGT) != 0) {
	diff --git a/sys/modules/vmm/Makefile b/sys/modules/vmm/Makefile
	--- a/sys/modules/vmm/Makefile
	+++ b/sys/modules/vmm/Makefile
	@@ -23,7 +23,7 @@

	.if ${MACHINE_CPUARCH} == "aarch64"
	CFLAGS+= -I${SRCTOP}/sys/${MACHINE}/vmm/io
	-DPSRCS+= assym.inc
	+DPSRCS+= assym.inc hyp_genassym.c hyp_assym.h

	# TODO: Add the new EL2 code
	SRCS+= vmm_arm64.c \
	@@ -47,6 +47,12 @@
	CLEANFILES+= vmm_hyp_blob.elf.full
	CLEANFILES+= vmm_hyp_blob.elf vmm_hyp_blob.bin

	+hyp_assym.h: hyp_genassym.o
	+ sh ${SYSDIR}/kern/genassym.sh hyp_genassym.o > ${.TARGET}
	+
	+hyp_genassym.o: offset.inc
	+ ${CC} -c ${NOSAN_CFLAGS:N-flto*:N-fno-common} -fcommon ${.IMPSRC}
	+
	vmm_nvhe_exception.o: vmm_nvhe_exception.S vmm_hyp_exception.S
	${CC} -c -x assembler-with-cpp -DLOCORE \
	${NOSAN_CFLAGS:N-mbranch-protection*} ${.IMPSRC} -o ${.TARGET} -fpie
	@@ -142,9 +148,6 @@
	${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \
	${.IMPSRC} -o ${.TARGET}

	-hyp_genassym.o: offset.inc
	- ${CC} -c ${NOSAN_CFLAGS:N-flto*:N-fno-common} -fcommon ${.IMPSRC}
	-
	vmx_genassym.o: offset.inc
	${CC} -c ${NOSAN_CFLAGS:N-flto*:N-fno-common} -fcommon ${.IMPSRC}

File Metadata

Mime Type: text/plain
Expires: Sat, Jun 27, 3:07 PM (15 h, 20 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 34355528
Default Alt Text: D56556.id179463.diff (37 KB)

D56556.id179463.diffNo OneTemporaryActions

D56556.id179463.diffView Options

File Metadata

Event Timeline

D56556.id179463.diff
No OneTemporary
Actions

D56556.id179463.diff
View Options