diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv index 36eea03f29a1..e84f1367680f 100644 --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -1,98 +1,99 @@ cddl/dev/dtrace/riscv/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/riscv/dtrace_isa.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/dtrace/riscv/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/dtrace/riscv/instr_size.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/riscv/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" crypto/des/des_enc.c optional netsmb dev/cpufreq/cpufreq_dt.c optional cpufreq fdt dev/ofw/ofw_cpu.c optional fdt dev/ofw/ofw_pcib.c optional pci fdt dev/pci/pci_dw.c optional pci fdt dev/pci/pci_dw_if.m optional pci fdt dev/pci/pci_host_generic.c optional pci dev/pci/pci_host_generic_fdt.c optional pci fdt dev/uart/uart_cpu_fdt.c optional uart fdt dev/uart/uart_dev_lowrisc.c optional uart_lowrisc dev/usb/controller/generic_ehci.c optional ehci dev/usb/controller/generic_ehci_fdt.c optional ehci fdt dev/usb/controller/generic_ohci.c optional ohci fdt dev/usb/controller/generic_usb_if.m optional ohci fdt dev/usb/controller/generic_xhci.c optional xhci dev/usb/controller/generic_xhci_fdt.c optional xhci fdt dev/vmm/vmm_dev.c optional vmm dev/vmm/vmm_stat.c optional vmm dev/xilinx/axi_quad_spi.c optional xilinx_spi dev/xilinx/axidma.c optional axidma xdma dev/xilinx/if_xae.c optional xae dev/xilinx/xlnx_pcib.c optional pci fdt xlnx_pcib kern/msi_if.m standard kern/pic_if.m standard kern/subr_devmap.c standard kern/subr_dummy_vdso_tc.c standard kern/subr_intr.c standard kern/subr_physmem.c standard libkern/bcopy.c standard libkern/memcmp.c standard libkern/memset.c standard libkern/strcmp.c standard libkern/strlen.c standard libkern/strncmp.c standard riscv/riscv/aplic.c standard riscv/riscv/autoconf.c standard riscv/riscv/bus_machdep.c standard riscv/riscv/bus_space_asm.S standard riscv/riscv/busdma_bounce.c standard riscv/riscv/busdma_machdep.c standard riscv/riscv/cache.c standard riscv/riscv/clock.c standard riscv/riscv/copyinout.S standard riscv/riscv/cpufunc_asm.S standard riscv/riscv/db_disasm.c optional ddb riscv/riscv/db_interface.c optional ddb riscv/riscv/db_trace.c optional ddb riscv/riscv/dump_machdep.c standard riscv/riscv/elf_machdep.c standard riscv/riscv/exception.S standard riscv/riscv/exec_machdep.c standard riscv/riscv/fpe.c standard riscv/riscv/gdb_machdep.c optional gdb riscv/riscv/intc.c standard riscv/riscv/identcpu.c standard riscv/riscv/locore.S standard no-obj riscv/riscv/machdep.c standard riscv/riscv/minidump_machdep.c standard riscv/riscv/mp_machdep.c optional smp riscv/riscv/mem.c standard riscv/riscv/nexus.c standard riscv/riscv/ofw_machdep.c optional fdt riscv/riscv/plic.c standard riscv/riscv/pmap.c standard riscv/riscv/ptrace_machdep.c standard riscv/riscv/riscv_console.c optional rcons riscv/riscv/riscv_syscon.c optional syscon riscv_syscon fdt riscv/riscv/sigtramp.S standard riscv/riscv/sbi.c standard riscv/riscv/sbi_ipi.c optional smp riscv/riscv/sdt_machdep.c optional kdtrace_hooks riscv/riscv/stack_machdep.c optional ddb | stack riscv/riscv/support.S standard riscv/riscv/swtch.S standard riscv/riscv/sys_machdep.c standard riscv/riscv/trap.c standard riscv/riscv/timer.c standard riscv/riscv/uio_machdep.c standard riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack riscv/riscv/vm_machdep.c standard riscv/vmm/vmm.c optional vmm riscv/vmm/vmm_aplic.c optional vmm riscv/vmm/vmm_dev_machdep.c optional vmm +riscv/vmm/vmm_fence.c optional vmm riscv/vmm/vmm_instruction_emul.c optional vmm riscv/vmm/vmm_riscv.c optional vmm riscv/vmm/vmm_sbi.c optional vmm riscv/vmm/vmm_switch.S optional vmm riscv/vmm/vmm_vtimer.c optional vmm riscv/thead/thead.c standard # Zstd contrib/zstd/lib/freebsd/zstd_kfreebsd.c optional zstdio compile-with ${ZSTD_C} diff --git a/sys/riscv/include/cpufunc.h b/sys/riscv/include/cpufunc.h index 8f5b87d24ce3..75b22632c546 100644 --- a/sys/riscv/include/cpufunc.h +++ b/sys/riscv/include/cpufunc.h @@ -1,154 +1,169 @@ /*- * Copyright (c) 2015-2016 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_CPUFUNC_H_ #define _MACHINE_CPUFUNC_H_ static __inline void breakpoint(void) { __asm("ebreak"); } #ifdef _KERNEL #include #include static __inline register_t intr_disable(void) { uint64_t ret; __asm __volatile( "csrrci %0, sstatus, %1" : "=&r" (ret) : "i" (SSTATUS_SIE) ); return (ret & (SSTATUS_SIE)); } static __inline void intr_restore(register_t s) { __asm __volatile( "csrs sstatus, %0" :: "r" (s) ); } static __inline void intr_enable(void) { __asm __volatile( "csrsi sstatus, %0" :: "i" (SSTATUS_SIE) ); } /* NB: fence() is defined as a macro in . */ static __inline void fence_i(void) { __asm __volatile("fence.i" ::: "memory"); } static __inline void sfence_vma(void) { __asm __volatile("sfence.vma" ::: "memory"); } static __inline void sfence_vma_page(uintptr_t addr) { __asm __volatile("sfence.vma %0" :: "r" (addr) : "memory"); } +static __inline void +sfence_vma_asid(uint64_t asid) +{ + + __asm __volatile("sfence.vma x0, %0" :: "r" (asid) : "memory"); +} + +static __inline void +sfence_vma_asid_page(uint64_t asid, uintptr_t addr) +{ + + __asm __volatile("sfence.vma %0, %1" :: "r" (addr), "r" (asid) + : "memory"); +} + #define rdcycle() csr_read64(cycle) #define rdtime() csr_read64(time) #define rdinstret() csr_read64(instret) #define rdhpmcounter(n) csr_read64(hpmcounter##n) /* Cache hooks. */ extern int64_t dcache_line_size; typedef void (*cache_op_t)(vm_offset_t start, vm_size_t size); struct riscv_cache_ops { cache_op_t dcache_wbinv_range; cache_op_t dcache_inv_range; cache_op_t dcache_wb_range; }; extern struct riscv_cache_ops cache_ops; static __inline void cpu_dcache_wbinv_range(vm_offset_t addr, vm_size_t size) { if (cache_ops.dcache_wbinv_range != NULL) cache_ops.dcache_wbinv_range(addr, size); } static __inline void cpu_dcache_inv_range(vm_offset_t addr, vm_size_t size) { if (cache_ops.dcache_inv_range != NULL) cache_ops.dcache_inv_range(addr, size); } static __inline void cpu_dcache_wb_range(vm_offset_t addr, vm_size_t size) { if (cache_ops.dcache_wb_range != NULL) cache_ops.dcache_wb_range(addr, size); } void riscv_cache_install_hooks(struct riscv_cache_ops *, u_int); #define cpufunc_nullop() riscv_nullop() void riscv_nullop(void); #endif /* _KERNEL */ #endif /* _MACHINE_CPUFUNC_H_ */ diff --git a/sys/riscv/vmm/riscv.h b/sys/riscv/vmm/riscv.h index f3665d33a386..793c61534cee 100644 --- a/sys/riscv/vmm/riscv.h +++ b/sys/riscv/vmm/riscv.h @@ -1,137 +1,159 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Mihai Carabas * Copyright (c) 2024 Ruslan Bukin * * This software was developed by the University of Cambridge Computer * Laboratory (Department of Computer Science and Technology) under Innovate * UK project 105694, "Digital Security by Design (DSbD) Technology Platform * Prototype". * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _VMM_RISCV_H_ #define _VMM_RISCV_H_ #include #include #include #include struct hypregs { uint64_t hyp_ra; uint64_t hyp_sp; uint64_t hyp_gp; uint64_t hyp_tp; uint64_t hyp_t[7]; uint64_t hyp_s[12]; uint64_t hyp_a[8]; uint64_t hyp_sepc; uint64_t hyp_sstatus; uint64_t hyp_hstatus; }; struct hypcsr { uint64_t hvip; uint64_t vsstatus; uint64_t vsie; uint64_t vstvec; uint64_t vsscratch; uint64_t vsepc; uint64_t vscause; uint64_t vstval; uint64_t vsatp; uint64_t scounteren; uint64_t senvcfg; }; +enum vmm_fence_type { + VMM_RISCV_FENCE_INVALID = 0, + VMM_RISCV_FENCE_I, + VMM_RISCV_FENCE_VMA, + VMM_RISCV_FENCE_VMA_ASID, +}; + +struct vmm_fence { + enum vmm_fence_type type; + size_t start; + size_t size; + uint64_t asid; +}; + struct hypctx { struct hypregs host_regs; struct hypregs guest_regs; struct hypcsr guest_csrs; uint64_t host_sscratch; uint64_t host_stvec; uint64_t host_scounteren; uint64_t guest_scounteren; struct hyp *hyp; struct vcpu *vcpu; bool has_exception; int cpu_id; int ipi_pending; int interrupts_pending; struct vtimer vtimer; + + struct vmm_fence *fence_queue; + struct mtx fence_queue_mtx; + int fence_queue_head; + int fence_queue_tail; +#define FENCE_REQ_I (1 << 0) +#define FENCE_REQ_VMA (1 << 1) + int fence_req; }; struct hyp { struct vm *vm; uint64_t vmid_generation; bool aplic_attached; struct aplic *aplic; struct hypctx *ctx[]; }; struct hyptrap { uint64_t sepc; uint64_t scause; uint64_t stval; uint64_t htval; uint64_t htinst; }; #define DEFINE_VMMOPS_IFUNC(ret_type, opname, args) \ ret_type vmmops_##opname args; DEFINE_VMMOPS_IFUNC(int, modinit, (void)) DEFINE_VMMOPS_IFUNC(int, modcleanup, (void)) DEFINE_VMMOPS_IFUNC(void *, init, (struct vm *vm, struct pmap *pmap)) DEFINE_VMMOPS_IFUNC(int, gla2gpa, (void *vcpui, struct vm_guest_paging *paging, uint64_t gla, int prot, uint64_t *gpa, int *is_fault)) DEFINE_VMMOPS_IFUNC(int, run, (void *vcpui, register_t pc, struct pmap *pmap, struct vm_eventinfo *info)) DEFINE_VMMOPS_IFUNC(void, cleanup, (void *vmi)) DEFINE_VMMOPS_IFUNC(void *, vcpu_init, (void *vmi, struct vcpu *vcpu, int vcpu_id)) DEFINE_VMMOPS_IFUNC(void, vcpu_cleanup, (void *vcpui)) DEFINE_VMMOPS_IFUNC(int, exception, (void *vcpui, uint64_t scause)) DEFINE_VMMOPS_IFUNC(int, getreg, (void *vcpui, int num, uint64_t *retval)) DEFINE_VMMOPS_IFUNC(int, setreg, (void *vcpui, int num, uint64_t val)) DEFINE_VMMOPS_IFUNC(int, getcap, (void *vcpui, int num, int *retval)) DEFINE_VMMOPS_IFUNC(int, setcap, (void *vcpui, int num, int val)) DEFINE_VMMOPS_IFUNC(struct vmspace *, vmspace_alloc, (vm_offset_t min, vm_offset_t max)) DEFINE_VMMOPS_IFUNC(void, vmspace_free, (struct vmspace *vmspace)) #define dprintf(fmt, ...) struct hypctx *riscv_get_active_vcpu(void); void vmm_switch(struct hypctx *); void vmm_unpriv_trap(struct hyptrap *, uint64_t tmp); int vmm_sbi_ecall(struct vcpu *, bool *); void riscv_send_ipi(struct hypctx *hypctx, int hart_id); int riscv_check_ipi(struct hypctx *hypctx, bool clear); bool riscv_check_interrupts_pending(struct hypctx *hypctx); #endif /* !_VMM_RISCV_H_ */ diff --git a/sys/riscv/vmm/vmm_fence.c b/sys/riscv/vmm/vmm_fence.c new file mode 100644 index 000000000000..6bba3f4b5dfe --- /dev/null +++ b/sys/riscv/vmm/vmm_fence.c @@ -0,0 +1,208 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Ruslan Bukin + * + * This software was developed by the University of Cambridge Computer + * Laboratory (Department of Computer Science and Technology) under Innovate + * UK project 105694, "Digital Security by Design (DSbD) Technology Platform + * Prototype". + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "riscv.h" +#include "vmm_fence.h" + +static bool +vmm_fence_dequeue(struct hypctx *hypctx, struct vmm_fence *new_fence) +{ + struct vmm_fence *queue; + struct vmm_fence *fence; + + mtx_lock_spin(&hypctx->fence_queue_mtx); + queue = hypctx->fence_queue; + fence = &queue[hypctx->fence_queue_head]; + if (fence->type != VMM_RISCV_FENCE_INVALID) { + *new_fence = *fence; + fence->type = VMM_RISCV_FENCE_INVALID; + hypctx->fence_queue_head = + (hypctx->fence_queue_head + 1) % VMM_FENCE_QUEUE_SIZE; + } else { + mtx_unlock_spin(&hypctx->fence_queue_mtx); + return (false); + } + mtx_unlock_spin(&hypctx->fence_queue_mtx); + + return (true); +} + +static bool +vmm_fence_enqueue(struct hypctx *hypctx, struct vmm_fence *new_fence) +{ + struct vmm_fence *queue; + struct vmm_fence *fence; + + mtx_lock_spin(&hypctx->fence_queue_mtx); + queue = hypctx->fence_queue; + fence = &queue[hypctx->fence_queue_tail]; + if (fence->type == VMM_RISCV_FENCE_INVALID) { + *fence = *new_fence; + hypctx->fence_queue_tail = + (hypctx->fence_queue_tail + 1) % VMM_FENCE_QUEUE_SIZE; + } else { + mtx_unlock_spin(&hypctx->fence_queue_mtx); + return (false); + } + mtx_unlock_spin(&hypctx->fence_queue_mtx); + + return (true); +} + +static void +vmm_fence_process_one(struct vmm_fence *fence) +{ + uint64_t va; + + KASSERT(fence->type == VMM_RISCV_FENCE_VMA || + fence->type == VMM_RISCV_FENCE_VMA_ASID, + ("%s: wrong fence type %d", __func__, fence->type)); + + switch (fence->type) { + case VMM_RISCV_FENCE_VMA: + for (va = fence->start; va < fence->start + fence->size; + va += PAGE_SIZE) + sfence_vma_page(va); + break; + case VMM_RISCV_FENCE_VMA_ASID: + if (fence->start == 0 && fence->size == 0) + sfence_vma_asid(fence->asid); + else + for (va = fence->start; va < fence->start + fence->size; + va += PAGE_SIZE) + sfence_vma_asid_page(fence->asid, va); + break; + default: + break; + } +} + +void +vmm_fence_process(struct hypctx *hypctx) +{ + struct vmm_fence fence; + int pending; + + pending = atomic_readandclear_32(&hypctx->fence_req); + + KASSERT((pending & ~(FENCE_REQ_I | FENCE_REQ_VMA)) == 0, + ("wrong fence bit mask")); + + if (pending & FENCE_REQ_I) + fence_i(); + + if (pending & FENCE_REQ_VMA) + sfence_vma(); + + while (vmm_fence_dequeue(hypctx, &fence) == true) + vmm_fence_process_one(&fence); +} + +void +vmm_fence_add(struct vm *vm, cpuset_t *cpus, struct vmm_fence *fence) +{ + struct hypctx *hypctx; + cpuset_t running_cpus; + struct vcpu *vcpu; + uint16_t maxcpus; + int hostcpu; + int state; + bool enq; + int i; + + CPU_ZERO(&running_cpus); + + maxcpus = vm_get_maxcpus(vm); + for (i = 0; i < maxcpus; i++) { + if (!CPU_ISSET(i, cpus)) + continue; + vcpu = vm_vcpu(vm, i); + hypctx = vcpu_get_cookie(vcpu); + + enq = false; + + /* No need to enqueue fences i and vma global. */ + switch (fence->type) { + case VMM_RISCV_FENCE_I: + atomic_set_32(&hypctx->fence_req, FENCE_REQ_I); + break; + case VMM_RISCV_FENCE_VMA: + if (fence->start == 0 && fence->size == 0) + atomic_set_32(&hypctx->fence_req, + FENCE_REQ_VMA); + else + enq = true; + break; + case VMM_RISCV_FENCE_VMA_ASID: + enq = true; + break; + default: + KASSERT(0, ("%s: wrong fence type %d", __func__, + fence->type)); + break; + } + + /* + * Try to enqueue. In case of failure use more conservative + * request. + */ + if (enq) + if (vmm_fence_enqueue(hypctx, fence) == false) + atomic_set_32(&hypctx->fence_req, + FENCE_REQ_VMA); + + mb(); + + state = vcpu_get_state(vcpu, &hostcpu); + if (state == VCPU_RUNNING) + CPU_SET(hostcpu, &running_cpus); + } + + /* + * Interrupt other cores. On reception of IPI they will leave guest. + * On entry back to the guest they will process fence request. + * + * If vcpu migrates to another cpu right here, it should process + * all fences on entry to the guest as well. + */ + if (!CPU_EMPTY(&running_cpus)) + smp_rendezvous_cpus(running_cpus, NULL, NULL, NULL, NULL); +} diff --git a/sys/riscv/vmm/vmm_fence.h b/sys/riscv/vmm/vmm_fence.h new file mode 100644 index 000000000000..05d4466fd634 --- /dev/null +++ b/sys/riscv/vmm/vmm_fence.h @@ -0,0 +1,43 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Ruslan Bukin + * + * This software was developed by the University of Cambridge Computer + * Laboratory (Department of Computer Science and Technology) under Innovate + * UK project 105694, "Digital Security by Design (DSbD) Technology Platform + * Prototype". + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _VMM_FENCE_H_ +#define _VMM_FENCE_H_ + +struct hypctx; + +#define VMM_FENCE_QUEUE_SIZE 128 + +void vmm_fence_process(struct hypctx *hypctx); +void vmm_fence_add(struct vm *vm, cpuset_t *cpus, struct vmm_fence *fence); + +#endif /* !_VMM_FENCE_H_ */ diff --git a/sys/riscv/vmm/vmm_riscv.c b/sys/riscv/vmm/vmm_riscv.c index 6ac945dfa1d0..78250ae7c440 100644 --- a/sys/riscv/vmm/vmm_riscv.c +++ b/sys/riscv/vmm/vmm_riscv.c @@ -1,939 +1,948 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2024 Ruslan Bukin * * This software was developed by the University of Cambridge Computer * Laboratory (Department of Computer Science and Technology) under Innovate * UK project 105694, "Digital Security by Design (DSbD) Technology Platform * Prototype". * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "riscv.h" #include "vmm_aplic.h" +#include "vmm_fence.h" #include "vmm_stat.h" MALLOC_DEFINE(M_HYP, "RISC-V VMM HYP", "RISC-V VMM HYP"); DPCPU_DEFINE_STATIC(struct hypctx *, vcpu); static int m_op(uint32_t insn, int match, int mask) { if (((insn ^ match) & mask) == 0) return (1); return (0); } static inline void riscv_set_active_vcpu(struct hypctx *hypctx) { DPCPU_SET(vcpu, hypctx); } struct hypctx * riscv_get_active_vcpu(void) { return (DPCPU_GET(vcpu)); } int vmmops_modinit(void) { if (!has_hyp) { printf("vmm: riscv hart doesn't support H-extension.\n"); return (ENXIO); } return (0); } int vmmops_modcleanup(void) { return (0); } void * vmmops_init(struct vm *vm, pmap_t pmap) { struct hyp *hyp; vm_size_t size; size = round_page(sizeof(struct hyp) + sizeof(struct hypctx *) * vm_get_maxcpus(vm)); hyp = malloc_aligned(size, PAGE_SIZE, M_HYP, M_WAITOK | M_ZERO); hyp->vm = vm; hyp->aplic_attached = false; aplic_vminit(hyp); return (hyp); } static void vmmops_delegate(void) { uint64_t hedeleg; uint64_t hideleg; hedeleg = (1UL << SCAUSE_INST_MISALIGNED); hedeleg |= (1UL << SCAUSE_ILLEGAL_INSTRUCTION); hedeleg |= (1UL << SCAUSE_BREAKPOINT); hedeleg |= (1UL << SCAUSE_ECALL_USER); hedeleg |= (1UL << SCAUSE_INST_PAGE_FAULT); hedeleg |= (1UL << SCAUSE_LOAD_PAGE_FAULT); hedeleg |= (1UL << SCAUSE_STORE_PAGE_FAULT); csr_write(hedeleg, hedeleg); hideleg = (1UL << IRQ_SOFTWARE_HYPERVISOR); hideleg |= (1UL << IRQ_TIMER_HYPERVISOR); hideleg |= (1UL << IRQ_EXTERNAL_HYPERVISOR); csr_write(hideleg, hideleg); } static void vmmops_vcpu_restore_csrs(struct hypctx *hypctx) { struct hypcsr *csrs; csrs = &hypctx->guest_csrs; csr_write(vsstatus, csrs->vsstatus); csr_write(vsie, csrs->vsie); csr_write(vstvec, csrs->vstvec); csr_write(vsscratch, csrs->vsscratch); csr_write(vsepc, csrs->vsepc); csr_write(vscause, csrs->vscause); csr_write(vstval, csrs->vstval); csr_write(hvip, csrs->hvip); csr_write(vsatp, csrs->vsatp); } static void vmmops_vcpu_save_csrs(struct hypctx *hypctx) { struct hypcsr *csrs; csrs = &hypctx->guest_csrs; csrs->vsstatus = csr_read(vsstatus); csrs->vsie = csr_read(vsie); csrs->vstvec = csr_read(vstvec); csrs->vsscratch = csr_read(vsscratch); csrs->vsepc = csr_read(vsepc); csrs->vscause = csr_read(vscause); csrs->vstval = csr_read(vstval); csrs->hvip = csr_read(hvip); csrs->vsatp = csr_read(vsatp); } void * vmmops_vcpu_init(void *vmi, struct vcpu *vcpu1, int vcpuid) { struct hypctx *hypctx; struct hyp *hyp; vm_size_t size; hyp = vmi; dprintf("%s: hyp %p\n", __func__, hyp); KASSERT(vcpuid >= 0 && vcpuid < vm_get_maxcpus(hyp->vm), ("%s: Invalid vcpuid %d", __func__, vcpuid)); size = round_page(sizeof(struct hypctx)); hypctx = malloc_aligned(size, PAGE_SIZE, M_HYP, M_WAITOK | M_ZERO); hypctx->hyp = hyp; hypctx->vcpu = vcpu1; hypctx->guest_scounteren = HCOUNTEREN_CY | HCOUNTEREN_TM; + /* Fence queue. */ + hypctx->fence_queue = mallocarray(VMM_FENCE_QUEUE_SIZE, + sizeof(struct vmm_fence), M_HYP, M_WAITOK | M_ZERO); + mtx_init(&hypctx->fence_queue_mtx, "fence queue", NULL, MTX_SPIN); + /* sstatus */ hypctx->guest_regs.hyp_sstatus = SSTATUS_SPP | SSTATUS_SPIE; hypctx->guest_regs.hyp_sstatus |= SSTATUS_FS_INITIAL; /* hstatus */ hypctx->guest_regs.hyp_hstatus = HSTATUS_SPV | HSTATUS_VTW; hypctx->guest_regs.hyp_hstatus |= HSTATUS_SPVP; hypctx->cpu_id = vcpuid; hyp->ctx[vcpuid] = hypctx; aplic_cpuinit(hypctx); vtimer_cpuinit(hypctx); return (hypctx); } static int riscv_vmm_pinit(pmap_t pmap) { dprintf("%s: pmap %p\n", __func__, pmap); pmap_pinit_stage(pmap, PM_STAGE2); return (1); } struct vmspace * vmmops_vmspace_alloc(vm_offset_t min, vm_offset_t max) { return (vmspace_alloc(min, max, riscv_vmm_pinit)); } void vmmops_vmspace_free(struct vmspace *vmspace) { pmap_remove_pages(vmspace_pmap(vmspace)); vmspace_free(vmspace); } static void riscv_unpriv_read(struct hypctx *hypctx, uintptr_t guest_addr, uint64_t *data, struct hyptrap *trap) { register struct hyptrap * htrap asm("a0"); uintptr_t old_hstatus; uintptr_t old_stvec; uintptr_t entry; uint64_t val; uint64_t tmp; int intr; entry = (uintptr_t)&vmm_unpriv_trap; htrap = trap; intr = intr_disable(); old_hstatus = csr_swap(hstatus, hypctx->guest_regs.hyp_hstatus); /* * Setup a temporary exception vector, so that if hlvx.hu raises * an exception we catch it in the vmm_unpriv_trap(). */ old_stvec = csr_swap(stvec, entry); /* * Read first two bytes of instruction assuming it could be a * compressed one. */ __asm __volatile(".option push\n" ".option norvc\n" "hlvx.hu %[val], (%[addr])\n" ".option pop\n" : [val] "=r" (val) : [addr] "r" (guest_addr), "r" (htrap) : "a1", "memory"); /* * Check if previous hlvx.hu did not raise an exception, and then * read the rest of instruction if it is a full-length one. */ if (trap->scause == -1 && (val & 0x3) == 0x3) { guest_addr += 2; __asm __volatile(".option push\n" ".option norvc\n" "hlvx.hu %[tmp], (%[addr])\n" ".option pop\n" : [tmp] "=r" (tmp) : [addr] "r" (guest_addr), "r" (htrap) : "a1", "memory"); val |= (tmp << 16); } csr_write(hstatus, old_hstatus); csr_write(stvec, old_stvec); intr_restore(intr); *data = val; } static int riscv_gen_inst_emul_data(struct hypctx *hypctx, struct vm_exit *vme_ret, struct hyptrap *trap) { uintptr_t guest_addr; struct vie *vie; uint64_t insn; int reg_num; int rs2, rd; int direction; int sign_extend; int access_size; guest_addr = vme_ret->sepc; KASSERT(vme_ret->scause == SCAUSE_FETCH_GUEST_PAGE_FAULT || vme_ret->scause == SCAUSE_LOAD_GUEST_PAGE_FAULT || vme_ret->scause == SCAUSE_STORE_GUEST_PAGE_FAULT, ("Invalid scause")); direction = vme_ret->scause == SCAUSE_STORE_GUEST_PAGE_FAULT ? VM_DIR_WRITE : VM_DIR_READ; sign_extend = 1; bzero(trap, sizeof(struct hyptrap)); trap->scause = -1; riscv_unpriv_read(hypctx, guest_addr, &insn, trap); if (trap->scause != -1) return (-1); if ((insn & 0x3) == 0x3) { rs2 = (insn & RS2_MASK) >> RS2_SHIFT; rd = (insn & RD_MASK) >> RD_SHIFT; if (direction == VM_DIR_WRITE) { if (m_op(insn, MATCH_SB, MASK_SB)) access_size = 1; else if (m_op(insn, MATCH_SH, MASK_SH)) access_size = 2; else if (m_op(insn, MATCH_SW, MASK_SW)) access_size = 4; else if (m_op(insn, MATCH_SD, MASK_SD)) access_size = 8; else { printf("unknown store instr at %lx", guest_addr); return (-2); } reg_num = rs2; } else { if (m_op(insn, MATCH_LB, MASK_LB)) access_size = 1; else if (m_op(insn, MATCH_LH, MASK_LH)) access_size = 2; else if (m_op(insn, MATCH_LW, MASK_LW)) access_size = 4; else if (m_op(insn, MATCH_LD, MASK_LD)) access_size = 8; else if (m_op(insn, MATCH_LBU, MASK_LBU)) { access_size = 1; sign_extend = 0; } else if (m_op(insn, MATCH_LHU, MASK_LHU)) { access_size = 2; sign_extend = 0; } else if (m_op(insn, MATCH_LWU, MASK_LWU)) { access_size = 4; sign_extend = 0; } else { printf("unknown load instr at %lx", guest_addr); return (-3); } reg_num = rd; } vme_ret->inst_length = 4; } else { rs2 = (insn >> 7) & 0x7; rs2 += 0x8; rd = (insn >> 2) & 0x7; rd += 0x8; if (direction == VM_DIR_WRITE) { if (m_op(insn, MATCH_C_SW, MASK_C_SW)) access_size = 4; else if (m_op(insn, MATCH_C_SD, MASK_C_SD)) access_size = 8; else { printf("unknown compressed store instr at %lx", guest_addr); return (-4); } } else { if (m_op(insn, MATCH_C_LW, MASK_C_LW)) access_size = 4; else if (m_op(insn, MATCH_C_LD, MASK_C_LD)) access_size = 8; else { printf("unknown load instr at %lx", guest_addr); return (-5); } } reg_num = rd; vme_ret->inst_length = 2; } vme_ret->u.inst_emul.gpa = (vme_ret->htval << 2) | (vme_ret->stval & 0x3); dprintf("guest_addr %lx insn %lx, reg %d, gpa %lx\n", guest_addr, insn, reg_num, vme_ret->u.inst_emul.gpa); vie = &vme_ret->u.inst_emul.vie; vie->dir = direction; vie->reg = reg_num; vie->sign_extend = sign_extend; vie->access_size = access_size; return (0); } static bool riscv_handle_world_switch(struct hypctx *hypctx, struct vm_exit *vme, pmap_t pmap) { struct hyptrap trap; uint64_t insn; uint64_t gpa; bool handled; bool retu; int ret; int i; handled = false; if (vme->scause & SCAUSE_INTR) { /* * Host interrupt? Leave critical section to handle. */ vmm_stat_incr(hypctx->vcpu, VMEXIT_IRQ, 1); vme->exitcode = VM_EXITCODE_BOGUS; vme->inst_length = 0; return (handled); } switch (vme->scause) { case SCAUSE_FETCH_GUEST_PAGE_FAULT: case SCAUSE_LOAD_GUEST_PAGE_FAULT: case SCAUSE_STORE_GUEST_PAGE_FAULT: gpa = (vme->htval << 2) | (vme->stval & 0x3); if (vm_mem_allocated(hypctx->vcpu, gpa)) { vme->exitcode = VM_EXITCODE_PAGING; vme->inst_length = 0; vme->u.paging.gpa = gpa; } else { ret = riscv_gen_inst_emul_data(hypctx, vme, &trap); if (ret != 0) { vme->exitcode = VM_EXITCODE_HYP; vme->u.hyp.scause = trap.scause; break; } vme->exitcode = VM_EXITCODE_INST_EMUL; } break; case SCAUSE_ILLEGAL_INSTRUCTION: /* * TODO: handle illegal instruction properly. */ printf("%s: Illegal instruction at %lx stval 0x%lx htval " "0x%lx\n", __func__, vme->sepc, vme->stval, vme->htval); vmm_stat_incr(hypctx->vcpu, VMEXIT_UNHANDLED, 1); vme->exitcode = VM_EXITCODE_BOGUS; handled = false; break; case SCAUSE_VIRTUAL_SUPERVISOR_ECALL: retu = false; vmm_sbi_ecall(hypctx->vcpu, &retu); if (retu == false) { handled = true; break; } for (i = 0; i < nitems(vme->u.ecall.args); i++) vme->u.ecall.args[i] = hypctx->guest_regs.hyp_a[i]; vme->exitcode = VM_EXITCODE_ECALL; handled = false; break; case SCAUSE_VIRTUAL_INSTRUCTION: insn = vme->stval; if (m_op(insn, MATCH_WFI, MASK_WFI)) vme->exitcode = VM_EXITCODE_WFI; else vme->exitcode = VM_EXITCODE_BOGUS; handled = false; break; default: printf("unknown scause %lx\n", vme->scause); vmm_stat_incr(hypctx->vcpu, VMEXIT_UNHANDLED, 1); vme->exitcode = VM_EXITCODE_BOGUS; handled = false; break; } return (handled); } int vmmops_gla2gpa(void *vcpui, struct vm_guest_paging *paging, uint64_t gla, int prot, uint64_t *gpa, int *is_fault) { /* Implement me. */ return (ENOSYS); } void riscv_send_ipi(struct hypctx *hypctx, int hart_id) { struct hyp *hyp; struct vm *vm; hyp = hypctx->hyp; vm = hyp->vm; atomic_set_32(&hypctx->ipi_pending, 1); vcpu_notify_event(vm_vcpu(vm, hart_id)); } int riscv_check_ipi(struct hypctx *hypctx, bool clear) { int val; if (clear) val = atomic_swap_32(&hypctx->ipi_pending, 0); else val = hypctx->ipi_pending; return (val); } bool riscv_check_interrupts_pending(struct hypctx *hypctx) { if (hypctx->interrupts_pending) return (true); return (false); } static void riscv_sync_interrupts(struct hypctx *hypctx) { int pending; pending = aplic_check_pending(hypctx); if (pending) hypctx->guest_csrs.hvip |= HVIP_VSEIP; else hypctx->guest_csrs.hvip &= ~HVIP_VSEIP; /* Guest clears VSSIP bit manually. */ if (riscv_check_ipi(hypctx, true)) hypctx->guest_csrs.hvip |= HVIP_VSSIP; if (riscv_check_interrupts_pending(hypctx)) hypctx->guest_csrs.hvip |= HVIP_VSTIP; else hypctx->guest_csrs.hvip &= ~HVIP_VSTIP; csr_write(hvip, hypctx->guest_csrs.hvip); } int vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo) { struct hypctx *hypctx; struct vm_exit *vme; struct vcpu *vcpu; register_t val; uint64_t hvip; bool handled; hypctx = (struct hypctx *)vcpui; vcpu = hypctx->vcpu; vme = vm_exitinfo(vcpu); hypctx->guest_regs.hyp_sepc = (uint64_t)pc; vmmops_delegate(); /* * From The RISC-V Instruction Set Manual * Volume II: RISC-V Privileged Architectures * * If the new virtual machine's guest physical page tables * have been modified, it may be necessary to execute an HFENCE.GVMA * instruction (see Section 5.3.2) before or after writing hgatp. */ __asm __volatile("hfence.gvma" ::: "memory"); csr_write(hgatp, pmap->pm_satp); if (has_sstc) csr_write(henvcfg, HENVCFG_STCE); csr_write(hie, HIE_VSEIE | HIE_VSSIE | HIE_SGEIE); /* TODO: should we trap rdcycle / rdtime? */ csr_write(hcounteren, HCOUNTEREN_CY | HCOUNTEREN_TM); vmmops_vcpu_restore_csrs(hypctx); for (;;) { dprintf("%s: pc %lx\n", __func__, pc); if (hypctx->has_exception) { hypctx->has_exception = false; /* * TODO: implement exception injection. */ } val = intr_disable(); /* Check if the vcpu is suspended */ if (vcpu_suspended(evinfo)) { intr_restore(val); vm_exit_suspended(vcpu, pc); break; } if (vcpu_debugged(vcpu)) { intr_restore(val); vm_exit_debug(vcpu, pc); break; } /* * TODO: What happens if a timer interrupt is asserted exactly * here, but for the previous VM? */ riscv_set_active_vcpu(hypctx); aplic_flush_hwstate(hypctx); riscv_sync_interrupts(hypctx); + vmm_fence_process(hypctx); dprintf("%s: Entering guest VM, vsatp %lx, ss %lx hs %lx\n", __func__, csr_read(vsatp), hypctx->guest_regs.hyp_sstatus, hypctx->guest_regs.hyp_hstatus); vmm_switch(hypctx); dprintf("%s: Leaving guest VM, hstatus %lx\n", __func__, hypctx->guest_regs.hyp_hstatus); /* Guest can clear VSSIP. It can't clear VSTIP or VSEIP. */ hvip = csr_read(hvip); if ((hypctx->guest_csrs.hvip ^ hvip) & HVIP_VSSIP) { if (hvip & HVIP_VSSIP) { /* TODO: VSSIP was set by guest. */ } else { /* VSSIP was cleared by guest. */ hypctx->guest_csrs.hvip &= ~HVIP_VSSIP; } } aplic_sync_hwstate(hypctx); /* * TODO: deactivate stage 2 pmap here if needed. */ vme->scause = csr_read(scause); vme->sepc = csr_read(sepc); vme->stval = csr_read(stval); vme->htval = csr_read(htval); vme->htinst = csr_read(htinst); intr_restore(val); vmm_stat_incr(vcpu, VMEXIT_COUNT, 1); vme->pc = hypctx->guest_regs.hyp_sepc; vme->inst_length = INSN_SIZE; handled = riscv_handle_world_switch(hypctx, vme, pmap); if (handled == false) /* Exit loop to emulate instruction. */ break; else { /* Resume guest execution from the next instruction. */ hypctx->guest_regs.hyp_sepc += vme->inst_length; } } vmmops_vcpu_save_csrs(hypctx); return (0); } static void riscv_pcpu_vmcleanup(void *arg) { struct hyp *hyp; int i, maxcpus; hyp = arg; maxcpus = vm_get_maxcpus(hyp->vm); for (i = 0; i < maxcpus; i++) { if (riscv_get_active_vcpu() == hyp->ctx[i]) { riscv_set_active_vcpu(NULL); break; } } } void vmmops_vcpu_cleanup(void *vcpui) { struct hypctx *hypctx; hypctx = vcpui; dprintf("%s\n", __func__); aplic_cpucleanup(hypctx); + mtx_destroy(&hypctx->fence_queue_mtx); + free(hypctx->fence_queue, M_HYP); free(hypctx, M_HYP); } void vmmops_cleanup(void *vmi) { struct hyp *hyp; hyp = vmi; dprintf("%s\n", __func__); aplic_vmcleanup(hyp); smp_rendezvous(NULL, riscv_pcpu_vmcleanup, NULL, hyp); free(hyp, M_HYP); } /* * Return register value. Registers have different sizes and an explicit cast * must be made to ensure proper conversion. */ static uint64_t * hypctx_regptr(struct hypctx *hypctx, int reg) { switch (reg) { case VM_REG_GUEST_RA: return (&hypctx->guest_regs.hyp_ra); case VM_REG_GUEST_SP: return (&hypctx->guest_regs.hyp_sp); case VM_REG_GUEST_GP: return (&hypctx->guest_regs.hyp_gp); case VM_REG_GUEST_TP: return (&hypctx->guest_regs.hyp_tp); case VM_REG_GUEST_T0: return (&hypctx->guest_regs.hyp_t[0]); case VM_REG_GUEST_T1: return (&hypctx->guest_regs.hyp_t[1]); case VM_REG_GUEST_T2: return (&hypctx->guest_regs.hyp_t[2]); case VM_REG_GUEST_S0: return (&hypctx->guest_regs.hyp_s[0]); case VM_REG_GUEST_S1: return (&hypctx->guest_regs.hyp_s[1]); case VM_REG_GUEST_A0: return (&hypctx->guest_regs.hyp_a[0]); case VM_REG_GUEST_A1: return (&hypctx->guest_regs.hyp_a[1]); case VM_REG_GUEST_A2: return (&hypctx->guest_regs.hyp_a[2]); case VM_REG_GUEST_A3: return (&hypctx->guest_regs.hyp_a[3]); case VM_REG_GUEST_A4: return (&hypctx->guest_regs.hyp_a[4]); case VM_REG_GUEST_A5: return (&hypctx->guest_regs.hyp_a[5]); case VM_REG_GUEST_A6: return (&hypctx->guest_regs.hyp_a[6]); case VM_REG_GUEST_A7: return (&hypctx->guest_regs.hyp_a[7]); case VM_REG_GUEST_S2: return (&hypctx->guest_regs.hyp_s[2]); case VM_REG_GUEST_S3: return (&hypctx->guest_regs.hyp_s[3]); case VM_REG_GUEST_S4: return (&hypctx->guest_regs.hyp_s[4]); case VM_REG_GUEST_S5: return (&hypctx->guest_regs.hyp_s[5]); case VM_REG_GUEST_S6: return (&hypctx->guest_regs.hyp_s[6]); case VM_REG_GUEST_S7: return (&hypctx->guest_regs.hyp_s[7]); case VM_REG_GUEST_S8: return (&hypctx->guest_regs.hyp_s[8]); case VM_REG_GUEST_S9: return (&hypctx->guest_regs.hyp_s[9]); case VM_REG_GUEST_S10: return (&hypctx->guest_regs.hyp_s[10]); case VM_REG_GUEST_S11: return (&hypctx->guest_regs.hyp_s[11]); case VM_REG_GUEST_T3: return (&hypctx->guest_regs.hyp_t[3]); case VM_REG_GUEST_T4: return (&hypctx->guest_regs.hyp_t[4]); case VM_REG_GUEST_T5: return (&hypctx->guest_regs.hyp_t[5]); case VM_REG_GUEST_T6: return (&hypctx->guest_regs.hyp_t[6]); case VM_REG_GUEST_SEPC: return (&hypctx->guest_regs.hyp_sepc); default: break; } return (NULL); } int vmmops_getreg(void *vcpui, int reg, uint64_t *retval) { uint64_t *regp; int running, hostcpu; struct hypctx *hypctx; hypctx = vcpui; running = vcpu_is_running(hypctx->vcpu, &hostcpu); if (running && hostcpu != curcpu) panic("%s: %s%d is running", __func__, vm_name(hypctx->hyp->vm), vcpu_vcpuid(hypctx->vcpu)); if (reg == VM_REG_GUEST_ZERO) { *retval = 0; return (0); } regp = hypctx_regptr(hypctx, reg); if (regp == NULL) return (EINVAL); *retval = *regp; return (0); } int vmmops_setreg(void *vcpui, int reg, uint64_t val) { struct hypctx *hypctx; int running, hostcpu; uint64_t *regp; hypctx = vcpui; running = vcpu_is_running(hypctx->vcpu, &hostcpu); if (running && hostcpu != curcpu) panic("%s: %s%d is running", __func__, vm_name(hypctx->hyp->vm), vcpu_vcpuid(hypctx->vcpu)); regp = hypctx_regptr(hypctx, reg); if (regp == NULL) return (EINVAL); *regp = val; return (0); } int vmmops_exception(void *vcpui, uint64_t scause) { struct hypctx *hypctx; int running, hostcpu; hypctx = vcpui; running = vcpu_is_running(hypctx->vcpu, &hostcpu); if (running && hostcpu != curcpu) panic("%s: %s%d is running", __func__, vm_name(hypctx->hyp->vm), vcpu_vcpuid(hypctx->vcpu)); /* TODO: implement me. */ return (ENOSYS); } int vmmops_getcap(void *vcpui, int num, int *retval) { int ret; ret = ENOENT; switch (num) { case VM_CAP_SSTC: *retval = has_sstc; ret = 0; break; case VM_CAP_UNRESTRICTED_GUEST: *retval = 1; ret = 0; break; default: break; } return (ret); } int vmmops_setcap(void *vcpui, int num, int val) { return (ENOENT); } diff --git a/sys/riscv/vmm/vmm_sbi.c b/sys/riscv/vmm/vmm_sbi.c index 63dcf9b4a7ae..586eb7c4d41c 100644 --- a/sys/riscv/vmm/vmm_sbi.c +++ b/sys/riscv/vmm/vmm_sbi.c @@ -1,205 +1,233 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2024 Ruslan Bukin * * This software was developed by the University of Cambridge Computer * Laboratory (Department of Computer Science and Technology) under Innovate * UK project 105694, "Digital Security by Design (DSbD) Technology Platform * Prototype". * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "riscv.h" +#include "vmm_fence.h" static int vmm_sbi_handle_rfnc(struct vcpu *vcpu, struct hypctx *hypctx) { - uint64_t hart_mask __unused; - uint64_t start __unused; - uint64_t size __unused; - uint64_t asid __unused; + struct vmm_fence fence; + uint64_t hart_mask; + uint64_t hart_mask_base; uint64_t func_id; + struct hyp *hyp; + uint16_t maxcpus; + cpuset_t cpus; + int vcpu_id; + int i; func_id = hypctx->guest_regs.hyp_a[6]; hart_mask = hypctx->guest_regs.hyp_a[0]; - start = hypctx->guest_regs.hyp_a[2]; - size = hypctx->guest_regs.hyp_a[3]; - asid = hypctx->guest_regs.hyp_a[4]; + hart_mask_base = hypctx->guest_regs.hyp_a[1]; - dprintf("%s: %ld hart_mask %lx start %lx size %lx\n", __func__, - func_id, hart_mask, start, size); + /* Construct vma_fence. */ - /* TODO: implement remote sfence. */ + fence.start = hypctx->guest_regs.hyp_a[2]; + fence.size = hypctx->guest_regs.hyp_a[3]; + fence.asid = hypctx->guest_regs.hyp_a[4]; switch (func_id) { case SBI_RFNC_REMOTE_FENCE_I: + fence.type = VMM_RISCV_FENCE_I; break; case SBI_RFNC_REMOTE_SFENCE_VMA: + fence.type = VMM_RISCV_FENCE_VMA; break; case SBI_RFNC_REMOTE_SFENCE_VMA_ASID: + fence.type = VMM_RISCV_FENCE_VMA_ASID; break; default: - break; + return (-1); + } + + /* Construct cpuset_t from the mask supplied. */ + + CPU_ZERO(&cpus); + hyp = hypctx->hyp; + maxcpus = vm_get_maxcpus(hyp->vm); + for (i = 0; i < maxcpus; i++) { + vcpu = vm_vcpu(hyp->vm, i); + if (vcpu == NULL) + continue; + vcpu_id = vcpu_vcpuid(vcpu); + if (hart_mask_base != -1UL) { + if (vcpu_id < hart_mask_base) + continue; + if (!(hart_mask & (1UL << (vcpu_id - hart_mask_base)))) + continue; + } + CPU_SET(i, &cpus); } - hypctx->guest_regs.hyp_a[0] = 0; + vmm_fence_add(hyp->vm, &cpus, &fence); return (0); } static int vmm_sbi_handle_time(struct vcpu *vcpu, struct hypctx *hypctx) { uint64_t func_id; uint64_t next_val; int ret; func_id = hypctx->guest_regs.hyp_a[6]; next_val = hypctx->guest_regs.hyp_a[0]; switch (func_id) { case SBI_TIME_SET_TIMER: vtimer_set_timer(hypctx, next_val); ret = 0; break; default: ret = -1; break; } hypctx->guest_regs.hyp_a[0] = ret; return (0); } static int vmm_sbi_handle_ipi(struct vcpu *vcpu, struct hypctx *hypctx) { struct hypctx *target_hypctx; struct vcpu *target_vcpu __unused; cpuset_t active_cpus; struct hyp *hyp; uint64_t hart_mask; uint64_t func_id; int hart_id; int bit; int ret; func_id = hypctx->guest_regs.hyp_a[6]; hart_mask = hypctx->guest_regs.hyp_a[0]; dprintf("%s: hart_mask %lx\n", __func__, hart_mask); hyp = hypctx->hyp; active_cpus = vm_active_cpus(hyp->vm); switch (func_id) { case SBI_IPI_SEND_IPI: while ((bit = ffs(hart_mask))) { hart_id = (bit - 1); hart_mask &= ~(1u << hart_id); if (CPU_ISSET(hart_id, &active_cpus)) { /* TODO. */ target_vcpu = vm_vcpu(hyp->vm, hart_id); target_hypctx = hypctx->hyp->ctx[hart_id]; riscv_send_ipi(target_hypctx, hart_id); } } ret = 0; break; default: printf("%s: unknown func %ld\n", __func__, func_id); ret = -1; break; } hypctx->guest_regs.hyp_a[0] = ret; return (0); } int vmm_sbi_ecall(struct vcpu *vcpu, bool *retu) { int sbi_extension_id __unused; struct hypctx *hypctx; + int error; hypctx = riscv_get_active_vcpu(); sbi_extension_id = hypctx->guest_regs.hyp_a[7]; dprintf("%s: args %lx %lx %lx %lx %lx %lx %lx %lx\n", __func__, hypctx->guest_regs.hyp_a[0], hypctx->guest_regs.hyp_a[1], hypctx->guest_regs.hyp_a[2], hypctx->guest_regs.hyp_a[3], hypctx->guest_regs.hyp_a[4], hypctx->guest_regs.hyp_a[5], hypctx->guest_regs.hyp_a[6], hypctx->guest_regs.hyp_a[7]); switch (sbi_extension_id) { case SBI_EXT_ID_RFNC: - vmm_sbi_handle_rfnc(vcpu, hypctx); + error = vmm_sbi_handle_rfnc(vcpu, hypctx); + hypctx->guest_regs.hyp_a[0] = error; break; case SBI_EXT_ID_TIME: vmm_sbi_handle_time(vcpu, hypctx); break; case SBI_EXT_ID_IPI: vmm_sbi_handle_ipi(vcpu, hypctx); break; default: *retu = true; break; } return (0); }