Index: head/sys/riscv/include/asm.h =================================================================== --- head/sys/riscv/include/asm.h (revision 338466) +++ head/sys/riscv/include/asm.h (revision 338467) @@ -1,66 +1,74 @@ /*- - * Copyright (c) 2015 Ruslan Bukin + * Copyright (c) 2015-2018 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ASM_H_ #define _MACHINE_ASM_H_ #undef __FBSDID #if !defined(lint) && !defined(STRIP_FBSDID) #define __FBSDID(s) .ident s #else #define __FBSDID(s) /* nothing */ #endif /* not lint and not STRIP_FBSDID */ #define _C_LABEL(x) x #define ENTRY(sym) \ .text; .globl sym; .type sym,@function; .align 4; sym: #define END(sym) .size sym, . - sym #define EENTRY(sym) \ .globl sym; sym: #define EEND(sym) #define WEAK_REFERENCE(sym, alias) \ .weak alias; \ .set alias,sym #define SET_FAULT_HANDLER(handler, tmp) \ ld tmp, PC_CURTHREAD(gp); \ ld tmp, TD_PCB(tmp); /* Load the pcb */ \ sd handler, PCB_ONFAULT(tmp) /* Set the handler */ + +#define ENTER_USER_ACCESS(tmp) \ + li tmp, SSTATUS_SUM; \ + csrs sstatus, tmp + +#define EXIT_USER_ACCESS(tmp) \ + li tmp, SSTATUS_SUM; \ + csrc sstatus, tmp #endif /* _MACHINE_ASM_H_ */ Index: head/sys/riscv/riscv/copyinout.S =================================================================== --- head/sys/riscv/riscv/copyinout.S (revision 338466) +++ head/sys/riscv/riscv/copyinout.S (revision 338467) @@ -1,140 +1,148 @@ /*- * Copyright (c) 2015-2018 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); +#include #include #include "assym.inc" /* * Fault handler for the copy{in,out} functions below. */ ENTRY(copyio_fault) SET_FAULT_HANDLER(x0, a1) /* Clear the handler */ + EXIT_USER_ACCESS(a1) copyio_fault_nopcb: li a0, EFAULT ret END(copyio_fault) /* * Copies from a kernel to user address * * int copyout(const void *kaddr, void *udaddr, size_t len) */ ENTRY(copyout) beqz a2, 2f /* If len == 0 then skip loop */ add a3, a1, a2 li a4, VM_MAXUSER_ADDRESS bgt a3, a4, copyio_fault_nopcb la a6, copyio_fault /* Get the handler address */ SET_FAULT_HANDLER(a6, a7) /* Set the handler */ + ENTER_USER_ACCESS(a7) 1: lb a4, 0(a0) /* Load from kaddr */ addi a0, a0, 1 sb a4, 0(a1) /* Store in uaddr */ addi a1, a1, 1 addi a2, a2, -1 /* len-- */ bnez a2, 1b + EXIT_USER_ACCESS(a7) SET_FAULT_HANDLER(x0, a7) /* Clear the handler */ 2: li a0, 0 /* return 0 */ ret END(copyout) /* * Copies from a user to kernel address * * int copyin(const void *uaddr, void *kdaddr, size_t len) */ ENTRY(copyin) beqz a2, 2f /* If len == 0 then skip loop */ add a3, a0, a2 li a4, VM_MAXUSER_ADDRESS bgt a3, a4, copyio_fault_nopcb la a6, copyio_fault /* Get the handler address */ SET_FAULT_HANDLER(a6, a7) /* Set the handler */ + ENTER_USER_ACCESS(a7) 1: lb a4, 0(a0) /* Load from uaddr */ addi a0, a0, 1 sb a4, 0(a1) /* Store in kaddr */ addi a1, a1, 1 addi a2, a2, -1 /* len-- */ bnez a2, 1b + EXIT_USER_ACCESS(a7) SET_FAULT_HANDLER(x0, a7) /* Clear the handler */ 2: li a0, 0 /* return 0 */ ret END(copyin) /* * Copies a string from a user to kernel address * * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) */ ENTRY(copyinstr) mv a5, x0 /* count = 0 */ beqz a2, 3f /* If len == 0 then skip loop */ la a6, copyio_fault /* Get the handler address */ SET_FAULT_HANDLER(a6, a7) /* Set the handler */ + ENTER_USER_ACCESS(a7) li a7, VM_MAXUSER_ADDRESS 1: bgt a0, a7, copyio_fault lb a4, 0(a0) /* Load from uaddr */ addi a0, a0, 1 sb a4, 0(a1) /* Store in kaddr */ addi a1, a1, 1 beqz a4, 2f addi a2, a2, -1 /* len-- */ addi a5, a5, 1 /* count++ */ bnez a2, 1b - -2: SET_FAULT_HANDLER(x0, a7) /* Clear the handler */ + +2: EXIT_USER_ACCESS(a7) + SET_FAULT_HANDLER(x0, a7) /* Clear the handler */ 3: beqz a3, 4f /* Check if done != NULL */ addi a5, a5, 1 /* count++ */ sd a5, 0(a3) /* done = count */ 4: mv a0, x0 /* return 0 */ beqz a4, 5f li a0, ENAMETOOLONG 5: ret END(copyinstr) Index: head/sys/riscv/riscv/exception.S =================================================================== --- head/sys/riscv/riscv/exception.S (revision 338466) +++ head/sys/riscv/riscv/exception.S (revision 338467) @@ -1,240 +1,237 @@ /*- - * Copyright (c) 2015-2017 Ruslan Bukin + * Copyright (c) 2015-2018 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "assym.inc" #include #include .macro save_registers el addi sp, sp, -(TF_SIZE) sd ra, (TF_RA)(sp) sd tp, (TF_TP)(sp) .if \el == 0 /* We came from userspace. Load our pcpu */ sd gp, (TF_GP)(sp) ld gp, (TF_SIZE)(sp) .endif sd t0, (TF_T + 0 * 8)(sp) sd t1, (TF_T + 1 * 8)(sp) sd t2, (TF_T + 2 * 8)(sp) sd t3, (TF_T + 3 * 8)(sp) sd t4, (TF_T + 4 * 8)(sp) sd t5, (TF_T + 5 * 8)(sp) sd t6, (TF_T + 6 * 8)(sp) sd s0, (TF_S + 0 * 8)(sp) sd s1, (TF_S + 1 * 8)(sp) sd s2, (TF_S + 2 * 8)(sp) sd s3, (TF_S + 3 * 8)(sp) sd s4, (TF_S + 4 * 8)(sp) sd s5, (TF_S + 5 * 8)(sp) sd s6, (TF_S + 6 * 8)(sp) sd s7, (TF_S + 7 * 8)(sp) sd s8, (TF_S + 8 * 8)(sp) sd s9, (TF_S + 9 * 8)(sp) sd s10, (TF_S + 10 * 8)(sp) sd s11, (TF_S + 11 * 8)(sp) sd a0, (TF_A + 0 * 8)(sp) sd a1, (TF_A + 1 * 8)(sp) sd a2, (TF_A + 2 * 8)(sp) sd a3, (TF_A + 3 * 8)(sp) sd a4, (TF_A + 4 * 8)(sp) sd a5, (TF_A + 5 * 8)(sp) sd a6, (TF_A + 6 * 8)(sp) sd a7, (TF_A + 7 * 8)(sp) #if 0 /* XXX: temporary test: spin if stack is not kernel one */ .if \el == 1 /* kernel */ mv t0, sp srli t0, t0, 63 1: beqz t0, 1b .endif #endif .if \el == 1 /* Store kernel sp */ li t1, TF_SIZE add t0, sp, t1 sd t0, (TF_SP)(sp) .else /* Store user sp */ csrr t0, sscratch sd t0, (TF_SP)(sp) .endif li t0, 0 csrw sscratch, t0 csrr t0, sepc sd t0, (TF_SEPC)(sp) csrr t0, sstatus sd t0, (TF_SSTATUS)(sp) csrr t0, sbadaddr sd t0, (TF_SBADADDR)(sp) csrr t0, scause sd t0, (TF_SCAUSE)(sp) .endm .macro load_registers el ld t0, (TF_SSTATUS)(sp) .if \el == 0 - /* - * Ensure user interrupts will be enabled on eret - * and supervisor mode can access userspace on trap. - */ - li t1, (SSTATUS_SPIE | SSTATUS_SUM) + /* Ensure user interrupts will be enabled on eret */ + li t1, SSTATUS_SPIE or t0, t0, t1 .else /* * Disable interrupts for supervisor mode exceptions. * For user mode exceptions we have already done this * in do_ast. */ li t1, ~SSTATUS_SIE and t0, t0, t1 .endif csrw sstatus, t0 ld t0, (TF_SEPC)(sp) csrw sepc, t0 .if \el == 0 /* We go to userspace. Load user sp */ ld t0, (TF_SP)(sp) csrw sscratch, t0 /* And store our pcpu */ sd gp, (TF_SIZE)(sp) ld gp, (TF_GP)(sp) .endif ld ra, (TF_RA)(sp) ld tp, (TF_TP)(sp) ld t0, (TF_T + 0 * 8)(sp) ld t1, (TF_T + 1 * 8)(sp) ld t2, (TF_T + 2 * 8)(sp) ld t3, (TF_T + 3 * 8)(sp) ld t4, (TF_T + 4 * 8)(sp) ld t5, (TF_T + 5 * 8)(sp) ld t6, (TF_T + 6 * 8)(sp) ld s0, (TF_S + 0 * 8)(sp) ld s1, (TF_S + 1 * 8)(sp) ld s2, (TF_S + 2 * 8)(sp) ld s3, (TF_S + 3 * 8)(sp) ld s4, (TF_S + 4 * 8)(sp) ld s5, (TF_S + 5 * 8)(sp) ld s6, (TF_S + 6 * 8)(sp) ld s7, (TF_S + 7 * 8)(sp) ld s8, (TF_S + 8 * 8)(sp) ld s9, (TF_S + 9 * 8)(sp) ld s10, (TF_S + 10 * 8)(sp) ld s11, (TF_S + 11 * 8)(sp) ld a0, (TF_A + 0 * 8)(sp) ld a1, (TF_A + 1 * 8)(sp) ld a2, (TF_A + 2 * 8)(sp) ld a3, (TF_A + 3 * 8)(sp) ld a4, (TF_A + 4 * 8)(sp) ld a5, (TF_A + 5 * 8)(sp) ld a6, (TF_A + 6 * 8)(sp) ld a7, (TF_A + 7 * 8)(sp) addi sp, sp, (TF_SIZE) .endm .macro do_ast /* Disable interrupts */ csrr a4, sstatus 1: csrci sstatus, (SSTATUS_SIE) ld a1, PC_CURTHREAD(gp) lw a2, TD_FLAGS(a1) li a3, (TDF_ASTPENDING|TDF_NEEDRESCHED) and a2, a2, a3 beqz a2, 2f /* Restore interrupts */ andi a4, a4, (SSTATUS_SIE) csrs sstatus, a4 /* Handle the ast */ mv a0, sp call _C_LABEL(ast) /* Re-check for new ast scheduled */ j 1b 2: .endm ENTRY(cpu_exception_handler) csrrw sp, sscratch, sp beqz sp, 1f /* User mode detected */ csrrw sp, sscratch, sp j cpu_exception_handler_user 1: /* Supervisor mode detected */ csrrw sp, sscratch, sp j cpu_exception_handler_supervisor END(cpu_exception_handler) ENTRY(cpu_exception_handler_supervisor) save_registers 1 mv a0, sp call _C_LABEL(do_trap_supervisor) load_registers 1 sret END(cpu_exception_handler_supervisor) ENTRY(cpu_exception_handler_user) csrrw sp, sscratch, sp save_registers 0 mv a0, sp call _C_LABEL(do_trap_user) do_ast load_registers 0 csrrw sp, sscratch, sp sret END(cpu_exception_handler_user) Index: head/sys/riscv/riscv/locore.S =================================================================== --- head/sys/riscv/riscv/locore.S (revision 338466) +++ head/sys/riscv/riscv/locore.S (revision 338467) @@ -1,317 +1,314 @@ /*- - * Copyright (c) 2015-2017 Ruslan Bukin + * Copyright (c) 2015-2018 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "assym.inc" #include #include #include #include #include #include .globl kernbase .set kernbase, KERNBASE /* Trap entries */ .text /* Reset vector */ .text .globl _start _start: /* Get the physical address kernel loaded to */ la t0, virt_map ld t1, 0(t0) sub t1, t1, t0 li t2, KERNBASE sub s9, t2, t1 /* s9 = physmem base */ mv s10, a0 /* s10 = hart id */ mv s11, a1 /* s11 = dtbp */ - - li t0, SSTATUS_SUM - csrs sstatus, t0 /* Direct secondary cores to mpentry */ bnez s10, mpentry /* * Page tables */ /* Add L1 entry for kernel */ la s1, pagetable_l1 la s2, pagetable_l2 /* Link to next level PN */ srli s2, s2, PAGE_SHIFT li a5, KERNBASE srli a5, a5, L1_SHIFT /* >> L1_SHIFT */ andi a5, a5, 0x1ff /* & 0x1ff */ li t4, PTE_V slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */ or t6, t4, t5 /* Store L1 PTE entry to position */ li a6, PTE_SIZE mulw a5, a5, a6 add t0, s1, a5 sd t6, (t0) /* Level 2 superpages (512 x 2MiB) */ la s1, pagetable_l2 srli t4, s9, 21 /* Div physmem base by 2 MiB */ li t2, 512 /* Build 512 entries */ add t3, t4, t2 li t5, 0 2: li t0, (PTE_V | PTE_RWX | PTE_D) slli t2, t4, PTE_PPN1_S /* << PTE_PPN1_S */ or t5, t0, t2 sd t5, (s1) /* Store PTE entry to position */ addi s1, s1, PTE_SIZE addi t4, t4, 1 bltu t4, t3, 2b /* Create an L1 page for early devmap */ la s1, pagetable_l1 la s2, pagetable_l2_devmap /* Link to next level PN */ srli s2, s2, PAGE_SHIFT li a5, (VM_MAX_KERNEL_ADDRESS - L2_SIZE) srli a5, a5, L1_SHIFT /* >> L1_SHIFT */ andi a5, a5, 0x1ff /* & 0x1ff */ li t4, PTE_V slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */ or t6, t4, t5 /* Store single level1 PTE entry to position */ li a6, PTE_SIZE mulw a5, a5, a6 add t0, s1, a5 sd t6, (t0) /* Create an L2 page superpage for DTB */ la s1, pagetable_l2_devmap mv s2, s11 srli s2, s2, PAGE_SHIFT li t0, (PTE_V | PTE_RWX | PTE_D) slli t2, s2, PTE_PPN0_S /* << PTE_PPN0_S */ or t0, t0, t2 /* Store PTE entry to position */ li a6, PTE_SIZE li a5, 510 mulw a5, a5, a6 add t1, s1, a5 sd t0, (t1) /* Page tables END */ /* Setup supervisor trap vector */ la t0, va sub t0, t0, s9 li t1, KERNBASE add t0, t0, t1 csrw stvec, t0 /* Set page tables base register */ la s2, pagetable_l1 srli s2, s2, PAGE_SHIFT li t0, SATP_MODE_SV39 or s2, s2, t0 sfence.vma csrw sptbr, s2 .align 2 va: /* Setup supervisor trap vector */ la t0, cpu_exception_handler csrw stvec, t0 /* Ensure sscratch is zero */ li t0, 0 csrw sscratch, t0 /* Initialize stack pointer */ la s3, initstack_end mv sp, s3 addi sp, sp, -PCB_SIZE /* Clear BSS */ la a0, _C_LABEL(__bss_start) la s1, _C_LABEL(_end) 1: sd zero, 0(a0) addi a0, a0, 8 bltu a0, s1, 1b /* Fill riscv_bootparams */ addi sp, sp, -40 la t0, pagetable_l1 sd t0, 0(sp) /* kern_l1pt */ sd s9, 8(sp) /* kern_phys */ la t0, initstack_end sd t0, 16(sp) /* kern_stack */ li t0, (VM_MAX_KERNEL_ADDRESS - 2 * L2_SIZE) sd t0, 24(sp) /* dtbp_virt */ sd s11, 32(sp) /* dtbp_phys */ mv a0, sp call _C_LABEL(initriscv) /* Off we go */ call _C_LABEL(mi_startup) .align 4 initstack: .space (PAGE_SIZE * KSTACK_PAGES) initstack_end: ENTRY(sigcode) mv a0, sp addi a0, a0, SF_UC 1: li t0, SYS_sigreturn ecall /* sigreturn failed, exit */ li t0, SYS_exit ecall j 1b END(sigcode) /* This may be copied to the stack, keep it 16-byte aligned */ .align 3 esigcode: .data .align 3 .global szsigcode szsigcode: .quad esigcode - sigcode .align 12 pagetable_l1: .space PAGE_SIZE pagetable_l2: .space PAGE_SIZE pagetable_l2_devmap: .space PAGE_SIZE .align 3 virt_map: .quad virt_map /* Not in use, but required for linking. */ .align 3 .globl __global_pointer$ __global_pointer$: .space 8 .globl init_pt_va init_pt_va: .quad pagetable_l2 /* XXX: Keep page tables VA */ #ifndef SMP ENTRY(mpentry) 1: wfi j 1b END(mpentry) #else /* * mpentry(unsigned long) * * Called by a core when it is being brought online. */ ENTRY(mpentry) /* * Calculate the offset to __riscv_boot_ap * for the current core, cpuid is in a0. */ li t1, 4 mulw t1, t1, a0 /* Get the pointer */ la t0, __riscv_boot_ap add t0, t0, t1 1: /* Wait the kernel to be ready */ lw t1, 0(t0) beqz t1, 1b /* Setup stack pointer */ la t0, secondary_stacks li t1, (PAGE_SIZE * KSTACK_PAGES) mulw t1, t1, s10 add t0, t0, t1 sub t0, t0, s9 li t1, KERNBASE add sp, t0, t1 /* Setup supervisor trap vector */ la t0, mpva sub t0, t0, s9 li t1, KERNBASE add t0, t0, t1 csrw stvec, t0 /* Set page tables base register */ la s2, pagetable_l1 srli s2, s2, PAGE_SHIFT li t0, SATP_MODE_SV39 or s2, s2, t0 sfence.vma csrw sptbr, s2 .align 2 mpva: /* Setup supervisor trap vector */ la t0, cpu_exception_handler csrw stvec, t0 /* Ensure sscratch is zero */ li t0, 0 csrw sscratch, t0 call init_secondary END(mpentry) #endif Index: head/sys/riscv/riscv/support.S =================================================================== --- head/sys/riscv/riscv/support.S (revision 338466) +++ head/sys/riscv/riscv/support.S (revision 338467) @@ -1,253 +1,275 @@ /*- - * Copyright (c) 2015 Ruslan Bukin + * Copyright (c) 2015-2018 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include +#include #include "assym.inc" /* * One of the fu* or su* functions failed, return -1. */ ENTRY(fsu_fault) SET_FAULT_HANDLER(x0, a1) /* Reset the handler function */ + EXIT_USER_ACCESS(a1) fsu_fault_nopcb: li a0, -1 ret END(fsu_fault) /* * int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) */ ENTRY(casueword32) li a4, (VM_MAXUSER_ADDRESS-3) bgt a0, a4, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a4) /* And set it */ + ENTER_USER_ACCESS(a4) 1: lr.w a4, 0(a0) /* Load-exclusive the data */ bne a4, a1, 2f /* If not equal then exit */ sc.w a5, a3, 0(a0) /* Store the new data */ bnez a5, 1b /* Retry on failure */ -2: SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */ +2: EXIT_USER_ACCESS(a5) + SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */ sw a4, 0(a2) /* Store the read data */ li a0, 0 /* Success */ ret /* Return */ END(casueword32) /* * int casueword(volatile u_long *, u_long, u_long *, u_long) */ ENTRY(casueword) li a4, (VM_MAXUSER_ADDRESS-7) bgt a0, a4, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a4) /* And set it */ + ENTER_USER_ACCESS(a4) 1: lr.d a4, 0(a0) /* Load-exclusive the data */ bne a4, a1, 2f /* If not equal then exit */ sc.d a5, a3, 0(a0) /* Store the new data */ bnez a5, 1b /* Retry on failure */ -2: SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */ +2: EXIT_USER_ACCESS(a5) + SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */ sd a4, 0(a2) /* Store the read data */ li a0, 0 /* Success */ ret /* Return */ END(casueword) /* * int fubyte(volatile const void *) */ ENTRY(fubyte) li a1, VM_MAXUSER_ADDRESS bgt a0, a1, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a1) /* And set it */ + ENTER_USER_ACCESS(a1) lb a0, 0(a0) /* Try loading the data */ + EXIT_USER_ACCESS(a1) SET_FAULT_HANDLER(x0, a1) /* Reset the fault handler */ ret /* Return */ END(fubyte) /* * int fuword(volatile const void *) */ ENTRY(fuword16) li a1, (VM_MAXUSER_ADDRESS-1) bgt a0, a1, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a1) /* And set it */ + ENTER_USER_ACCESS(a1) lh a0, 0(a0) /* Try loading the data */ + EXIT_USER_ACCESS(a1) SET_FAULT_HANDLER(x0, a1) /* Reset the fault handler */ ret /* Return */ END(fuword16) /* * int32_t fueword32(volatile const void *, int32_t *) */ ENTRY(fueword32) li a2, (VM_MAXUSER_ADDRESS-3) bgt a0, a2, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ + ENTER_USER_ACCESS(a2) lw a0, 0(a0) /* Try loading the data */ + EXIT_USER_ACCESS(a2) SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ sw a0, 0(a1) /* Save the data in kernel space */ li a0, 0 /* Success */ ret /* Return */ END(fueword32) /* * long fueword(volatile const void *, int64_t *) * int64_t fueword64(volatile const void *, int64_t *) */ ENTRY(fueword) EENTRY(fueword64) li a2, (VM_MAXUSER_ADDRESS-7) bgt a0, a2, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ + ENTER_USER_ACCESS(a2) ld a0, 0(a0) /* Try loading the data */ + EXIT_USER_ACCESS(a2) SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ sd a0, 0(a1) /* Save the data in kernel space */ li a0, 0 /* Success */ ret /* Return */ EEND(fueword64) END(fueword) /* * int subyte(volatile void *, int) */ ENTRY(subyte) li a2, VM_MAXUSER_ADDRESS bgt a0, a2, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ + ENTER_USER_ACCESS(a2) sb a1, 0(a0) /* Try storing the data */ + EXIT_USER_ACCESS(a2) SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ li a0, 0 /* Success */ ret /* Return */ END(subyte) /* * int suword16(volatile void *, int) */ ENTRY(suword16) li a2, (VM_MAXUSER_ADDRESS-1) bgt a0, a2, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ + ENTER_USER_ACCESS(a2) sh a1, 0(a0) /* Try storing the data */ + EXIT_USER_ACCESS(a2) SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ li a0, 0 /* Success */ ret /* Return */ END(suword16) /* * int suword32(volatile void *, int) */ ENTRY(suword32) li a2, (VM_MAXUSER_ADDRESS-3) bgt a0, a2, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ + ENTER_USER_ACCESS(a2) sw a1, 0(a0) /* Try storing the data */ + EXIT_USER_ACCESS(a2) SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ li a0, 0 /* Success */ ret /* Return */ END(suword32) /* * int suword(volatile void *, long) */ ENTRY(suword) EENTRY(suword64) li a2, (VM_MAXUSER_ADDRESS-7) bgt a0, a2, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ + ENTER_USER_ACCESS(a2) sd a1, 0(a0) /* Try storing the data */ + EXIT_USER_ACCESS(a2) SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ li a0, 0 /* Success */ ret /* Return */ EEND(suword64) END(suword) ENTRY(setjmp) /* Store the stack pointer */ sd sp, 0(a0) addi a0, a0, 8 /* Store the general purpose registers and ra */ sd s0, (0 * 8)(a0) sd s1, (1 * 8)(a0) sd s2, (2 * 8)(a0) sd s3, (3 * 8)(a0) sd s4, (4 * 8)(a0) sd s5, (5 * 8)(a0) sd s6, (6 * 8)(a0) sd s7, (7 * 8)(a0) sd s8, (8 * 8)(a0) sd s9, (9 * 8)(a0) sd s10, (10 * 8)(a0) sd s11, (11 * 8)(a0) sd ra, (12 * 8)(a0) /* Return value */ li a0, 0 ret END(setjmp) ENTRY(longjmp) /* Restore the stack pointer */ ld sp, 0(a0) addi a0, a0, 8 /* Restore the general purpose registers and ra */ ld s0, (0 * 8)(a0) ld s1, (1 * 8)(a0) ld s2, (2 * 8)(a0) ld s3, (3 * 8)(a0) ld s4, (4 * 8)(a0) ld s5, (5 * 8)(a0) ld s6, (6 * 8)(a0) ld s7, (7 * 8)(a0) ld s8, (8 * 8)(a0) ld s9, (9 * 8)(a0) ld s10, (10 * 8)(a0) ld s11, (11 * 8)(a0) ld ra, (12 * 8)(a0) /* Load the return value */ mv a0, a1 ret END(longjmp) Index: head/sys/riscv/riscv/vm_machdep.c =================================================================== --- head/sys/riscv/riscv/vm_machdep.c (revision 338466) +++ head/sys/riscv/riscv/vm_machdep.c (revision 338467) @@ -1,276 +1,275 @@ /*- - * Copyright (c) 2015-2017 Ruslan Bukin + * Copyright (c) 2015-2018 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if __riscv_xlen == 64 #define TP_OFFSET 16 /* sizeof(struct tcb) */ #endif /* * Finish a fork operation, with process p2 nearly set up. * Copy and update the pcb, set up the stack so that the child * ready to run and return to user mode. */ void cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) { struct pcb *pcb2; struct trapframe *tf; register_t val; if ((flags & RFPROC) == 0) return; if (td1 == curthread) { /* * Save the tp. These normally happen in cpu_switch, * but if userland changes this then forks this may * not have happened. */ __asm __volatile("mv %0, tp" : "=&r"(val)); td1->td_pcb->pcb_tp = val; /* RISCVTODO: save the FPU state here */ } pcb2 = (struct pcb *)(td2->td_kstack + td2->td_kstack_pages * PAGE_SIZE) - 1; td2->td_pcb = pcb2; bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); td2->td_pcb->pcb_l1addr = vtophys(vmspace_pmap(td2->td_proc->p_vmspace)->pm_l1); tf = (struct trapframe *)STACKALIGN((struct trapframe *)pcb2 - 1); bcopy(td1->td_frame, tf, sizeof(*tf)); /* Clear syscall error flag */ tf->tf_t[0] = 0; /* Arguments for child */ tf->tf_a[0] = 0; tf->tf_a[1] = 0; tf->tf_sstatus |= (SSTATUS_SPIE); /* Enable interrupts. */ - tf->tf_sstatus |= (SSTATUS_SUM); /* Supervisor can access userspace. */ tf->tf_sstatus &= ~(SSTATUS_SPP); /* User mode. */ td2->td_frame = tf; /* Set the return value registers for fork() */ td2->td_pcb->pcb_s[0] = (uintptr_t)fork_return; td2->td_pcb->pcb_s[1] = (uintptr_t)td2; td2->td_pcb->pcb_ra = (uintptr_t)fork_trampoline; td2->td_pcb->pcb_sp = (uintptr_t)td2->td_frame; /* Setup to release spin count in fork_exit(). */ td2->td_md.md_spinlock_count = 1; td2->td_md.md_saved_sstatus_ie = (SSTATUS_SIE); } void cpu_reset(void) { sbi_shutdown(); while(1); } void cpu_thread_swapin(struct thread *td) { } void cpu_thread_swapout(struct thread *td) { } void cpu_set_syscall_retval(struct thread *td, int error) { struct trapframe *frame; frame = td->td_frame; switch (error) { case 0: frame->tf_a[0] = td->td_retval[0]; frame->tf_a[1] = td->td_retval[1]; frame->tf_t[0] = 0; /* syscall succeeded */ break; case ERESTART: frame->tf_sepc -= 4; /* prev instruction */ break; case EJUSTRETURN: break; default: frame->tf_a[0] = error; frame->tf_t[0] = 1; /* syscall error */ break; } } /* * Initialize machine state, mostly pcb and trap frame for a new * thread, about to return to userspace. Put enough state in the new * thread's PCB to get it to go back to the fork_return(), which * finalizes the thread state and handles peculiarities of the first * return to userspace for the new thread. */ void cpu_copy_thread(struct thread *td, struct thread *td0) { bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe)); bcopy(td0->td_pcb, td->td_pcb, sizeof(struct pcb)); td->td_pcb->pcb_s[0] = (uintptr_t)fork_return; td->td_pcb->pcb_s[1] = (uintptr_t)td; td->td_pcb->pcb_ra = (uintptr_t)fork_trampoline; td->td_pcb->pcb_sp = (uintptr_t)td->td_frame; /* Setup to release spin count in fork_exit(). */ td->td_md.md_spinlock_count = 1; td->td_md.md_saved_sstatus_ie = (SSTATUS_SIE); } /* * Set that machine state for performing an upcall that starts * the entry function with the given argument. */ void cpu_set_upcall(struct thread *td, void (*entry)(void *), void *arg, stack_t *stack) { struct trapframe *tf; tf = td->td_frame; tf->tf_sp = STACKALIGN((uintptr_t)stack->ss_sp + stack->ss_size); tf->tf_sepc = (register_t)entry; tf->tf_a[0] = (register_t)arg; } int cpu_set_user_tls(struct thread *td, void *tls_base) { struct pcb *pcb; if ((uintptr_t)tls_base >= VM_MAXUSER_ADDRESS) return (EINVAL); pcb = td->td_pcb; pcb->pcb_tp = (register_t)tls_base + TP_OFFSET; if (td == curthread) __asm __volatile("mv tp, %0" :: "r"(pcb->pcb_tp)); return (0); } void cpu_thread_exit(struct thread *td) { } void cpu_thread_alloc(struct thread *td) { td->td_pcb = (struct pcb *)(td->td_kstack + td->td_kstack_pages * PAGE_SIZE) - 1; td->td_frame = (struct trapframe *)STACKALIGN( (caddr_t)td->td_pcb - 8 - sizeof(struct trapframe)); } void cpu_thread_free(struct thread *td) { } void cpu_thread_clean(struct thread *td) { } /* * Intercept the return address from a freshly forked process that has NOT * been scheduled yet. * * This is needed to make kernel threads stay in kernel mode. */ void cpu_fork_kthread_handler(struct thread *td, void (*func)(void *), void *arg) { td->td_pcb->pcb_s[0] = (uintptr_t)func; td->td_pcb->pcb_s[1] = (uintptr_t)arg; td->td_pcb->pcb_ra = (uintptr_t)fork_trampoline; td->td_pcb->pcb_sp = (uintptr_t)td->td_frame; } void cpu_exit(struct thread *td) { } void swi_vm(void *v) { /* Nothing to do here - busdma bounce buffers are not implemented. */ }