Index: head/sys/riscv/include/asm.h
===================================================================
--- head/sys/riscv/include/asm.h (revision 338466)
+++ head/sys/riscv/include/asm.h (revision 338467)
@@ -1,66 +1,74 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin
+ * Copyright (c) 2015-2018 Ruslan Bukin
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
* University of Cambridge Computer Laboratory under DARPA/AFRL contract
* FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
*
* Portions of this software were developed by the University of Cambridge
* Computer Laboratory as part of the CTSRD Project, with support from the
* UK Higher Education Innovation Fund (HEIF).
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _MACHINE_ASM_H_
#define _MACHINE_ASM_H_
#undef __FBSDID
#if !defined(lint) && !defined(STRIP_FBSDID)
#define __FBSDID(s) .ident s
#else
#define __FBSDID(s) /* nothing */
#endif /* not lint and not STRIP_FBSDID */
#define _C_LABEL(x) x
#define ENTRY(sym) \
.text; .globl sym; .type sym,@function; .align 4; sym:
#define END(sym) .size sym, . - sym
#define EENTRY(sym) \
.globl sym; sym:
#define EEND(sym)
#define WEAK_REFERENCE(sym, alias) \
.weak alias; \
.set alias,sym
#define SET_FAULT_HANDLER(handler, tmp) \
ld tmp, PC_CURTHREAD(gp); \
ld tmp, TD_PCB(tmp); /* Load the pcb */ \
sd handler, PCB_ONFAULT(tmp) /* Set the handler */
+
+#define ENTER_USER_ACCESS(tmp) \
+ li tmp, SSTATUS_SUM; \
+ csrs sstatus, tmp
+
+#define EXIT_USER_ACCESS(tmp) \
+ li tmp, SSTATUS_SUM; \
+ csrc sstatus, tmp
#endif /* _MACHINE_ASM_H_ */
Index: head/sys/riscv/riscv/copyinout.S
===================================================================
--- head/sys/riscv/riscv/copyinout.S (revision 338466)
+++ head/sys/riscv/riscv/copyinout.S (revision 338467)
@@ -1,140 +1,148 @@
/*-
* Copyright (c) 2015-2018 Ruslan Bukin
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
* University of Cambridge Computer Laboratory under DARPA/AFRL contract
* FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
*
* Portions of this software were developed by the University of Cambridge
* Computer Laboratory as part of the CTSRD Project, with support from the
* UK Higher Education Innovation Fund (HEIF).
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include
__FBSDID("$FreeBSD$");
+#include
#include
#include "assym.inc"
/*
* Fault handler for the copy{in,out} functions below.
*/
ENTRY(copyio_fault)
SET_FAULT_HANDLER(x0, a1) /* Clear the handler */
+ EXIT_USER_ACCESS(a1)
copyio_fault_nopcb:
li a0, EFAULT
ret
END(copyio_fault)
/*
* Copies from a kernel to user address
*
* int copyout(const void *kaddr, void *udaddr, size_t len)
*/
ENTRY(copyout)
beqz a2, 2f /* If len == 0 then skip loop */
add a3, a1, a2
li a4, VM_MAXUSER_ADDRESS
bgt a3, a4, copyio_fault_nopcb
la a6, copyio_fault /* Get the handler address */
SET_FAULT_HANDLER(a6, a7) /* Set the handler */
+ ENTER_USER_ACCESS(a7)
1: lb a4, 0(a0) /* Load from kaddr */
addi a0, a0, 1
sb a4, 0(a1) /* Store in uaddr */
addi a1, a1, 1
addi a2, a2, -1 /* len-- */
bnez a2, 1b
+ EXIT_USER_ACCESS(a7)
SET_FAULT_HANDLER(x0, a7) /* Clear the handler */
2: li a0, 0 /* return 0 */
ret
END(copyout)
/*
* Copies from a user to kernel address
*
* int copyin(const void *uaddr, void *kdaddr, size_t len)
*/
ENTRY(copyin)
beqz a2, 2f /* If len == 0 then skip loop */
add a3, a0, a2
li a4, VM_MAXUSER_ADDRESS
bgt a3, a4, copyio_fault_nopcb
la a6, copyio_fault /* Get the handler address */
SET_FAULT_HANDLER(a6, a7) /* Set the handler */
+ ENTER_USER_ACCESS(a7)
1: lb a4, 0(a0) /* Load from uaddr */
addi a0, a0, 1
sb a4, 0(a1) /* Store in kaddr */
addi a1, a1, 1
addi a2, a2, -1 /* len-- */
bnez a2, 1b
+ EXIT_USER_ACCESS(a7)
SET_FAULT_HANDLER(x0, a7) /* Clear the handler */
2: li a0, 0 /* return 0 */
ret
END(copyin)
/*
* Copies a string from a user to kernel address
*
* int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done)
*/
ENTRY(copyinstr)
mv a5, x0 /* count = 0 */
beqz a2, 3f /* If len == 0 then skip loop */
la a6, copyio_fault /* Get the handler address */
SET_FAULT_HANDLER(a6, a7) /* Set the handler */
+ ENTER_USER_ACCESS(a7)
li a7, VM_MAXUSER_ADDRESS
1: bgt a0, a7, copyio_fault
lb a4, 0(a0) /* Load from uaddr */
addi a0, a0, 1
sb a4, 0(a1) /* Store in kaddr */
addi a1, a1, 1
beqz a4, 2f
addi a2, a2, -1 /* len-- */
addi a5, a5, 1 /* count++ */
bnez a2, 1b
-
-2: SET_FAULT_HANDLER(x0, a7) /* Clear the handler */
+
+2: EXIT_USER_ACCESS(a7)
+ SET_FAULT_HANDLER(x0, a7) /* Clear the handler */
3: beqz a3, 4f /* Check if done != NULL */
addi a5, a5, 1 /* count++ */
sd a5, 0(a3) /* done = count */
4: mv a0, x0 /* return 0 */
beqz a4, 5f
li a0, ENAMETOOLONG
5:
ret
END(copyinstr)
Index: head/sys/riscv/riscv/exception.S
===================================================================
--- head/sys/riscv/riscv/exception.S (revision 338466)
+++ head/sys/riscv/riscv/exception.S (revision 338467)
@@ -1,240 +1,237 @@
/*-
- * Copyright (c) 2015-2017 Ruslan Bukin
+ * Copyright (c) 2015-2018 Ruslan Bukin
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
* University of Cambridge Computer Laboratory under DARPA/AFRL contract
* FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
*
* Portions of this software were developed by the University of Cambridge
* Computer Laboratory as part of the CTSRD Project, with support from the
* UK Higher Education Innovation Fund (HEIF).
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include
__FBSDID("$FreeBSD$");
#include "assym.inc"
#include
#include
.macro save_registers el
addi sp, sp, -(TF_SIZE)
sd ra, (TF_RA)(sp)
sd tp, (TF_TP)(sp)
.if \el == 0 /* We came from userspace. Load our pcpu */
sd gp, (TF_GP)(sp)
ld gp, (TF_SIZE)(sp)
.endif
sd t0, (TF_T + 0 * 8)(sp)
sd t1, (TF_T + 1 * 8)(sp)
sd t2, (TF_T + 2 * 8)(sp)
sd t3, (TF_T + 3 * 8)(sp)
sd t4, (TF_T + 4 * 8)(sp)
sd t5, (TF_T + 5 * 8)(sp)
sd t6, (TF_T + 6 * 8)(sp)
sd s0, (TF_S + 0 * 8)(sp)
sd s1, (TF_S + 1 * 8)(sp)
sd s2, (TF_S + 2 * 8)(sp)
sd s3, (TF_S + 3 * 8)(sp)
sd s4, (TF_S + 4 * 8)(sp)
sd s5, (TF_S + 5 * 8)(sp)
sd s6, (TF_S + 6 * 8)(sp)
sd s7, (TF_S + 7 * 8)(sp)
sd s8, (TF_S + 8 * 8)(sp)
sd s9, (TF_S + 9 * 8)(sp)
sd s10, (TF_S + 10 * 8)(sp)
sd s11, (TF_S + 11 * 8)(sp)
sd a0, (TF_A + 0 * 8)(sp)
sd a1, (TF_A + 1 * 8)(sp)
sd a2, (TF_A + 2 * 8)(sp)
sd a3, (TF_A + 3 * 8)(sp)
sd a4, (TF_A + 4 * 8)(sp)
sd a5, (TF_A + 5 * 8)(sp)
sd a6, (TF_A + 6 * 8)(sp)
sd a7, (TF_A + 7 * 8)(sp)
#if 0
/* XXX: temporary test: spin if stack is not kernel one */
.if \el == 1 /* kernel */
mv t0, sp
srli t0, t0, 63
1:
beqz t0, 1b
.endif
#endif
.if \el == 1
/* Store kernel sp */
li t1, TF_SIZE
add t0, sp, t1
sd t0, (TF_SP)(sp)
.else
/* Store user sp */
csrr t0, sscratch
sd t0, (TF_SP)(sp)
.endif
li t0, 0
csrw sscratch, t0
csrr t0, sepc
sd t0, (TF_SEPC)(sp)
csrr t0, sstatus
sd t0, (TF_SSTATUS)(sp)
csrr t0, sbadaddr
sd t0, (TF_SBADADDR)(sp)
csrr t0, scause
sd t0, (TF_SCAUSE)(sp)
.endm
.macro load_registers el
ld t0, (TF_SSTATUS)(sp)
.if \el == 0
- /*
- * Ensure user interrupts will be enabled on eret
- * and supervisor mode can access userspace on trap.
- */
- li t1, (SSTATUS_SPIE | SSTATUS_SUM)
+ /* Ensure user interrupts will be enabled on eret */
+ li t1, SSTATUS_SPIE
or t0, t0, t1
.else
/*
* Disable interrupts for supervisor mode exceptions.
* For user mode exceptions we have already done this
* in do_ast.
*/
li t1, ~SSTATUS_SIE
and t0, t0, t1
.endif
csrw sstatus, t0
ld t0, (TF_SEPC)(sp)
csrw sepc, t0
.if \el == 0
/* We go to userspace. Load user sp */
ld t0, (TF_SP)(sp)
csrw sscratch, t0
/* And store our pcpu */
sd gp, (TF_SIZE)(sp)
ld gp, (TF_GP)(sp)
.endif
ld ra, (TF_RA)(sp)
ld tp, (TF_TP)(sp)
ld t0, (TF_T + 0 * 8)(sp)
ld t1, (TF_T + 1 * 8)(sp)
ld t2, (TF_T + 2 * 8)(sp)
ld t3, (TF_T + 3 * 8)(sp)
ld t4, (TF_T + 4 * 8)(sp)
ld t5, (TF_T + 5 * 8)(sp)
ld t6, (TF_T + 6 * 8)(sp)
ld s0, (TF_S + 0 * 8)(sp)
ld s1, (TF_S + 1 * 8)(sp)
ld s2, (TF_S + 2 * 8)(sp)
ld s3, (TF_S + 3 * 8)(sp)
ld s4, (TF_S + 4 * 8)(sp)
ld s5, (TF_S + 5 * 8)(sp)
ld s6, (TF_S + 6 * 8)(sp)
ld s7, (TF_S + 7 * 8)(sp)
ld s8, (TF_S + 8 * 8)(sp)
ld s9, (TF_S + 9 * 8)(sp)
ld s10, (TF_S + 10 * 8)(sp)
ld s11, (TF_S + 11 * 8)(sp)
ld a0, (TF_A + 0 * 8)(sp)
ld a1, (TF_A + 1 * 8)(sp)
ld a2, (TF_A + 2 * 8)(sp)
ld a3, (TF_A + 3 * 8)(sp)
ld a4, (TF_A + 4 * 8)(sp)
ld a5, (TF_A + 5 * 8)(sp)
ld a6, (TF_A + 6 * 8)(sp)
ld a7, (TF_A + 7 * 8)(sp)
addi sp, sp, (TF_SIZE)
.endm
.macro do_ast
/* Disable interrupts */
csrr a4, sstatus
1:
csrci sstatus, (SSTATUS_SIE)
ld a1, PC_CURTHREAD(gp)
lw a2, TD_FLAGS(a1)
li a3, (TDF_ASTPENDING|TDF_NEEDRESCHED)
and a2, a2, a3
beqz a2, 2f
/* Restore interrupts */
andi a4, a4, (SSTATUS_SIE)
csrs sstatus, a4
/* Handle the ast */
mv a0, sp
call _C_LABEL(ast)
/* Re-check for new ast scheduled */
j 1b
2:
.endm
ENTRY(cpu_exception_handler)
csrrw sp, sscratch, sp
beqz sp, 1f
/* User mode detected */
csrrw sp, sscratch, sp
j cpu_exception_handler_user
1:
/* Supervisor mode detected */
csrrw sp, sscratch, sp
j cpu_exception_handler_supervisor
END(cpu_exception_handler)
ENTRY(cpu_exception_handler_supervisor)
save_registers 1
mv a0, sp
call _C_LABEL(do_trap_supervisor)
load_registers 1
sret
END(cpu_exception_handler_supervisor)
ENTRY(cpu_exception_handler_user)
csrrw sp, sscratch, sp
save_registers 0
mv a0, sp
call _C_LABEL(do_trap_user)
do_ast
load_registers 0
csrrw sp, sscratch, sp
sret
END(cpu_exception_handler_user)
Index: head/sys/riscv/riscv/locore.S
===================================================================
--- head/sys/riscv/riscv/locore.S (revision 338466)
+++ head/sys/riscv/riscv/locore.S (revision 338467)
@@ -1,317 +1,314 @@
/*-
- * Copyright (c) 2015-2017 Ruslan Bukin
+ * Copyright (c) 2015-2018 Ruslan Bukin
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
* University of Cambridge Computer Laboratory under DARPA/AFRL contract
* FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
*
* Portions of this software were developed by the University of Cambridge
* Computer Laboratory as part of the CTSRD Project, with support from the
* UK Higher Education Innovation Fund (HEIF).
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#include "assym.inc"
#include
#include
#include
#include
#include
#include
.globl kernbase
.set kernbase, KERNBASE
/* Trap entries */
.text
/* Reset vector */
.text
.globl _start
_start:
/* Get the physical address kernel loaded to */
la t0, virt_map
ld t1, 0(t0)
sub t1, t1, t0
li t2, KERNBASE
sub s9, t2, t1 /* s9 = physmem base */
mv s10, a0 /* s10 = hart id */
mv s11, a1 /* s11 = dtbp */
-
- li t0, SSTATUS_SUM
- csrs sstatus, t0
/* Direct secondary cores to mpentry */
bnez s10, mpentry
/*
* Page tables
*/
/* Add L1 entry for kernel */
la s1, pagetable_l1
la s2, pagetable_l2 /* Link to next level PN */
srli s2, s2, PAGE_SHIFT
li a5, KERNBASE
srli a5, a5, L1_SHIFT /* >> L1_SHIFT */
andi a5, a5, 0x1ff /* & 0x1ff */
li t4, PTE_V
slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */
or t6, t4, t5
/* Store L1 PTE entry to position */
li a6, PTE_SIZE
mulw a5, a5, a6
add t0, s1, a5
sd t6, (t0)
/* Level 2 superpages (512 x 2MiB) */
la s1, pagetable_l2
srli t4, s9, 21 /* Div physmem base by 2 MiB */
li t2, 512 /* Build 512 entries */
add t3, t4, t2
li t5, 0
2:
li t0, (PTE_V | PTE_RWX | PTE_D)
slli t2, t4, PTE_PPN1_S /* << PTE_PPN1_S */
or t5, t0, t2
sd t5, (s1) /* Store PTE entry to position */
addi s1, s1, PTE_SIZE
addi t4, t4, 1
bltu t4, t3, 2b
/* Create an L1 page for early devmap */
la s1, pagetable_l1
la s2, pagetable_l2_devmap /* Link to next level PN */
srli s2, s2, PAGE_SHIFT
li a5, (VM_MAX_KERNEL_ADDRESS - L2_SIZE)
srli a5, a5, L1_SHIFT /* >> L1_SHIFT */
andi a5, a5, 0x1ff /* & 0x1ff */
li t4, PTE_V
slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */
or t6, t4, t5
/* Store single level1 PTE entry to position */
li a6, PTE_SIZE
mulw a5, a5, a6
add t0, s1, a5
sd t6, (t0)
/* Create an L2 page superpage for DTB */
la s1, pagetable_l2_devmap
mv s2, s11
srli s2, s2, PAGE_SHIFT
li t0, (PTE_V | PTE_RWX | PTE_D)
slli t2, s2, PTE_PPN0_S /* << PTE_PPN0_S */
or t0, t0, t2
/* Store PTE entry to position */
li a6, PTE_SIZE
li a5, 510
mulw a5, a5, a6
add t1, s1, a5
sd t0, (t1)
/* Page tables END */
/* Setup supervisor trap vector */
la t0, va
sub t0, t0, s9
li t1, KERNBASE
add t0, t0, t1
csrw stvec, t0
/* Set page tables base register */
la s2, pagetable_l1
srli s2, s2, PAGE_SHIFT
li t0, SATP_MODE_SV39
or s2, s2, t0
sfence.vma
csrw sptbr, s2
.align 2
va:
/* Setup supervisor trap vector */
la t0, cpu_exception_handler
csrw stvec, t0
/* Ensure sscratch is zero */
li t0, 0
csrw sscratch, t0
/* Initialize stack pointer */
la s3, initstack_end
mv sp, s3
addi sp, sp, -PCB_SIZE
/* Clear BSS */
la a0, _C_LABEL(__bss_start)
la s1, _C_LABEL(_end)
1:
sd zero, 0(a0)
addi a0, a0, 8
bltu a0, s1, 1b
/* Fill riscv_bootparams */
addi sp, sp, -40
la t0, pagetable_l1
sd t0, 0(sp) /* kern_l1pt */
sd s9, 8(sp) /* kern_phys */
la t0, initstack_end
sd t0, 16(sp) /* kern_stack */
li t0, (VM_MAX_KERNEL_ADDRESS - 2 * L2_SIZE)
sd t0, 24(sp) /* dtbp_virt */
sd s11, 32(sp) /* dtbp_phys */
mv a0, sp
call _C_LABEL(initriscv) /* Off we go */
call _C_LABEL(mi_startup)
.align 4
initstack:
.space (PAGE_SIZE * KSTACK_PAGES)
initstack_end:
ENTRY(sigcode)
mv a0, sp
addi a0, a0, SF_UC
1:
li t0, SYS_sigreturn
ecall
/* sigreturn failed, exit */
li t0, SYS_exit
ecall
j 1b
END(sigcode)
/* This may be copied to the stack, keep it 16-byte aligned */
.align 3
esigcode:
.data
.align 3
.global szsigcode
szsigcode:
.quad esigcode - sigcode
.align 12
pagetable_l1:
.space PAGE_SIZE
pagetable_l2:
.space PAGE_SIZE
pagetable_l2_devmap:
.space PAGE_SIZE
.align 3
virt_map:
.quad virt_map
/* Not in use, but required for linking. */
.align 3
.globl __global_pointer$
__global_pointer$:
.space 8
.globl init_pt_va
init_pt_va:
.quad pagetable_l2 /* XXX: Keep page tables VA */
#ifndef SMP
ENTRY(mpentry)
1:
wfi
j 1b
END(mpentry)
#else
/*
* mpentry(unsigned long)
*
* Called by a core when it is being brought online.
*/
ENTRY(mpentry)
/*
* Calculate the offset to __riscv_boot_ap
* for the current core, cpuid is in a0.
*/
li t1, 4
mulw t1, t1, a0
/* Get the pointer */
la t0, __riscv_boot_ap
add t0, t0, t1
1:
/* Wait the kernel to be ready */
lw t1, 0(t0)
beqz t1, 1b
/* Setup stack pointer */
la t0, secondary_stacks
li t1, (PAGE_SIZE * KSTACK_PAGES)
mulw t1, t1, s10
add t0, t0, t1
sub t0, t0, s9
li t1, KERNBASE
add sp, t0, t1
/* Setup supervisor trap vector */
la t0, mpva
sub t0, t0, s9
li t1, KERNBASE
add t0, t0, t1
csrw stvec, t0
/* Set page tables base register */
la s2, pagetable_l1
srli s2, s2, PAGE_SHIFT
li t0, SATP_MODE_SV39
or s2, s2, t0
sfence.vma
csrw sptbr, s2
.align 2
mpva:
/* Setup supervisor trap vector */
la t0, cpu_exception_handler
csrw stvec, t0
/* Ensure sscratch is zero */
li t0, 0
csrw sscratch, t0
call init_secondary
END(mpentry)
#endif
Index: head/sys/riscv/riscv/support.S
===================================================================
--- head/sys/riscv/riscv/support.S (revision 338466)
+++ head/sys/riscv/riscv/support.S (revision 338467)
@@ -1,253 +1,275 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin
+ * Copyright (c) 2015-2018 Ruslan Bukin
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
* University of Cambridge Computer Laboratory under DARPA/AFRL contract
* FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
*
* Portions of this software were developed by the University of Cambridge
* Computer Laboratory as part of the CTSRD Project, with support from the
* UK Higher Education Innovation Fund (HEIF).
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include
__FBSDID("$FreeBSD$");
#include
+#include
#include "assym.inc"
/*
* One of the fu* or su* functions failed, return -1.
*/
ENTRY(fsu_fault)
SET_FAULT_HANDLER(x0, a1) /* Reset the handler function */
+ EXIT_USER_ACCESS(a1)
fsu_fault_nopcb:
li a0, -1
ret
END(fsu_fault)
/*
* int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t)
*/
ENTRY(casueword32)
li a4, (VM_MAXUSER_ADDRESS-3)
bgt a0, a4, fsu_fault_nopcb
la a6, fsu_fault /* Load the fault handler */
SET_FAULT_HANDLER(a6, a4) /* And set it */
+ ENTER_USER_ACCESS(a4)
1: lr.w a4, 0(a0) /* Load-exclusive the data */
bne a4, a1, 2f /* If not equal then exit */
sc.w a5, a3, 0(a0) /* Store the new data */
bnez a5, 1b /* Retry on failure */
-2: SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */
+2: EXIT_USER_ACCESS(a5)
+ SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */
sw a4, 0(a2) /* Store the read data */
li a0, 0 /* Success */
ret /* Return */
END(casueword32)
/*
* int casueword(volatile u_long *, u_long, u_long *, u_long)
*/
ENTRY(casueword)
li a4, (VM_MAXUSER_ADDRESS-7)
bgt a0, a4, fsu_fault_nopcb
la a6, fsu_fault /* Load the fault handler */
SET_FAULT_HANDLER(a6, a4) /* And set it */
+ ENTER_USER_ACCESS(a4)
1: lr.d a4, 0(a0) /* Load-exclusive the data */
bne a4, a1, 2f /* If not equal then exit */
sc.d a5, a3, 0(a0) /* Store the new data */
bnez a5, 1b /* Retry on failure */
-2: SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */
+2: EXIT_USER_ACCESS(a5)
+ SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */
sd a4, 0(a2) /* Store the read data */
li a0, 0 /* Success */
ret /* Return */
END(casueword)
/*
* int fubyte(volatile const void *)
*/
ENTRY(fubyte)
li a1, VM_MAXUSER_ADDRESS
bgt a0, a1, fsu_fault_nopcb
la a6, fsu_fault /* Load the fault handler */
SET_FAULT_HANDLER(a6, a1) /* And set it */
+ ENTER_USER_ACCESS(a1)
lb a0, 0(a0) /* Try loading the data */
+ EXIT_USER_ACCESS(a1)
SET_FAULT_HANDLER(x0, a1) /* Reset the fault handler */
ret /* Return */
END(fubyte)
/*
* int fuword(volatile const void *)
*/
ENTRY(fuword16)
li a1, (VM_MAXUSER_ADDRESS-1)
bgt a0, a1, fsu_fault_nopcb
la a6, fsu_fault /* Load the fault handler */
SET_FAULT_HANDLER(a6, a1) /* And set it */
+ ENTER_USER_ACCESS(a1)
lh a0, 0(a0) /* Try loading the data */
+ EXIT_USER_ACCESS(a1)
SET_FAULT_HANDLER(x0, a1) /* Reset the fault handler */
ret /* Return */
END(fuword16)
/*
* int32_t fueword32(volatile const void *, int32_t *)
*/
ENTRY(fueword32)
li a2, (VM_MAXUSER_ADDRESS-3)
bgt a0, a2, fsu_fault_nopcb
la a6, fsu_fault /* Load the fault handler */
SET_FAULT_HANDLER(a6, a2) /* And set it */
+ ENTER_USER_ACCESS(a2)
lw a0, 0(a0) /* Try loading the data */
+ EXIT_USER_ACCESS(a2)
SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */
sw a0, 0(a1) /* Save the data in kernel space */
li a0, 0 /* Success */
ret /* Return */
END(fueword32)
/*
* long fueword(volatile const void *, int64_t *)
* int64_t fueword64(volatile const void *, int64_t *)
*/
ENTRY(fueword)
EENTRY(fueword64)
li a2, (VM_MAXUSER_ADDRESS-7)
bgt a0, a2, fsu_fault_nopcb
la a6, fsu_fault /* Load the fault handler */
SET_FAULT_HANDLER(a6, a2) /* And set it */
+ ENTER_USER_ACCESS(a2)
ld a0, 0(a0) /* Try loading the data */
+ EXIT_USER_ACCESS(a2)
SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */
sd a0, 0(a1) /* Save the data in kernel space */
li a0, 0 /* Success */
ret /* Return */
EEND(fueword64)
END(fueword)
/*
* int subyte(volatile void *, int)
*/
ENTRY(subyte)
li a2, VM_MAXUSER_ADDRESS
bgt a0, a2, fsu_fault_nopcb
la a6, fsu_fault /* Load the fault handler */
SET_FAULT_HANDLER(a6, a2) /* And set it */
+ ENTER_USER_ACCESS(a2)
sb a1, 0(a0) /* Try storing the data */
+ EXIT_USER_ACCESS(a2)
SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */
li a0, 0 /* Success */
ret /* Return */
END(subyte)
/*
* int suword16(volatile void *, int)
*/
ENTRY(suword16)
li a2, (VM_MAXUSER_ADDRESS-1)
bgt a0, a2, fsu_fault_nopcb
la a6, fsu_fault /* Load the fault handler */
SET_FAULT_HANDLER(a6, a2) /* And set it */
+ ENTER_USER_ACCESS(a2)
sh a1, 0(a0) /* Try storing the data */
+ EXIT_USER_ACCESS(a2)
SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */
li a0, 0 /* Success */
ret /* Return */
END(suword16)
/*
* int suword32(volatile void *, int)
*/
ENTRY(suword32)
li a2, (VM_MAXUSER_ADDRESS-3)
bgt a0, a2, fsu_fault_nopcb
la a6, fsu_fault /* Load the fault handler */
SET_FAULT_HANDLER(a6, a2) /* And set it */
+ ENTER_USER_ACCESS(a2)
sw a1, 0(a0) /* Try storing the data */
+ EXIT_USER_ACCESS(a2)
SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */
li a0, 0 /* Success */
ret /* Return */
END(suword32)
/*
* int suword(volatile void *, long)
*/
ENTRY(suword)
EENTRY(suword64)
li a2, (VM_MAXUSER_ADDRESS-7)
bgt a0, a2, fsu_fault_nopcb
la a6, fsu_fault /* Load the fault handler */
SET_FAULT_HANDLER(a6, a2) /* And set it */
+ ENTER_USER_ACCESS(a2)
sd a1, 0(a0) /* Try storing the data */
+ EXIT_USER_ACCESS(a2)
SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */
li a0, 0 /* Success */
ret /* Return */
EEND(suword64)
END(suword)
ENTRY(setjmp)
/* Store the stack pointer */
sd sp, 0(a0)
addi a0, a0, 8
/* Store the general purpose registers and ra */
sd s0, (0 * 8)(a0)
sd s1, (1 * 8)(a0)
sd s2, (2 * 8)(a0)
sd s3, (3 * 8)(a0)
sd s4, (4 * 8)(a0)
sd s5, (5 * 8)(a0)
sd s6, (6 * 8)(a0)
sd s7, (7 * 8)(a0)
sd s8, (8 * 8)(a0)
sd s9, (9 * 8)(a0)
sd s10, (10 * 8)(a0)
sd s11, (11 * 8)(a0)
sd ra, (12 * 8)(a0)
/* Return value */
li a0, 0
ret
END(setjmp)
ENTRY(longjmp)
/* Restore the stack pointer */
ld sp, 0(a0)
addi a0, a0, 8
/* Restore the general purpose registers and ra */
ld s0, (0 * 8)(a0)
ld s1, (1 * 8)(a0)
ld s2, (2 * 8)(a0)
ld s3, (3 * 8)(a0)
ld s4, (4 * 8)(a0)
ld s5, (5 * 8)(a0)
ld s6, (6 * 8)(a0)
ld s7, (7 * 8)(a0)
ld s8, (8 * 8)(a0)
ld s9, (9 * 8)(a0)
ld s10, (10 * 8)(a0)
ld s11, (11 * 8)(a0)
ld ra, (12 * 8)(a0)
/* Load the return value */
mv a0, a1
ret
END(longjmp)
Index: head/sys/riscv/riscv/vm_machdep.c
===================================================================
--- head/sys/riscv/riscv/vm_machdep.c (revision 338466)
+++ head/sys/riscv/riscv/vm_machdep.c (revision 338467)
@@ -1,276 +1,275 @@
/*-
- * Copyright (c) 2015-2017 Ruslan Bukin
+ * Copyright (c) 2015-2018 Ruslan Bukin
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
* University of Cambridge Computer Laboratory under DARPA/AFRL contract
* FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
*
* Portions of this software were developed by the University of Cambridge
* Computer Laboratory as part of the CTSRD Project, with support from the
* UK Higher Education Innovation Fund (HEIF).
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include
__FBSDID("$FreeBSD$");
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#if __riscv_xlen == 64
#define TP_OFFSET 16 /* sizeof(struct tcb) */
#endif
/*
* Finish a fork operation, with process p2 nearly set up.
* Copy and update the pcb, set up the stack so that the child
* ready to run and return to user mode.
*/
void
cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
{
struct pcb *pcb2;
struct trapframe *tf;
register_t val;
if ((flags & RFPROC) == 0)
return;
if (td1 == curthread) {
/*
* Save the tp. These normally happen in cpu_switch,
* but if userland changes this then forks this may
* not have happened.
*/
__asm __volatile("mv %0, tp" : "=&r"(val));
td1->td_pcb->pcb_tp = val;
/* RISCVTODO: save the FPU state here */
}
pcb2 = (struct pcb *)(td2->td_kstack +
td2->td_kstack_pages * PAGE_SIZE) - 1;
td2->td_pcb = pcb2;
bcopy(td1->td_pcb, pcb2, sizeof(*pcb2));
td2->td_pcb->pcb_l1addr =
vtophys(vmspace_pmap(td2->td_proc->p_vmspace)->pm_l1);
tf = (struct trapframe *)STACKALIGN((struct trapframe *)pcb2 - 1);
bcopy(td1->td_frame, tf, sizeof(*tf));
/* Clear syscall error flag */
tf->tf_t[0] = 0;
/* Arguments for child */
tf->tf_a[0] = 0;
tf->tf_a[1] = 0;
tf->tf_sstatus |= (SSTATUS_SPIE); /* Enable interrupts. */
- tf->tf_sstatus |= (SSTATUS_SUM); /* Supervisor can access userspace. */
tf->tf_sstatus &= ~(SSTATUS_SPP); /* User mode. */
td2->td_frame = tf;
/* Set the return value registers for fork() */
td2->td_pcb->pcb_s[0] = (uintptr_t)fork_return;
td2->td_pcb->pcb_s[1] = (uintptr_t)td2;
td2->td_pcb->pcb_ra = (uintptr_t)fork_trampoline;
td2->td_pcb->pcb_sp = (uintptr_t)td2->td_frame;
/* Setup to release spin count in fork_exit(). */
td2->td_md.md_spinlock_count = 1;
td2->td_md.md_saved_sstatus_ie = (SSTATUS_SIE);
}
void
cpu_reset(void)
{
sbi_shutdown();
while(1);
}
void
cpu_thread_swapin(struct thread *td)
{
}
void
cpu_thread_swapout(struct thread *td)
{
}
void
cpu_set_syscall_retval(struct thread *td, int error)
{
struct trapframe *frame;
frame = td->td_frame;
switch (error) {
case 0:
frame->tf_a[0] = td->td_retval[0];
frame->tf_a[1] = td->td_retval[1];
frame->tf_t[0] = 0; /* syscall succeeded */
break;
case ERESTART:
frame->tf_sepc -= 4; /* prev instruction */
break;
case EJUSTRETURN:
break;
default:
frame->tf_a[0] = error;
frame->tf_t[0] = 1; /* syscall error */
break;
}
}
/*
* Initialize machine state, mostly pcb and trap frame for a new
* thread, about to return to userspace. Put enough state in the new
* thread's PCB to get it to go back to the fork_return(), which
* finalizes the thread state and handles peculiarities of the first
* return to userspace for the new thread.
*/
void
cpu_copy_thread(struct thread *td, struct thread *td0)
{
bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe));
bcopy(td0->td_pcb, td->td_pcb, sizeof(struct pcb));
td->td_pcb->pcb_s[0] = (uintptr_t)fork_return;
td->td_pcb->pcb_s[1] = (uintptr_t)td;
td->td_pcb->pcb_ra = (uintptr_t)fork_trampoline;
td->td_pcb->pcb_sp = (uintptr_t)td->td_frame;
/* Setup to release spin count in fork_exit(). */
td->td_md.md_spinlock_count = 1;
td->td_md.md_saved_sstatus_ie = (SSTATUS_SIE);
}
/*
* Set that machine state for performing an upcall that starts
* the entry function with the given argument.
*/
void
cpu_set_upcall(struct thread *td, void (*entry)(void *), void *arg,
stack_t *stack)
{
struct trapframe *tf;
tf = td->td_frame;
tf->tf_sp = STACKALIGN((uintptr_t)stack->ss_sp + stack->ss_size);
tf->tf_sepc = (register_t)entry;
tf->tf_a[0] = (register_t)arg;
}
int
cpu_set_user_tls(struct thread *td, void *tls_base)
{
struct pcb *pcb;
if ((uintptr_t)tls_base >= VM_MAXUSER_ADDRESS)
return (EINVAL);
pcb = td->td_pcb;
pcb->pcb_tp = (register_t)tls_base + TP_OFFSET;
if (td == curthread)
__asm __volatile("mv tp, %0" :: "r"(pcb->pcb_tp));
return (0);
}
void
cpu_thread_exit(struct thread *td)
{
}
void
cpu_thread_alloc(struct thread *td)
{
td->td_pcb = (struct pcb *)(td->td_kstack +
td->td_kstack_pages * PAGE_SIZE) - 1;
td->td_frame = (struct trapframe *)STACKALIGN(
(caddr_t)td->td_pcb - 8 - sizeof(struct trapframe));
}
void
cpu_thread_free(struct thread *td)
{
}
void
cpu_thread_clean(struct thread *td)
{
}
/*
* Intercept the return address from a freshly forked process that has NOT
* been scheduled yet.
*
* This is needed to make kernel threads stay in kernel mode.
*/
void
cpu_fork_kthread_handler(struct thread *td, void (*func)(void *), void *arg)
{
td->td_pcb->pcb_s[0] = (uintptr_t)func;
td->td_pcb->pcb_s[1] = (uintptr_t)arg;
td->td_pcb->pcb_ra = (uintptr_t)fork_trampoline;
td->td_pcb->pcb_sp = (uintptr_t)td->td_frame;
}
void
cpu_exit(struct thread *td)
{
}
void
swi_vm(void *v)
{
/* Nothing to do here - busdma bounce buffers are not implemented. */
}