Index: head/lib/csu/riscv/crt1.c =================================================================== --- head/lib/csu/riscv/crt1.c (revision 322360) +++ head/lib/csu/riscv/crt1.c (revision 322361) @@ -1,89 +1,92 @@ /* LINTLIBRARY */ /*- * Copyright 1996-1998 John D. Polstra. - * Copyright (c) 2015 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include "libc_private.h" #include "crtbrand.c" #include "ignore_init.c" typedef void (*fptr)(void); #ifdef GCRT extern void _mcleanup(void); extern void monstartup(void *, void *); extern int eprol; extern int etext; #endif void __start(int argc, char **argv, char **env, void (*cleanup)(void)); /* The entry function. */ __asm(" .text \n" " .align 0 \n" " .globl _start \n" " _start: \n" " mv a3, a2 \n" /* cleanup */ " addi a1, a0, 8 \n" /* get argv */ " ld a0, 0(a0) \n" /* load argc */ " slli t0, a0, 3 \n" /* mult by arg size */ " add a2, a1, t0 \n" /* env is after argv */ " addi a2, a2, 8 \n" /* argv is null terminated */ -" lla gp, _gp \n" /* load global pointer */ +" .option push \n" +" .option norelax \n" +" lla gp, __global_pointer$\n" +" .option pop \n" " call __start"); void __start(int argc, char **argv, char **env, void (*cleanup)(void)) { handle_argv(argc, argv, env); if (&_DYNAMIC != NULL) atexit(cleanup); else _init_tls(); #ifdef GCRT atexit(_mcleanup); monstartup(&eprol, &etext); __asm__("eprol:"); #endif handle_static_init(argc, argv, env); exit(main(argc, argv, env)); } Index: head/lib/csu/riscv/crti.S =================================================================== --- head/lib/csu/riscv/crti.S (revision 322360) +++ head/lib/csu/riscv/crti.S (revision 322361) @@ -1,58 +1,59 @@ /*- - * Copyright (c) 2015 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); -# this puts _gp into .dynsym, so symlook_obj can now find that (see reloc.c) - .weak _gp -_gp: +# this puts __global_pointer$ into .dynsym, so symlook_obj can now find that +# (see reloc.c) + .weak __global_pointer$ +__global_pointer$: .section .init,"ax",@progbits - .align 2 + .align 0 .globl _init .type _init,@function _init: addi sp, sp, -16 sd ra, 0(sp) .section .fini,"ax",@progbits - .align 2 + .align 0 .globl _fini .type _fini,@function _fini: addi sp, sp, -16 sd ra, 0(sp) .section .note.GNU-stack,"",%progbits Index: head/lib/libc/riscv/gen/fabs.S =================================================================== --- head/lib/libc/riscv/gen/fabs.S (revision 322360) +++ head/lib/libc/riscv/gen/fabs.S (revision 322361) @@ -1,41 +1,43 @@ /*- - * Copyright (c) 2015 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); +#ifndef SOFTFLOAT ENTRY(fabs) fabs.d fa0, fa0 ret END(fabs) +#endif Index: head/libexec/rtld-elf/riscv/reloc.c =================================================================== --- head/libexec/rtld-elf/riscv/reloc.c (revision 322360) +++ head/libexec/rtld-elf/riscv/reloc.c (revision 322361) @@ -1,404 +1,404 @@ /*- - * Copyright (c) 2015 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 * ("CTSRD"), as part of the DARPA CRASH research programme. * * This software was developed by the University of Cambridge Computer * Laboratory as part of the CTSRD Project, with support from the UK Higher * Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include "debug.h" #include "rtld.h" #include "rtld_printf.h" /* * It is possible for the compiler to emit relocations for unaligned data. * We handle this situation with these inlines. */ #define RELOC_ALIGNED_P(x) \ (((uintptr_t)(x) & (sizeof(void *) - 1)) == 0) void _exit(int); uint64_t set_gp(Obj_Entry *obj) { uint64_t old; SymLook req; uint64_t gp; int res; __asm __volatile("mv %0, gp" : "=r"(old)); - symlook_init(&req, "_gp"); + symlook_init(&req, "__global_pointer$"); req.ventry = NULL; req.flags = SYMLOOK_EARLY; res = symlook_obj(&req, obj); if (res == 0) { gp = req.sym_out->st_value; __asm __volatile("mv gp, %0" :: "r"(gp)); } return (old); } void init_pltgot(Obj_Entry *obj) { if (obj->pltgot != NULL) { obj->pltgot[0] = (Elf_Addr)&_rtld_bind_start; obj->pltgot[1] = (Elf_Addr)obj; } } int do_copy_relocations(Obj_Entry *dstobj) { const Obj_Entry *srcobj, *defobj; const Elf_Rela *relalim; const Elf_Rela *rela; const Elf_Sym *srcsym; const Elf_Sym *dstsym; const void *srcaddr; const char *name; void *dstaddr; SymLook req; size_t size; int res; /* * COPY relocs are invalid outside of the main program */ assert(dstobj->mainprog); relalim = (const Elf_Rela *)((char *)dstobj->rela + dstobj->relasize); for (rela = dstobj->rela; rela < relalim; rela++) { if (ELF_R_TYPE(rela->r_info) != R_RISCV_COPY) continue; dstaddr = (void *)(dstobj->relocbase + rela->r_offset); dstsym = dstobj->symtab + ELF_R_SYM(rela->r_info); name = dstobj->strtab + dstsym->st_name; size = dstsym->st_size; symlook_init(&req, name); req.ventry = fetch_ventry(dstobj, ELF_R_SYM(rela->r_info)); req.flags = SYMLOOK_EARLY; for (srcobj = globallist_next(dstobj); srcobj != NULL; srcobj = globallist_next(srcobj)) { res = symlook_obj(&req, srcobj); if (res == 0) { srcsym = req.sym_out; defobj = req.defobj_out; break; } } if (srcobj == NULL) { _rtld_error( "Undefined symbol \"%s\" referenced from COPY relocation in %s", name, dstobj->path); return (-1); } srcaddr = (const void *)(defobj->relocbase + srcsym->st_value); memcpy(dstaddr, srcaddr, size); } return (0); } /* * Process the PLT relocations. */ int reloc_plt(Obj_Entry *obj) { const Elf_Rela *relalim; const Elf_Rela *rela; relalim = (const Elf_Rela *)((char *)obj->pltrela + obj->pltrelasize); for (rela = obj->pltrela; rela < relalim; rela++) { Elf_Addr *where; assert(ELF_R_TYPE(rela->r_info) == R_RISCV_JUMP_SLOT); where = (Elf_Addr *)(obj->relocbase + rela->r_offset); *where += (Elf_Addr)obj->relocbase; } return (0); } /* * LD_BIND_NOW was set - force relocation for all jump slots */ int reloc_jmpslots(Obj_Entry *obj, int flags, RtldLockState *lockstate) { const Obj_Entry *defobj; const Elf_Rela *relalim; const Elf_Rela *rela; const Elf_Sym *def; relalim = (const Elf_Rela *)((char *)obj->pltrela + obj->pltrelasize); for (rela = obj->pltrela; rela < relalim; rela++) { Elf_Addr *where; where = (Elf_Addr *)(obj->relocbase + rela->r_offset); switch(ELF_R_TYPE(rela->r_info)) { case R_RISCV_JUMP_SLOT: def = find_symdef(ELF_R_SYM(rela->r_info), obj, &defobj, SYMLOOK_IN_PLT | flags, NULL, lockstate); if (def == NULL) { dbg("reloc_jmpslots: sym not found"); return (-1); } *where = (Elf_Addr)(defobj->relocbase + def->st_value); break; default: _rtld_error("Unknown relocation type %x in jmpslot", (unsigned int)ELF_R_TYPE(rela->r_info)); return (-1); } } return (0); } int reloc_iresolve(Obj_Entry *obj, struct Struct_RtldLockState *lockstate) { /* XXX not implemented */ return (0); } int reloc_gnu_ifunc(Obj_Entry *obj, int flags, struct Struct_RtldLockState *lockstate) { /* XXX not implemented */ return (0); } Elf_Addr reloc_jmpslot(Elf_Addr *where, Elf_Addr target, const Obj_Entry *defobj, const Obj_Entry *obj, const Elf_Rel *rel) { assert(ELF_R_TYPE(rel->r_info) == R_RISCV_JUMP_SLOT); if (*where != target && !ld_bind_not) *where = target; return (target); } /* * Process non-PLT relocations */ int reloc_non_plt(Obj_Entry *obj, Obj_Entry *obj_rtld, int flags, RtldLockState *lockstate) { const Obj_Entry *defobj; const Elf_Rela *relalim; const Elf_Rela *rela; const Elf_Sym *def; SymCache *cache; Elf_Addr *where; unsigned long symnum; if ((flags & SYMLOOK_IFUNC) != 0) /* XXX not implemented */ return (0); /* * The dynamic loader may be called from a thread, we have * limited amounts of stack available so we cannot use alloca(). */ if (obj == obj_rtld) cache = NULL; else cache = calloc(obj->dynsymcount, sizeof(SymCache)); /* No need to check for NULL here */ relalim = (const Elf_Rela *)((caddr_t)obj->rela + obj->relasize); for (rela = obj->rela; rela < relalim; rela++) { where = (Elf_Addr *)(obj->relocbase + rela->r_offset); symnum = ELF_R_SYM(rela->r_info); switch (ELF_R_TYPE(rela->r_info)) { case R_RISCV_JUMP_SLOT: /* This will be handled by the plt/jmpslot routines */ break; case R_RISCV_NONE: break; case R_RISCV_64: def = find_symdef(symnum, obj, &defobj, flags, cache, lockstate); if (def == NULL) return (-1); *where = (Elf_Addr)(defobj->relocbase + def->st_value + rela->r_addend); break; case R_RISCV_TLS_DTPMOD64: def = find_symdef(symnum, obj, &defobj, flags, cache, lockstate); if (def == NULL) return -1; *where += (Elf_Addr)defobj->tlsindex; break; case R_RISCV_COPY: /* * These are deferred until all other relocations have * been done. All we do here is make sure that the * COPY relocation is not in a shared library. They * are allowed only in executable files. */ if (!obj->mainprog) { _rtld_error("%s: Unexpected R_RISCV_COPY " "relocation in shared library", obj->path); return (-1); } break; case R_RISCV_TLS_DTPREL64: def = find_symdef(symnum, obj, &defobj, flags, cache, lockstate); if (def == NULL) return (-1); /* * We lazily allocate offsets for static TLS as we * see the first relocation that references the * TLS block. This allows us to support (small * amounts of) static TLS in dynamically loaded * modules. If we run out of space, we generate an * error. */ if (!defobj->tls_done) { if (!allocate_tls_offset((Obj_Entry*) defobj)) { _rtld_error( "%s: No space available for static " "Thread Local Storage", obj->path); return (-1); } } *where += (Elf_Addr)(def->st_value + rela->r_addend - TLS_DTV_OFFSET); break; case R_RISCV_TLS_TPREL64: def = find_symdef(symnum, obj, &defobj, flags, cache, lockstate); if (def == NULL) return (-1); /* * We lazily allocate offsets for static TLS as we * see the first relocation that references the * TLS block. This allows us to support (small * amounts of) static TLS in dynamically loaded * modules. If we run out of space, we generate an * error. */ if (!defobj->tls_done) { if (!allocate_tls_offset((Obj_Entry*) defobj)) { _rtld_error( "%s: No space available for static " "Thread Local Storage", obj->path); return (-1); } } *where = (def->st_value + rela->r_addend + defobj->tlsoffset - TLS_TP_OFFSET); break; case R_RISCV_RELATIVE: *where = (Elf_Addr)(obj->relocbase + rela->r_addend); break; default: rtld_printf("%s: Unhandled relocation %lu\n", obj->path, ELF_R_TYPE(rela->r_info)); return (-1); } } return (0); } void ifunc_init(Elf_Auxinfo aux_info[__min_size(AT_COUNT)] __unused) { } void allocate_initial_tls(Obj_Entry *objs) { Elf_Addr **tp; /* * Fix the size of the static TLS block by using the maximum * offset allocated so far and adding a bit for dynamic modules to * use. */ tls_static_space = tls_last_offset + tls_last_size + RTLD_STATIC_TLS_EXTRA; tp = (Elf_Addr **) ((char *)allocate_tls(objs, NULL, TLS_TCB_SIZE, 16) + TLS_TP_OFFSET + TLS_TCB_SIZE); __asm __volatile("mv tp, %0" :: "r"(tp)); } void * __tls_get_addr(tls_index* ti) { char *_tp; void *p; __asm __volatile("mv %0, tp" : "=r" (_tp)); p = tls_get_addr_common((Elf_Addr**)((Elf_Addr)_tp - TLS_TP_OFFSET - TLS_TCB_SIZE), ti->ti_module, ti->ti_offset); return (p + TLS_DTV_OFFSET); } Index: head/share/mk/bsd.cpu.mk =================================================================== --- head/share/mk/bsd.cpu.mk (revision 322360) +++ head/share/mk/bsd.cpu.mk (revision 322361) @@ -1,402 +1,405 @@ # $FreeBSD$ # Set default CPU compile flags and baseline CPUTYPE for each arch. The # compile flags must support the minimum CPU type for each architecture but # may tune support for more advanced processors. .if !defined(CPUTYPE) || empty(CPUTYPE) _CPUCFLAGS = . if ${MACHINE_CPUARCH} == "aarch64" MACHINE_CPU = arm64 . elif ${MACHINE_CPUARCH} == "amd64" MACHINE_CPU = amd64 sse2 sse mmx . elif ${MACHINE_CPUARCH} == "arm" MACHINE_CPU = arm . elif ${MACHINE_CPUARCH} == "i386" MACHINE_CPU = i486 . elif ${MACHINE_CPUARCH} == "mips" MACHINE_CPU = mips . elif ${MACHINE_CPUARCH} == "powerpc" MACHINE_CPU = aim . elif ${MACHINE_CPUARCH} == "riscv" MACHINE_CPU = riscv . elif ${MACHINE_CPUARCH} == "sparc64" MACHINE_CPU = ultrasparc . endif .else # Handle aliases (not documented in make.conf to avoid user confusion # between e.g. i586 and pentium) . if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386" . if ${CPUTYPE} == "barcelona" CPUTYPE = amdfam10 . elif ${CPUTYPE} == "core-avx2" CPUTYPE = haswell . elif ${CPUTYPE} == "core-avx-i" CPUTYPE = ivybridge . elif ${CPUTYPE} == "corei7-avx" CPUTYPE = sandybridge . elif ${CPUTYPE} == "corei7" CPUTYPE = nehalem . elif ${CPUTYPE} == "slm" CPUTYPE = silvermont . elif ${CPUTYPE} == "atom" CPUTYPE = bonnell . elif ${CPUTYPE} == "core" CPUTYPE = prescott . endif . if ${MACHINE_CPUARCH} == "amd64" . if ${CPUTYPE} == "prescott" CPUTYPE = nocona . endif . else . if ${CPUTYPE} == "k7" CPUTYPE = athlon . elif ${CPUTYPE} == "p4" CPUTYPE = pentium4 . elif ${CPUTYPE} == "p4m" CPUTYPE = pentium4m . elif ${CPUTYPE} == "p3" CPUTYPE = pentium3 . elif ${CPUTYPE} == "p3m" CPUTYPE = pentium3m . elif ${CPUTYPE} == "p-m" CPUTYPE = pentium-m . elif ${CPUTYPE} == "p2" CPUTYPE = pentium2 . elif ${CPUTYPE} == "i686" CPUTYPE = pentiumpro . elif ${CPUTYPE} == "i586/mmx" CPUTYPE = pentium-mmx . elif ${CPUTYPE} == "i586" CPUTYPE = pentium . endif . endif . elif ${MACHINE_ARCH} == "sparc64" . if ${CPUTYPE} == "us" CPUTYPE = ultrasparc . elif ${CPUTYPE} == "us3" CPUTYPE = ultrasparc3 . endif . endif ############################################################################### # Logic to set up correct gcc optimization flag. This must be included # after /etc/make.conf so it can react to the local value of CPUTYPE # defined therein. Consult: # http://gcc.gnu.org/onlinedocs/gcc/ARM-Options.html # http://gcc.gnu.org/onlinedocs/gcc/RS-6000-and-PowerPC-Options.html # http://gcc.gnu.org/onlinedocs/gcc/MIPS-Options.html # http://gcc.gnu.org/onlinedocs/gcc/SPARC-Options.html # http://gcc.gnu.org/onlinedocs/gcc/i386-and-x86_002d64-Options.html . if ${MACHINE_CPUARCH} == "i386" . if ${CPUTYPE} == "crusoe" _CPUCFLAGS = -march=i686 -falign-functions=0 -falign-jumps=0 -falign-loops=0 . elif ${CPUTYPE} == "k5" _CPUCFLAGS = -march=pentium . elif ${CPUTYPE} == "c7" _CPUCFLAGS = -march=c3-2 . else _CPUCFLAGS = -march=${CPUTYPE} . endif . elif ${MACHINE_CPUARCH} == "amd64" _CPUCFLAGS = -march=${CPUTYPE} . elif ${MACHINE_CPUARCH} == "arm" . if ${CPUTYPE} == "xscale" #XXX: gcc doesn't seem to like -mcpu=xscale, and dies while rebuilding itself #_CPUCFLAGS = -mcpu=xscale _CPUCFLAGS = -march=armv5te -D__XSCALE__ . elif ${CPUTYPE:M*soft*} != "" _CPUCFLAGS = -mfloat-abi=softfp . elif ${CPUTYPE} == "armv6" # Not sure we still need ARM_ARCH_6=1 here. _CPUCFLAGS = -march=${CPUTYPE} -DARM_ARCH_6=1 . elif ${CPUTYPE} == "cortexa" _CPUCFLAGS = -march=armv7 -DARM_ARCH_6=1 -mfpu=vfp . elif ${CPUTYPE:Marmv[4567]*} != "" # Handle all the armvX types that FreeBSD runs: # armv4, armv4t, armv5, armv5te, armv6, armv6t2, armv7, armv7-a, armv7ve # they require -march=. All the others require -mcpu=. _CPUCFLAGS = -march=${CPUTYPE} . else # Common values for FreeBSD # arm: (any arm v4 or v5 processor you are targeting) # arm920t, arm926ej-s, marvell-pj4, fa526, fa626, # fa606te, fa626te, fa726te # armv6: (any arm v7 or v8 processor you are targeting and the arm1176jzf-s) # arm1176jzf-s, generic-armv7-a, cortex-a5, cortex-a7, cortex-a8, # cortex-a9, cortex-a12, cortex-a15, cortex-a17, cortex-a53, cortex-a57, # cortex-a72, exynos-m1 _CPUCFLAGS = -mcpu=${CPUTYPE} . endif . elif ${MACHINE_ARCH} == "powerpc" . if ${CPUTYPE} == "e500" _CPUCFLAGS = -Wa,-me500 -msoft-float . else _CPUCFLAGS = -mcpu=${CPUTYPE} -mno-powerpc64 . endif . elif ${MACHINE_ARCH} == "powerpcspe" _CPUCFLAGS = -Wa,-me500 -mspe=yes -mabi=spe -mfloat-gprs=double . elif ${MACHINE_ARCH} == "powerpc64" _CPUCFLAGS = -mcpu=${CPUTYPE} . elif ${MACHINE_CPUARCH} == "mips" # mips[1234], mips32, mips64, and all later releases need to have mips # preserved (releases later than r2 require external toolchain) . if ${CPUTYPE:Mmips32*} != "" || ${CPUTYPE:Mmips64*} != "" || \ ${CPUTYPE:Mmips[1234]} != "" _CPUCFLAGS = -march=${CPUTYPE} . else # Default -march to the CPUTYPE passed in, with mips stripped off so we # accept either mips4kc or 4kc, mostly for historical reasons # Typical values for cores: # 4kc, 24kc, 34kc, 74kc, 1004kc, octeon, octeon+, octeon2, octeon3, # sb1, xlp, xlr _CPUCFLAGS = -march=${CPUTYPE:S/^mips//} . endif . elif ${MACHINE_ARCH} == "sparc64" . if ${CPUTYPE} == "v9" _CPUCFLAGS = -mcpu=v9 . elif ${CPUTYPE} == "ultrasparc" _CPUCFLAGS = -mcpu=ultrasparc . elif ${CPUTYPE} == "ultrasparc3" _CPUCFLAGS = -mcpu=ultrasparc3 . endif . elif ${MACHINE_CPUARCH} == "aarch64" _CPUCFLAGS = -mcpu=${CPUTYPE} . endif # Set up the list of CPU features based on the CPU type. This is an # unordered list to make it easy for client makefiles to test for the # presence of a CPU feature. ########## i386 . if ${MACHINE_CPUARCH} == "i386" . if ${CPUTYPE} == "znver1" MACHINE_CPU = avx2 avx sse42 sse41 ssse3 sse4a sse3 sse2 sse mmx k6 k5 i586 . elif ${CPUTYPE} == "bdver4" MACHINE_CPU = xop avx2 avx sse42 sse41 ssse3 sse4a sse3 sse2 sse mmx k6 k5 i586 . elif ${CPUTYPE} == "bdver3" || ${CPUTYPE} == "bdver2" || \ ${CPUTYPE} == "bdver1" MACHINE_CPU = xop avx sse42 sse41 ssse3 sse4a sse3 sse2 sse mmx k6 k5 i586 . elif ${CPUTYPE} == "btver2" MACHINE_CPU = avx sse42 sse41 ssse3 sse4a sse3 sse2 sse mmx k6 k5 i586 . elif ${CPUTYPE} == "btver1" MACHINE_CPU = ssse3 sse4a sse3 sse2 sse mmx k6 k5 i586 . elif ${CPUTYPE} == "amdfam10" MACHINE_CPU = athlon-xp athlon k7 3dnow sse4a sse3 sse2 sse mmx k6 k5 i586 . elif ${CPUTYPE} == "opteron-sse3" || ${CPUTYPE} == "athlon64-sse3" MACHINE_CPU = athlon-xp athlon k7 3dnow sse3 sse2 sse mmx k6 k5 i586 . elif ${CPUTYPE} == "opteron" || ${CPUTYPE} == "athlon64" || \ ${CPUTYPE} == "athlon-fx" MACHINE_CPU = athlon-xp athlon k7 3dnow sse2 sse mmx k6 k5 i586 . elif ${CPUTYPE} == "athlon-mp" || ${CPUTYPE} == "athlon-xp" || \ ${CPUTYPE} == "athlon-4" MACHINE_CPU = athlon-xp athlon k7 3dnow sse mmx k6 k5 i586 . elif ${CPUTYPE} == "athlon" || ${CPUTYPE} == "athlon-tbird" MACHINE_CPU = athlon k7 3dnow mmx k6 k5 i586 . elif ${CPUTYPE} == "k6-3" || ${CPUTYPE} == "k6-2" || ${CPUTYPE} == "geode" MACHINE_CPU = 3dnow mmx k6 k5 i586 . elif ${CPUTYPE} == "k6" MACHINE_CPU = mmx k6 k5 i586 . elif ${CPUTYPE} == "k5" MACHINE_CPU = k5 i586 . elif ${CPUTYPE} == "skylake" || ${CPUTYPE} == "knl" MACHINE_CPU = avx512 avx2 avx sse42 sse41 ssse3 sse3 sse2 sse i686 mmx i586 . elif ${CPUTYPE} == "broadwell" || ${CPUTYPE} == "haswell" MACHINE_CPU = avx2 avx sse42 sse41 ssse3 sse3 sse2 sse i686 mmx i586 . elif ${CPUTYPE} == "ivybridge" || ${CPUTYPE} == "sandybridge" MACHINE_CPU = avx sse42 sse41 ssse3 sse3 sse2 sse i686 mmx i586 . elif ${CPUTYPE} == "westmere" || ${CPUTYPE} == "nehalem" || \ ${CPUTYPE} == "silvermont" MACHINE_CPU = sse42 sse41 ssse3 sse3 sse2 sse i686 mmx i586 . elif ${CPUTYPE} == "penryn" MACHINE_CPU = sse41 ssse3 sse3 sse2 sse i686 mmx i586 . elif ${CPUTYPE} == "core2" || ${CPUTYPE} == "bonnell" MACHINE_CPU = ssse3 sse3 sse2 sse i686 mmx i586 . elif ${CPUTYPE} == "yonah" || ${CPUTYPE} == "prescott" MACHINE_CPU = sse3 sse2 sse i686 mmx i586 . elif ${CPUTYPE} == "pentium4" || ${CPUTYPE} == "pentium4m" || \ ${CPUTYPE} == "pentium-m" MACHINE_CPU = sse2 sse i686 mmx i586 . elif ${CPUTYPE} == "pentium3" || ${CPUTYPE} == "pentium3m" MACHINE_CPU = sse i686 mmx i586 . elif ${CPUTYPE} == "pentium2" MACHINE_CPU = i686 mmx i586 . elif ${CPUTYPE} == "pentiumpro" MACHINE_CPU = i686 i586 . elif ${CPUTYPE} == "pentium-mmx" MACHINE_CPU = mmx i586 . elif ${CPUTYPE} == "pentium" MACHINE_CPU = i586 . elif ${CPUTYPE} == "c7" MACHINE_CPU = sse3 sse2 sse i686 mmx i586 . elif ${CPUTYPE} == "c3-2" MACHINE_CPU = sse i686 mmx i586 . elif ${CPUTYPE} == "c3" MACHINE_CPU = 3dnow mmx i586 . elif ${CPUTYPE} == "winchip2" MACHINE_CPU = 3dnow mmx . elif ${CPUTYPE} == "winchip-c6" MACHINE_CPU = mmx . endif MACHINE_CPU += i486 ########## amd64 . elif ${MACHINE_CPUARCH} == "amd64" . if ${CPUTYPE} == "znver1" MACHINE_CPU = avx2 avx sse42 sse41 ssse3 sse4a sse3 . elif ${CPUTYPE} == "bdver4" MACHINE_CPU = xop avx2 avx sse42 sse41 ssse3 sse4a sse3 . elif ${CPUTYPE} == "bdver3" || ${CPUTYPE} == "bdver2" || \ ${CPUTYPE} == "bdver1" MACHINE_CPU = xop avx sse42 sse41 ssse3 sse4a sse3 . elif ${CPUTYPE} == "btver2" MACHINE_CPU = avx sse42 sse41 ssse3 sse4a sse3 . elif ${CPUTYPE} == "btver1" MACHINE_CPU = ssse3 sse4a sse3 . elif ${CPUTYPE} == "amdfam10" MACHINE_CPU = k8 3dnow sse4a sse3 . elif ${CPUTYPE} == "opteron-sse3" || ${CPUTYPE} == "athlon64-sse3" || \ ${CPUTYPE} == "k8-sse3" MACHINE_CPU = k8 3dnow sse3 . elif ${CPUTYPE} == "opteron" || ${CPUTYPE} == "athlon64" || \ ${CPUTYPE} == "athlon-fx" || ${CPUTYPE} == "k8" MACHINE_CPU = k8 3dnow . elif ${CPUTYPE} == "skylake" || ${CPUTYPE} == "knl" MACHINE_CPU = avx512 avx2 avx sse42 sse41 ssse3 sse3 . elif ${CPUTYPE} == "broadwell" || ${CPUTYPE} == "haswell" MACHINE_CPU = avx2 avx sse42 sse41 ssse3 sse3 . elif ${CPUTYPE} == "ivybridge" || ${CPUTYPE} == "sandybridge" MACHINE_CPU = avx sse42 sse41 ssse3 sse3 . elif ${CPUTYPE} == "westmere" || ${CPUTYPE} == "nehalem" || \ ${CPUTYPE} == "silvermont" MACHINE_CPU = sse42 sse41 ssse3 sse3 . elif ${CPUTYPE} == "penryn" MACHINE_CPU = sse41 ssse3 sse3 . elif ${CPUTYPE} == "core2" || ${CPUTYPE} == "bonnell" MACHINE_CPU = ssse3 sse3 . elif ${CPUTYPE} == "nocona" MACHINE_CPU = sse3 . endif MACHINE_CPU += amd64 sse2 sse mmx ########## Mips . elif ${MACHINE_CPUARCH} == "mips" MACHINE_CPU = mips ########## powerpc . elif ${MACHINE_ARCH} == "powerpc" . if ${CPUTYPE} == "e500" MACHINE_CPU = booke softfp . endif ########## riscv . elif ${MACHINE_CPUARCH} == "riscv" MACHINE_CPU = riscv ########## sparc64 . elif ${MACHINE_ARCH} == "sparc64" . if ${CPUTYPE} == "v9" MACHINE_CPU = v9 . elif ${CPUTYPE} == "ultrasparc" MACHINE_CPU = v9 ultrasparc . elif ${CPUTYPE} == "ultrasparc3" MACHINE_CPU = v9 ultrasparc ultrasparc3 . endif . endif .endif .if ${MACHINE_CPUARCH} == "mips" CFLAGS += -G0 . if ${MACHINE_ARCH:Mmips*el*} != "" AFLAGS += -EL CFLAGS += -EL LDFLAGS += -EL . else AFLAGS += -EB CFLAGS += -EB LDFLAGS += -EB . endif . if ${MACHINE_ARCH:Mmips64*} != "" AFLAGS+= -mabi=64 CFLAGS+= -mabi=64 LDFLAGS+= -mabi=64 . elif ${MACHINE_ARCH:Mmipsn32*} != "" AFLAGS+= -mabi=n32 CFLAGS+= -mabi=n32 LDFLAGS+= -mabi=n32 . else AFLAGS+= -mabi=32 CFLAGS+= -mabi=32 LDFLAGS+= -mabi=32 . endif . if ${MACHINE_ARCH:Mmips*hf} CFLAGS += -mhard-float . else CFLAGS += -msoft-float . endif .endif ########## arm .if ${MACHINE_CPUARCH} == "arm" MACHINE_CPU += arm . if ${MACHINE_ARCH:Marmv6*} != "" MACHINE_CPU += armv6 . endif # armv6 is a hybrid. It can use the softfp ABI, but doesn't emulate # floating point in the general case, so don't define softfp for # it at this time. arm and armeb are pure softfp, so define it # for them. . if ${MACHINE_ARCH:Marmv6*} == "" MACHINE_CPU += softfp . endif # Normally armv6 is hard float ABI from FreeBSD 11 onwards. However # when CPUTYPE has 'soft' in it, we use the soft-float ABI to allow # building of soft-float ABI libraries. In this case, we have to # add the -mfloat-abi=softfp to force that. .if ${MACHINE_ARCH:Marmv6*} && defined(CPUTYPE) && ${CPUTYPE:M*soft*} != "" # Needs to be CFLAGS not _CPUCFLAGS because it's needed for the ABI # not a nice optimization. CFLAGS += -mfloat-abi=softfp .endif .endif .if ${MACHINE_ARCH} == "powerpcspe" CFLAGS += -mcpu=8540 -Wa,-me500 -mspe=yes -mabi=spe -mfloat-gprs=double .endif .if ${MACHINE_CPUARCH} == "riscv" .if ${TARGET_ARCH:Mriscv*sf} -CFLAGS += -mno-float -ACFLAGS += -mno-float +CFLAGS += -march=rv64imac -mabi=lp64 +ACFLAGS += -march=rv64imac -mabi=lp64 +.else +CFLAGS += -march=rv64imafdc -mabi=lp64 +ACFLAGS += -march=rv64imafdc -mabi=lp64 .endif .endif # NB: COPTFLAGS is handled in /usr/src/sys/conf/kern.pre.mk .if !defined(NO_CPU_CFLAGS) CFLAGS += ${_CPUCFLAGS} .endif # # Prohibit the compiler from emitting SIMD instructions. # These flags are added to CFLAGS in areas where the extra context-switch # cost outweighs the advantages of SIMD instructions. # # gcc: # Setting -mno-mmx implies -mno-3dnow # Setting -mno-sse implies -mno-sse2, -mno-sse3, -mno-ssse3 and -mfpmath=387 # # clang: # Setting -mno-mmx implies -mno-3dnow and -mno-3dnowa # Setting -mno-sse implies -mno-sse2, -mno-sse3, -mno-ssse3, -mno-sse41 and # -mno-sse42 # (-mfpmath= is not supported) # .if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64" CFLAGS_NO_SIMD.clang= -mno-avx CFLAGS_NO_SIMD= -mno-mmx -mno-sse .endif CFLAGS_NO_SIMD += ${CFLAGS_NO_SIMD.${COMPILER_TYPE}} # Add in any architecture-specific CFLAGS. # These come from make.conf or the command line or the environment. CFLAGS += ${CFLAGS.${MACHINE_ARCH}} CXXFLAGS += ${CXXFLAGS.${MACHINE_ARCH}} Index: head/share/mk/bsd.stand.mk =================================================================== --- head/share/mk/bsd.stand.mk (revision 322360) +++ head/share/mk/bsd.stand.mk (revision 322361) @@ -1,29 +1,29 @@ # $FreeBSD$ # # Common definitons for programs building in the stand-alone environment # and/or using libstand. # CFLAGS+= -ffreestanding -Wformat CFLAGS+= ${CFLAGS_NO_SIMD} -D_STANDALONE .if ${MACHINE_CPUARCH} == "riscv" -CFLAGS+= -mno-float +CFLAGS+= -march=rv64imac -mabi=lp64 .elif ${MACHINE_CPUARCH} != "aarch64" CFLAGS+= -msoft-float .endif .if ${MACHINE_CPUARCH} == "i386" CFLAGS.gcc+= -mpreferred-stack-boundary=2 .endif .if ${MACHINE_CPUARCH} == "amd64" CFLAGS+= -fPIC -mno-red-zone .endif .if ${MACHINE_CPUARCH} == "aarch64" CFLAGS+= -fPIC -mgeneral-regs-only .endif .if ${MACHINE_CPUARCH} == "arm" CFLAGS+= -fPIC .endif .if ${MACHINE_CPUARCH} == "mips" CFLAGS+= -G0 -fno-pic -mno-abicalls .endif Index: head/sys/boot/fdt/dts/riscv/spike.dts =================================================================== --- head/sys/boot/fdt/dts/riscv/spike.dts (revision 322360) +++ head/sys/boot/fdt/dts/riscv/spike.dts (nonexistent) @@ -1,111 +0,0 @@ -/*- - * Copyright (c) 2015-2016 Ruslan Bukin - * All rights reserved. - * - * Portions of this software were developed by SRI International and the - * University of Cambridge Computer Laboratory under DARPA/AFRL contract - * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. - * - * Portions of this software were developed by the University of Cambridge - * Computer Laboratory as part of the CTSRD Project, with support from the - * UK Higher Education Innovation Fund (HEIF). - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -/dts-v1/; - -/ { - model = "UC Berkeley Spike Simulator RV64"; - compatible = "riscv,rv64"; - #address-cells = <1>; - #size-cells = <1>; - #interrupt-cells = <1>; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - cpu@0 { - device_type = "cpu"; - compatible = "riscv,rv64"; - reg = <0x0>; - }; - - cpu@1 { - device_type = "cpu"; - compatible = "riscv,rv64"; - reg = <0x0>; - }; - }; - - aliases { - console0 = &console0; - }; - - memory { - /* - * This is not used currently. - * We take information from sbi_query_memory. - */ - device_type = "memory"; - reg = <0x80000000 0x40000000>; /* 1GB at 0x80000000 */ - }; - - soc { - #address-cells = <1>; - #size-cells = <1>; - #interrupt-cells = <1>; - - compatible = "simple-bus"; - ranges; - - pic0: pic@0 { - compatible = "riscv,pic"; - interrupt-controller; - }; - - timer0: timer@0 { - compatible = "riscv,timer"; - reg = < 0x40000000 0x0008 >, /* rtc */ - < 0x40000008 0x1000 >; /* timecmp */ - interrupts = < 5 >; - interrupt-parent = < &pic0 >; - clock-frequency = < 1000000 >; - }; - - console0: console@0 { - compatible = "riscv,console"; - status = "okay"; - interrupts = < 1 >; - interrupt-parent = < &pic0 >; - }; - }; - - chosen { - bootargs = "-v"; - stdin = "console0"; - stdout = "console0"; - }; -}; Property changes on: head/sys/boot/fdt/dts/riscv/spike.dts ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/sys/boot/fdt/dts/riscv/rocket.dts =================================================================== --- head/sys/boot/fdt/dts/riscv/rocket.dts (revision 322360) +++ head/sys/boot/fdt/dts/riscv/rocket.dts (nonexistent) @@ -1,105 +0,0 @@ -/*- - * Copyright (c) 2015-2016 Ruslan Bukin - * All rights reserved. - * - * Portions of this software were developed by SRI International and the - * University of Cambridge Computer Laboratory under DARPA/AFRL contract - * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. - * - * Portions of this software were developed by the University of Cambridge - * Computer Laboratory as part of the CTSRD Project, with support from the - * UK Higher Education Innovation Fund (HEIF). - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -/dts-v1/; - -/ { - model = "RocketChip RV64"; - compatible = "riscv,rv64"; - #address-cells = <1>; - #size-cells = <1>; - #interrupt-cells = <1>; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - cpu@0 { - device_type = "cpu"; - compatible = "riscv,rv64"; - reg = <0x0>; - }; - }; - - aliases { - console0 = &console0; - }; - - memory { - /* - * This is not used currently. - * We take information from sbi_query_memory. - */ - device_type = "memory"; - reg = <0x80000000 0x10000000>; /* 256MB at 0x80000000 */ - }; - - soc { - #address-cells = <1>; - #size-cells = <1>; - #interrupt-cells = <1>; - - compatible = "simple-bus"; - ranges; - - pic0: pic@0 { - compatible = "riscv,pic"; - interrupt-controller; - }; - - timer0: timer@0 { - compatible = "riscv,timer"; - reg = < 0x4400bff8 0x0008 >, /* rtc */ - < 0x44004000 0x1000 >; /* timecmp */ - interrupts = < 5 >; - interrupt-parent = < &pic0 >; - clock-frequency = < 1000000 >; - }; - - console0: console@0 { - compatible = "riscv,console"; - status = "okay"; - interrupts = < 1 >; - interrupt-parent = < &pic0 >; - }; - }; - - chosen { - bootargs = "-v"; - stdin = "console0"; - stdout = "console0"; - }; -}; Property changes on: head/sys/boot/fdt/dts/riscv/rocket.dts ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/sys/boot/fdt/dts/riscv/lowrisc.dts =================================================================== --- head/sys/boot/fdt/dts/riscv/lowrisc.dts (revision 322360) +++ head/sys/boot/fdt/dts/riscv/lowrisc.dts (nonexistent) @@ -1,108 +0,0 @@ -/*- - * Copyright (c) 2016 Ruslan Bukin - * All rights reserved. - * - * Portions of this software were developed by SRI International and the - * University of Cambridge Computer Laboratory under DARPA/AFRL contract - * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. - * - * Portions of this software were developed by the University of Cambridge - * Computer Laboratory as part of the CTSRD Project, with support from the - * UK Higher Education Innovation Fund (HEIF). - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -/dts-v1/; - -/ { - model = "UC Berkeley Spike Simulator RV64I"; - compatible = "riscv,rv64i"; - #address-cells = <1>; - #size-cells = <1>; - #interrupt-cells = <1>; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - cpu@0 { - device_type = "cpu"; - compatible = "riscv,rv64i"; - reg = <0x40002000>; - }; - }; - - aliases { - serial0 = &serial0; - }; - - memory { - device_type = "memory"; - reg = <0x0 0x8000000>; /* 128MB at 0x0 */ - }; - - soc { - #address-cells = <1>; - #size-cells = <1>; - #interrupt-cells = <1>; - - compatible = "simple-bus"; - ranges; - - pic0: pic@0 { - compatible = "riscv,pic"; - interrupt-controller; - }; - - serial0: serial@80001000 { - compatible = "ns16550"; - reg = <0x80001000 0x1000>; - reg-shift = <2>; - interrupts = < 4 >; - interrupt-parent = <&pic0>; - current-speed = <115200>; - clock-frequency = < 1000000 >; - status = "okay"; - }; - - spi0: spi@80010000 { - compatible = "xlnx,xps-spi-3.2"; - reg = <0x80010000 0x1000>; - }; - - timer0: timer@0 { - compatible = "riscv,timer"; - interrupts = < 1 >; - interrupt-parent = < &pic0 >; - clock-frequency = < 1000000 >; - }; - }; - - chosen { - bootargs = "-v"; - stdin = "serial0"; - stdout = "serial0"; - }; -}; Property changes on: head/sys/boot/fdt/dts/riscv/lowrisc.dts ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/sys/boot/fdt/dts/riscv/qemu.dts =================================================================== --- head/sys/boot/fdt/dts/riscv/qemu.dts (revision 322360) +++ head/sys/boot/fdt/dts/riscv/qemu.dts (nonexistent) @@ -1,105 +0,0 @@ -/*- - * Copyright (c) 2016 Ruslan Bukin - * All rights reserved. - * - * Portions of this software were developed by SRI International and the - * University of Cambridge Computer Laboratory under DARPA/AFRL contract - * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. - * - * Portions of this software were developed by the University of Cambridge - * Computer Laboratory as part of the CTSRD Project, with support from the - * UK Higher Education Innovation Fund (HEIF). - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -/dts-v1/; - -/ { - model = "QEMU RV64"; - compatible = "riscv,rv64"; - #address-cells = <1>; - #size-cells = <1>; - #interrupt-cells = <1>; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - cpu@0 { - device_type = "cpu"; - compatible = "riscv,rv64"; - reg = <0x0>; - }; - }; - - aliases { - console0 = &console0; - }; - - memory { - /* - * This is not used currently. - * We take information from sbi_query_memory. - */ - device_type = "memory"; - reg = <0x80000000 0x40000000>; /* 1GB at 0x80000000 */ - }; - - soc { - #address-cells = <1>; - #size-cells = <1>; - #interrupt-cells = <1>; - - compatible = "simple-bus"; - ranges; - - pic0: pic@0 { - compatible = "riscv,pic"; - interrupt-controller; - }; - - timer0: timer@0 { - compatible = "riscv,timer"; - reg = < 0x40000000 0x0008 >, /* rtc */ - < 0x40000008 0x1000 >; /* timecmp */ - interrupts = < 5 >; - interrupt-parent = < &pic0 >; - clock-frequency = < 400000000 >; - }; - - console0: console@0 { - compatible = "riscv,console"; - status = "okay"; - interrupts = < 1 >; - interrupt-parent = < &pic0 >; - }; - }; - - chosen { - bootargs = "-v"; - stdin = "console0"; - stdout = "console0"; - }; -}; Property changes on: head/sys/boot/fdt/dts/riscv/qemu.dts ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/sys/conf/Makefile.riscv =================================================================== --- head/sys/conf/Makefile.riscv (revision 322360) +++ head/sys/conf/Makefile.riscv (revision 322361) @@ -1,52 +1,57 @@ # Makefile.riscv -- with config changes. # Copyright 1990 W. Jolitz # from: @(#)Makefile.i386 7.1 5/10/91 # from FreeBSD: src/sys/conf/Makefile.i386,v 1.255 2002/02/20 23:35:49 # $FreeBSD$ # # Makefile for FreeBSD # # RISCVTODO: copy pasted from aarch64, needs to be # constructed from a machine description: # config machineid # Most changes should be made in the machine description # /sys/riscv/conf/``machineid'' # after which you should do # config machineid # Generic makefile changes should be made in # /sys/conf/Makefile.riscv # after which config should be rerun for all machines. # # Which version of config(8) is required. %VERSREQ= 600012 .if !defined(S) S= ../../.. .endif .include "$S/conf/kern.pre.mk" INCLUDES+= -I$S/contrib/libfdt +SYSTEM_LD= @${LD} -N -m ${LD_EMULATION} -Bdynamic -T ${LDSCRIPT} ${_LDFLAGS} \ + --no-warn-mismatch --warn-common --export-dynamic \ + --dynamic-linker /red/herring \ + -o ${.TARGET} -X ${SYSTEM_OBJS} vers.o + .if !empty(DDB_ENABLED) CFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls .endif # hack because genassym.c includes sys/bus.h which includes these. genassym.o: bus_if.h device_if.h %BEFORE_DEPEND %OBJS %FILES.c %FILES.s %FILES.m %CLEAN %RULES .include "$S/conf/kern.post.mk" Index: head/sys/conf/files.riscv =================================================================== --- head/sys/conf/files.riscv (revision 322360) +++ head/sys/conf/files.riscv (revision 322361) @@ -1,59 +1,58 @@ # $FreeBSD$ cddl/compat/opensolaris/kern/opensolaris_atomic.c optional zfs | dtrace compile-with "${CDDL_C}" cddl/dev/dtrace/riscv/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/riscv/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/riscv/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" crypto/blowfish/bf_enc.c optional crypto | ipsec | ipsec_support crypto/des/des_enc.c optional crypto | ipsec | ipsec_support | netsmb dev/ofw/ofw_cpu.c optional fdt dev/uart/uart_cpu_fdt.c optional uart fdt dev/xilinx/axi_quad_spi.c optional xilinx_spi kern/kern_clocksource.c standard kern/subr_devmap.c standard kern/subr_dummy_vdso_tc.c standard libkern/bcmp.c standard libkern/ffs.c standard libkern/ffsl.c standard libkern/fls.c standard libkern/flsl.c standard libkern/flsll.c standard libkern/memmove.c standard libkern/memset.c standard riscv/riscv/autoconf.c standard riscv/riscv/bcopy.c standard riscv/riscv/bus_machdep.c standard riscv/riscv/bus_space_asm.S standard riscv/riscv/busdma_machdep.c standard riscv/riscv/clock.c standard riscv/riscv/copyinout.S standard riscv/riscv/copystr.c standard riscv/riscv/cpufunc_asm.S standard riscv/riscv/db_disasm.c optional ddb riscv/riscv/db_interface.c optional ddb riscv/riscv/db_trace.c optional ddb riscv/riscv/dump_machdep.c standard riscv/riscv/elf_machdep.c standard riscv/riscv/exception.S standard riscv/riscv/intr_machdep.c standard riscv/riscv/in_cksum.c optional inet | inet6 riscv/riscv/identcpu.c standard riscv/riscv/locore.S standard no-obj riscv/riscv/machdep.c standard riscv/riscv/minidump_machdep.c standard riscv/riscv/mp_machdep.c optional smp riscv/riscv/mem.c standard riscv/riscv/nexus.c standard riscv/riscv/ofw_machdep.c optional fdt riscv/riscv/pmap.c standard riscv/riscv/riscv_console.c optional rcons -riscv/riscv/sbi.S standard riscv/riscv/stack_machdep.c optional ddb | stack riscv/riscv/support.S standard riscv/riscv/swtch.S standard riscv/riscv/sys_machdep.c standard riscv/riscv/trap.c standard riscv/riscv/timer.c standard riscv/riscv/uio_machdep.c standard riscv/riscv/uma_machdep.c standard riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack riscv/riscv/vm_machdep.c standard Index: head/sys/conf/kern.mk =================================================================== --- head/sys/conf/kern.mk (revision 322360) +++ head/sys/conf/kern.mk (revision 322361) @@ -1,275 +1,275 @@ # $FreeBSD$ # # Warning flags for compiling the kernel and components of the kernel: # CWARNFLAGS?= -Wall -Wredundant-decls -Wnested-externs -Wstrict-prototypes \ -Wmissing-prototypes -Wpointer-arith -Winline -Wcast-qual \ -Wundef -Wno-pointer-sign ${FORMAT_EXTENSIONS} \ -Wmissing-include-dirs -fdiagnostics-show-option \ -Wno-unknown-pragmas \ ${CWARNEXTRA} # # The following flags are next up for working on: # -Wextra # Disable a few warnings for clang, since there are several places in the # kernel where fixing them is more trouble than it is worth, or where there is # a false positive. .if ${COMPILER_TYPE} == "clang" NO_WCONSTANT_CONVERSION= -Wno-error-constant-conversion NO_WSHIFT_COUNT_NEGATIVE= -Wno-error-shift-count-negative NO_WSHIFT_COUNT_OVERFLOW= -Wno-error-shift-count-overflow NO_WSELF_ASSIGN= -Wno-error-self-assign NO_WUNNEEDED_INTERNAL_DECL= -Wno-error-unneeded-internal-declaration NO_WSOMETIMES_UNINITIALIZED= -Wno-error-sometimes-uninitialized NO_WCAST_QUAL= -Wno-error-cast-qual # Several other warnings which might be useful in some cases, but not severe # enough to error out the whole kernel build. Display them anyway, so there is # some incentive to fix them eventually. CWARNEXTRA?= -Wno-error-tautological-compare -Wno-error-empty-body \ -Wno-error-parentheses-equality -Wno-error-unused-function \ -Wno-error-pointer-sign .if ${COMPILER_VERSION} >= 30700 CWARNEXTRA+= -Wno-error-shift-negative-value .endif .if ${COMPILER_VERSION} >= 40000 CWARNEXTRA+= -Wno-error-address-of-packed-member .endif CLANG_NO_IAS= -no-integrated-as .if ${COMPILER_VERSION} < 30500 # XXX: clang < 3.5 integrated-as doesn't grok .codeNN directives CLANG_NO_IAS34= -no-integrated-as .endif .endif .if ${COMPILER_TYPE} == "gcc" .if ${COMPILER_VERSION} >= 40800 # Catch-all for all the things that are in our tree, but for which we're # not yet ready for this compiler. CWARNEXTRA?= -Wno-error=address \ -Wno-error=aggressive-loop-optimizations \ -Wno-error=array-bounds \ -Wno-error=attributes \ -Wno-error=cast-qual \ -Wno-error=enum-compare \ -Wno-error=inline \ -Wno-error=maybe-uninitialized \ -Wno-error=overflow \ -Wno-error=sequence-point \ -Wno-error=strict-overflow \ -Wno-error=unused-but-set-variable .if ${COMPILER_VERSION} >= 60100 CWARNEXTRA+= -Wno-error=misleading-indentation \ -Wno-error=nonnull-compare \ -Wno-error=shift-overflow \ -Wno-error=tautological-compare .endif .else # For gcc 4.2, eliminate the too-often-wrong warnings about uninitialized vars. CWARNEXTRA?= -Wno-uninitialized .endif .endif # External compilers may not support our format extensions. Allow them # to be disabled. WARNING: format checking is disabled in this case. .if ${MK_FORMAT_EXTENSIONS} == "no" FORMAT_EXTENSIONS= -Wno-format .elif ${COMPILER_TYPE} == "clang" && ${COMPILER_VERSION} >= 30600 FORMAT_EXTENSIONS= -D__printf__=__freebsd_kprintf__ .else FORMAT_EXTENSIONS= -fformat-extensions .endif # # On i386, do not align the stack to 16-byte boundaries. Otherwise GCC 2.95 # and above adds code to the entry and exit point of every function to align the # stack to 16-byte boundaries -- thus wasting approximately 12 bytes of stack # per function call. While the 16-byte alignment may benefit micro benchmarks, # it is probably an overall loss as it makes the code bigger (less efficient # use of code cache tag lines) and uses more stack (less efficient use of data # cache tag lines). Explicitly prohibit the use of FPU, SSE and other SIMD # operations inside the kernel itself. These operations are exclusively # reserved for user applications. # # gcc: # Setting -mno-mmx implies -mno-3dnow # Setting -mno-sse implies -mno-sse2, -mno-sse3 and -mno-ssse3 # # clang: # Setting -mno-mmx implies -mno-3dnow and -mno-3dnowa # Setting -mno-sse implies -mno-sse2, -mno-sse3, -mno-ssse3, -mno-sse41 and -mno-sse42 # .if ${MACHINE_CPUARCH} == "i386" CFLAGS.gcc+= -mno-align-long-strings -mpreferred-stack-boundary=2 CFLAGS.clang+= -mno-aes -mno-avx CFLAGS+= -mno-mmx -mno-sse -msoft-float INLINE_LIMIT?= 8000 .endif .if ${MACHINE_CPUARCH} == "arm" INLINE_LIMIT?= 8000 .endif .if ${MACHINE_CPUARCH} == "aarch64" # We generally don't want fpu instructions in the kernel. CFLAGS += -mgeneral-regs-only # Reserve x18 for pcpu data CFLAGS += -ffixed-x18 INLINE_LIMIT?= 8000 .endif .if ${MACHINE_CPUARCH} == "riscv" -CFLAGS.gcc+= -mcmodel=medany +CFLAGS.gcc+= -mcmodel=medany -march=rv64imafdc -mabi=lp64 INLINE_LIMIT?= 8000 .endif # # For sparc64 we want the medany code model so modules may be located # anywhere in the 64-bit address space. We also tell GCC to use floating # point emulation. This avoids using floating point registers for integer # operations which it has a tendency to do. # .if ${MACHINE_CPUARCH} == "sparc64" CFLAGS.clang+= -mcmodel=large -fno-dwarf2-cfi-asm CFLAGS.gcc+= -mcmodel=medany -msoft-float INLINE_LIMIT?= 15000 .endif # # For AMD64, we explicitly prohibit the use of FPU, SSE and other SIMD # operations inside the kernel itself. These operations are exclusively # reserved for user applications. # # gcc: # Setting -mno-mmx implies -mno-3dnow # Setting -mno-sse implies -mno-sse2, -mno-sse3, -mno-ssse3 and -mfpmath=387 # # clang: # Setting -mno-mmx implies -mno-3dnow and -mno-3dnowa # Setting -mno-sse implies -mno-sse2, -mno-sse3, -mno-ssse3, -mno-sse41 and -mno-sse42 # (-mfpmath= is not supported) # .if ${MACHINE_CPUARCH} == "amd64" CFLAGS.clang+= -mno-aes -mno-avx CFLAGS+= -mcmodel=kernel -mno-red-zone -mno-mmx -mno-sse -msoft-float \ -fno-asynchronous-unwind-tables INLINE_LIMIT?= 8000 .endif # # For PowerPC we tell gcc to use floating point emulation. This avoids using # floating point registers for integer operations which it has a tendency to do. # Also explicitly disable Altivec instructions inside the kernel. # .if ${MACHINE_CPUARCH} == "powerpc" CFLAGS+= -mno-altivec -msoft-float INLINE_LIMIT?= 15000 .endif .if ${MACHINE_ARCH} == "powerpcspe" CFLAGS.gcc+= -mno-spe .endif # # Use dot symbols on powerpc64 to make ddb happy # .if ${MACHINE_ARCH} == "powerpc64" CFLAGS.gcc+= -mcall-aixdesc .endif # # For MIPS we also tell gcc to use floating point emulation # .if ${MACHINE_CPUARCH} == "mips" CFLAGS+= -msoft-float INLINE_LIMIT?= 8000 .if ${MACHINE_ARCH:Mmips*hf} != "" CFLAGS+= -DCPU_HAVEFPU .endif .endif # # GCC 3.0 and above like to do certain optimizations based on the # assumption that the program is linked against libc. Stop this. # CFLAGS+= -ffreestanding # # The C standard leaves signed integer overflow behavior undefined. # gcc and clang opimizers take advantage of this. The kernel makes # use of signed integer wraparound mechanics so we need the compiler # to treat it as a wraparound and not take shortcuts. # CFLAGS+= -fwrapv # # GCC SSP support # .if ${MK_SSP} != "no" && \ ${MACHINE_CPUARCH} != "arm" && ${MACHINE_CPUARCH} != "mips" CFLAGS+= -fstack-protector .endif # # Add -gdwarf-2 when compiling -g. The default starting in clang v3.4 # and gcc 4.8 is to generate DWARF version 4. However, our tools don't # cope well with DWARF 4, so force it to genereate DWARF2, which they # understand. Do this unconditionally as it is harmless when not needed, # but critical for these newer versions. # .if ${CFLAGS:M-g} != "" && ${CFLAGS:M-gdwarf*} == "" CFLAGS+= -gdwarf-2 .endif CFLAGS+= ${CWARNFLAGS:M*} ${CWARNFLAGS.${.IMPSRC:T}} CFLAGS+= ${CFLAGS.${COMPILER_TYPE}} ${CFLAGS.${.IMPSRC:T}} # Tell bmake not to mistake standard targets for things to be searched for # or expect to ever be up-to-date. PHONY_NOTMAIN = afterdepend afterinstall all beforedepend beforeinstall \ beforelinking build build-tools buildfiles buildincludes \ checkdpadd clean cleandepend cleandir cleanobj configure \ depend distclean distribute exe \ html includes install installfiles installincludes lint \ obj objlink objs objwarn \ realinstall regress \ tags whereobj .PHONY: ${PHONY_NOTMAIN} .NOTMAIN: ${PHONY_NOTMAIN} CSTD= c99 .if ${CSTD} == "k&r" CFLAGS+= -traditional .elif ${CSTD} == "c89" || ${CSTD} == "c90" CFLAGS+= -std=iso9899:1990 .elif ${CSTD} == "c94" || ${CSTD} == "c95" CFLAGS+= -std=iso9899:199409 .elif ${CSTD} == "c99" CFLAGS+= -std=iso9899:1999 .else # CSTD CFLAGS+= -std=${CSTD} .endif # CSTD # Set target-specific linker emulation name. LD_EMULATION_aarch64=aarch64elf LD_EMULATION_amd64=elf_x86_64_fbsd LD_EMULATION_arm=armelf_fbsd LD_EMULATION_armeb=armelfb_fbsd LD_EMULATION_armv6=armelf_fbsd LD_EMULATION_i386=elf_i386_fbsd LD_EMULATION_mips= elf32btsmip_fbsd LD_EMULATION_mips64= elf64btsmip_fbsd LD_EMULATION_mipsel= elf32ltsmip_fbsd LD_EMULATION_mips64el= elf64ltsmip_fbsd LD_EMULATION_mipsn32= elf32btsmipn32_fbsd LD_EMULATION_mipsn32el= elf32btsmipn32_fbsd # I don't think this is a thing that works LD_EMULATION_powerpc= elf32ppc_fbsd LD_EMULATION_powerpcspe= elf32ppc_fbsd LD_EMULATION_powerpc64= elf64ppc_fbsd LD_EMULATION_riscv64= elf64lriscv LD_EMULATION_sparc64= elf64_sparc_fbsd LD_EMULATION=${LD_EMULATION_${MACHINE_ARCH}} Index: head/sys/riscv/conf/LOWRISC.hints =================================================================== --- head/sys/riscv/conf/LOWRISC.hints (revision 322360) +++ head/sys/riscv/conf/LOWRISC.hints (nonexistent) @@ -1,5 +0,0 @@ -# $FreeBSD$ - -# MMC device -hint.mmc_spi.0.at="spibus0" -hint.mmc_spi.0.cs=0 Property changes on: head/sys/riscv/conf/LOWRISC.hints ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/sys/riscv/conf/LOWRISC =================================================================== --- head/sys/riscv/conf/LOWRISC (revision 322360) +++ head/sys/riscv/conf/LOWRISC (nonexistent) @@ -1,43 +0,0 @@ -# -# Kernel configuration file for lowRISC. -# -# For more information on this file, please read the config(5) manual page, -# and/or the handbook section on Kernel Configuration Files: -# -# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html -# -# The handbook is also available locally in /usr/share/doc/handbook -# if you've installed the doc distribution, otherwise always see the -# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the -# latest information. -# -# An exhaustive list of options and more detailed explanations of the -# device lines is also present in the ../../conf/NOTES and NOTES files. -# If you are in doubt as to the purpose or necessity of a line, check first -# in NOTES. -# -# $FreeBSD$ - -include GENERIC -ident LOWRISC - -hints "LOWRISC.hints" - -options ROOTDEVNAME=\"ufs:mmcsd0s2\" - -# MMC/SD -device mmc -device mmcsd -# device mmc_spi - -# SPI -device spibus -device xilinx_spi - -# Serial (COM) ports -device uart # Generic UART driver -device uart_ns8250 # ns8250-type UART driver - -# RISCVTODO: This needs to be done via loader (when it's available). -options FDT_DTB_STATIC -makeoptions FDT_DTS_FILE=lowrisc.dts Property changes on: head/sys/riscv/conf/LOWRISC ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/sys/riscv/conf/QEMU =================================================================== --- head/sys/riscv/conf/QEMU (revision 322360) +++ head/sys/riscv/conf/QEMU (nonexistent) @@ -1,29 +0,0 @@ -# -# Kernel configuration file for QEMU emulator. -# -# For more information on this file, please read the config(5) manual page, -# and/or the handbook section on Kernel Configuration Files: -# -# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html -# -# The handbook is also available locally in /usr/share/doc/handbook -# if you've installed the doc distribution, otherwise always see the -# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the -# latest information. -# -# An exhaustive list of options and more detailed explanations of the -# device lines is also present in the ../../conf/NOTES and NOTES files. -# If you are in doubt as to the purpose or necessity of a line, check first -# in NOTES. -# -# $FreeBSD$ - -include GENERIC -ident QEMU - -device rcons -options ROOTDEVNAME=\"ufs:/dev/md0\" - -# RISCVTODO: This needs to be done via loader (when it's available). -options FDT_DTB_STATIC -makeoptions FDT_DTS_FILE=qemu.dts Property changes on: head/sys/riscv/conf/QEMU ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/sys/riscv/conf/ROCKET =================================================================== --- head/sys/riscv/conf/ROCKET (revision 322360) +++ head/sys/riscv/conf/ROCKET (nonexistent) @@ -1,29 +0,0 @@ -# -# Kernel configuration file for Rocket Core. -# -# For more information on this file, please read the config(5) manual page, -# and/or the handbook section on Kernel Configuration Files: -# -# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html -# -# The handbook is also available locally in /usr/share/doc/handbook -# if you've installed the doc distribution, otherwise always see the -# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the -# latest information. -# -# An exhaustive list of options and more detailed explanations of the -# device lines is also present in the ../../conf/NOTES and NOTES files. -# If you are in doubt as to the purpose or necessity of a line, check first -# in NOTES. -# -# $FreeBSD$ - -include GENERIC -ident ROCKET - -device rcons -options ROOTDEVNAME=\"ufs:/dev/md0\" - -# RISCVTODO: This needs to be done via loader (when it's available). -options FDT_DTB_STATIC -makeoptions FDT_DTS_FILE=rocket.dts Property changes on: head/sys/riscv/conf/ROCKET ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/sys/riscv/conf/SPIKE =================================================================== --- head/sys/riscv/conf/SPIKE (revision 322360) +++ head/sys/riscv/conf/SPIKE (nonexistent) @@ -1,29 +0,0 @@ -# -# Kernel configuration file for UCB Spike simulator. -# -# For more information on this file, please read the config(5) manual page, -# and/or the handbook section on Kernel Configuration Files: -# -# http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html -# -# The handbook is also available locally in /usr/share/doc/handbook -# if you've installed the doc distribution, otherwise always see the -# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the -# latest information. -# -# An exhaustive list of options and more detailed explanations of the -# device lines is also present in the ../../conf/NOTES and NOTES files. -# If you are in doubt as to the purpose or necessity of a line, check first -# in NOTES. -# -# $FreeBSD$ - -include GENERIC -ident SPIKE - -device rcons -options ROOTDEVNAME=\"ufs:/dev/md0\" - -# RISCVTODO: This needs to be done via loader (when it's available). -options FDT_DTB_STATIC -makeoptions FDT_DTS_FILE=spike.dts Property changes on: head/sys/riscv/conf/SPIKE ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/sys/riscv/conf/GENERIC =================================================================== --- head/sys/riscv/conf/GENERIC (revision 322360) +++ head/sys/riscv/conf/GENERIC (revision 322361) @@ -1,114 +1,117 @@ # # GENERIC -- Generic kernel configuration file for FreeBSD/RISC-V # # For more information on this file, please read the config(5) manual page, # and/or the handbook section on Kernel Configuration Files: # # http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html # # The handbook is also available locally in /usr/share/doc/handbook # if you've installed the doc distribution, otherwise always see the # FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the # latest information. # # An exhaustive list of options and more detailed explanations of the # device lines is also present in the ../../conf/NOTES and NOTES files. # If you are in doubt as to the purpose or necessity of a line, check first # in NOTES. # # $FreeBSD$ cpu RISCV ident GENERIC makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols # makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support # FIXME: linker error. "--relax and -r may not be used together" makeoptions WITHOUT_MODULES="usb otusfw mwlfw ispfw mwlfw ralfw rtwnfw" # makeoptions NO_MODULES options SCHED_ULE # ULE scheduler options PREEMPTION # Enable kernel thread preemption options INET # InterNETworking options INET6 # IPv6 communications protocols options TCP_HHOOK # hhook(9) framework for TCP options IPSEC # IP (v4/v6) security options TCP_OFFLOAD # TCP offload options SCTP # Stream Control Transmission Protocol options FFS # Berkeley Fast Filesystem options SOFTUPDATES # Enable FFS soft updates support options UFS_ACL # Support for access control lists options UFS_DIRHASH # Improve performance on big directories options UFS_GJOURNAL # Enable gjournal-based UFS journaling options QUOTA # Enable disk quotas for UFS options NFSCL # Network Filesystem Client options NFSD # Network Filesystem Server options NFSLOCKD # Network Lock Manager options NFS_ROOT # NFS usable as /, requires NFSCL options MSDOSFS # MSDOS Filesystem options CD9660 # ISO 9660 Filesystem options PROCFS # Process filesystem (requires PSEUDOFS) options PSEUDOFS # Pseudo-filesystem framework options GEOM_PART_GPT # GUID Partition Tables. # options GEOM_RAID # Soft RAID functionality. options GEOM_LABEL # Provides labelization options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI options KTRACE # ktrace(1) support # options STACK # stack(9) support options SYSVSHM # SYSV-style shared memory options SYSVMSG # SYSV-style message queues options SYSVSEM # SYSV-style semaphores options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions options PRINTF_BUFR_SIZE=128 # Prevent printf output being interspersed. options KBD_INSTALL_CDEV # install a CDEV entry in /dev # options HWPMC_HOOKS # Necessary kernel hooks for hwpmc(4) options AUDIT # Security event auditing options CAPABILITY_MODE # Capsicum capability mode options CAPABILITIES # Capsicum capabilities options MAC # TrustedBSD MAC Framework options KDTRACE_FRAME # Ensure frames are compiled in options KDTRACE_HOOKS # Kernel DTrace hooks options FPE # Floating-point extension support options RACCT # Resource accounting framework options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default options RCTL # Resource limits options SMP +# RISC-V SBI console +device rcons + # Uncomment for memory disk # options MD_ROOT # options MD_ROOT_SIZE=32768 # 32MB ram disk # makeoptions MFS_IMAGE=/path/to/img # options ROOTDEVNAME=\"ufs:/dev/md0\" # Debugging support. Always need this: options KDB # Enable kernel debugger support. options KDB_TRACE # Print a stack trace for a panic. # For full debugger support use (turn off in stable branch): options DDB # Support DDB. # options GDB # Support remote GDB. options DEADLKRES # Enable the deadlock resolver options INVARIANTS # Enable calls of extra sanity checking options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS # options WITNESS # Enable checks to detect deadlocks and cycles # options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # options EARLY_PRINTF # options VERBOSE_SYSINIT # Pseudo devices. device loop # Network loopback device random # Entropy device device ether # Ethernet support device vlan # 802.1Q VLAN support device tun # Packet tunnel. device md # Memory "disks" device gif # IPv6 and IPv4 tunneling device firmware # firmware assist module # The `bpf' device enables the Berkeley Packet Filter. # Be aware of the administrative consequences of enabling this! # Note that 'bpf' is required for DHCP. device bpf # Berkeley packet filter options FDT Index: head/sys/riscv/include/machdep.h =================================================================== --- head/sys/riscv/include/machdep.h (revision 322360) +++ head/sys/riscv/include/machdep.h (revision 322361) @@ -1,51 +1,54 @@ /*- - * Copyright (c) 2015 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_MACHDEP_H_ #define _MACHINE_MACHDEP_H_ struct riscv_bootparams { vm_offset_t kern_l1pt; /* Kernel L1 base */ + vm_offset_t kern_phys; /* Kernel base (physical) addr */ vm_offset_t kern_stack; + vm_offset_t dtbp_virt; /* Device tree blob virtual addr */ + vm_offset_t dtbp_phys; /* Device tree blob physical addr */ }; extern vm_paddr_t physmap[]; extern u_int physmap_idx; vm_offset_t fake_preload_metadata(struct riscv_bootparams *rbp); void initriscv(struct riscv_bootparams *); #endif /* _MACHINE_MACHDEP_H_ */ Index: head/sys/riscv/include/riscvreg.h =================================================================== --- head/sys/riscv/include/riscvreg.h (revision 322360) +++ head/sys/riscv/include/riscvreg.h (revision 322361) @@ -1,198 +1,210 @@ /*- - * Copyright (c) 2015-2016 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_RISCVREG_H_ #define _MACHINE_RISCVREG_H_ #define EXCP_SHIFT 0 #define EXCP_MASK (0xf << EXCP_SHIFT) #define EXCP_MISALIGNED_FETCH 0 #define EXCP_FAULT_FETCH 1 #define EXCP_ILLEGAL_INSTRUCTION 2 #define EXCP_BREAKPOINT 3 #define EXCP_MISALIGNED_LOAD 4 #define EXCP_FAULT_LOAD 5 #define EXCP_MISALIGNED_STORE 6 #define EXCP_FAULT_STORE 7 #define EXCP_USER_ECALL 8 #define EXCP_SUPERVISOR_ECALL 9 #define EXCP_HYPERVISOR_ECALL 10 #define EXCP_MACHINE_ECALL 11 +#define EXCP_INST_PAGE_FAULT 12 +#define EXCP_LOAD_PAGE_FAULT 13 +#define EXCP_STORE_PAGE_FAULT 15 #define EXCP_INTR (1ul << 63) #define SSTATUS_UIE (1 << 0) #define SSTATUS_SIE (1 << 1) #define SSTATUS_UPIE (1 << 4) #define SSTATUS_SPIE (1 << 5) #define SSTATUS_SPIE_SHIFT 5 #define SSTATUS_SPP (1 << 8) #define SSTATUS_SPP_SHIFT 8 #define SSTATUS_FS_SHIFT 13 #define SSTATUS_FS_OFF (0x0 << SSTATUS_FS_SHIFT) #define SSTATUS_FS_INITIAL (0x1 << SSTATUS_FS_SHIFT) #define SSTATUS_FS_CLEAN (0x2 << SSTATUS_FS_SHIFT) #define SSTATUS_FS_DIRTY (0x3 << SSTATUS_FS_SHIFT) #define SSTATUS_FS_MASK (0x3 << SSTATUS_FS_SHIFT) #define SSTATUS_XS_SHIFT 15 #define SSTATUS_XS_MASK (0x3 << SSTATUS_XS_SHIFT) -#define SSTATUS_PUM (1 << 18) +#define SSTATUS_SUM (1 << 18) #define SSTATUS32_SD (1 << 63) #define SSTATUS64_SD (1 << 31) #define MSTATUS_UIE (1 << 0) #define MSTATUS_SIE (1 << 1) #define MSTATUS_HIE (1 << 2) #define MSTATUS_MIE (1 << 3) #define MSTATUS_UPIE (1 << 4) #define MSTATUS_SPIE (1 << 5) #define MSTATUS_SPIE_SHIFT 5 #define MSTATUS_HPIE (1 << 6) #define MSTATUS_MPIE (1 << 7) #define MSTATUS_MPIE_SHIFT 7 #define MSTATUS_SPP (1 << 8) #define MSTATUS_SPP_SHIFT 8 #define MSTATUS_HPP_MASK 0x3 #define MSTATUS_HPP_SHIFT 9 #define MSTATUS_MPP_MASK 0x3 #define MSTATUS_MPP_SHIFT 11 #define MSTATUS_FS_MASK 0x3 #define MSTATUS_FS_SHIFT 13 #define MSTATUS_XS_MASK 0x3 #define MSTATUS_XS_SHIFT 15 #define MSTATUS_MPRV (1 << 17) #define MSTATUS_PUM (1 << 18) #define MSTATUS_VM_MASK 0x1f #define MSTATUS_VM_SHIFT 24 #define MSTATUS_VM_MBARE 0 #define MSTATUS_VM_MBB 1 #define MSTATUS_VM_MBBID 2 #define MSTATUS_VM_SV32 8 #define MSTATUS_VM_SV39 9 #define MSTATUS_VM_SV48 10 #define MSTATUS_VM_SV57 11 #define MSTATUS_VM_SV64 12 #define MSTATUS32_SD (1 << 63) #define MSTATUS64_SD (1 << 31) #define MSTATUS_PRV_U 0 /* user */ #define MSTATUS_PRV_S 1 /* supervisor */ #define MSTATUS_PRV_H 2 /* hypervisor */ #define MSTATUS_PRV_M 3 /* machine */ #define MIE_USIE (1 << 0) #define MIE_SSIE (1 << 1) #define MIE_HSIE (1 << 2) #define MIE_MSIE (1 << 3) #define MIE_UTIE (1 << 4) #define MIE_STIE (1 << 5) #define MIE_HTIE (1 << 6) #define MIE_MTIE (1 << 7) #define MIP_USIP (1 << 0) #define MIP_SSIP (1 << 1) #define MIP_HSIP (1 << 2) #define MIP_MSIP (1 << 3) #define MIP_UTIP (1 << 4) #define MIP_STIP (1 << 5) #define MIP_HTIP (1 << 6) #define MIP_MTIP (1 << 7) #define SIE_USIE (1 << 0) #define SIE_SSIE (1 << 1) #define SIE_UTIE (1 << 4) #define SIE_STIE (1 << 5) #define MIP_SEIP (1 << 9) /* Note: sip register has no SIP_STIP bit in Spike simulator */ #define SIP_SSIP (1 << 1) #define SIP_STIP (1 << 5) + +#define SATP_PPN_S 0 +#define SATP_PPN_M (0xfffffffffff << SATP_PPN_S) +#define SATP_ASID_S 44 +#define SATP_ASID_M (0xffff << SATP_ASID_S) +#define SATP_MODE_S 60 +#define SATP_MODE_M (0xf << SATP_MODE_S) +#define SATP_MODE_SV39 (8ULL << SATP_MODE_S) +#define SATP_MODE_SV48 (9ULL << SATP_MODE_S) #if 0 /* lowRISC TODO */ #define NCSRS 4096 #define CSR_IPI 0x783 #define CSR_IO_IRQ 0x7c0 /* lowRISC only? */ #endif #define XLEN 8 #define INSN_SIZE 4 #define RISCV_INSN_NOP 0x00000013 #define RISCV_INSN_BREAK 0x00100073 #define RISCV_INSN_RET 0x00008067 #define CSR_ZIMM(val) \ (__builtin_constant_p(val) && ((u_long)(val) < 32)) #define csr_swap(csr, val) \ ({ if (CSR_ZIMM(val)) \ __asm __volatile("csrrwi %0, " #csr ", %1" \ : "=r" (val) : "i" (val)); \ else \ __asm __volatile("csrrw %0, " #csr ", %1" \ : "=r" (val) : "r" (val)); \ val; \ }) #define csr_write(csr, val) \ ({ if (CSR_ZIMM(val)) \ __asm __volatile("csrwi " #csr ", %0" :: "i" (val)); \ else \ __asm __volatile("csrw " #csr ", %0" :: "r" (val)); \ }) #define csr_set(csr, val) \ ({ if (CSR_ZIMM(val)) \ __asm __volatile("csrsi " #csr ", %0" :: "i" (val)); \ else \ __asm __volatile("csrs " #csr ", %0" :: "r" (val)); \ }) #define csr_clear(csr, val) \ ({ if (CSR_ZIMM(val)) \ __asm __volatile("csrci " #csr ", %0" :: "i" (val)); \ else \ __asm __volatile("csrc " #csr ", %0" :: "r" (val)); \ }) #define csr_read(csr) \ ({ u_long val; \ __asm __volatile("csrr %0, " #csr : "=r" (val)); \ val; \ }) #endif /* !_MACHINE_RISCVREG_H_ */ Index: head/sys/riscv/include/sbi.h =================================================================== --- head/sys/riscv/include/sbi.h (revision 322360) +++ head/sys/riscv/include/sbi.h (revision 322361) @@ -1,65 +1,133 @@ /*- - * Copyright (c) 2016 Ruslan Bukin + * Copyright (c) 2016-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_SBI_H_ #define _MACHINE_SBI_H_ -typedef struct { - uint64_t base; - uint64_t size; - uint64_t node_id; -} memory_block_info; +#define SBI_SET_TIMER 0 +#define SBI_CONSOLE_PUTCHAR 1 +#define SBI_CONSOLE_GETCHAR 2 +#define SBI_CLEAR_IPI 3 +#define SBI_SEND_IPI 4 +#define SBI_REMOTE_FENCE_I 5 +#define SBI_REMOTE_SFENCE_VMA 6 +#define SBI_REMOTE_SFENCE_VMA_ASID 7 +#define SBI_SHUTDOWN 8 -uint64_t sbi_query_memory(uint64_t id, memory_block_info *p); -uint64_t sbi_hart_id(void); -uint64_t sbi_num_harts(void); -uint64_t sbi_timebase(void); -void sbi_set_timer(uint64_t stime_value); -void sbi_send_ipi(uint64_t hart_id); -uint64_t sbi_clear_ipi(void); -void sbi_shutdown(void); +static __inline uint64_t +sbi_call(uint64_t arg7, uint64_t arg0, uint64_t arg1, uint64_t arg2) +{ -void sbi_console_putchar(unsigned char ch); -int sbi_console_getchar(void); + register uintptr_t a0 __asm ("a0") = (uintptr_t)(arg0); + register uintptr_t a1 __asm ("a1") = (uintptr_t)(arg1); + register uintptr_t a2 __asm ("a2") = (uintptr_t)(arg2); + register uintptr_t a7 __asm ("a7") = (uintptr_t)(arg7); + __asm __volatile( \ + "ecall" \ + :"+r"(a0) \ + :"r"(a1), "r"(a2), "r"(a7) \ + :"memory"); -void sbi_remote_sfence_vm(uint64_t hart_mask_ptr, uint64_t asid); -void sbi_remote_sfence_vm_range(uint64_t hart_mask_ptr, uint64_t asid, uint64_t start, uint64_t size); -void sbi_remote_fence_i(uint64_t hart_mask_ptr); + return (a0); +} -uint64_t sbi_mask_interrupt(uint64_t which); -uint64_t sbi_unmask_interrupt(uint64_t which); +static __inline void +sbi_console_putchar(int ch) +{ + + sbi_call(SBI_CONSOLE_PUTCHAR, ch, 0, 0); +} + +static __inline int +sbi_console_getchar(void) +{ + + return (sbi_call(SBI_CONSOLE_GETCHAR, 0, 0, 0)); +} + +static __inline void +sbi_set_timer(uint64_t val) +{ + + sbi_call(SBI_SET_TIMER, val, 0, 0); +} + +static __inline void +sbi_shutdown(void) +{ + + sbi_call(SBI_SHUTDOWN, 0, 0, 0); +} + +static __inline void +sbi_clear_ipi(void) +{ + + sbi_call(SBI_CLEAR_IPI, 0, 0, 0); +} + +static __inline void +sbi_send_ipi(const unsigned long *hart_mask) +{ + + sbi_call(SBI_SEND_IPI, (uint64_t)hart_mask, 0, 0); +} + +static __inline void +sbi_remote_fence_i(const unsigned long *hart_mask) +{ + + sbi_call(SBI_REMOTE_FENCE_I, (uint64_t)hart_mask, 0, 0); +} + +static __inline void +sbi_remote_sfence_vma(const unsigned long *hart_mask, + unsigned long start, unsigned long size) +{ + + sbi_call(SBI_REMOTE_SFENCE_VMA, (uint64_t)hart_mask, 0, 0); +} + +static __inline void +sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, + unsigned long start, unsigned long size, + unsigned long asid) +{ + + sbi_call(SBI_REMOTE_SFENCE_VMA_ASID, (uint64_t)hart_mask, 0, 0); +} #endif /* !_MACHINE_SBI_H_ */ Index: head/sys/riscv/riscv/sbi.S =================================================================== --- head/sys/riscv/riscv/sbi.S (revision 322360) +++ head/sys/riscv/riscv/sbi.S (nonexistent) @@ -1,52 +0,0 @@ -/*- - * Copyright (c) 2016 Ruslan Bukin - * All rights reserved. - * - * Portions of this software were developed by SRI International and the - * University of Cambridge Computer Laboratory under DARPA/AFRL contract - * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. - * - * Portions of this software were developed by the University of Cambridge - * Computer Laboratory as part of the CTSRD Project, with support from the - * UK Higher Education Innovation Fund (HEIF). - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -__FBSDID("$FreeBSD$"); - -.globl sbi_hart_id; sbi_hart_id = -2048 -.globl sbi_num_harts; sbi_num_harts = -2032 -.globl sbi_query_memory; sbi_query_memory = -2016 -.globl sbi_console_putchar; sbi_console_putchar = -2000 -.globl sbi_console_getchar; sbi_console_getchar = -1984 -.globl sbi_send_ipi; sbi_send_ipi = -1952 -.globl sbi_clear_ipi; sbi_clear_ipi = -1936 -.globl sbi_timebase; sbi_timebase = -1920 -.globl sbi_shutdown; sbi_shutdown = -1904 -.globl sbi_set_timer; sbi_set_timer = -1888 -.globl sbi_mask_interrupt; sbi_mask_interrupt = -1872 -.globl sbi_unmask_interrupt; sbi_unmask_interrupt = -1856 -.globl sbi_remote_sfence_vm; sbi_remote_sfence_vm = -1840 -.globl sbi_remote_sfence_vm_range; sbi_remote_sfence_vm_range = -1824 -.globl sbi_remote_fence_i; sbi_remote_fence_i = -1808 Property changes on: head/sys/riscv/riscv/sbi.S ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -native \ No newline at end of property Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property Deleted: svn:mime-type ## -1 +0,0 ## -text/plain \ No newline at end of property Index: head/sys/riscv/riscv/cpufunc_asm.S =================================================================== --- head/sys/riscv/riscv/cpufunc_asm.S (revision 322360) +++ head/sys/riscv/riscv/cpufunc_asm.S (revision 322361) @@ -1,102 +1,102 @@ /*- - * Copyright (c) 2015-2016 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include __FBSDID("$FreeBSD$"); .text .align 2 .Lpage_mask: .word PAGE_MASK ENTRY(riscv_nullop) ret END(riscv_nullop) /* * Generic functions to read/modify/write the internal coprocessor registers */ ENTRY(riscv_tlb_flushID) - sfence.vm + sfence.vma ret END(riscv_tlb_flushID) ENTRY(riscv_tlb_flushID_SE) - sfence.vm + sfence.vma ret END(riscv_tlb_flushID_SE) /* * void riscv_dcache_wb_range(vm_offset_t, vm_size_t) */ ENTRY(riscv_dcache_wb_range) - sfence.vm + sfence.vma ret END(riscv_dcache_wb_range) /* * void riscv_dcache_wbinv_range(vm_offset_t, vm_size_t) */ ENTRY(riscv_dcache_wbinv_range) - sfence.vm + sfence.vma ret END(riscv_dcache_wbinv_range) /* * void riscv_dcache_inv_range(vm_offset_t, vm_size_t) */ ENTRY(riscv_dcache_inv_range) - sfence.vm + sfence.vma ret END(riscv_dcache_inv_range) /* * void riscv_idcache_wbinv_range(vm_offset_t, vm_size_t) */ ENTRY(riscv_idcache_wbinv_range) fence.i - sfence.vm + sfence.vma ret END(riscv_idcache_wbinv_range) /* * void riscv_icache_sync_range(vm_offset_t, vm_size_t) */ ENTRY(riscv_icache_sync_range) fence.i ret END(riscv_icache_sync_range) Index: head/sys/riscv/riscv/exception.S =================================================================== --- head/sys/riscv/riscv/exception.S (revision 322360) +++ head/sys/riscv/riscv/exception.S (revision 322361) @@ -1,237 +1,240 @@ /*- - * Copyright (c) 2015-2016 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "assym.s" #include #include .macro save_registers el addi sp, sp, -(TF_SIZE) sd ra, (TF_RA)(sp) sd tp, (TF_TP)(sp) .if \el == 0 /* We came from userspace. Load our pcpu */ sd gp, (TF_GP)(sp) ld gp, (TF_SIZE)(sp) .endif sd t0, (TF_T + 0 * 8)(sp) sd t1, (TF_T + 1 * 8)(sp) sd t2, (TF_T + 2 * 8)(sp) sd t3, (TF_T + 3 * 8)(sp) sd t4, (TF_T + 4 * 8)(sp) sd t5, (TF_T + 5 * 8)(sp) sd t6, (TF_T + 6 * 8)(sp) sd s0, (TF_S + 0 * 8)(sp) sd s1, (TF_S + 1 * 8)(sp) sd s2, (TF_S + 2 * 8)(sp) sd s3, (TF_S + 3 * 8)(sp) sd s4, (TF_S + 4 * 8)(sp) sd s5, (TF_S + 5 * 8)(sp) sd s6, (TF_S + 6 * 8)(sp) sd s7, (TF_S + 7 * 8)(sp) sd s8, (TF_S + 8 * 8)(sp) sd s9, (TF_S + 9 * 8)(sp) sd s10, (TF_S + 10 * 8)(sp) sd s11, (TF_S + 11 * 8)(sp) sd a0, (TF_A + 0 * 8)(sp) sd a1, (TF_A + 1 * 8)(sp) sd a2, (TF_A + 2 * 8)(sp) sd a3, (TF_A + 3 * 8)(sp) sd a4, (TF_A + 4 * 8)(sp) sd a5, (TF_A + 5 * 8)(sp) sd a6, (TF_A + 6 * 8)(sp) sd a7, (TF_A + 7 * 8)(sp) #if 0 /* XXX: temporary test: spin if stack is not kernel one */ .if \el == 1 /* kernel */ mv t0, sp srli t0, t0, 63 1: beqz t0, 1b .endif #endif .if \el == 1 /* Store kernel sp */ li t1, TF_SIZE add t0, sp, t1 sd t0, (TF_SP)(sp) .else /* Store user sp */ csrr t0, sscratch sd t0, (TF_SP)(sp) .endif li t0, 0 csrw sscratch, t0 csrr t0, sepc sd t0, (TF_SEPC)(sp) csrr t0, sstatus sd t0, (TF_SSTATUS)(sp) csrr t0, sbadaddr sd t0, (TF_SBADADDR)(sp) csrr t0, scause sd t0, (TF_SCAUSE)(sp) .endm .macro load_registers el ld t0, (TF_SSTATUS)(sp) .if \el == 0 - /* Ensure user interrupts will be enabled on eret. */ - li t1, SSTATUS_SPIE + /* + * Ensure user interrupts will be enabled on eret + * and supervisor mode can access userspace on trap. + */ + li t1, (SSTATUS_SPIE | SSTATUS_SUM) or t0, t0, t1 .else /* * Disable interrupts for supervisor mode exceptions. * For user mode exceptions we have already done this * in do_ast. */ li t1, ~SSTATUS_SIE and t0, t0, t1 .endif csrw sstatus, t0 ld t0, (TF_SEPC)(sp) csrw sepc, t0 .if \el == 0 /* We go to userspace. Load user sp */ ld t0, (TF_SP)(sp) csrw sscratch, t0 /* And store our pcpu */ sd gp, (TF_SIZE)(sp) ld gp, (TF_GP)(sp) .endif ld ra, (TF_RA)(sp) ld tp, (TF_TP)(sp) ld t0, (TF_T + 0 * 8)(sp) ld t1, (TF_T + 1 * 8)(sp) ld t2, (TF_T + 2 * 8)(sp) ld t3, (TF_T + 3 * 8)(sp) ld t4, (TF_T + 4 * 8)(sp) ld t5, (TF_T + 5 * 8)(sp) ld t6, (TF_T + 6 * 8)(sp) ld s0, (TF_S + 0 * 8)(sp) ld s1, (TF_S + 1 * 8)(sp) ld s2, (TF_S + 2 * 8)(sp) ld s3, (TF_S + 3 * 8)(sp) ld s4, (TF_S + 4 * 8)(sp) ld s5, (TF_S + 5 * 8)(sp) ld s6, (TF_S + 6 * 8)(sp) ld s7, (TF_S + 7 * 8)(sp) ld s8, (TF_S + 8 * 8)(sp) ld s9, (TF_S + 9 * 8)(sp) ld s10, (TF_S + 10 * 8)(sp) ld s11, (TF_S + 11 * 8)(sp) ld a0, (TF_A + 0 * 8)(sp) ld a1, (TF_A + 1 * 8)(sp) ld a2, (TF_A + 2 * 8)(sp) ld a3, (TF_A + 3 * 8)(sp) ld a4, (TF_A + 4 * 8)(sp) ld a5, (TF_A + 5 * 8)(sp) ld a6, (TF_A + 6 * 8)(sp) ld a7, (TF_A + 7 * 8)(sp) addi sp, sp, (TF_SIZE) .endm .macro do_ast /* Disable interrupts */ csrr a4, sstatus 1: csrci sstatus, (SSTATUS_SIE) ld a1, PC_CURTHREAD(gp) lw a2, TD_FLAGS(a1) li a3, (TDF_ASTPENDING|TDF_NEEDRESCHED) and a2, a2, a3 beqz a2, 2f /* Restore interrupts */ andi a4, a4, (SSTATUS_SIE) csrs sstatus, a4 /* Handle the ast */ mv a0, sp call _C_LABEL(ast) /* Re-check for new ast scheduled */ j 1b 2: .endm ENTRY(cpu_exception_handler) csrrw sp, sscratch, sp beqz sp, 1f /* User mode detected */ csrrw sp, sscratch, sp j cpu_exception_handler_user 1: /* Supervisor mode detected */ csrrw sp, sscratch, sp j cpu_exception_handler_supervisor END(cpu_exception_handler) ENTRY(cpu_exception_handler_supervisor) save_registers 1 mv a0, sp call _C_LABEL(do_trap_supervisor) load_registers 1 sret END(cpu_exception_handler_supervisor) ENTRY(cpu_exception_handler_user) csrrw sp, sscratch, sp save_registers 0 mv a0, sp call _C_LABEL(do_trap_user) do_ast load_registers 0 csrrw sp, sscratch, sp sret END(cpu_exception_handler_user) Index: head/sys/riscv/riscv/intr_machdep.c =================================================================== --- head/sys/riscv/riscv/intr_machdep.c (revision 322360) +++ head/sys/riscv/riscv/intr_machdep.c (revision 322361) @@ -1,317 +1,324 @@ /*- - * Copyright (c) 2015-2016 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SMP #include #endif u_long intrcnt[NIRQS]; size_t sintrcnt = sizeof(intrcnt); char intrnames[NIRQS * (MAXCOMLEN + 1) * 2]; size_t sintrnames = sizeof(intrnames); static struct intr_event *intr_events[NIRQS]; static riscv_intrcnt_t riscv_intr_counters[NIRQS]; static int intrcnt_index; riscv_intrcnt_t riscv_intrcnt_create(const char* name) { riscv_intrcnt_t counter; counter = &intrcnt[intrcnt_index++]; riscv_intrcnt_setname(counter, name); return (counter); } void riscv_intrcnt_setname(riscv_intrcnt_t counter, const char *name) { int i; i = (counter - intrcnt); KASSERT(counter != NULL, ("riscv_intrcnt_setname: NULL counter")); snprintf(intrnames + (MAXCOMLEN + 1) * i, MAXCOMLEN + 1, "%-*s", MAXCOMLEN, name); } static void riscv_mask_irq(void *source) { uintptr_t irq; irq = (uintptr_t)source; switch (irq) { case IRQ_TIMER_SUPERVISOR: csr_clear(sie, SIE_STIE); break; case IRQ_SOFTWARE_USER: csr_clear(sie, SIE_USIE); case IRQ_SOFTWARE_SUPERVISOR: csr_clear(sie, SIE_SSIE); break; #if 0 /* lowRISC TODO */ case IRQ_UART: machine_command(ECALL_IO_IRQ_MASK, 0); break; #endif default: panic("Unknown irq %d\n", irq); } } static void riscv_unmask_irq(void *source) { uintptr_t irq; irq = (uintptr_t)source; switch (irq) { case IRQ_TIMER_SUPERVISOR: csr_set(sie, SIE_STIE); break; case IRQ_SOFTWARE_USER: csr_set(sie, SIE_USIE); break; case IRQ_SOFTWARE_SUPERVISOR: csr_set(sie, SIE_SSIE); break; #if 0 /* lowRISC TODO */ case IRQ_UART: machine_command(ECALL_IO_IRQ_MASK, 1); break; #endif default: panic("Unknown irq %d\n", irq); } } void riscv_init_interrupts(void) { char name[MAXCOMLEN + 1]; int i; for (i = 0; i < NIRQS; i++) { snprintf(name, MAXCOMLEN + 1, "int%d:", i); riscv_intr_counters[i] = riscv_intrcnt_create(name); } } int riscv_setup_intr(const char *name, driver_filter_t *filt, void (*handler)(void*), void *arg, int irq, int flags, void **cookiep) { struct intr_event *event; int error; if (irq < 0 || irq >= NIRQS) panic("%s: unknown intr %d", __func__, irq); event = intr_events[irq]; if (event == NULL) { error = intr_event_create(&event, (void *)(uintptr_t)irq, 0, irq, riscv_mask_irq, riscv_unmask_irq, NULL, NULL, "int%d", irq); if (error) return (error); intr_events[irq] = event; riscv_unmask_irq((void*)(uintptr_t)irq); } error = intr_event_add_handler(event, name, filt, handler, arg, intr_priority(flags), flags, cookiep); if (error) { printf("Failed to setup intr: %d\n", irq); return (error); } riscv_intrcnt_setname(riscv_intr_counters[irq], event->ie_fullname); return (0); } int riscv_teardown_intr(void *ih) { /* TODO */ return (0); } int riscv_config_intr(u_int irq, enum intr_trigger trig, enum intr_polarity pol) { /* There is no configuration for interrupts */ return (0); } void riscv_cpu_intr(struct trapframe *frame) { struct intr_event *event; int active_irq; critical_enter(); KASSERT(frame->tf_scause & EXCP_INTR, ("riscv_cpu_intr: wrong frame passed")); active_irq = (frame->tf_scause & EXCP_MASK); switch (active_irq) { #if 0 /* lowRISC TODO */ case IRQ_UART: #endif case IRQ_SOFTWARE_USER: case IRQ_SOFTWARE_SUPERVISOR: case IRQ_TIMER_SUPERVISOR: event = intr_events[active_irq]; /* Update counters */ atomic_add_long(riscv_intr_counters[active_irq], 1); VM_CNT_INC(v_intr); break; default: event = NULL; } if (!event || TAILQ_EMPTY(&event->ie_handlers) || (intr_event_handle(event, frame) != 0)) printf("stray interrupt %d\n", active_irq); critical_exit(); } #ifdef SMP void riscv_setup_ipihandler(driver_filter_t *filt) { riscv_setup_intr("ipi", filt, NULL, NULL, IRQ_SOFTWARE_SUPERVISOR, INTR_TYPE_MISC, NULL); } void riscv_unmask_ipi(void) { csr_set(sie, SIE_SSIE); } /* Sending IPI */ static void ipi_send(struct pcpu *pc, int ipi) { + uintptr_t mask; CTR3(KTR_SMP, "%s: cpu=%d, ipi=%x", __func__, pc->pc_cpuid, ipi); atomic_set_32(&pc->pc_pending_ipis, ipi); - sbi_send_ipi(pc->pc_cpuid); + mask = (1 << (pc->pc_cpuid)); + sbi_send_ipi(&mask); + CTR1(KTR_SMP, "%s: sent", __func__); } void ipi_all_but_self(u_int ipi) { cpuset_t other_cpus; other_cpus = all_cpus; CPU_CLR(PCPU_GET(cpuid), &other_cpus); CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); ipi_selected(other_cpus, ipi); } void ipi_cpu(int cpu, u_int ipi) { cpuset_t cpus; CPU_ZERO(&cpus); CPU_SET(cpu, &cpus); CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x\n", __func__, cpu, ipi); ipi_send(cpuid_to_pcpu[cpu], ipi); } void ipi_selected(cpuset_t cpus, u_int ipi) { struct pcpu *pc; + uintptr_t mask; CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi); + mask = 0; STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { if (CPU_ISSET(pc->pc_cpuid, &cpus)) { CTR3(KTR_SMP, "%s: pc: %p, ipi: %x\n", __func__, pc, ipi); - ipi_send(pc, ipi); + atomic_set_32(&pc->pc_pending_ipis, ipi); + mask |= (1 << (pc->pc_cpuid)); } } + sbi_send_ipi(&mask); } #endif Index: head/sys/riscv/riscv/locore.S =================================================================== --- head/sys/riscv/riscv/locore.S (revision 322360) +++ head/sys/riscv/riscv/locore.S (revision 322361) @@ -1,318 +1,312 @@ /*- - * Copyright (c) 2015-2016 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "assym.s" #include #include #include #include #include #include .globl kernbase .set kernbase, KERNBASE /* Trap entries */ .text /* Reset vector */ .text .globl _start _start: - /* Setup supervisor trap vector */ - la t0, cpu_exception_handler - csrw stvec, t0 + /* Get the physical address kernel loaded to */ + la t0, virt_map + ld t1, 0(t0) + sub t1, t1, t0 + li t2, KERNBASE + sub s9, t2, t1 /* s9 = physmem base */ + mv s10, a0 /* s10 = hart id */ + mv s11, a1 /* s11 = dtbp */ - /* Ensure sscratch is zero */ - li t0, 0 - csrw sscratch, t0 + li t0, SSTATUS_SUM + csrs sstatus, t0 - /* Load physical memory information */ - li a0, 0 - la a1, memory_info - call sbi_query_memory - - /* Store base to s6 */ - la s6, memory_info - ld s6, 0(s6) /* s6 = physmem base */ - /* Direct secondary cores to mpentry */ - call sbi_hart_id - bnez a0, mpentry + bnez s10, mpentry /* * Page tables */ - /* Create an L1 page for early devmap */ + /* Add L1 entry for kernel */ la s1, pagetable_l1 - la s2, pagetable_l2_devmap /* Link to next level PN */ - li t0, KERNBASE - sub s2, s2, t0 - add s2, s2, s6 + la s2, pagetable_l2 /* Link to next level PN */ srli s2, s2, PAGE_SHIFT - li a5, (VM_MAX_KERNEL_ADDRESS - L2_SIZE) + li a5, KERNBASE srli a5, a5, L1_SHIFT /* >> L1_SHIFT */ andi a5, a5, 0x1ff /* & 0x1ff */ li t4, PTE_V slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */ or t6, t4, t5 - /* Store single level1 PTE entry to position */ + /* Store L1 PTE entry to position */ li a6, PTE_SIZE mulw a5, a5, a6 add t0, s1, a5 sd t6, (t0) - /* Create an L1 page for SBI */ - la s1, pagetable_l1 - la s2, pagetable_l2_sbi /* Link to next level PN */ - li t0, KERNBASE - sub s2, s2, t0 - add s2, s2, s6 - srli s2, s2, PAGE_SHIFT - li a5, 511 - li t4, PTE_V - slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */ - or t6, t4, t5 + /* Level 2 superpages (512 x 2MiB) */ + la s1, pagetable_l2 + srli t4, s9, 21 /* Div physmem base by 2 MiB */ + li t2, 512 /* Build 512 entries */ + add t3, t4, t2 + li t5, 0 +2: + li t0, (PTE_V | PTE_RWX | PTE_D) + slli t2, t4, PTE_PPN1_S /* << PTE_PPN1_S */ + or t5, t0, t2 + sd t5, (s1) /* Store PTE entry to position */ + addi s1, s1, PTE_SIZE - /* Store SBI L1 PTE entry to position */ - li a6, PTE_SIZE - mulw a5, a5, a6 - add t0, s1, a5 - sd t6, (t0) + addi t4, t4, 1 + bltu t4, t3, 2b - /* Create an L2 page for SBI */ - la s1, pagetable_l2_sbi - la s2, pagetable_l3_sbi /* Link to next level PN */ - li t0, KERNBASE - sub s2, s2, t0 - add s2, s2, s6 + /* Create an L1 page for early devmap */ + la s1, pagetable_l1 + la s2, pagetable_l2_devmap /* Link to next level PN */ srli s2, s2, PAGE_SHIFT - li a5, 511 + + li a5, (VM_MAX_KERNEL_ADDRESS - L2_SIZE) + srli a5, a5, L1_SHIFT /* >> L1_SHIFT */ + andi a5, a5, 0x1ff /* & 0x1ff */ li t4, PTE_V slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */ or t6, t4, t5 - /* Store SBI L2 PTE entry to position */ + /* Store single level1 PTE entry to position */ li a6, PTE_SIZE mulw a5, a5, a6 add t0, s1, a5 sd t6, (t0) - /* Create an L3 page for SBI */ - la s1, pagetable_l3_sbi - li s2, 0x8000b000 + /* Create an L2 page superpage for DTB */ + la s1, pagetable_l2_devmap + mv s2, s11 srli s2, s2, PAGE_SHIFT - li a5, 511 - li t4, PTE_V | PTE_RX | PTE_W - slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */ - or t6, t4, t5 - /* Store SBI L3 PTE entry to position */ - li a6, PTE_SIZE - mulw a5, a5, a6 - add t0, s1, a5 - sd t6, (t0) - /* END SBI page creation */ + li t0, (PTE_V | PTE_RWX | PTE_D) + slli t2, s2, PTE_PPN0_S /* << PTE_PPN0_S */ + or t0, t0, t2 - /* Add L1 entry for kernel */ - la s1, pagetable_l1 - la s2, pagetable_l2 /* Link to next level PN */ - li t0, KERNBASE - sub s2, s2, t0 - add s2, s2, s6 - srli s2, s2, PAGE_SHIFT - - li a5, KERNBASE - srli a5, a5, L1_SHIFT /* >> L1_SHIFT */ - andi a5, a5, 0x1ff /* & 0x1ff */ - li t4, PTE_V - slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */ - or t6, t4, t5 - - /* Store L1 PTE entry to position */ + /* Store PTE entry to position */ li a6, PTE_SIZE + li a5, 510 mulw a5, a5, a6 - add t0, s1, a5 - sd t6, (t0) + add t1, s1, a5 + sd t0, (t1) - /* Level 2 superpages (512 x 2MiB) */ - la s1, pagetable_l2 - srli t4, s6, 21 /* Div physmem base by 2 MiB */ - li t2, 512 /* Build 512 entries */ - add t3, t4, t2 - li t5, 0 -2: - li t0, (PTE_V | PTE_RWX) - slli t2, t4, PTE_PPN1_S /* << PTE_PPN1_S */ - or t5, t0, t2 - sd t5, (s1) /* Store PTE entry to position */ - addi s1, s1, PTE_SIZE + /* Page tables END */ - addi t4, t4, 1 - bltu t4, t3, 2b + /* Setup supervisor trap vector */ + la t0, va + sub t0, t0, s9 + li t1, KERNBASE + add t0, t0, t1 + csrw stvec, t0 /* Set page tables base register */ la s2, pagetable_l1 - li t0, KERNBASE - sub s2, s2, t0 - add s2, s2, s6 srli s2, s2, PAGE_SHIFT + li t0, SATP_MODE_SV39 + or s2, s2, t0 + sfence.vma csrw sptbr, s2 +va: + /* Setup supervisor trap vector */ + la t0, cpu_exception_handler + csrw stvec, t0 + + /* Ensure sscratch is zero */ + li t0, 0 + csrw sscratch, t0 + /* Initialize stack pointer */ la s3, initstack_end mv sp, s3 addi sp, sp, -PCB_SIZE /* Clear BSS */ la a0, _C_LABEL(__bss_start) la s1, _C_LABEL(_end) 1: sd zero, 0(a0) addi a0, a0, 8 bltu a0, s1, 1b /* Fill riscv_bootparams */ - addi sp, sp, -16 + addi sp, sp, -40 la t0, pagetable_l1 sd t0, 0(sp) /* kern_l1pt */ + sd s9, 8(sp) /* kern_phys */ la t0, initstack_end - sd t0, 8(sp) /* kern_stack */ + sd t0, 16(sp) /* kern_stack */ + li t0, (VM_MAX_KERNEL_ADDRESS - 2 * L2_SIZE) + sd t0, 24(sp) /* dtbp_virt */ + sd s11, 32(sp) /* dtbp_phys */ + mv a0, sp call _C_LABEL(initriscv) /* Off we go */ call _C_LABEL(mi_startup) .align 4 initstack: .space (PAGE_SIZE * KSTACK_PAGES) initstack_end: ENTRY(sigcode) mv a0, sp addi a0, a0, SF_UC 1: li t0, SYS_sigreturn ecall /* sigreturn failed, exit */ li t0, SYS_exit ecall j 1b END(sigcode) /* This may be copied to the stack, keep it 16-byte aligned */ .align 3 esigcode: .data .align 3 .global szsigcode szsigcode: .quad esigcode - sigcode .align 12 pagetable_l1: .space PAGE_SIZE pagetable_l2: .space PAGE_SIZE pagetable_l2_devmap: .space PAGE_SIZE -pagetable_l2_sbi: - .space PAGE_SIZE -pagetable_l3_sbi: - .space PAGE_SIZE - .globl memory_info -memory_info: - .space (24) + .align 3 +virt_map: + .quad virt_map + /* Not in use, but required for linking. */ + .align 3 + .globl __global_pointer$ +__global_pointer$: + .space 8 + .globl init_pt_va init_pt_va: .quad pagetable_l2 /* XXX: Keep page tables VA */ #ifndef SMP ENTRY(mpentry) 1: wfi j 1b END(mpentry) #else /* * mpentry(unsigned long) * * Called by a core when it is being brought online. */ ENTRY(mpentry) + /* Setup stack pointer */ + la t0, secondary_stacks + li t1, (PAGE_SIZE * KSTACK_PAGES) + mulw t1, t1, s10 + add t0, t0, t1 + sub t0, t0, s9 + li t1, KERNBASE + add sp, t0, t1 + + /* Setup supervisor trap vector */ + la t0, mpva + sub t0, t0, s9 + li t1, KERNBASE + add t0, t0, t1 + csrw stvec, t0 + + /* Set page tables base register */ + la s2, pagetable_l1 + srli s2, s2, PAGE_SHIFT + li t0, SATP_MODE_SV39 + or s2, s2, t0 + sfence.vma + csrw sptbr, s2 +mpva: + /* Setup supervisor trap vector */ + la t0, cpu_exception_handler + csrw stvec, t0 + + /* Ensure sscratch is zero */ + li t0, 0 + csrw sscratch, t0 /* * Calculate the offset to __riscv_boot_ap * for current core, cpuid in a0. */ li t1, 4 mulw t1, t1, a0 /* Get pointer */ la t0, __riscv_boot_ap add t0, t0, t1 1: /* Wait the kernel to be ready */ lw t1, 0(t0) beqz t1, 1b - - /* Set page tables base register */ - la s2, pagetable_l1 - li t0, KERNBASE - sub s2, s2, t0 - add s2, s2, s6 - srli s2, s2, PAGE_SHIFT - csrw sptbr, s2 - - /* Setup stack pointer */ - la t0, secondary_stacks - li t1, (PAGE_SIZE * KSTACK_PAGES) - mulw t1, t1, a0 - add sp, t0, t1 call init_secondary END(mpentry) #endif Index: head/sys/riscv/riscv/machdep.c =================================================================== --- head/sys/riscv/riscv/machdep.c (revision 322360) +++ head/sys/riscv/riscv/machdep.c (revision 322361) @@ -1,879 +1,893 @@ /*- * Copyright (c) 2014 Andrew Turner - * Copyright (c) 2015-2016 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FPE #include #endif #ifdef FDT #include #include #endif struct pcpu __pcpu[MAXCPU]; static struct trapframe proc0_tf; vm_paddr_t phys_avail[PHYS_AVAIL_SIZE + 2]; vm_paddr_t dump_avail[PHYS_AVAIL_SIZE + 2]; int early_boot = 1; int cold = 1; long realmem = 0; long Maxmem = 0; +#define DTB_SIZE_MAX (1024 * 1024) + #define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1)) vm_paddr_t physmap[PHYSMAP_SIZE]; u_int physmap_idx; struct kva_md_info kmi; int64_t dcache_line_size; /* The minimum D cache line size */ int64_t icache_line_size; /* The minimum I cache line size */ int64_t idcache_line_size; /* The minimum cache line size */ extern int *end; extern int *initstack_end; -extern memory_block_info memory_info; struct pcpu *pcpup; uintptr_t mcall_trap(uintptr_t mcause, uintptr_t* regs); uintptr_t mcall_trap(uintptr_t mcause, uintptr_t* regs) { return (0); } static void cpu_startup(void *dummy) { identify_cpu(); vm_ksubmap_init(&kmi); bufinit(); vm_pager_bufferinit(); } SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); int cpu_idle_wakeup(int cpu) { return (0); } void bzero(void *buf, size_t len) { uint8_t *p; p = buf; while(len-- > 0) *p++ = 0; } int fill_regs(struct thread *td, struct reg *regs) { struct trapframe *frame; frame = td->td_frame; regs->sepc = frame->tf_sepc; regs->sstatus = frame->tf_sstatus; regs->ra = frame->tf_ra; regs->sp = frame->tf_sp; regs->gp = frame->tf_gp; regs->tp = frame->tf_tp; memcpy(regs->t, frame->tf_t, sizeof(regs->t)); memcpy(regs->s, frame->tf_s, sizeof(regs->s)); memcpy(regs->a, frame->tf_a, sizeof(regs->a)); return (0); } int set_regs(struct thread *td, struct reg *regs) { struct trapframe *frame; frame = td->td_frame; frame->tf_sepc = regs->sepc; frame->tf_sstatus = regs->sstatus; frame->tf_ra = regs->ra; frame->tf_sp = regs->sp; frame->tf_gp = regs->gp; frame->tf_tp = regs->tp; memcpy(frame->tf_t, regs->t, sizeof(frame->tf_t)); memcpy(frame->tf_s, regs->s, sizeof(frame->tf_s)); memcpy(frame->tf_a, regs->a, sizeof(frame->tf_a)); return (0); } int fill_fpregs(struct thread *td, struct fpreg *regs) { #ifdef FPE struct pcb *pcb; pcb = td->td_pcb; if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) { /* * If we have just been running FPE instructions we will * need to save the state to memcpy it below. */ fpe_state_save(td); memcpy(regs->fp_x, pcb->pcb_x, sizeof(regs->fp_x)); regs->fp_fcsr = pcb->pcb_fcsr; } else #endif memset(regs->fp_x, 0, sizeof(regs->fp_x)); return (0); } int set_fpregs(struct thread *td, struct fpreg *regs) { #ifdef FPE struct pcb *pcb; pcb = td->td_pcb; memcpy(pcb->pcb_x, regs->fp_x, sizeof(regs->fp_x)); pcb->pcb_fcsr = regs->fp_fcsr; #endif return (0); } int fill_dbregs(struct thread *td, struct dbreg *regs) { panic("fill_dbregs"); } int set_dbregs(struct thread *td, struct dbreg *regs) { panic("set_dbregs"); } int ptrace_set_pc(struct thread *td, u_long addr) { panic("ptrace_set_pc"); return (0); } int ptrace_single_step(struct thread *td) { /* TODO; */ return (0); } int ptrace_clear_single_step(struct thread *td) { /* TODO; */ return (0); } void exec_setregs(struct thread *td, struct image_params *imgp, u_long stack) { struct trapframe *tf; struct pcb *pcb; tf = td->td_frame; pcb = td->td_pcb; memset(tf, 0, sizeof(struct trapframe)); /* * We need to set a0 for init as it doesn't call * cpu_set_syscall_retval to copy the value. We also * need to set td_retval for the cases where we do. */ tf->tf_a[0] = td->td_retval[0] = stack; tf->tf_sp = STACKALIGN(stack); tf->tf_ra = imgp->entry_addr; tf->tf_sepc = imgp->entry_addr; pcb->pcb_fpflags &= ~PCB_FP_STARTED; } /* Sanity check these are the same size, they will be memcpy'd to and fro */ CTASSERT(sizeof(((struct trapframe *)0)->tf_a) == sizeof((struct gpregs *)0)->gp_a); CTASSERT(sizeof(((struct trapframe *)0)->tf_s) == sizeof((struct gpregs *)0)->gp_s); CTASSERT(sizeof(((struct trapframe *)0)->tf_t) == sizeof((struct gpregs *)0)->gp_t); CTASSERT(sizeof(((struct trapframe *)0)->tf_a) == sizeof((struct reg *)0)->a); CTASSERT(sizeof(((struct trapframe *)0)->tf_s) == sizeof((struct reg *)0)->s); CTASSERT(sizeof(((struct trapframe *)0)->tf_t) == sizeof((struct reg *)0)->t); +/* Support for FDT configurations only. */ +CTASSERT(FDT); + int get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) { struct trapframe *tf = td->td_frame; memcpy(mcp->mc_gpregs.gp_t, tf->tf_t, sizeof(mcp->mc_gpregs.gp_t)); memcpy(mcp->mc_gpregs.gp_s, tf->tf_s, sizeof(mcp->mc_gpregs.gp_s)); memcpy(mcp->mc_gpregs.gp_a, tf->tf_a, sizeof(mcp->mc_gpregs.gp_a)); if (clear_ret & GET_MC_CLEAR_RET) { mcp->mc_gpregs.gp_a[0] = 0; mcp->mc_gpregs.gp_t[0] = 0; /* clear syscall error */ } mcp->mc_gpregs.gp_ra = tf->tf_ra; mcp->mc_gpregs.gp_sp = tf->tf_sp; mcp->mc_gpregs.gp_gp = tf->tf_gp; mcp->mc_gpregs.gp_tp = tf->tf_tp; mcp->mc_gpregs.gp_sepc = tf->tf_sepc; mcp->mc_gpregs.gp_sstatus = tf->tf_sstatus; return (0); } int set_mcontext(struct thread *td, mcontext_t *mcp) { struct trapframe *tf; tf = td->td_frame; memcpy(tf->tf_t, mcp->mc_gpregs.gp_t, sizeof(tf->tf_t)); memcpy(tf->tf_s, mcp->mc_gpregs.gp_s, sizeof(tf->tf_s)); memcpy(tf->tf_a, mcp->mc_gpregs.gp_a, sizeof(tf->tf_a)); tf->tf_ra = mcp->mc_gpregs.gp_ra; tf->tf_sp = mcp->mc_gpregs.gp_sp; tf->tf_gp = mcp->mc_gpregs.gp_gp; tf->tf_tp = mcp->mc_gpregs.gp_tp; tf->tf_sepc = mcp->mc_gpregs.gp_sepc; tf->tf_sstatus = mcp->mc_gpregs.gp_sstatus; return (0); } static void get_fpcontext(struct thread *td, mcontext_t *mcp) { #ifdef FPE struct pcb *curpcb; critical_enter(); curpcb = curthread->td_pcb; KASSERT(td->td_pcb == curpcb, ("Invalid fpe pcb")); if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) { /* * If we have just been running FPE instructions we will * need to save the state to memcpy it below. */ fpe_state_save(td); KASSERT((curpcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0, ("Non-userspace FPE flags set in get_fpcontext")); memcpy(mcp->mc_fpregs.fp_x, curpcb->pcb_x, sizeof(mcp->mc_fpregs)); mcp->mc_fpregs.fp_fcsr = curpcb->pcb_fcsr; mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags; mcp->mc_flags |= _MC_FP_VALID; } critical_exit(); #endif } static void set_fpcontext(struct thread *td, mcontext_t *mcp) { #ifdef FPE struct pcb *curpcb; critical_enter(); if ((mcp->mc_flags & _MC_FP_VALID) != 0) { curpcb = curthread->td_pcb; /* FPE usage is enabled, override registers. */ memcpy(curpcb->pcb_x, mcp->mc_fpregs.fp_x, sizeof(mcp->mc_fpregs)); curpcb->pcb_fcsr = mcp->mc_fpregs.fp_fcsr; curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags & PCB_FP_USERMASK; } critical_exit(); #endif } void cpu_idle(int busy) { spinlock_enter(); if (!busy) cpu_idleclock(); if (!sched_runnable()) __asm __volatile( "fence \n" "wfi \n"); if (!busy) cpu_activeclock(); spinlock_exit(); } void cpu_halt(void) { panic("cpu_halt"); } /* * Flush the D-cache for non-DMA I/O so that the I-cache can * be made coherent later. */ void cpu_flush_dcache(void *ptr, size_t len) { /* TBD */ } /* Get current clock frequency for the given CPU ID. */ int cpu_est_clockrate(int cpu_id, uint64_t *rate) { panic("cpu_est_clockrate"); } void cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) { } void spinlock_enter(void) { struct thread *td; td = curthread; if (td->td_md.md_spinlock_count == 0) { td->td_md.md_spinlock_count = 1; td->td_md.md_saved_sstatus_ie = intr_disable(); } else td->td_md.md_spinlock_count++; critical_enter(); } void spinlock_exit(void) { struct thread *td; register_t sstatus_ie; td = curthread; critical_exit(); sstatus_ie = td->td_md.md_saved_sstatus_ie; td->td_md.md_spinlock_count--; if (td->td_md.md_spinlock_count == 0) intr_restore(sstatus_ie); } #ifndef _SYS_SYSPROTO_H_ struct sigreturn_args { ucontext_t *ucp; }; #endif int sys_sigreturn(struct thread *td, struct sigreturn_args *uap) { uint64_t sstatus; ucontext_t uc; int error; if (uap == NULL) return (EFAULT); if (copyin(uap->sigcntxp, &uc, sizeof(uc))) return (EFAULT); /* * Make sure the processor mode has not been tampered with and * interrupts have not been disabled. * Supervisor interrupts in user mode are always enabled. */ sstatus = uc.uc_mcontext.mc_gpregs.gp_sstatus; if ((sstatus & SSTATUS_SPP) != 0) return (EINVAL); error = set_mcontext(td, &uc.uc_mcontext); if (error != 0) return (error); set_fpcontext(td, &uc.uc_mcontext); /* Restore signal mask. */ kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); return (EJUSTRETURN); } /* * Construct a PCB from a trapframe. This is called from kdb_trap() where * we want to start a backtrace from the function that caused us to enter * the debugger. We have the context in the trapframe, but base the trace * on the PCB. The PCB doesn't have to be perfect, as long as it contains * enough for a backtrace. */ void makectx(struct trapframe *tf, struct pcb *pcb) { memcpy(pcb->pcb_t, tf->tf_t, sizeof(tf->tf_t)); memcpy(pcb->pcb_s, tf->tf_s, sizeof(tf->tf_s)); memcpy(pcb->pcb_a, tf->tf_a, sizeof(tf->tf_a)); pcb->pcb_ra = tf->tf_ra; pcb->pcb_sp = tf->tf_sp; pcb->pcb_gp = tf->tf_gp; pcb->pcb_tp = tf->tf_tp; pcb->pcb_sepc = tf->tf_sepc; } void sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) { struct sigframe *fp, frame; struct sysentvec *sysent; struct trapframe *tf; struct sigacts *psp; struct thread *td; struct proc *p; int onstack; int code; int sig; td = curthread; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); sig = ksi->ksi_signo; code = ksi->ksi_code; psp = p->p_sigacts; mtx_assert(&psp->ps_mtx, MA_OWNED); tf = td->td_frame; onstack = sigonstack(tf->tf_sp); CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, catcher, sig); /* Allocate and validate space for the signal handler context. */ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack && SIGISMEMBER(psp->ps_sigonstack, sig)) { fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + td->td_sigstk.ss_size); } else { fp = (struct sigframe *)td->td_frame->tf_sp; } /* Make room, keeping the stack aligned */ fp--; fp = (struct sigframe *)STACKALIGN(fp); /* Fill in the frame to copy out */ get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); get_fpcontext(td, &frame.sf_uc.uc_mcontext); frame.sf_si = ksi->ksi_info; frame.sf_uc.uc_sigmask = *mask; frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) ? ((onstack) ? SS_ONSTACK : 0) : SS_DISABLE; frame.sf_uc.uc_stack = td->td_sigstk; mtx_unlock(&psp->ps_mtx); PROC_UNLOCK(td->td_proc); /* Copy the sigframe out to the user's stack. */ if (copyout(&frame, fp, sizeof(*fp)) != 0) { /* Process has trashed its stack. Kill it. */ CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp); PROC_LOCK(p); sigexit(td, SIGILL); } tf->tf_a[0] = sig; tf->tf_a[1] = (register_t)&fp->sf_si; tf->tf_a[2] = (register_t)&fp->sf_uc; tf->tf_sepc = (register_t)catcher; tf->tf_sp = (register_t)fp; sysent = p->p_sysent; if (sysent->sv_sigcode_base != 0) tf->tf_ra = (register_t)sysent->sv_sigcode_base; else tf->tf_ra = (register_t)(sysent->sv_psstrings - *(sysent->sv_szsigcode)); CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_sepc, tf->tf_sp); PROC_LOCK(p); mtx_lock(&psp->ps_mtx); } static void init_proc0(vm_offset_t kstack) { pcpup = &__pcpu[0]; proc_linkup0(&proc0, &thread0); thread0.td_kstack = kstack; thread0.td_pcb = (struct pcb *)(thread0.td_kstack) - 1; thread0.td_pcb->pcb_fpflags = 0; thread0.td_frame = &proc0_tf; pcpup->pc_curpcb = thread0.td_pcb; } static int add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap, u_int *physmap_idxp) { u_int i, insert_idx, _physmap_idx; _physmap_idx = *physmap_idxp; if (length == 0) return (1); /* * Find insertion point while checking for overlap. Start off by * assuming the new entry will be added to the end. */ insert_idx = _physmap_idx; for (i = 0; i <= _physmap_idx; i += 2) { if (base < physmap[i + 1]) { if (base + length <= physmap[i]) { insert_idx = i; break; } if (boothowto & RB_VERBOSE) printf( "Overlapping memory regions, ignoring second region\n"); return (1); } } /* See if we can prepend to the next entry. */ if (insert_idx <= _physmap_idx && base + length == physmap[insert_idx]) { physmap[insert_idx] = base; return (1); } /* See if we can append to the previous entry. */ if (insert_idx > 0 && base == physmap[insert_idx - 1]) { physmap[insert_idx - 1] += length; return (1); } _physmap_idx += 2; *physmap_idxp = _physmap_idx; if (_physmap_idx == PHYSMAP_SIZE) { printf( "Too many segments in the physical address map, giving up\n"); return (0); } /* * Move the last 'N' entries down to make room for the new * entry if needed. */ for (i = _physmap_idx; i > insert_idx; i -= 2) { physmap[i] = physmap[i - 2]; physmap[i + 1] = physmap[i - 1]; } /* Insert the new entry. */ physmap[insert_idx] = base; physmap[insert_idx + 1] = base + length; printf("physmap[%d] = 0x%016lx\n", insert_idx, base); printf("physmap[%d] = 0x%016lx\n", insert_idx + 1, base + length); return (1); } #ifdef FDT static void -try_load_dtb(caddr_t kmdp) +try_load_dtb(caddr_t kmdp, vm_offset_t dtbp) { - vm_offset_t dtbp; #if defined(FDT_DTB_STATIC) dtbp = (vm_offset_t)&fdt_static_dtb; -#else - /* TODO */ - dtbp = (vm_offset_t)NULL; #endif + if (dtbp == (vm_offset_t)NULL) { printf("ERROR loading DTB\n"); return; } if (OF_install(OFW_FDT, 0) == FALSE) panic("Cannot install FDT"); if (OF_init((void *)dtbp) != 0) panic("OF_init failed with the found device tree"); } #endif static void cache_setup(void) { /* TODO */ } /* * Fake up a boot descriptor table. * RISCVTODO: This needs to be done via loader (when it's available). */ vm_offset_t fake_preload_metadata(struct riscv_bootparams *rvbp __unused) { #ifdef DDB vm_offset_t zstart = 0, zend = 0; #endif vm_offset_t lastaddr; int i = 0; static uint32_t fake_preload[35]; fake_preload[i++] = MODINFO_NAME; fake_preload[i++] = strlen("kernel") + 1; strcpy((char*)&fake_preload[i++], "kernel"); i += 1; fake_preload[i++] = MODINFO_TYPE; fake_preload[i++] = strlen("elf64 kernel") + 1; strcpy((char*)&fake_preload[i++], "elf64 kernel"); i += 3; fake_preload[i++] = MODINFO_ADDR; fake_preload[i++] = sizeof(vm_offset_t); fake_preload[i++] = (uint64_t)(KERNBASE + KERNENTRY); i += 1; fake_preload[i++] = MODINFO_SIZE; fake_preload[i++] = sizeof(uint64_t); printf("end is 0x%016lx\n", (uint64_t)&end); fake_preload[i++] = (uint64_t)&end - (uint64_t)(KERNBASE + KERNENTRY); i += 1; #ifdef DDB #if 0 /* RISCVTODO */ if (*(uint32_t *)KERNVIRTADDR == MAGIC_TRAMP_NUMBER) { fake_preload[i++] = MODINFO_METADATA|MODINFOMD_SSYM; fake_preload[i++] = sizeof(vm_offset_t); fake_preload[i++] = *(uint32_t *)(KERNVIRTADDR + 4); fake_preload[i++] = MODINFO_METADATA|MODINFOMD_ESYM; fake_preload[i++] = sizeof(vm_offset_t); fake_preload[i++] = *(uint32_t *)(KERNVIRTADDR + 8); lastaddr = *(uint32_t *)(KERNVIRTADDR + 8); zend = lastaddr; zstart = *(uint32_t *)(KERNVIRTADDR + 4); db_fetch_ksymtab(zstart, zend); } else #endif #endif lastaddr = (vm_offset_t)&end; fake_preload[i++] = 0; fake_preload[i] = 0; preload_metadata = (void *)fake_preload; return (lastaddr); } void initriscv(struct riscv_bootparams *rvbp) { + struct mem_region mem_regions[FDT_MEM_REGIONS]; + vm_offset_t rstart, rend; + vm_offset_t s, e; + int mem_regions_sz; vm_offset_t lastaddr; vm_size_t kernlen; caddr_t kmdp; + int i; /* Set the module data location */ lastaddr = fake_preload_metadata(rvbp); /* Find the kernel address */ kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); boothowto = RB_VERBOSE | RB_SINGLE; boothowto = RB_VERBOSE; kern_envp = NULL; #ifdef FDT - try_load_dtb(kmdp); + try_load_dtb(kmdp, rvbp->dtbp_virt); #endif /* Load the physical memory ranges */ physmap_idx = 0; -#if 0 - struct mem_region mem_regions[FDT_MEM_REGIONS]; - int mem_regions_sz; - int i; +#ifdef FDT /* Grab physical memory regions information from device tree. */ if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, NULL) != 0) panic("Cannot get physical memory regions"); - for (i = 0; i < mem_regions_sz; i++) - add_physmap_entry(mem_regions[i].mr_start, - mem_regions[i].mr_size, physmap, &physmap_idx); -#endif - add_physmap_entry(memory_info.base, memory_info.size, - physmap, &physmap_idx); + s = rvbp->dtbp_phys; + e = s + DTB_SIZE_MAX; + for (i = 0; i < mem_regions_sz; i++) { + rstart = mem_regions[i].mr_start; + rend = (mem_regions[i].mr_start + mem_regions[i].mr_size); + + if ((rstart < s) && (rend > e)) { + /* Exclude DTB region. */ + add_physmap_entry(rstart, (s - rstart), physmap, &physmap_idx); + add_physmap_entry(e, (rend - e), physmap, &physmap_idx); + } else { + add_physmap_entry(mem_regions[i].mr_start, + mem_regions[i].mr_size, physmap, &physmap_idx); + } + } +#endif + /* Set the pcpu data, this is needed by pmap_bootstrap */ pcpup = &__pcpu[0]; pcpu_init(pcpup, 0, sizeof(struct pcpu)); /* Set the pcpu pointer */ __asm __volatile("mv gp, %0" :: "r"(pcpup)); PCPU_SET(curthread, &thread0); /* Do basic tuning, hz etc */ init_param1(); cache_setup(); /* Bootstrap enough of pmap to enter the kernel proper */ kernlen = (lastaddr - KERNBASE); - pmap_bootstrap(rvbp->kern_l1pt, memory_info.base, kernlen); + pmap_bootstrap(rvbp->kern_l1pt, mem_regions[0].mr_start, kernlen); cninit(); init_proc0(rvbp->kern_stack); /* set page table base register for thread0 */ thread0.td_pcb->pcb_l1addr = \ - (rvbp->kern_l1pt - KERNBASE + memory_info.base); + (rvbp->kern_l1pt - KERNBASE + rvbp->kern_phys); msgbufinit(msgbufp, msgbufsize); mutex_init(); init_param2(physmem); kdb_init(); riscv_init_interrupts(); early_boot = 0; } Index: head/sys/riscv/riscv/nexus.c =================================================================== --- head/sys/riscv/riscv/nexus.c (revision 322360) +++ head/sys/riscv/riscv/nexus.c (revision 322361) @@ -1,388 +1,390 @@ /*- * Copyright 1998 Massachusetts Institute of Technology * * Permission to use, copy, modify, and distribute this software and * its documentation for any purpose and without fee is hereby * granted, provided that both the above copyright notice and this * permission notice appear in all copies, that both the above * copyright notice and this permission notice appear in all * supporting documentation, and that the name of M.I.T. not be used * in advertising or publicity pertaining to distribution of the * software without specific, written prior permission. M.I.T. makes * no representations about the suitability of this software for any * purpose. It is provided "as is" without express or implied * warranty. * * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * This code implements a `root nexus' for RISC-V Architecture * machines. The function of the root nexus is to serve as an * attachment point for both processors and buses, and to manage * resources which are common to all of them. In particular, * this code implements the core resource managers for interrupt * requests, DMA requests (which rightfully should be a part of the * ISA code but it's easier to do it here for now), I/O port addresses, * and I/O memory address space. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "opt_platform.h" #include #include "ofw_bus_if.h" extern struct bus_space memmap_bus; static MALLOC_DEFINE(M_NEXUSDEV, "nexusdev", "Nexus device"); struct nexus_device { struct resource_list nx_resources; }; #define DEVTONX(dev) ((struct nexus_device *)device_get_ivars(dev)) static struct rman mem_rman; static struct rman irq_rman; static device_probe_t nexus_fdt_probe; static int nexus_attach(device_t); static int nexus_print_child(device_t, device_t); static device_t nexus_add_child(device_t, u_int, const char *, int); static struct resource *nexus_alloc_resource(device_t, device_t, int, int *, u_long, u_long, u_long, u_int); static int nexus_activate_resource(device_t, device_t, int, int, struct resource *); static int nexus_config_intr(device_t dev, int irq, enum intr_trigger trig, enum intr_polarity pol); static struct resource_list *nexus_get_reslist(device_t, device_t); static int nexus_set_resource(device_t, device_t, int, int, u_long, u_long); static int nexus_deactivate_resource(device_t, device_t, int, int, struct resource *); static int nexus_setup_intr(device_t dev, device_t child, struct resource *res, int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, void **cookiep); static int nexus_teardown_intr(device_t, device_t, struct resource *, void *); static int nexus_ofw_map_intr(device_t dev, device_t child, phandle_t iparent, int icells, pcell_t *intr); static device_method_t nexus_methods[] = { /* Device interface */ DEVMETHOD(device_probe, nexus_fdt_probe), DEVMETHOD(device_attach, nexus_attach), /* OFW interface */ DEVMETHOD(ofw_bus_map_intr, nexus_ofw_map_intr), /* Bus interface */ DEVMETHOD(bus_print_child, nexus_print_child), DEVMETHOD(bus_add_child, nexus_add_child), DEVMETHOD(bus_alloc_resource, nexus_alloc_resource), DEVMETHOD(bus_activate_resource, nexus_activate_resource), DEVMETHOD(bus_config_intr, nexus_config_intr), DEVMETHOD(bus_get_resource_list, nexus_get_reslist), DEVMETHOD(bus_set_resource, nexus_set_resource), DEVMETHOD(bus_deactivate_resource, nexus_deactivate_resource), DEVMETHOD(bus_setup_intr, nexus_setup_intr), DEVMETHOD(bus_teardown_intr, nexus_teardown_intr), { 0, 0 } }; static driver_t nexus_fdt_driver = { "nexus", nexus_methods, 1 /* no softc */ }; static int nexus_fdt_probe(device_t dev) { device_quiet(dev); return (BUS_PROBE_DEFAULT); } static int nexus_attach(device_t dev) { mem_rman.rm_start = 0; mem_rman.rm_end = BUS_SPACE_MAXADDR; mem_rman.rm_type = RMAN_ARRAY; mem_rman.rm_descr = "I/O memory addresses"; if (rman_init(&mem_rman) || rman_manage_region(&mem_rman, 0, BUS_SPACE_MAXADDR)) panic("nexus_attach mem_rman"); irq_rman.rm_start = 0; irq_rman.rm_end = ~0; irq_rman.rm_type = RMAN_ARRAY; irq_rman.rm_descr = "Interrupts"; if (rman_init(&irq_rman) || rman_manage_region(&irq_rman, 0, ~0)) panic("nexus_attach irq_rman"); + nexus_add_child(dev, 8, "timer", 0); + nexus_add_child(dev, 9, "rcons", 0); nexus_add_child(dev, 10, "ofwbus", 0); bus_generic_probe(dev); bus_generic_attach(dev); return (0); } static int nexus_print_child(device_t bus, device_t child) { int retval = 0; retval += bus_print_child_header(bus, child); retval += printf("\n"); return (retval); } static device_t nexus_add_child(device_t bus, u_int order, const char *name, int unit) { device_t child; struct nexus_device *ndev; ndev = malloc(sizeof(struct nexus_device), M_NEXUSDEV, M_NOWAIT|M_ZERO); if (!ndev) return (0); resource_list_init(&ndev->nx_resources); child = device_add_child_ordered(bus, order, name, unit); /* should we free this in nexus_child_detached? */ device_set_ivars(child, ndev); return (child); } /* * Allocate a resource on behalf of child. NB: child is usually going to be a * child of one of our descendants, not a direct child of nexus0. * (Exceptions include footbridge.) */ static struct resource * nexus_alloc_resource(device_t bus, device_t child, int type, int *rid, u_long start, u_long end, u_long count, u_int flags) { struct nexus_device *ndev = DEVTONX(child); struct resource *rv; struct resource_list_entry *rle; struct rman *rm; int needactivate = flags & RF_ACTIVE; /* * If this is an allocation of the "default" range for a given * RID, and we know what the resources for this device are * (ie. they aren't maintained by a child bus), then work out * the start/end values. */ if (RMAN_IS_DEFAULT_RANGE(start, end) && (count == 1)) { if (device_get_parent(child) != bus || ndev == NULL) return(NULL); rle = resource_list_find(&ndev->nx_resources, type, *rid); if (rle == NULL) return(NULL); start = rle->start; end = rle->end; count = rle->count; } switch (type) { case SYS_RES_IRQ: rm = &irq_rman; break; case SYS_RES_MEMORY: case SYS_RES_IOPORT: rm = &mem_rman; break; default: return (NULL); } rv = rman_reserve_resource(rm, start, end, count, flags, child); if (rv == NULL) return (NULL); rman_set_rid(rv, *rid); rman_set_bushandle(rv, rman_get_start(rv)); if (needactivate) { if (bus_activate_resource(child, type, *rid, rv)) { rman_release_resource(rv); return (NULL); } } return (rv); } static int nexus_config_intr(device_t dev, int irq, enum intr_trigger trig, enum intr_polarity pol) { return (riscv_config_intr(irq, trig, pol)); } static int nexus_setup_intr(device_t dev, device_t child, struct resource *res, int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, void **cookiep) { int error; if ((rman_get_flags(res) & RF_SHAREABLE) == 0) flags |= INTR_EXCL; /* We depend here on rman_activate_resource() being idempotent. */ error = rman_activate_resource(res); if (error) return (error); error = riscv_setup_intr(device_get_nameunit(child), filt, intr, arg, rman_get_start(res), flags, cookiep); return (error); } static int nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih) { return (riscv_teardown_intr(ih)); } static int nexus_activate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { int err; bus_addr_t paddr; bus_size_t psize; bus_space_handle_t vaddr; if ((err = rman_activate_resource(r)) != 0) return (err); /* * If this is a memory resource, map it into the kernel. */ if (type == SYS_RES_MEMORY || type == SYS_RES_IOPORT) { paddr = (bus_addr_t)rman_get_start(r); psize = (bus_size_t)rman_get_size(r); err = bus_space_map(&memmap_bus, paddr, psize, 0, &vaddr); if (err != 0) { rman_deactivate_resource(r); return (err); } rman_set_bustag(r, &memmap_bus); rman_set_virtual(r, (void *)vaddr); rman_set_bushandle(r, vaddr); } return (0); } static struct resource_list * nexus_get_reslist(device_t dev, device_t child) { struct nexus_device *ndev = DEVTONX(child); return (&ndev->nx_resources); } static int nexus_set_resource(device_t dev, device_t child, int type, int rid, u_long start, u_long count) { struct nexus_device *ndev = DEVTONX(child); struct resource_list *rl = &ndev->nx_resources; /* XXX this should return a success/failure indicator */ resource_list_add(rl, type, rid, start, start + count - 1, count); return(0); } static int nexus_deactivate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { bus_size_t psize; bus_space_handle_t vaddr; psize = (bus_size_t)rman_get_size(r); vaddr = rman_get_bushandle(r); if (vaddr != 0) { bus_space_unmap(&memmap_bus, vaddr, psize); rman_set_virtual(r, NULL); rman_set_bushandle(r, 0); } return (rman_deactivate_resource(r)); } static devclass_t nexus_fdt_devclass; EARLY_DRIVER_MODULE(nexus_fdt, root, nexus_fdt_driver, nexus_fdt_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_FIRST); static int nexus_ofw_map_intr(device_t dev, device_t child, phandle_t iparent, int icells, pcell_t *intr) { int irq; if (icells == 3) { irq = intr[1]; if (intr[0] == 0) irq += 32; /* SPI */ else irq += 16; /* PPI */ } else irq = intr[0]; return (irq); } Index: head/sys/riscv/riscv/pmap.c =================================================================== --- head/sys/riscv/riscv/pmap.c (revision 322360) +++ head/sys/riscv/riscv/pmap.c (revision 322361) @@ -1,3286 +1,3291 @@ /*- * Copyright (c) 1991 Regents of the University of California. * All rights reserved. * Copyright (c) 1994 John S. Dyson * All rights reserved. * Copyright (c) 1994 David Greenman * All rights reserved. * Copyright (c) 2003 Peter Wemm * All rights reserved. * Copyright (c) 2005-2010 Alan L. Cox * All rights reserved. * Copyright (c) 2014 Andrew Turner * All rights reserved. * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. - * Copyright (c) 2015-2016 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department and William Jolitz of UUNET Technologies Inc. * * Portions of this software were developed by Andrew Turner under * sponsorship from The FreeBSD Foundation. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 */ /*- * Copyright (c) 2003 Networks Associates Technology, Inc. * All rights reserved. * * This software was developed for the FreeBSD Project by Jake Burkholder, * Safeport Network Services, and Network Associates Laboratories, the * Security Research Division of Network Associates, Inc. under * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA * CHATS research program. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Manages physical address maps. * * Since the information managed by this module is * also stored by the logical address mapping module, * this module may throw away valid virtual-to-physical * mappings at almost any time. However, invalidations * of virtual-to-physical mappings must be done as * requested. * * In order to cope with hardware architectures which * make virtual-to-physical map invalidates expensive, * this module may delay invalidate or reduced protection * operations until such time as they are actually * necessary. This module is given full information as * to which processors are currently using which maps, * and to when physical maps must be made correct. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define NPDEPG (PAGE_SIZE/(sizeof (pd_entry_t))) #define NUPDE (NPDEPG * NPDEPG) #define NUSERPGTBLS (NUPDE + NPDEPG) #if !defined(DIAGNOSTIC) #ifdef __GNUC_GNU_INLINE__ #define PMAP_INLINE __attribute__((__gnu_inline__)) inline #else #define PMAP_INLINE extern inline #endif #else #define PMAP_INLINE #endif #ifdef PV_STATS #define PV_STAT(x) do { x ; } while (0) #else #define PV_STAT(x) do { } while (0) #endif #define pmap_l2_pindex(v) ((v) >> L2_SHIFT) #define NPV_LIST_LOCKS MAXCPU #define PHYS_TO_PV_LIST_LOCK(pa) \ (&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS]) #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ struct rwlock **_lockp = (lockp); \ struct rwlock *_new_lock; \ \ _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ if (_new_lock != *_lockp) { \ if (*_lockp != NULL) \ rw_wunlock(*_lockp); \ *_lockp = _new_lock; \ rw_wlock(*_lockp); \ } \ } while (0) #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) #define RELEASE_PV_LIST_LOCK(lockp) do { \ struct rwlock **_lockp = (lockp); \ \ if (*_lockp != NULL) { \ rw_wunlock(*_lockp); \ *_lockp = NULL; \ } \ } while (0) #define VM_PAGE_TO_PV_LIST_LOCK(m) \ PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) /* The list of all the user pmaps */ LIST_HEAD(pmaplist, pmap); static struct pmaplist allpmaps; static MALLOC_DEFINE(M_VMPMAP, "pmap", "PMAP L1"); struct pmap kernel_pmap_store; vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ vm_offset_t kernel_vm_end = 0; struct msgbuf *msgbufp = NULL; vm_paddr_t dmap_phys_base; /* The start of the dmap region */ vm_paddr_t dmap_phys_max; /* The limit of the dmap region */ vm_offset_t dmap_max_addr; /* The virtual address limit of the dmap */ /* This code assumes all L1 DMAP entries will be used */ CTASSERT((DMAP_MIN_ADDRESS & ~L1_OFFSET) == DMAP_MIN_ADDRESS); CTASSERT((DMAP_MAX_ADDRESS & ~L1_OFFSET) == DMAP_MAX_ADDRESS); static struct rwlock_padalign pvh_global_lock; /* * Data for the pv entry allocation mechanism */ static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); static struct mtx pv_chunks_mutex; static struct rwlock pv_list_locks[NPV_LIST_LOCKS]; static void free_pv_chunk(struct pv_chunk *pc); static void free_pv_entry(pmap_t pmap, pv_entry_t pv); static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va); static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva, pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp); static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, struct rwlock **lockp); static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp); static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free); static int pmap_unuse_l3(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); /* * These load the old table data and store the new value. * They need to be atomic as the System MMU may write to the table at * the same time as the CPU. */ #define pmap_load_store(table, entry) atomic_swap_64(table, entry) #define pmap_set(table, mask) atomic_set_64(table, mask) #define pmap_load_clear(table) atomic_swap_64(table, 0) #define pmap_load(table) (*table) /********************/ /* Inline functions */ /********************/ static __inline void pagecopy(void *s, void *d) { memcpy(d, s, PAGE_SIZE); } static __inline void pagezero(void *p) { bzero(p, PAGE_SIZE); } #define pmap_l1_index(va) (((va) >> L1_SHIFT) & Ln_ADDR_MASK) #define pmap_l2_index(va) (((va) >> L2_SHIFT) & Ln_ADDR_MASK) #define pmap_l3_index(va) (((va) >> L3_SHIFT) & Ln_ADDR_MASK) #define PTE_TO_PHYS(pte) ((pte >> PTE_PPN0_S) * PAGE_SIZE) static __inline pd_entry_t * pmap_l1(pmap_t pmap, vm_offset_t va) { return (&pmap->pm_l1[pmap_l1_index(va)]); } static __inline pd_entry_t * pmap_l1_to_l2(pd_entry_t *l1, vm_offset_t va) { vm_paddr_t phys; pd_entry_t *l2; phys = PTE_TO_PHYS(pmap_load(l1)); l2 = (pd_entry_t *)PHYS_TO_DMAP(phys); return (&l2[pmap_l2_index(va)]); } static __inline pd_entry_t * pmap_l2(pmap_t pmap, vm_offset_t va) { pd_entry_t *l1; l1 = pmap_l1(pmap, va); if (l1 == NULL) return (NULL); if ((pmap_load(l1) & PTE_V) == 0) return (NULL); if ((pmap_load(l1) & PTE_RX) != 0) return (NULL); return (pmap_l1_to_l2(l1, va)); } static __inline pt_entry_t * pmap_l2_to_l3(pd_entry_t *l2, vm_offset_t va) { vm_paddr_t phys; pt_entry_t *l3; phys = PTE_TO_PHYS(pmap_load(l2)); l3 = (pd_entry_t *)PHYS_TO_DMAP(phys); return (&l3[pmap_l3_index(va)]); } static __inline pt_entry_t * pmap_l3(pmap_t pmap, vm_offset_t va) { pd_entry_t *l2; l2 = pmap_l2(pmap, va); if (l2 == NULL) return (NULL); if ((pmap_load(l2) & PTE_V) == 0) return (NULL); if ((pmap_load(l2) & PTE_RX) != 0) return (NULL); return (pmap_l2_to_l3(l2, va)); } static __inline int pmap_is_write(pt_entry_t entry) { return (entry & PTE_W); } static __inline int pmap_is_current(pmap_t pmap) { return ((pmap == pmap_kernel()) || (pmap == curthread->td_proc->p_vmspace->vm_map.pmap)); } static __inline int pmap_l3_valid(pt_entry_t l3) { return (l3 & PTE_V); } static __inline int pmap_l3_valid_cacheable(pt_entry_t l3) { /* TODO */ return (0); } #define PTE_SYNC(pte) cpu_dcache_wb_range((vm_offset_t)pte, sizeof(*pte)) /* Checks if the page is dirty. */ static inline int pmap_page_dirty(pt_entry_t pte) { return (pte & PTE_D); } static __inline void pmap_resident_count_inc(pmap_t pmap, int count) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); pmap->pm_stats.resident_count += count; } static __inline void pmap_resident_count_dec(pmap_t pmap, int count) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT(pmap->pm_stats.resident_count >= count, ("pmap %p resident count underflow %ld %d", pmap, pmap->pm_stats.resident_count, count)); pmap->pm_stats.resident_count -= count; } static void pmap_distribute_l1(struct pmap *pmap, vm_pindex_t l1index, pt_entry_t entry) { struct pmap *user_pmap; pd_entry_t *l1; /* Distribute new kernel L1 entry to all the user pmaps */ if (pmap != kernel_pmap) return; LIST_FOREACH(user_pmap, &allpmaps, pm_list) { l1 = &user_pmap->pm_l1[l1index]; if (entry) pmap_load_store(l1, entry); else pmap_load_clear(l1); } } static pt_entry_t * pmap_early_page_idx(vm_offset_t l1pt, vm_offset_t va, u_int *l1_slot, u_int *l2_slot) { pt_entry_t *l2; pd_entry_t *l1; l1 = (pd_entry_t *)l1pt; *l1_slot = (va >> L1_SHIFT) & Ln_ADDR_MASK; /* Check locore has used a table L1 map */ KASSERT((l1[*l1_slot] & PTE_RX) == 0, ("Invalid bootstrap L1 table")); /* Find the address of the L2 table */ l2 = (pt_entry_t *)init_pt_va; *l2_slot = pmap_l2_index(va); return (l2); } static vm_paddr_t pmap_early_vtophys(vm_offset_t l1pt, vm_offset_t va) { u_int l1_slot, l2_slot; pt_entry_t *l2; u_int ret; l2 = pmap_early_page_idx(l1pt, va, &l1_slot, &l2_slot); /* Check locore has used L2 superpages */ KASSERT((l2[l2_slot] & PTE_RX) != 0, ("Invalid bootstrap L2 table")); /* L2 is superpages */ ret = (l2[l2_slot] >> PTE_PPN1_S) << L2_SHIFT; ret += (va & L2_OFFSET); return (ret); } static void pmap_bootstrap_dmap(vm_offset_t kern_l1, vm_paddr_t min_pa, vm_paddr_t max_pa) { vm_offset_t va; vm_paddr_t pa; pd_entry_t *l1; u_int l1_slot; pt_entry_t entry; pn_t pn; pa = dmap_phys_base = min_pa & ~L1_OFFSET; va = DMAP_MIN_ADDRESS; l1 = (pd_entry_t *)kern_l1; l1_slot = pmap_l1_index(DMAP_MIN_ADDRESS); for (; va < DMAP_MAX_ADDRESS && pa < max_pa; pa += L1_SIZE, va += L1_SIZE, l1_slot++) { KASSERT(l1_slot < Ln_ENTRIES, ("Invalid L1 index")); /* superpages */ pn = (pa / PAGE_SIZE); entry = (PTE_V | PTE_RWX); entry |= (pn << PTE_PPN0_S); pmap_load_store(&l1[l1_slot], entry); } /* Set the upper limit of the DMAP region */ dmap_phys_max = pa; dmap_max_addr = va; cpu_dcache_wb_range((vm_offset_t)l1, PAGE_SIZE); cpu_tlb_flushID(); } static vm_offset_t pmap_bootstrap_l3(vm_offset_t l1pt, vm_offset_t va, vm_offset_t l3_start) { vm_offset_t l2pt, l3pt; pt_entry_t entry; pd_entry_t *l2; vm_paddr_t pa; u_int l2_slot; pn_t pn; KASSERT((va & L2_OFFSET) == 0, ("Invalid virtual address")); l2 = pmap_l2(kernel_pmap, va); l2 = (pd_entry_t *)((uintptr_t)l2 & ~(PAGE_SIZE - 1)); l2pt = (vm_offset_t)l2; l2_slot = pmap_l2_index(va); l3pt = l3_start; for (; va < VM_MAX_KERNEL_ADDRESS; l2_slot++, va += L2_SIZE) { KASSERT(l2_slot < Ln_ENTRIES, ("Invalid L2 index")); pa = pmap_early_vtophys(l1pt, l3pt); pn = (pa / PAGE_SIZE); entry = (PTE_V); entry |= (pn << PTE_PPN0_S); pmap_load_store(&l2[l2_slot], entry); l3pt += PAGE_SIZE; } /* Clean the L2 page table */ memset((void *)l3_start, 0, l3pt - l3_start); cpu_dcache_wb_range(l3_start, l3pt - l3_start); cpu_dcache_wb_range((vm_offset_t)l2, PAGE_SIZE); return (l3pt); } /* * Bootstrap the system enough to run with virtual memory. */ void pmap_bootstrap(vm_offset_t l1pt, vm_paddr_t kernstart, vm_size_t kernlen) { u_int l1_slot, l2_slot, avail_slot, map_slot, used_map_slot; uint64_t kern_delta; pt_entry_t *l2; vm_offset_t va, freemempos; vm_offset_t dpcpu, msgbufpv; vm_paddr_t pa, min_pa, max_pa; int i; kern_delta = KERNBASE - kernstart; physmem = 0; printf("pmap_bootstrap %lx %lx %lx\n", l1pt, kernstart, kernlen); printf("%lx\n", l1pt); printf("%lx\n", (KERNBASE >> L1_SHIFT) & Ln_ADDR_MASK); /* Set this early so we can use the pagetable walking functions */ kernel_pmap_store.pm_l1 = (pd_entry_t *)l1pt; PMAP_LOCK_INIT(kernel_pmap); /* * Initialize the global pv list lock. */ rw_init(&pvh_global_lock, "pmap pv global"); LIST_INIT(&allpmaps); /* Assume the address we were loaded to is a valid physical address */ min_pa = max_pa = KERNBASE - kern_delta; /* * Find the minimum physical address. physmap is sorted, * but may contain empty ranges. */ for (i = 0; i < (physmap_idx * 2); i += 2) { if (physmap[i] == physmap[i + 1]) continue; if (physmap[i] <= min_pa) min_pa = physmap[i]; if (physmap[i + 1] > max_pa) max_pa = physmap[i + 1]; - break; } + printf("physmap_idx %lx\n", physmap_idx); + printf("min_pa %lx\n", min_pa); + printf("max_pa %lx\n", max_pa); /* Create a direct map region early so we can use it for pa -> va */ pmap_bootstrap_dmap(l1pt, min_pa, max_pa); va = KERNBASE; pa = KERNBASE - kern_delta; /* * Start to initialize phys_avail by copying from physmap * up to the physical address KERNBASE points at. */ map_slot = avail_slot = 0; for (; map_slot < (physmap_idx * 2); map_slot += 2) { if (physmap[map_slot] == physmap[map_slot + 1]) continue; if (physmap[map_slot] <= pa && physmap[map_slot + 1] > pa) break; phys_avail[avail_slot] = physmap[map_slot]; phys_avail[avail_slot + 1] = physmap[map_slot + 1]; physmem += (phys_avail[avail_slot + 1] - phys_avail[avail_slot]) >> PAGE_SHIFT; avail_slot += 2; } /* Add the memory before the kernel */ if (physmap[avail_slot] < pa) { phys_avail[avail_slot] = physmap[map_slot]; phys_avail[avail_slot + 1] = pa; physmem += (phys_avail[avail_slot + 1] - phys_avail[avail_slot]) >> PAGE_SHIFT; avail_slot += 2; } used_map_slot = map_slot; /* * Read the page table to find out what is already mapped. * This assumes we have mapped a block of memory from KERNBASE * using a single L1 entry. */ l2 = pmap_early_page_idx(l1pt, KERNBASE, &l1_slot, &l2_slot); /* Sanity check the index, KERNBASE should be the first VA */ KASSERT(l2_slot == 0, ("The L2 index is non-zero")); /* Find how many pages we have mapped */ for (; l2_slot < Ln_ENTRIES; l2_slot++) { if ((l2[l2_slot] & PTE_V) == 0) break; /* Check locore used L2 superpages */ KASSERT((l2[l2_slot] & PTE_RX) != 0, ("Invalid bootstrap L2 table")); va += L2_SIZE; pa += L2_SIZE; } va = roundup2(va, L2_SIZE); freemempos = KERNBASE + kernlen; freemempos = roundup2(freemempos, PAGE_SIZE); /* Create the l3 tables for the early devmap */ freemempos = pmap_bootstrap_l3(l1pt, VM_MAX_KERNEL_ADDRESS - L2_SIZE, freemempos); cpu_tlb_flushID(); #define alloc_pages(var, np) \ (var) = freemempos; \ freemempos += (np * PAGE_SIZE); \ memset((char *)(var), 0, ((np) * PAGE_SIZE)); /* Allocate dynamic per-cpu area. */ alloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu, 0); /* Allocate memory for the msgbuf, e.g. for /sbin/dmesg */ alloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); msgbufp = (void *)msgbufpv; virtual_avail = roundup2(freemempos, L2_SIZE); virtual_end = VM_MAX_KERNEL_ADDRESS - L2_SIZE; kernel_vm_end = virtual_avail; pa = pmap_early_vtophys(l1pt, freemempos); /* Finish initialising physmap */ map_slot = used_map_slot; for (; avail_slot < (PHYS_AVAIL_SIZE - 2) && map_slot < (physmap_idx * 2); map_slot += 2) { if (physmap[map_slot] == physmap[map_slot + 1]) { continue; } /* Have we used the current range? */ if (physmap[map_slot + 1] <= pa) { continue; } /* Do we need to split the entry? */ if (physmap[map_slot] < pa) { phys_avail[avail_slot] = pa; phys_avail[avail_slot + 1] = physmap[map_slot + 1]; } else { phys_avail[avail_slot] = physmap[map_slot]; phys_avail[avail_slot + 1] = physmap[map_slot + 1]; } physmem += (phys_avail[avail_slot + 1] - phys_avail[avail_slot]) >> PAGE_SHIFT; avail_slot += 2; } phys_avail[avail_slot] = 0; phys_avail[avail_slot + 1] = 0; /* * Maxmem isn't the "maximum memory", it's one larger than the * highest page of the physical address space. It should be * called something like "Maxphyspage". */ Maxmem = atop(phys_avail[avail_slot - 1]); cpu_tlb_flushID(); } /* * Initialize a vm_page's machine-dependent fields. */ void pmap_page_init(vm_page_t m) { TAILQ_INIT(&m->md.pv_list); m->md.pv_memattr = VM_MEMATTR_WRITE_BACK; } /* * Initialize the pmap module. * Called by vm_init, to initialize any structures that the pmap * system needs to map virtual memory. */ void pmap_init(void) { int i; /* * Initialize the pv chunk list mutex. */ mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); /* * Initialize the pool of pv list locks. */ for (i = 0; i < NPV_LIST_LOCKS; i++) rw_init(&pv_list_locks[i], "pmap pv list"); } /* * Normal, non-SMP, invalidation functions. * We inline these within pmap.c for speed. */ PMAP_INLINE void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { /* TODO */ sched_pin(); - __asm __volatile("sfence.vm"); + __asm __volatile("sfence.vma %0" :: "r" (va) : "memory"); sched_unpin(); } PMAP_INLINE void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { /* TODO */ sched_pin(); - __asm __volatile("sfence.vm"); + __asm __volatile("sfence.vma"); sched_unpin(); } PMAP_INLINE void pmap_invalidate_all(pmap_t pmap) { /* TODO */ sched_pin(); - __asm __volatile("sfence.vm"); + __asm __volatile("sfence.vma"); sched_unpin(); } /* * Routine: pmap_extract * Function: * Extract the physical page address associated * with the given map/virtual_address pair. */ vm_paddr_t pmap_extract(pmap_t pmap, vm_offset_t va) { pd_entry_t *l2p, l2; pt_entry_t *l3p, l3; vm_paddr_t pa; pa = 0; PMAP_LOCK(pmap); /* * Start with the l2 tabel. We are unable to allocate * pages in the l1 table. */ l2p = pmap_l2(pmap, va); if (l2p != NULL) { l2 = pmap_load(l2p); if ((l2 & PTE_RX) == 0) { l3p = pmap_l2_to_l3(l2p, va); if (l3p != NULL) { l3 = pmap_load(l3p); pa = PTE_TO_PHYS(l3); pa |= (va & L3_OFFSET); } } else { /* L2 is superpages */ pa = (l2 >> PTE_PPN1_S) << L2_SHIFT; pa |= (va & L2_OFFSET); } } PMAP_UNLOCK(pmap); return (pa); } /* * Routine: pmap_extract_and_hold * Function: * Atomically extract and hold the physical page * with the given pmap and virtual address pair * if that mapping permits the given protection. */ vm_page_t pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) { pt_entry_t *l3p, l3; vm_paddr_t phys; vm_paddr_t pa; vm_page_t m; pa = 0; m = NULL; PMAP_LOCK(pmap); retry: l3p = pmap_l3(pmap, va); if (l3p != NULL && (l3 = pmap_load(l3p)) != 0) { if ((pmap_is_write(l3)) || ((prot & VM_PROT_WRITE) == 0)) { phys = PTE_TO_PHYS(l3); if (vm_page_pa_tryrelock(pmap, phys, &pa)) goto retry; m = PHYS_TO_VM_PAGE(phys); vm_page_hold(m); } } PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); } vm_paddr_t pmap_kextract(vm_offset_t va) { pd_entry_t *l2; pt_entry_t *l3; vm_paddr_t pa; if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { pa = DMAP_TO_PHYS(va); } else { l2 = pmap_l2(kernel_pmap, va); if (l2 == NULL) panic("pmap_kextract: No l2"); if ((pmap_load(l2) & PTE_RX) != 0) { /* superpages */ pa = (pmap_load(l2) >> PTE_PPN1_S) << L2_SHIFT; pa |= (va & L2_OFFSET); return (pa); } l3 = pmap_l2_to_l3(l2, va); if (l3 == NULL) panic("pmap_kextract: No l3..."); pa = PTE_TO_PHYS(pmap_load(l3)); pa |= (va & PAGE_MASK); } return (pa); } /*************************************************** * Low level mapping routines..... ***************************************************/ void pmap_kenter_device(vm_offset_t sva, vm_size_t size, vm_paddr_t pa) { pt_entry_t entry; pt_entry_t *l3; vm_offset_t va; pn_t pn; KASSERT((pa & L3_OFFSET) == 0, ("pmap_kenter_device: Invalid physical address")); KASSERT((sva & L3_OFFSET) == 0, ("pmap_kenter_device: Invalid virtual address")); KASSERT((size & PAGE_MASK) == 0, ("pmap_kenter_device: Mapping is not page-sized")); va = sva; while (size != 0) { l3 = pmap_l3(kernel_pmap, va); KASSERT(l3 != NULL, ("Invalid page table, va: 0x%lx", va)); pn = (pa / PAGE_SIZE); entry = (PTE_V | PTE_RWX); entry |= (pn << PTE_PPN0_S); pmap_load_store(l3, entry); PTE_SYNC(l3); va += PAGE_SIZE; pa += PAGE_SIZE; size -= PAGE_SIZE; } pmap_invalidate_range(kernel_pmap, sva, va); } /* * Remove a page from the kernel pagetables. * Note: not SMP coherent. */ PMAP_INLINE void pmap_kremove(vm_offset_t va) { pt_entry_t *l3; l3 = pmap_l3(kernel_pmap, va); KASSERT(l3 != NULL, ("pmap_kremove: Invalid address")); if (pmap_l3_valid_cacheable(pmap_load(l3))) cpu_dcache_wb_range(va, L3_SIZE); pmap_load_clear(l3); PTE_SYNC(l3); pmap_invalidate_page(kernel_pmap, va); } void pmap_kremove_device(vm_offset_t sva, vm_size_t size) { pt_entry_t *l3; vm_offset_t va; KASSERT((sva & L3_OFFSET) == 0, ("pmap_kremove_device: Invalid virtual address")); KASSERT((size & PAGE_MASK) == 0, ("pmap_kremove_device: Mapping is not page-sized")); va = sva; while (size != 0) { l3 = pmap_l3(kernel_pmap, va); KASSERT(l3 != NULL, ("Invalid page table, va: 0x%lx", va)); pmap_load_clear(l3); PTE_SYNC(l3); va += PAGE_SIZE; size -= PAGE_SIZE; } pmap_invalidate_range(kernel_pmap, sva, va); } /* * Used to map a range of physical addresses into kernel * virtual address space. * * The value passed in '*virt' is a suggested virtual address for * the mapping. Architectures which can support a direct-mapped * physical to virtual region can return the appropriate address * within that region, leaving '*virt' unchanged. Other * architectures should map the pages starting at '*virt' and * update '*virt' with the first usable address after the mapped * region. */ vm_offset_t pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) { return PHYS_TO_DMAP(start); } /* * Add a list of wired pages to the kva * this routine is only used for temporary * kernel mappings that do not need to have * page modification or references recorded. * Note that old mappings are simply written * over. The page *must* be wired. * Note: SMP coherent. Uses a ranged shootdown IPI. */ void pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) { pt_entry_t *l3, pa; vm_offset_t va; vm_page_t m; pt_entry_t entry; pn_t pn; int i; va = sva; for (i = 0; i < count; i++) { m = ma[i]; pa = VM_PAGE_TO_PHYS(m); pn = (pa / PAGE_SIZE); l3 = pmap_l3(kernel_pmap, va); entry = (PTE_V | PTE_RWX); entry |= (pn << PTE_PPN0_S); pmap_load_store(l3, entry); PTE_SYNC(l3); va += L3_SIZE; } pmap_invalidate_range(kernel_pmap, sva, va); } /* * This routine tears out page mappings from the * kernel -- it is meant only for temporary mappings. * Note: SMP coherent. Uses a ranged shootdown IPI. */ void pmap_qremove(vm_offset_t sva, int count) { pt_entry_t *l3; vm_offset_t va; KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode va %lx", sva)); va = sva; while (count-- > 0) { l3 = pmap_l3(kernel_pmap, va); KASSERT(l3 != NULL, ("pmap_kremove: Invalid address")); if (pmap_l3_valid_cacheable(pmap_load(l3))) cpu_dcache_wb_range(va, L3_SIZE); pmap_load_clear(l3); PTE_SYNC(l3); va += PAGE_SIZE; } pmap_invalidate_range(kernel_pmap, sva, va); } /*************************************************** * Page table page management routines..... ***************************************************/ static __inline void pmap_free_zero_pages(struct spglist *free) { vm_page_t m; while ((m = SLIST_FIRST(free)) != NULL) { SLIST_REMOVE_HEAD(free, plinks.s.ss); /* Preserve the page's PG_ZERO setting. */ vm_page_free_toq(m); } } /* * Schedule the specified unused page table page to be freed. Specifically, * add the page to the specified list of pages that will be released to the * physical memory manager after the TLB has been updated. */ static __inline void pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, boolean_t set_PG_ZERO) { if (set_PG_ZERO) m->flags |= PG_ZERO; else m->flags &= ~PG_ZERO; SLIST_INSERT_HEAD(free, m, plinks.s.ss); } /* * Decrements a page table page's wire count, which is used to record the * number of valid page table entries within the page. If the wire count * drops to zero, then the page table page is unmapped. Returns TRUE if the * page table page was unmapped and FALSE otherwise. */ static inline boolean_t pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { --m->wire_count; if (m->wire_count == 0) { _pmap_unwire_l3(pmap, va, m, free); return (TRUE); } else { return (FALSE); } } static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { vm_paddr_t phys; PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * unmap the page table page */ if (m->pindex >= NUPDE) { /* PD page */ pd_entry_t *l1; l1 = pmap_l1(pmap, va); pmap_load_clear(l1); pmap_distribute_l1(pmap, pmap_l1_index(va), 0); PTE_SYNC(l1); } else { /* PTE page */ pd_entry_t *l2; l2 = pmap_l2(pmap, va); pmap_load_clear(l2); PTE_SYNC(l2); } pmap_resident_count_dec(pmap, 1); if (m->pindex < NUPDE) { pd_entry_t *l1; /* We just released a PT, unhold the matching PD */ vm_page_t pdpg; l1 = pmap_l1(pmap, va); phys = PTE_TO_PHYS(pmap_load(l1)); pdpg = PHYS_TO_VM_PAGE(phys); pmap_unwire_l3(pmap, va, pdpg, free); } pmap_invalidate_page(pmap, va); /* * This is a release store so that the ordinary store unmapping * the page table page is globally performed before TLB shoot- * down is begun. */ atomic_subtract_rel_int(&vm_cnt.v_wire_count, 1); /* * Put page on a list so that it is released after * *ALL* TLB shootdown is done */ pmap_add_delayed_free_list(m, free, TRUE); } /* * After removing an l3 entry, this routine is used to * conditionally free the page, and manage the hold/wire counts. */ static int pmap_unuse_l3(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, struct spglist *free) { vm_paddr_t phys; vm_page_t mpte; if (va >= VM_MAXUSER_ADDRESS) return (0); KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); phys = PTE_TO_PHYS(ptepde); mpte = PHYS_TO_VM_PAGE(phys); return (pmap_unwire_l3(pmap, va, mpte, free)); } void pmap_pinit0(pmap_t pmap) { PMAP_LOCK_INIT(pmap); bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); pmap->pm_l1 = kernel_pmap->pm_l1; } int pmap_pinit(pmap_t pmap) { vm_paddr_t l1phys; vm_page_t l1pt; /* * allocate the l1 page */ while ((l1pt = vm_page_alloc(NULL, 0xdeadbeef, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) VM_WAIT; l1phys = VM_PAGE_TO_PHYS(l1pt); pmap->pm_l1 = (pd_entry_t *)PHYS_TO_DMAP(l1phys); if ((l1pt->flags & PG_ZERO) == 0) pagezero(pmap->pm_l1); bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); /* Install kernel pagetables */ memcpy(pmap->pm_l1, kernel_pmap->pm_l1, PAGE_SIZE); /* Add to the list of all user pmaps */ LIST_INSERT_HEAD(&allpmaps, pmap, pm_list); return (1); } /* * This routine is called if the desired page table page does not exist. * * If page table page allocation fails, this routine may sleep before * returning NULL. It sleeps only if a lock pointer was given. * * Note: If a page allocation fails at page table level two or three, * one or two pages may be held during the wait, only to be released * afterwards. This conservative approach is easily argued to avoid * race conditions. */ static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) { vm_page_t m, /*pdppg, */pdpg; pt_entry_t entry; vm_paddr_t phys; pn_t pn; PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * Allocate a page table page. */ if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { if (lockp != NULL) { RELEASE_PV_LIST_LOCK(lockp); PMAP_UNLOCK(pmap); rw_runlock(&pvh_global_lock); VM_WAIT; rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); } /* * Indicate the need to retry. While waiting, the page table * page may have been allocated. */ return (NULL); } if ((m->flags & PG_ZERO) == 0) pmap_zero_page(m); /* * Map the pagetable page into the process address space, if * it isn't already there. */ if (ptepindex >= NUPDE) { pd_entry_t *l1; vm_pindex_t l1index; l1index = ptepindex - NUPDE; l1 = &pmap->pm_l1[l1index]; pn = (VM_PAGE_TO_PHYS(m) / PAGE_SIZE); entry = (PTE_V); entry |= (pn << PTE_PPN0_S); pmap_load_store(l1, entry); pmap_distribute_l1(pmap, l1index, entry); PTE_SYNC(l1); } else { vm_pindex_t l1index; pd_entry_t *l1, *l2; l1index = ptepindex >> (L1_SHIFT - L2_SHIFT); l1 = &pmap->pm_l1[l1index]; if (pmap_load(l1) == 0) { /* recurse for allocating page dir */ if (_pmap_alloc_l3(pmap, NUPDE + l1index, lockp) == NULL) { --m->wire_count; atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); return (NULL); } } else { phys = PTE_TO_PHYS(pmap_load(l1)); pdpg = PHYS_TO_VM_PAGE(phys); pdpg->wire_count++; } phys = PTE_TO_PHYS(pmap_load(l1)); l2 = (pd_entry_t *)PHYS_TO_DMAP(phys); l2 = &l2[ptepindex & Ln_ADDR_MASK]; pn = (VM_PAGE_TO_PHYS(m) / PAGE_SIZE); entry = (PTE_V); entry |= (pn << PTE_PPN0_S); pmap_load_store(l2, entry); PTE_SYNC(l2); } pmap_resident_count_inc(pmap, 1); return (m); } static vm_page_t pmap_alloc_l3(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) { vm_pindex_t ptepindex; pd_entry_t *l2; vm_paddr_t phys; vm_page_t m; /* * Calculate pagetable page index */ ptepindex = pmap_l2_pindex(va); retry: /* * Get the page directory entry */ l2 = pmap_l2(pmap, va); /* * If the page table page is mapped, we just increment the * hold count, and activate it. */ if (l2 != NULL && pmap_load(l2) != 0) { phys = PTE_TO_PHYS(pmap_load(l2)); m = PHYS_TO_VM_PAGE(phys); m->wire_count++; } else { /* * Here if the pte page isn't mapped, or if it has been * deallocated. */ m = _pmap_alloc_l3(pmap, ptepindex, lockp); if (m == NULL && lockp != NULL) goto retry; } return (m); } /*************************************************** * Pmap allocation/deallocation routines. ***************************************************/ /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap_t pmap) { vm_page_t m; KASSERT(pmap->pm_stats.resident_count == 0, ("pmap_release: pmap resident count %ld != 0", pmap->pm_stats.resident_count)); m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_l1)); m->wire_count--; atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); /* Remove pmap from the allpmaps list */ LIST_REMOVE(pmap, pm_list); /* Remove kernel pagetables */ bzero(pmap->pm_l1, PAGE_SIZE); } #if 0 static int kvm_size(SYSCTL_HANDLER_ARGS) { unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS; return sysctl_handle_long(oidp, &ksize, 0, req); } SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 0, 0, kvm_size, "LU", "Size of KVM"); static int kvm_free(SYSCTL_HANDLER_ARGS) { unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end; return sysctl_handle_long(oidp, &kfree, 0, req); } SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 0, 0, kvm_free, "LU", "Amount of KVM free"); #endif /* 0 */ /* * grow the number of kernel page table entries, if needed */ void pmap_growkernel(vm_offset_t addr) { vm_paddr_t paddr; vm_page_t nkpg; pd_entry_t *l1, *l2; pt_entry_t entry; pn_t pn; mtx_assert(&kernel_map->system_mtx, MA_OWNED); addr = roundup2(addr, L2_SIZE); if (addr - 1 >= kernel_map->max_offset) addr = kernel_map->max_offset; while (kernel_vm_end < addr) { l1 = pmap_l1(kernel_pmap, kernel_vm_end); if (pmap_load(l1) == 0) { /* We need a new PDP entry */ nkpg = vm_page_alloc(NULL, kernel_vm_end >> L1_SHIFT, VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (nkpg == NULL) panic("pmap_growkernel: no memory to grow kernel"); if ((nkpg->flags & PG_ZERO) == 0) pmap_zero_page(nkpg); paddr = VM_PAGE_TO_PHYS(nkpg); pn = (paddr / PAGE_SIZE); entry = (PTE_V); entry |= (pn << PTE_PPN0_S); pmap_load_store(l1, entry); pmap_distribute_l1(kernel_pmap, pmap_l1_index(kernel_vm_end), entry); PTE_SYNC(l1); continue; /* try again */ } l2 = pmap_l1_to_l2(l1, kernel_vm_end); if ((pmap_load(l2) & PTE_A) != 0) { kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; if (kernel_vm_end - 1 >= kernel_map->max_offset) { kernel_vm_end = kernel_map->max_offset; break; } continue; } nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_SHIFT, VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (nkpg == NULL) panic("pmap_growkernel: no memory to grow kernel"); if ((nkpg->flags & PG_ZERO) == 0) { pmap_zero_page(nkpg); } paddr = VM_PAGE_TO_PHYS(nkpg); pn = (paddr / PAGE_SIZE); entry = (PTE_V); entry |= (pn << PTE_PPN0_S); pmap_load_store(l2, entry); PTE_SYNC(l2); pmap_invalidate_page(kernel_pmap, kernel_vm_end); kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; if (kernel_vm_end - 1 >= kernel_map->max_offset) { kernel_vm_end = kernel_map->max_offset; break; } } } /*************************************************** * page management routines. ***************************************************/ CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); CTASSERT(_NPCM == 3); CTASSERT(_NPCPV == 168); static __inline struct pv_chunk * pv_to_chunk(pv_entry_t pv) { return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); } #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) #define PC_FREE0 0xfffffffffffffffful #define PC_FREE1 0xfffffffffffffffful #define PC_FREE2 0x000000fffffffffful static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 }; #if 0 #ifdef PV_STATS static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, "Current number of pv entry chunks"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, "Current number of pv entry chunks allocated"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, "Current number of pv entry chunks frees"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0, "Number of times tried to get a chunk page but failed."); static long pv_entry_frees, pv_entry_allocs, pv_entry_count; static int pv_entry_spare; SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0, "Current number of pv entry frees"); SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0, "Current number of pv entry allocs"); SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, "Current number of pv entries"); SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0, "Current number of spare pv entries"); #endif #endif /* 0 */ /* * We are in a serious low memory condition. Resort to * drastic measures to free some pages so we can allocate * another pv entry chunk. * * Returns NULL if PV entries were reclaimed from the specified pmap. * * We do not, however, unmap 2mpages because subsequent accesses will * allocate per-page pv entries until repromotion occurs, thereby * exacerbating the shortage of free pv entries. */ static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) { panic("RISCVTODO: reclaim_pv_chunk"); } /* * free the pv_entry back to the free list */ static void free_pv_entry(pmap_t pmap, pv_entry_t pv) { struct pv_chunk *pc; int idx, field, bit; rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); PV_STAT(atomic_add_long(&pv_entry_frees, 1)); PV_STAT(atomic_add_int(&pv_entry_spare, 1)); PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); pc = pv_to_chunk(pv); idx = pv - &pc->pc_pventry[0]; field = idx / 64; bit = idx % 64; pc->pc_map[field] |= 1ul << bit; if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 || pc->pc_map[2] != PC_FREE2) { /* 98% of the time, pc is already at the head of the list. */ if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); } return; } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } static void free_pv_chunk(struct pv_chunk *pc) { vm_page_t m; mtx_lock(&pv_chunks_mutex); TAILQ_REMOVE(&pv_chunks, pc, pc_lru); mtx_unlock(&pv_chunks_mutex); PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); /* entire chunk is free, return it */ m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); #if 0 /* TODO: For minidump */ dump_drop_page(m->phys_addr); #endif vm_page_unwire(m, PQ_INACTIVE); vm_page_free(m); } /* * Returns a new PV entry, allocating a new PV chunk from the system when * needed. If this PV chunk allocation fails and a PV list lock pointer was * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is * returned. * * The given PV list lock may be released. */ static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp) { int bit, field; pv_entry_t pv; struct pv_chunk *pc; vm_page_t m; rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); retry: pc = TAILQ_FIRST(&pmap->pm_pvchunk); if (pc != NULL) { for (field = 0; field < _NPCM; field++) { if (pc->pc_map[field]) { bit = ffsl(pc->pc_map[field]) - 1; break; } } if (field < _NPCM) { pv = &pc->pc_pventry[field * 64 + bit]; pc->pc_map[field] &= ~(1ul << bit); /* If this was the last item, move it to tail */ if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && pc->pc_map[2] == 0) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); } PV_STAT(atomic_add_long(&pv_entry_count, 1)); PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); return (pv); } } /* No free items, allocate another chunk */ m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); if (m == NULL) { if (lockp == NULL) { PV_STAT(pc_chunk_tryfail++); return (NULL); } m = reclaim_pv_chunk(pmap, lockp); if (m == NULL) goto retry; } PV_STAT(atomic_add_int(&pc_chunk_count, 1)); PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); #if 0 /* TODO: This is for minidump */ dump_add_page(m->phys_addr); #endif pc = (void *)PHYS_TO_DMAP(m->phys_addr); pc->pc_pmap = pmap; pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */ pc->pc_map[1] = PC_FREE1; pc->pc_map[2] = PC_FREE2; mtx_lock(&pv_chunks_mutex); TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); mtx_unlock(&pv_chunks_mutex); pv = &pc->pc_pventry[0]; TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); PV_STAT(atomic_add_long(&pv_entry_count, 1)); PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); return (pv); } /* * First find and then remove the pv entry for the specified pmap and virtual * address from the specified pv list. Returns the pv entry if found and NULL * otherwise. This operation can be performed on pv lists for either 4KB or * 2MB page mappings. */ static __inline pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) { pv_entry_t pv; rw_assert(&pvh_global_lock, RA_LOCKED); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (pmap == PV_PMAP(pv) && va == pv->pv_va) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); pvh->pv_gen++; break; } } return (pv); } /* * First find and then destroy the pv entry for the specified pmap and virtual * address. This operation can be performed on pv lists for either 4KB or 2MB * page mappings. */ static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) { pv_entry_t pv; pv = pmap_pvh_remove(pvh, pmap, va); KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); free_pv_entry(pmap, pv); } /* * Conditionally create the PV entry for a 4KB page mapping if the required * memory can be allocated without resorting to reclamation. */ static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, struct rwlock **lockp) { pv_entry_t pv; rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* Pass NULL instead of the lock pointer to disable reclamation. */ if ((pv = get_pv_entry(pmap, NULL)) != NULL) { pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; return (TRUE); } else return (FALSE); } /* * pmap_remove_l3: do the things to unmap a page in a process */ static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va, pd_entry_t l2e, struct spglist *free, struct rwlock **lockp) { pt_entry_t old_l3; vm_paddr_t phys; vm_page_t m; PMAP_LOCK_ASSERT(pmap, MA_OWNED); if (pmap_is_current(pmap) && pmap_l3_valid_cacheable(pmap_load(l3))) cpu_dcache_wb_range(va, L3_SIZE); old_l3 = pmap_load_clear(l3); PTE_SYNC(l3); pmap_invalidate_page(pmap, va); if (old_l3 & PTE_SW_WIRED) pmap->pm_stats.wired_count -= 1; pmap_resident_count_dec(pmap, 1); if (old_l3 & PTE_SW_MANAGED) { phys = PTE_TO_PHYS(old_l3); m = PHYS_TO_VM_PAGE(phys); if (pmap_page_dirty(old_l3)) vm_page_dirty(m); if (old_l3 & PTE_A) vm_page_aflag_set(m, PGA_REFERENCED); CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); pmap_pvh_free(&m->md, pmap, va); } return (pmap_unuse_l3(pmap, va, l2e, free)); } /* * Remove the given range of addresses from the specified map. * * It is assumed that the start and end are properly * rounded to the page size. */ void pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { struct rwlock *lock; vm_offset_t va, va_next; pd_entry_t *l1, *l2; pt_entry_t l3_pte, *l3; struct spglist free; int anyvalid; /* * Perform an unsynchronized read. This is, however, safe. */ if (pmap->pm_stats.resident_count == 0) return; anyvalid = 0; SLIST_INIT(&free); rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); lock = NULL; for (; sva < eva; sva = va_next) { if (pmap->pm_stats.resident_count == 0) break; l1 = pmap_l1(pmap, sva); if (pmap_load(l1) == 0) { va_next = (sva + L1_SIZE) & ~L1_OFFSET; if (va_next < sva) va_next = eva; continue; } /* * Calculate index for next page table. */ va_next = (sva + L2_SIZE) & ~L2_OFFSET; if (va_next < sva) va_next = eva; l2 = pmap_l1_to_l2(l1, sva); if (l2 == NULL) continue; l3_pte = pmap_load(l2); /* * Weed out invalid mappings. */ if (l3_pte == 0) continue; if ((pmap_load(l2) & PTE_RX) != 0) continue; /* * Limit our scan to either the end of the va represented * by the current page table page, or to the end of the * range being removed. */ if (va_next > eva) va_next = eva; va = va_next; for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, sva += L3_SIZE) { if (l3 == NULL) panic("l3 == NULL"); if (pmap_load(l3) == 0) { if (va != va_next) { pmap_invalidate_range(pmap, va, sva); va = va_next; } continue; } if (va == va_next) va = sva; if (pmap_remove_l3(pmap, l3, sva, l3_pte, &free, &lock)) { sva += L3_SIZE; break; } } if (va != va_next) pmap_invalidate_range(pmap, va, sva); } if (lock != NULL) rw_wunlock(lock); if (anyvalid) pmap_invalidate_all(pmap); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); } /* * Routine: pmap_remove_all * Function: * Removes this physical page from * all physical maps in which it resides. * Reflects back modify bits to the pager. * * Notes: * Original versions of this routine were very * inefficient because they iteratively called * pmap_remove (slow...) */ void pmap_remove_all(vm_page_t m) { pv_entry_t pv; pmap_t pmap; pt_entry_t *l3, tl3; pd_entry_t *l2, tl2; struct spglist free; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_all: page %p is not managed", m)); SLIST_INIT(&free); rw_wlock(&pvh_global_lock); while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pmap_resident_count_dec(pmap, 1); l2 = pmap_l2(pmap, pv->pv_va); KASSERT(l2 != NULL, ("pmap_remove_all: no l2 table found")); tl2 = pmap_load(l2); KASSERT((tl2 & PTE_RX) == 0, ("pmap_remove_all: found a table when expecting " "a block in %p's pv list", m)); l3 = pmap_l2_to_l3(l2, pv->pv_va); if (pmap_is_current(pmap) && pmap_l3_valid_cacheable(pmap_load(l3))) cpu_dcache_wb_range(pv->pv_va, L3_SIZE); tl3 = pmap_load_clear(l3); PTE_SYNC(l3); pmap_invalidate_page(pmap, pv->pv_va); if (tl3 & PTE_SW_WIRED) pmap->pm_stats.wired_count--; if ((tl3 & PTE_A) != 0) vm_page_aflag_set(m, PGA_REFERENCED); /* * Update the vm_page_t clean and reference bits. */ if (pmap_page_dirty(tl3)) vm_page_dirty(m); pmap_unuse_l3(pmap, pv->pv_va, pmap_load(l2), &free); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); } vm_page_aflag_clear(m, PGA_WRITEABLE); rw_wunlock(&pvh_global_lock); pmap_free_zero_pages(&free); } /* * Set the physical protection on the * specified range of this map as requested. */ void pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { vm_offset_t va, va_next; pd_entry_t *l1, *l2; pt_entry_t *l3p, l3; pt_entry_t entry; if ((prot & VM_PROT_READ) == VM_PROT_NONE) { pmap_remove(pmap, sva, eva); return; } if ((prot & VM_PROT_WRITE) == VM_PROT_WRITE) return; PMAP_LOCK(pmap); for (; sva < eva; sva = va_next) { l1 = pmap_l1(pmap, sva); if (pmap_load(l1) == 0) { va_next = (sva + L1_SIZE) & ~L1_OFFSET; if (va_next < sva) va_next = eva; continue; } va_next = (sva + L2_SIZE) & ~L2_OFFSET; if (va_next < sva) va_next = eva; l2 = pmap_l1_to_l2(l1, sva); if (l2 == NULL) continue; if (pmap_load(l2) == 0) continue; if ((pmap_load(l2) & PTE_RX) != 0) continue; if (va_next > eva) va_next = eva; va = va_next; for (l3p = pmap_l2_to_l3(l2, sva); sva != va_next; l3p++, sva += L3_SIZE) { l3 = pmap_load(l3p); if (pmap_l3_valid(l3)) { entry = pmap_load(l3p); entry &= ~(PTE_W); pmap_load_store(l3p, entry); PTE_SYNC(l3p); /* XXX: Use pmap_invalidate_range */ pmap_invalidate_page(pmap, va); } } } PMAP_UNLOCK(pmap); /* TODO: Only invalidate entries we are touching */ pmap_invalidate_all(pmap); } /* * Insert the given physical page (p) at * the specified virtual address (v) in the * target physical map with the protection requested. * * If specified, the page will be wired down, meaning * that the related pte can not be reclaimed. * * NB: This is the only routine which MAY NOT lazy-evaluate * or lose information. That is, this routine must actually * insert this page into the given map NOW. */ int pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind __unused) { struct rwlock *lock; pd_entry_t *l1, *l2; pt_entry_t new_l3, orig_l3; pt_entry_t *l3; pv_entry_t pv; vm_paddr_t opa, pa, l2_pa, l3_pa; vm_page_t mpte, om, l2_m, l3_m; boolean_t nosleep; pt_entry_t entry; pn_t l2_pn; pn_t l3_pn; pn_t pn; va = trunc_page(va); if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) VM_OBJECT_ASSERT_LOCKED(m->object); pa = VM_PAGE_TO_PHYS(m); pn = (pa / PAGE_SIZE); new_l3 = PTE_V | PTE_R | PTE_X; if (prot & VM_PROT_WRITE) new_l3 |= PTE_W; if ((va >> 63) == 0) new_l3 |= PTE_U; new_l3 |= (pn << PTE_PPN0_S); if ((flags & PMAP_ENTER_WIRED) != 0) new_l3 |= PTE_SW_WIRED; CTR2(KTR_PMAP, "pmap_enter: %.16lx -> %.16lx", va, pa); mpte = NULL; lock = NULL; rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); if (va < VM_MAXUSER_ADDRESS) { nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; mpte = pmap_alloc_l3(pmap, va, nosleep ? NULL : &lock); if (mpte == NULL && nosleep) { CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); if (lock != NULL) rw_wunlock(lock); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); return (KERN_RESOURCE_SHORTAGE); } l3 = pmap_l3(pmap, va); } else { l3 = pmap_l3(pmap, va); /* TODO: This is not optimal, but should mostly work */ if (l3 == NULL) { l2 = pmap_l2(pmap, va); if (l2 == NULL) { l2_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (l2_m == NULL) panic("pmap_enter: l2 pte_m == NULL"); if ((l2_m->flags & PG_ZERO) == 0) pmap_zero_page(l2_m); l2_pa = VM_PAGE_TO_PHYS(l2_m); l2_pn = (l2_pa / PAGE_SIZE); l1 = pmap_l1(pmap, va); entry = (PTE_V); entry |= (l2_pn << PTE_PPN0_S); pmap_load_store(l1, entry); pmap_distribute_l1(pmap, pmap_l1_index(va), entry); PTE_SYNC(l1); l2 = pmap_l1_to_l2(l1, va); } KASSERT(l2 != NULL, ("No l2 table after allocating one")); l3_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (l3_m == NULL) panic("pmap_enter: l3 pte_m == NULL"); if ((l3_m->flags & PG_ZERO) == 0) pmap_zero_page(l3_m); l3_pa = VM_PAGE_TO_PHYS(l3_m); l3_pn = (l3_pa / PAGE_SIZE); entry = (PTE_V); entry |= (l3_pn << PTE_PPN0_S); pmap_load_store(l2, entry); PTE_SYNC(l2); l3 = pmap_l2_to_l3(l2, va); } pmap_invalidate_page(pmap, va); } om = NULL; orig_l3 = pmap_load(l3); opa = PTE_TO_PHYS(orig_l3); /* * Is the specified virtual address already mapped? */ if (pmap_l3_valid(orig_l3)) { /* * Wiring change, just update stats. We don't worry about * wiring PT pages as they remain resident as long as there * are valid mappings in them. Hence, if a user page is wired, * the PT page will be also. */ if ((flags & PMAP_ENTER_WIRED) != 0 && (orig_l3 & PTE_SW_WIRED) == 0) pmap->pm_stats.wired_count++; else if ((flags & PMAP_ENTER_WIRED) == 0 && (orig_l3 & PTE_SW_WIRED) != 0) pmap->pm_stats.wired_count--; /* * Remove the extra PT page reference. */ if (mpte != NULL) { mpte->wire_count--; KASSERT(mpte->wire_count > 0, ("pmap_enter: missing reference to page table page," " va: 0x%lx", va)); } /* * Has the physical page changed? */ if (opa == pa) { /* * No, might be a protection or wiring change. */ if ((orig_l3 & PTE_SW_MANAGED) != 0) { new_l3 |= PTE_SW_MANAGED; if (pmap_is_write(new_l3)) vm_page_aflag_set(m, PGA_WRITEABLE); } goto validate; } /* Flush the cache, there might be uncommitted data in it */ if (pmap_is_current(pmap) && pmap_l3_valid_cacheable(orig_l3)) cpu_dcache_wb_range(va, L3_SIZE); } else { /* * Increment the counters. */ if ((new_l3 & PTE_SW_WIRED) != 0) pmap->pm_stats.wired_count++; pmap_resident_count_inc(pmap, 1); } /* * Enter on the PV list if part of our managed memory. */ if ((m->oflags & VPO_UNMANAGED) == 0) { new_l3 |= PTE_SW_MANAGED; pv = get_pv_entry(pmap, &lock); pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; if (pmap_is_write(new_l3)) vm_page_aflag_set(m, PGA_WRITEABLE); } /* * Update the L3 entry. */ if (orig_l3 != 0) { validate: orig_l3 = pmap_load_store(l3, new_l3); PTE_SYNC(l3); opa = PTE_TO_PHYS(orig_l3); if (opa != pa) { if ((orig_l3 & PTE_SW_MANAGED) != 0) { om = PHYS_TO_VM_PAGE(opa); if (pmap_page_dirty(orig_l3)) vm_page_dirty(om); if ((orig_l3 & PTE_A) != 0) vm_page_aflag_set(om, PGA_REFERENCED); CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); pmap_pvh_free(&om->md, pmap, va); } } else if (pmap_page_dirty(orig_l3)) { if ((orig_l3 & PTE_SW_MANAGED) != 0) vm_page_dirty(m); } } else { pmap_load_store(l3, new_l3); PTE_SYNC(l3); } pmap_invalidate_page(pmap, va); if ((pmap != pmap_kernel()) && (pmap == &curproc->p_vmspace->vm_pmap)) cpu_icache_sync_range(va, PAGE_SIZE); if (lock != NULL) rw_wunlock(lock); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } /* * Maps a sequence of resident pages belonging to the same object. * The sequence begins with the given page m_start. This page is * mapped at the given virtual address start. Each subsequent page is * mapped at a virtual address that is offset from start by the same * amount as the page is offset from m_start within the object. The * last page in the sequence is the page with the largest offset from * m_start that can be mapped at a virtual address less than the given * virtual address end. Not every virtual page between start and end * is mapped; only those for which a resident page exists with the * corresponding offset from m_start are mapped. */ void pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { struct rwlock *lock; vm_offset_t va; vm_page_t m, mpte; vm_pindex_t diff, psize; VM_OBJECT_ASSERT_LOCKED(m_start->object); psize = atop(end - start); mpte = NULL; m = m_start; lock = NULL; rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { va = start + ptoa(diff); mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte, &lock); m = TAILQ_NEXT(m, listq); } if (lock != NULL) rw_wunlock(lock); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * this code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. * 2. Not wired. * 3. Read access. * 4. No page table pages. * but is *MUCH* faster than pmap_enter... */ void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { struct rwlock *lock; lock = NULL; rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock); if (lock != NULL) rw_wunlock(lock); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp) { struct spglist free; vm_paddr_t phys; pd_entry_t *l2; pt_entry_t *l3; vm_paddr_t pa; pt_entry_t entry; pn_t pn; KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || (m->oflags & VPO_UNMANAGED) != 0, ("pmap_enter_quick_locked: managed mapping within the clean submap")); rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va); /* * In the case that a page table page is not * resident, we are creating it here. */ if (va < VM_MAXUSER_ADDRESS) { vm_pindex_t l2pindex; /* * Calculate pagetable page index */ l2pindex = pmap_l2_pindex(va); if (mpte && (mpte->pindex == l2pindex)) { mpte->wire_count++; } else { /* * Get the l2 entry */ l2 = pmap_l2(pmap, va); /* * If the page table page is mapped, we just increment * the hold count, and activate it. Otherwise, we * attempt to allocate a page table page. If this * attempt fails, we don't retry. Instead, we give up. */ if (l2 != NULL && pmap_load(l2) != 0) { phys = PTE_TO_PHYS(pmap_load(l2)); mpte = PHYS_TO_VM_PAGE(phys); mpte->wire_count++; } else { /* * Pass NULL instead of the PV list lock * pointer, because we don't intend to sleep. */ mpte = _pmap_alloc_l3(pmap, l2pindex, NULL); if (mpte == NULL) return (mpte); } } l3 = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); l3 = &l3[pmap_l3_index(va)]; } else { mpte = NULL; l3 = pmap_l3(kernel_pmap, va); } if (l3 == NULL) panic("pmap_enter_quick_locked: No l3"); if (pmap_load(l3) != 0) { if (mpte != NULL) { mpte->wire_count--; mpte = NULL; } return (mpte); } /* * Enter on the PV list if part of our managed memory. */ if ((m->oflags & VPO_UNMANAGED) == 0 && !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { if (mpte != NULL) { SLIST_INIT(&free); if (pmap_unwire_l3(pmap, va, mpte, &free)) { pmap_invalidate_page(pmap, va); pmap_free_zero_pages(&free); } mpte = NULL; } return (mpte); } /* * Increment counters */ pmap_resident_count_inc(pmap, 1); pa = VM_PAGE_TO_PHYS(m); pn = (pa / PAGE_SIZE); /* RISCVTODO: check permissions */ entry = (PTE_V | PTE_RWX); entry |= (pn << PTE_PPN0_S); /* * Now validate mapping with RO protection */ if ((m->oflags & VPO_UNMANAGED) == 0) entry |= PTE_SW_MANAGED; pmap_load_store(l3, entry); PTE_SYNC(l3); pmap_invalidate_page(pmap, va); return (mpte); } /* * This code maps large physical mmap regions into the * processor address space. Note that some shortcuts * are taken, but the code works. */ void pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); } /* * Clear the wired attribute from the mappings for the specified range of * addresses in the given pmap. Every valid mapping within that range * must have the wired attribute set. In contrast, invalid mappings * cannot have the wired attribute set, so they are ignored. * * The wired attribute of the page table entry is not a hardware feature, * so there is no need to invalidate any TLB entries. */ void pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { vm_offset_t va_next; pd_entry_t *l1, *l2; pt_entry_t *l3; boolean_t pv_lists_locked; pv_lists_locked = FALSE; PMAP_LOCK(pmap); for (; sva < eva; sva = va_next) { l1 = pmap_l1(pmap, sva); if (pmap_load(l1) == 0) { va_next = (sva + L1_SIZE) & ~L1_OFFSET; if (va_next < sva) va_next = eva; continue; } va_next = (sva + L2_SIZE) & ~L2_OFFSET; if (va_next < sva) va_next = eva; l2 = pmap_l1_to_l2(l1, sva); if (pmap_load(l2) == 0) continue; if (va_next > eva) va_next = eva; for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, sva += L3_SIZE) { if (pmap_load(l3) == 0) continue; if ((pmap_load(l3) & PTE_SW_WIRED) == 0) panic("pmap_unwire: l3 %#jx is missing " "PTE_SW_WIRED", (uintmax_t)*l3); /* * PG_W must be cleared atomically. Although the pmap * lock synchronizes access to PG_W, another processor * could be setting PG_M and/or PG_A concurrently. */ atomic_clear_long(l3, PTE_SW_WIRED); pmap->pm_stats.wired_count--; } } if (pv_lists_locked) rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len * in the destination map. * * This routine is only advisory and need not do anything. */ void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { } /* * pmap_zero_page zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. */ void pmap_zero_page(vm_page_t m) { vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); pagezero((void *)va); } /* * pmap_zero_page_area zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. * * off and size may not cover an area beyond a single hardware page. */ void pmap_zero_page_area(vm_page_t m, int off, int size) { vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); if (off == 0 && size == PAGE_SIZE) pagezero((void *)va); else bzero((char *)va + off, size); } /* * pmap_copy_page copies the specified (machine independent) * page by mapping the page into virtual memory and using * bcopy to copy the page, one machine dependent page at a * time. */ void pmap_copy_page(vm_page_t msrc, vm_page_t mdst) { vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc)); vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst)); pagecopy((void *)src, (void *)dst); } int unmapped_buf_allowed = 1; void pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], vm_offset_t b_offset, int xfersize) { void *a_cp, *b_cp; vm_page_t m_a, m_b; vm_paddr_t p_a, p_b; vm_offset_t a_pg_offset, b_pg_offset; int cnt; while (xfersize > 0) { a_pg_offset = a_offset & PAGE_MASK; m_a = ma[a_offset >> PAGE_SHIFT]; p_a = m_a->phys_addr; b_pg_offset = b_offset & PAGE_MASK; m_b = mb[b_offset >> PAGE_SHIFT]; p_b = m_b->phys_addr; cnt = min(xfersize, PAGE_SIZE - a_pg_offset); cnt = min(cnt, PAGE_SIZE - b_pg_offset); if (__predict_false(!PHYS_IN_DMAP(p_a))) { panic("!DMAP a %lx", p_a); } else { a_cp = (char *)PHYS_TO_DMAP(p_a) + a_pg_offset; } if (__predict_false(!PHYS_IN_DMAP(p_b))) { panic("!DMAP b %lx", p_b); } else { b_cp = (char *)PHYS_TO_DMAP(p_b) + b_pg_offset; } bcopy(a_cp, b_cp, cnt); a_offset += cnt; b_offset += cnt; xfersize -= cnt; } } vm_offset_t pmap_quick_enter_page(vm_page_t m) { return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m))); } void pmap_quick_remove_page(vm_offset_t addr) { } /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may * be changed upwards or downwards in the future; it * is only necessary that true be returned for a small * subset of pmaps for proper page aging. */ boolean_t pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { struct rwlock *lock; pv_entry_t pv; int loops = 0; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_page_exists_quick: page %p is not managed", m)); rv = FALSE; rw_rlock(&pvh_global_lock); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { if (PV_PMAP(pv) == pmap) { rv = TRUE; break; } loops++; if (loops >= 16) break; } rw_runlock(lock); rw_runlock(&pvh_global_lock); return (rv); } /* * pmap_page_wired_mappings: * * Return the number of managed mappings to the given physical page * that are wired. */ int pmap_page_wired_mappings(vm_page_t m) { struct rwlock *lock; pmap_t pmap; pt_entry_t *l3; pv_entry_t pv; int count, md_gen; if ((m->oflags & VPO_UNMANAGED) != 0) return (0); rw_rlock(&pvh_global_lock); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); restart: count = 0; TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; rw_runlock(lock); PMAP_LOCK(pmap); rw_rlock(lock); if (md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); goto restart; } } l3 = pmap_l3(pmap, pv->pv_va); if (l3 != NULL && (pmap_load(l3) & PTE_SW_WIRED) != 0) count++; PMAP_UNLOCK(pmap); } rw_runlock(lock); rw_runlock(&pvh_global_lock); return (count); } /* * Destroy all managed, non-wired mappings in the given user-space * pmap. This pmap cannot be active on any processor besides the * caller. * * This function cannot be applied to the kernel pmap. Moreover, it * is not intended for general use. It is only to be used during * process termination. Consequently, it can be implemented in ways * that make it faster than pmap_remove(). First, it can more quickly * destroy mappings by iterating over the pmap's collection of PV * entries, rather than searching the page table. Second, it doesn't * have to test and clear the page table entries atomically, because * no processor is currently accessing the user address space. In * particular, a page table entry's dirty bit won't change state once * this function starts. */ void pmap_remove_pages(pmap_t pmap) { pd_entry_t ptepde, *l2; pt_entry_t *l3, tl3; struct spglist free; vm_page_t m; pv_entry_t pv; struct pv_chunk *pc, *npc; struct rwlock *lock; int64_t bit; uint64_t inuse, bitmask; int allfree, field, freed, idx; vm_paddr_t pa; lock = NULL; SLIST_INIT(&free); rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { allfree = 1; freed = 0; for (field = 0; field < _NPCM; field++) { inuse = ~pc->pc_map[field] & pc_freemask[field]; while (inuse != 0) { bit = ffsl(inuse) - 1; bitmask = 1UL << bit; idx = field * 64 + bit; pv = &pc->pc_pventry[idx]; inuse &= ~bitmask; l2 = pmap_l2(pmap, pv->pv_va); ptepde = pmap_load(l2); l3 = pmap_l2_to_l3(l2, pv->pv_va); tl3 = pmap_load(l3); /* * We cannot remove wired pages from a process' mapping at this time */ if (tl3 & PTE_SW_WIRED) { allfree = 0; continue; } pa = PTE_TO_PHYS(tl3); m = PHYS_TO_VM_PAGE(pa); KASSERT(m->phys_addr == pa, ("vm_page_t %p phys_addr mismatch %016jx %016jx", m, (uintmax_t)m->phys_addr, (uintmax_t)tl3)); KASSERT((m->flags & PG_FICTITIOUS) != 0 || m < &vm_page_array[vm_page_array_size], ("pmap_remove_pages: bad l3 %#jx", (uintmax_t)tl3)); if (pmap_is_current(pmap) && pmap_l3_valid_cacheable(pmap_load(l3))) cpu_dcache_wb_range(pv->pv_va, L3_SIZE); pmap_load_clear(l3); PTE_SYNC(l3); pmap_invalidate_page(pmap, pv->pv_va); /* * Update the vm_page_t clean/reference bits. */ if (pmap_page_dirty(tl3)) vm_page_dirty(m); CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); /* Mark free */ pc->pc_map[field] |= bitmask; pmap_resident_count_dec(pmap, 1); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; pmap_unuse_l3(pmap, pv->pv_va, ptepde, &free); freed++; } } PV_STAT(atomic_add_long(&pv_entry_frees, freed)); PV_STAT(atomic_add_int(&pv_entry_spare, freed)); PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); if (allfree) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } } pmap_invalidate_all(pmap); if (lock != NULL) rw_wunlock(lock); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); } /* * This is used to check if a page has been accessed or modified. As we * don't have a bit to see if it has been modified we have to assume it * has been if the page is read/write. */ static boolean_t pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) { struct rwlock *lock; pv_entry_t pv; pt_entry_t *l3, mask, value; pmap_t pmap; int md_gen; boolean_t rv; rv = FALSE; rw_rlock(&pvh_global_lock); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); restart: TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; rw_runlock(lock); PMAP_LOCK(pmap); rw_rlock(lock); if (md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); goto restart; } } l3 = pmap_l3(pmap, pv->pv_va); mask = 0; value = 0; if (modified) { mask |= PTE_D; value |= PTE_D; } if (accessed) { mask |= PTE_A; value |= PTE_A; } #if 0 if (modified) { mask |= ATTR_AP_RW_BIT; value |= ATTR_AP(ATTR_AP_RW); } if (accessed) { mask |= ATTR_AF | ATTR_DESCR_MASK; value |= ATTR_AF | L3_PAGE; } #endif rv = (pmap_load(l3) & mask) == value; PMAP_UNLOCK(pmap); if (rv) goto out; } out: rw_runlock(lock); rw_runlock(&pvh_global_lock); return (rv); } /* * pmap_is_modified: * * Return whether or not the specified physical page was modified * in any physical maps. */ boolean_t pmap_is_modified(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_is_modified: page %p is not managed", m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTEs can have PG_M set. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return (FALSE); return (pmap_page_test_mappings(m, FALSE, TRUE)); } /* * pmap_is_prefaultable: * * Return whether or not the specified virtual address is eligible * for prefault. */ boolean_t pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) { pt_entry_t *l3; boolean_t rv; rv = FALSE; PMAP_LOCK(pmap); l3 = pmap_l3(pmap, addr); if (l3 != NULL && pmap_load(l3) != 0) { rv = TRUE; } PMAP_UNLOCK(pmap); return (rv); } /* * pmap_is_referenced: * * Return whether or not the specified physical page was referenced * in any physical maps. */ boolean_t pmap_is_referenced(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_is_referenced: page %p is not managed", m)); return (pmap_page_test_mappings(m, TRUE, FALSE)); } /* * Clear the write and modified bits in each of the given page's mappings. */ void pmap_remove_write(vm_page_t m) { pmap_t pmap; struct rwlock *lock; pv_entry_t pv; pt_entry_t *l3, oldl3; pt_entry_t newl3; int md_gen; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_write: page %p is not managed", m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * set by another thread while the object is locked. Thus, * if PGA_WRITEABLE is clear, no page table entries need updating. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return; rw_rlock(&pvh_global_lock); lock = VM_PAGE_TO_PV_LIST_LOCK(m); retry_pv_loop: rw_wlock(lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); rw_wunlock(lock); goto retry_pv_loop; } } l3 = pmap_l3(pmap, pv->pv_va); retry: oldl3 = pmap_load(l3); if (pmap_is_write(oldl3)) { newl3 = oldl3 & ~(PTE_W); if (!atomic_cmpset_long(l3, oldl3, newl3)) goto retry; /* TODO: use pmap_page_dirty(oldl3) ? */ if ((oldl3 & PTE_A) != 0) vm_page_dirty(m); pmap_invalidate_page(pmap, pv->pv_va); } PMAP_UNLOCK(pmap); } rw_wunlock(lock); vm_page_aflag_clear(m, PGA_WRITEABLE); rw_runlock(&pvh_global_lock); } static __inline boolean_t safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte) { return (FALSE); } /* * pmap_ts_referenced: * * Return a count of reference bits for a page, clearing those bits. * It is not necessary for every reference bit to be cleared, but it * is necessary that 0 only be returned when there are truly no * reference bits set. * * As an optimization, update the page's dirty field if a modified bit is * found while counting reference bits. This opportunistic update can be * performed at low cost and can eliminate the need for some future calls * to pmap_is_modified(). However, since this function stops after * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some * dirty pages. Those dirty pages will only be detected by a future call * to pmap_is_modified(). */ int pmap_ts_referenced(vm_page_t m) { pv_entry_t pv, pvf; pmap_t pmap; struct rwlock *lock; pd_entry_t *l2; pt_entry_t *l3, old_l3; vm_paddr_t pa; int cleared, md_gen, not_cleared; struct spglist free; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_ts_referenced: page %p is not managed", m)); SLIST_INIT(&free); cleared = 0; pa = VM_PAGE_TO_PHYS(m); lock = PHYS_TO_PV_LIST_LOCK(pa); rw_rlock(&pvh_global_lock); rw_wlock(lock); retry: not_cleared = 0; if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) goto out; pv = pvf; do { if (pvf == NULL) pvf = pv; pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); goto retry; } } l2 = pmap_l2(pmap, pv->pv_va); KASSERT((pmap_load(l2) & PTE_RX) == 0, ("pmap_ts_referenced: found an invalid l2 table")); l3 = pmap_l2_to_l3(l2, pv->pv_va); old_l3 = pmap_load(l3); if (pmap_page_dirty(old_l3)) vm_page_dirty(m); if ((old_l3 & PTE_A) != 0) { if (safe_to_clear_referenced(pmap, old_l3)) { /* * TODO: We don't handle the access flag * at all. We need to be able to set it in * the exception handler. */ panic("RISCVTODO: safe_to_clear_referenced\n"); } else if ((old_l3 & PTE_SW_WIRED) == 0) { /* * Wired pages cannot be paged out so * doing accessed bit emulation for * them is wasted effort. We do the * hard work for unwired pages only. */ pmap_remove_l3(pmap, l3, pv->pv_va, pmap_load(l2), &free, &lock); pmap_invalidate_page(pmap, pv->pv_va); cleared++; if (pvf == pv) pvf = NULL; pv = NULL; KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), ("inconsistent pv lock %p %p for page %p", lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); } else not_cleared++; } PMAP_UNLOCK(pmap); /* Rotate the PV list if it has more than one entry. */ if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) { TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; } } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + not_cleared < PMAP_TS_REFERENCED_MAX); out: rw_wunlock(lock); rw_runlock(&pvh_global_lock); pmap_free_zero_pages(&free); return (cleared + not_cleared); } /* * Apply the given advice to the specified range of addresses within the * given pmap. Depending on the advice, clear the referenced and/or * modified flags in each mapping and set the mapped page's dirty field. */ void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) { } /* * Clear the modify bits on the specified physical page. */ void pmap_clear_modify(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(!vm_page_xbusied(m), ("pmap_clear_modify: page %p is exclusive busied", m)); /* * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. * If the object containing the page is locked and the page is not * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. */ if ((m->aflags & PGA_WRITEABLE) == 0) return; /* RISCVTODO: We lack support for tracking if a page is modified */ } void * pmap_mapbios(vm_paddr_t pa, vm_size_t size) { return ((void *)PHYS_TO_DMAP(pa)); } void pmap_unmapbios(vm_paddr_t pa, vm_size_t size) { } /* * Sets the memory attribute for the specified page. */ void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) { m->md.pv_memattr = ma; /* * RISCVTODO: Implement the below (from the amd64 pmap) * If "m" is a normal page, update its direct mapping. This update * can be relied upon to perform any cache operations that are * required for data coherence. */ if ((m->flags & PG_FICTITIOUS) == 0 && PHYS_IN_DMAP(VM_PAGE_TO_PHYS(m))) panic("RISCVTODO: pmap_page_set_memattr"); } /* * perform the pmap work for mincore */ int pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) { panic("RISCVTODO: pmap_mincore"); } void pmap_activate(struct thread *td) { pmap_t pmap; + uint64_t reg; critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); td->td_pcb->pcb_l1addr = vtophys(pmap->pm_l1); - __asm __volatile("csrw sptbr, %0" :: "r"(td->td_pcb->pcb_l1addr >> PAGE_SHIFT)); + reg = SATP_MODE_SV39; + reg |= (td->td_pcb->pcb_l1addr >> PAGE_SHIFT); + __asm __volatile("csrw sptbr, %0" :: "r"(reg)); pmap_invalidate_all(pmap); critical_exit(); } void pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) { panic("RISCVTODO: pmap_sync_icache"); } /* * Increase the starting virtual address of the given mapping if a * different alignment might result in more superpage mappings. */ void pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t size) { } /** * Get the kernel virtual address of a set of physical pages. If there are * physical addresses not covered by the DMAP perform a transient mapping * that will be removed when calling pmap_unmap_io_transient. * * \param page The pages the caller wishes to obtain the virtual * address on the kernel memory map. * \param vaddr On return contains the kernel virtual memory address * of the pages passed in the page parameter. * \param count Number of pages passed in. * \param can_fault TRUE if the thread using the mapped pages can take * page faults, FALSE otherwise. * * \returns TRUE if the caller must call pmap_unmap_io_transient when * finished or FALSE otherwise. * */ boolean_t pmap_map_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count, boolean_t can_fault) { vm_paddr_t paddr; boolean_t needs_mapping; int error, i; /* * Allocate any KVA space that we need, this is done in a separate * loop to prevent calling vmem_alloc while pinned. */ needs_mapping = FALSE; for (i = 0; i < count; i++) { paddr = VM_PAGE_TO_PHYS(page[i]); if (__predict_false(paddr >= DMAP_MAX_PHYSADDR)) { error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, &vaddr[i]); KASSERT(error == 0, ("vmem_alloc failed: %d", error)); needs_mapping = TRUE; } else { vaddr[i] = PHYS_TO_DMAP(paddr); } } /* Exit early if everything is covered by the DMAP */ if (!needs_mapping) return (FALSE); if (!can_fault) sched_pin(); for (i = 0; i < count; i++) { paddr = VM_PAGE_TO_PHYS(page[i]); if (paddr >= DMAP_MAX_PHYSADDR) { panic( "pmap_map_io_transient: TODO: Map out of DMAP data"); } } return (needs_mapping); } void pmap_unmap_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count, boolean_t can_fault) { vm_paddr_t paddr; int i; if (!can_fault) sched_unpin(); for (i = 0; i < count; i++) { paddr = VM_PAGE_TO_PHYS(page[i]); if (paddr >= DMAP_MAX_PHYSADDR) { panic("RISCVTODO: pmap_unmap_io_transient: Unmap data"); } } } Index: head/sys/riscv/riscv/riscv_console.c =================================================================== --- head/sys/riscv/riscv/riscv_console.c (revision 322360) +++ head/sys/riscv/riscv/riscv_console.c (revision 322361) @@ -1,359 +1,285 @@ /*- - * Copyright (c) 2015-2016 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -static struct resource_spec rcons_spec[] = { - { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE}, - { -1, 0 } -}; - /* bus softc */ struct rcons_softc { struct resource *res[1]; void *ihl[1]; device_t dev; }; /* CN Console interface */ static tsw_outwakeup_t riscvtty_outwakeup; static struct ttydevsw riscv_ttydevsw = { .tsw_flags = TF_NOPREFIX, .tsw_outwakeup = riscvtty_outwakeup, }; static int polltime; static struct callout riscv_callout; static struct tty *tp = NULL; #if defined(KDB) static int alt_break_state; #endif static void riscv_timeout(void *); static cn_probe_t riscv_cnprobe; static cn_init_t riscv_cninit; static cn_term_t riscv_cnterm; static cn_getc_t riscv_cngetc; static cn_putc_t riscv_cnputc; static cn_grab_t riscv_cngrab; static cn_ungrab_t riscv_cnungrab; CONSOLE_DRIVER(riscv); #define MAX_BURST_LEN 1 -#define QUEUE_SIZE 256 -struct queue_entry { - uint64_t data; - uint64_t used; - struct queue_entry *next; -}; - -struct queue_entry cnqueue[QUEUE_SIZE]; -struct queue_entry *entry_last; -struct queue_entry *entry_served; - static void riscv_putc(int c) { sbi_console_putchar(c); } #ifdef EARLY_PRINTF early_putc_t *early_putc = riscv_putc; #endif static void cn_drvinit(void *unused) { if (riscv_consdev.cn_pri != CN_DEAD && riscv_consdev.cn_name[0] != '\0') { tp = tty_alloc(&riscv_ttydevsw, NULL); tty_init_console(tp, 0); tty_makedev(tp, NULL, "%s", "rcons"); polltime = 1; callout_init(&riscv_callout, 1); callout_reset(&riscv_callout, polltime, riscv_timeout, NULL); } } SYSINIT(cndev, SI_SUB_CONFIGURE, SI_ORDER_MIDDLE, cn_drvinit, NULL); static void riscvtty_outwakeup(struct tty *tp) { u_char buf[MAX_BURST_LEN]; int len; int i; for (;;) { len = ttydisc_getc(tp, buf, sizeof(buf)); if (len == 0) break; KASSERT(len == 1, ("tty error")); for (i = 0; i < len; i++) riscv_putc(buf[i]); } } static void riscv_timeout(void *v) { int c; tty_lock(tp); while ((c = riscv_cngetc(NULL)) != -1) ttydisc_rint(tp, c, 0); ttydisc_rint_done(tp); tty_unlock(tp); callout_reset(&riscv_callout, polltime, riscv_timeout, NULL); } static void riscv_cnprobe(struct consdev *cp) { cp->cn_pri = CN_NORMAL; } static void riscv_cninit(struct consdev *cp) { - int i; strcpy(cp->cn_name, "rcons"); - - for (i = 0; i < QUEUE_SIZE; i++) { - if (i == (QUEUE_SIZE - 1)) - cnqueue[i].next = &cnqueue[0]; - else - cnqueue[i].next = &cnqueue[i+1]; - cnqueue[i].data = 0; - cnqueue[i].used = 0; - } - - entry_last = &cnqueue[0]; - entry_served = &cnqueue[0]; } static void riscv_cnterm(struct consdev *cp) { } static void riscv_cngrab(struct consdev *cp) { } static void riscv_cnungrab(struct consdev *cp) { } static int riscv_cngetc(struct consdev *cp) { - uint8_t data; int ch; #if defined(KDB) /* * RISCVTODO: BBL polls for console data on timer interrupt, * but interrupts are turned off in KDB. * So we currently do not have console in KDB. */ if (kdb_active) { ch = sbi_console_getchar(); while (ch) { - entry_last->data = ch; - entry_last->used = 1; - entry_last = entry_last->next; - ch = sbi_console_getchar(); } } #endif - if (entry_served->used == 1) { - data = entry_served->data; - entry_served->used = 0; - entry_served = entry_served->next; - ch = (data & 0xff); - if (ch > 0 && ch < 0xff) { + ch = sbi_console_getchar(); + if (ch > 0 && ch < 0xff) { #if defined(KDB) - kdb_alt_break(ch, &alt_break_state); + kdb_alt_break(ch, &alt_break_state); #endif - return (ch); - } + return (ch); } return (-1); } static void riscv_cnputc(struct consdev *cp, int c) { riscv_putc(c); } /* Bus interface */ static int -rcons_intr(void *arg) -{ - int c; - - c = sbi_console_getchar(); - if (c > 0 && c < 0xff) { - entry_last->data = c; - entry_last->used = 1; - entry_last = entry_last->next; - } - - csr_clear(sip, SIP_SSIP); - - return (FILTER_HANDLED); -} - -static int rcons_probe(device_t dev) { - if (!ofw_bus_status_okay(dev)) - return (ENXIO); - - if (!ofw_bus_is_compatible(dev, "riscv,console")) - return (ENXIO); - device_set_desc(dev, "RISC-V console"); + return (BUS_PROBE_DEFAULT); } static int rcons_attach(device_t dev) { struct rcons_softc *sc; - int error; + if (device_get_unit(dev) != 0) + return (ENXIO); + sc = device_get_softc(dev); sc->dev = dev; - if (bus_alloc_resources(dev, rcons_spec, sc->res)) { - device_printf(dev, "could not allocate resources\n"); - return (ENXIO); - } - - /* Setup IRQs handler */ - error = bus_setup_intr(dev, sc->res[0], INTR_TYPE_CLK, - rcons_intr, NULL, sc, &sc->ihl[0]); - if (error) { - device_printf(dev, "Unable to alloc int resource.\n"); - return (ENXIO); - } - csr_set(sie, SIE_SSIE); bus_generic_attach(sc->dev); - sbi_console_getchar(); - return (0); } static device_method_t rcons_methods[] = { DEVMETHOD(device_probe, rcons_probe), DEVMETHOD(device_attach, rcons_attach), DEVMETHOD_END }; static driver_t rcons_driver = { "rcons", rcons_methods, sizeof(struct rcons_softc) }; static devclass_t rcons_devclass; -DRIVER_MODULE(rcons, simplebus, rcons_driver, rcons_devclass, 0, 0); +DRIVER_MODULE(rcons, nexus, rcons_driver, rcons_devclass, 0, 0); Index: head/sys/riscv/riscv/swtch.S =================================================================== --- head/sys/riscv/riscv/swtch.S (revision 322360) +++ head/sys/riscv/riscv/swtch.S (revision 322361) @@ -1,436 +1,440 @@ /*- - * Copyright (c) 2015-2016 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "assym.s" #include "opt_sched.h" #include #include #include #include __FBSDID("$FreeBSD$"); #ifdef FPE .macro __fpe_state_save p /* * Enable FPE usage in supervisor mode, * so we can access registers. */ li t0, SSTATUS_FS_INITIAL csrs sstatus, t0 /* Store registers */ frcsr t0 sd t0, (PCB_FCSR)(\p) fsd f0, (PCB_X + 0 * 16)(\p) fsd f1, (PCB_X + 1 * 16)(\p) fsd f2, (PCB_X + 2 * 16)(\p) fsd f3, (PCB_X + 3 * 16)(\p) fsd f4, (PCB_X + 4 * 16)(\p) fsd f5, (PCB_X + 5 * 16)(\p) fsd f6, (PCB_X + 6 * 16)(\p) fsd f7, (PCB_X + 7 * 16)(\p) fsd f8, (PCB_X + 8 * 16)(\p) fsd f9, (PCB_X + 9 * 16)(\p) fsd f10, (PCB_X + 10 * 16)(\p) fsd f11, (PCB_X + 11 * 16)(\p) fsd f12, (PCB_X + 12 * 16)(\p) fsd f13, (PCB_X + 13 * 16)(\p) fsd f14, (PCB_X + 14 * 16)(\p) fsd f15, (PCB_X + 15 * 16)(\p) fsd f16, (PCB_X + 16 * 16)(\p) fsd f17, (PCB_X + 17 * 16)(\p) fsd f18, (PCB_X + 18 * 16)(\p) fsd f19, (PCB_X + 19 * 16)(\p) fsd f20, (PCB_X + 20 * 16)(\p) fsd f21, (PCB_X + 21 * 16)(\p) fsd f22, (PCB_X + 22 * 16)(\p) fsd f23, (PCB_X + 23 * 16)(\p) fsd f24, (PCB_X + 24 * 16)(\p) fsd f25, (PCB_X + 25 * 16)(\p) fsd f26, (PCB_X + 26 * 16)(\p) fsd f27, (PCB_X + 27 * 16)(\p) fsd f28, (PCB_X + 28 * 16)(\p) fsd f29, (PCB_X + 29 * 16)(\p) fsd f30, (PCB_X + 30 * 16)(\p) fsd f31, (PCB_X + 31 * 16)(\p) /* Disable FPE usage in supervisor mode. */ li t0, SSTATUS_FS_MASK csrc sstatus, t0 .endm .macro __fpe_state_load p /* * Enable FPE usage in supervisor mode, * so we can access registers. */ li t0, SSTATUS_FS_INITIAL csrs sstatus, t0 /* Restore registers */ ld t0, (PCB_FCSR)(\p) fscsr t0 fld f0, (PCB_X + 0 * 16)(\p) fld f1, (PCB_X + 1 * 16)(\p) fld f2, (PCB_X + 2 * 16)(\p) fld f3, (PCB_X + 3 * 16)(\p) fld f4, (PCB_X + 4 * 16)(\p) fld f5, (PCB_X + 5 * 16)(\p) fld f6, (PCB_X + 6 * 16)(\p) fld f7, (PCB_X + 7 * 16)(\p) fld f8, (PCB_X + 8 * 16)(\p) fld f9, (PCB_X + 9 * 16)(\p) fld f10, (PCB_X + 10 * 16)(\p) fld f11, (PCB_X + 11 * 16)(\p) fld f12, (PCB_X + 12 * 16)(\p) fld f13, (PCB_X + 13 * 16)(\p) fld f14, (PCB_X + 14 * 16)(\p) fld f15, (PCB_X + 15 * 16)(\p) fld f16, (PCB_X + 16 * 16)(\p) fld f17, (PCB_X + 17 * 16)(\p) fld f18, (PCB_X + 18 * 16)(\p) fld f19, (PCB_X + 19 * 16)(\p) fld f20, (PCB_X + 20 * 16)(\p) fld f21, (PCB_X + 21 * 16)(\p) fld f22, (PCB_X + 22 * 16)(\p) fld f23, (PCB_X + 23 * 16)(\p) fld f24, (PCB_X + 24 * 16)(\p) fld f25, (PCB_X + 25 * 16)(\p) fld f26, (PCB_X + 26 * 16)(\p) fld f27, (PCB_X + 27 * 16)(\p) fld f28, (PCB_X + 28 * 16)(\p) fld f29, (PCB_X + 29 * 16)(\p) fld f30, (PCB_X + 30 * 16)(\p) fld f31, (PCB_X + 31 * 16)(\p) /* Disable FPE usage in supervisor mode. */ li t0, SSTATUS_FS_MASK csrc sstatus, t0 .endm /* * void * fpe_state_save(struct thread *td) */ ENTRY(fpe_state_save) /* Get pointer to PCB */ ld a0, TD_PCB(a0) __fpe_state_save a0 ret END(fpe_state_save) #endif /* FPE */ /* * void cpu_throw(struct thread *old, struct thread *new) */ ENTRY(cpu_throw) /* Store the new curthread */ sd a1, PC_CURTHREAD(gp) /* And the new pcb */ ld x13, TD_PCB(a1) sd x13, PC_CURPCB(gp) - sfence.vm + sfence.vma /* Switch to the new pmap */ ld t0, PCB_L1ADDR(x13) srli t0, t0, PAGE_SHIFT + li t1, SATP_MODE_SV39 + or t0, t0, t1 csrw sptbr, t0 /* TODO: Invalidate the TLB */ - sfence.vm + sfence.vma /* Load registers */ ld ra, (PCB_RA)(x13) ld sp, (PCB_SP)(x13) /* s[0-11] */ ld s0, (PCB_S + 0 * 8)(x13) ld s1, (PCB_S + 1 * 8)(x13) ld s2, (PCB_S + 2 * 8)(x13) ld s3, (PCB_S + 3 * 8)(x13) ld s4, (PCB_S + 4 * 8)(x13) ld s5, (PCB_S + 5 * 8)(x13) ld s6, (PCB_S + 6 * 8)(x13) ld s7, (PCB_S + 7 * 8)(x13) ld s8, (PCB_S + 8 * 8)(x13) ld s9, (PCB_S + 9 * 8)(x13) ld s10, (PCB_S + 10 * 8)(x13) ld s11, (PCB_S + 11 * 8)(x13) #ifdef FPE /* Is FPE enabled for new thread? */ ld t0, TD_FRAME(a1) ld t1, (TF_SSTATUS)(t0) li t2, SSTATUS_FS_MASK and t3, t1, t2 beqz t3, 1f /* No, skip. */ /* Restore registers. */ __fpe_state_load x13 1: #endif ret .Lcpu_throw_panic_str: .asciz "cpu_throw: %p\0" END(cpu_throw) /* * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) * * a0 = old * a1 = new * a2 = mtx * x3 to x7, x16 and x17 are caller saved */ ENTRY(cpu_switch) /* Store the new curthread */ sd a1, PC_CURTHREAD(gp) /* And the new pcb */ ld x13, TD_PCB(a1) sd x13, PC_CURPCB(gp) /* Save the old context. */ ld x13, TD_PCB(a0) /* Store ra, sp and the callee-saved registers */ sd ra, (PCB_RA)(x13) sd sp, (PCB_SP)(x13) /* s[0-11] */ sd s0, (PCB_S + 0 * 8)(x13) sd s1, (PCB_S + 1 * 8)(x13) sd s2, (PCB_S + 2 * 8)(x13) sd s3, (PCB_S + 3 * 8)(x13) sd s4, (PCB_S + 4 * 8)(x13) sd s5, (PCB_S + 5 * 8)(x13) sd s6, (PCB_S + 6 * 8)(x13) sd s7, (PCB_S + 7 * 8)(x13) sd s8, (PCB_S + 8 * 8)(x13) sd s9, (PCB_S + 9 * 8)(x13) sd s10, (PCB_S + 10 * 8)(x13) sd s11, (PCB_S + 11 * 8)(x13) #ifdef FPE /* * Is FPE enabled and is it in dirty state * for the old thread? */ ld t0, TD_FRAME(a0) ld t1, (TF_SSTATUS)(t0) li t2, SSTATUS_FS_MASK and t3, t1, t2 li t2, SSTATUS_FS_DIRTY bne t3, t2, 1f /* No, skip. */ /* Yes, mark FPE state clean and save registers. */ li t2, ~SSTATUS_FS_MASK and t3, t1, t2 li t2, SSTATUS_FS_CLEAN or t3, t3, t2 sd t3, (TF_SSTATUS)(t0) __fpe_state_save x13 1: #endif /* * Restore the saved context. */ ld x13, TD_PCB(a1) /* * TODO: We may need to flush the cache here if switching * to a user process. */ - sfence.vm + sfence.vma /* Switch to the new pmap */ ld t0, PCB_L1ADDR(x13) srli t0, t0, PAGE_SHIFT + li t1, SATP_MODE_SV39 + or t0, t0, t1 csrw sptbr, t0 /* TODO: Invalidate the TLB */ - sfence.vm + sfence.vma /* Release the old thread */ sd a2, TD_LOCK(a0) #if defined(SCHED_ULE) && defined(SMP) /* Spin if TD_LOCK points to a blocked_lock */ la a2, _C_LABEL(blocked_lock) 1: ld t0, TD_LOCK(a1) beq t0, a2, 1b #endif /* Restore the registers */ ld ra, (PCB_RA)(x13) ld sp, (PCB_SP)(x13) /* s[0-11] */ ld s0, (PCB_S + 0 * 8)(x13) ld s1, (PCB_S + 1 * 8)(x13) ld s2, (PCB_S + 2 * 8)(x13) ld s3, (PCB_S + 3 * 8)(x13) ld s4, (PCB_S + 4 * 8)(x13) ld s5, (PCB_S + 5 * 8)(x13) ld s6, (PCB_S + 6 * 8)(x13) ld s7, (PCB_S + 7 * 8)(x13) ld s8, (PCB_S + 8 * 8)(x13) ld s9, (PCB_S + 9 * 8)(x13) ld s10, (PCB_S + 10 * 8)(x13) ld s11, (PCB_S + 11 * 8)(x13) #ifdef FPE /* Is FPE enabled for new thread? */ ld t0, TD_FRAME(a1) ld t1, (TF_SSTATUS)(t0) li t2, SSTATUS_FS_MASK and t3, t1, t2 beqz t3, 1f /* No, skip. */ /* Restore registers. */ __fpe_state_load x13 1: #endif ret .Lcpu_switch_panic_str: .asciz "cpu_switch: %p\0" END(cpu_switch) /* * fork_exit(void (*callout)(void *, struct trapframe *), void *arg, * struct trapframe *frame) */ ENTRY(fork_trampoline) mv a0, s0 mv a1, s1 mv a2, sp call _C_LABEL(fork_exit) /* Restore sstatus */ ld t0, (TF_SSTATUS)(sp) /* Ensure interrupts disabled */ li t1, ~SSTATUS_SIE and t0, t0, t1 csrw sstatus, t0 /* Restore exception program counter */ ld t0, (TF_SEPC)(sp) csrw sepc, t0 /* Restore the registers */ ld t0, (TF_T + 0 * 8)(sp) ld t1, (TF_T + 1 * 8)(sp) ld t2, (TF_T + 2 * 8)(sp) ld t3, (TF_T + 3 * 8)(sp) ld t4, (TF_T + 4 * 8)(sp) ld t5, (TF_T + 5 * 8)(sp) ld t6, (TF_T + 6 * 8)(sp) ld s0, (TF_S + 0 * 8)(sp) ld s1, (TF_S + 1 * 8)(sp) ld s2, (TF_S + 2 * 8)(sp) ld s3, (TF_S + 3 * 8)(sp) ld s4, (TF_S + 4 * 8)(sp) ld s5, (TF_S + 5 * 8)(sp) ld s6, (TF_S + 6 * 8)(sp) ld s7, (TF_S + 7 * 8)(sp) ld s8, (TF_S + 8 * 8)(sp) ld s9, (TF_S + 9 * 8)(sp) ld s10, (TF_S + 10 * 8)(sp) ld s11, (TF_S + 11 * 8)(sp) ld a0, (TF_A + 0 * 8)(sp) ld a1, (TF_A + 1 * 8)(sp) ld a2, (TF_A + 2 * 8)(sp) ld a3, (TF_A + 3 * 8)(sp) ld a4, (TF_A + 4 * 8)(sp) ld a5, (TF_A + 5 * 8)(sp) ld a6, (TF_A + 6 * 8)(sp) ld a7, (TF_A + 7 * 8)(sp) /* Load user ra and sp */ ld tp, (TF_TP)(sp) ld ra, (TF_RA)(sp) /* * Store our pcpup on stack, we will load it back * on kernel mode trap. */ sd gp, (TF_SIZE)(sp) ld gp, (TF_GP)(sp) /* Save kernel stack so we can use it doing a user trap */ addi sp, sp, TF_SIZE csrw sscratch, sp /* Load user stack */ ld sp, (TF_SP - TF_SIZE)(sp) sret END(fork_trampoline) ENTRY(savectx) /* Store ra, sp and the callee-saved registers */ sd ra, (PCB_RA)(a0) sd sp, (PCB_SP)(a0) /* s[0-11] */ sd s0, (PCB_S + 0 * 8)(a0) sd s1, (PCB_S + 1 * 8)(a0) sd s2, (PCB_S + 2 * 8)(a0) sd s3, (PCB_S + 3 * 8)(a0) sd s4, (PCB_S + 4 * 8)(a0) sd s5, (PCB_S + 5 * 8)(a0) sd s6, (PCB_S + 6 * 8)(a0) sd s7, (PCB_S + 7 * 8)(a0) sd s8, (PCB_S + 8 * 8)(a0) sd s9, (PCB_S + 9 * 8)(a0) sd s10, (PCB_S + 10 * 8)(a0) sd s11, (PCB_S + 11 * 8)(a0) #ifdef FPE __fpe_state_save a0 #endif ret END(savectx) Index: head/sys/riscv/riscv/timer.c =================================================================== --- head/sys/riscv/riscv/timer.c (revision 322360) +++ head/sys/riscv/riscv/timer.c (revision 322361) @@ -1,328 +1,286 @@ /*- - * Copyright (c) 2015-2016 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * RISC-V Timer */ #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include -#include -#include -#include +#define DEFAULT_FREQ 10000000 -#define DEFAULT_FREQ 1000000 - #define TIMER_COUNTS 0x00 #define TIMER_MTIMECMP(cpu) (cpu * 8) -#define READ8(_sc, _reg) \ - bus_space_read_8(_sc->bst, _sc->bsh, _reg) -#define WRITE8(_sc, _reg, _val) \ - bus_space_write_8(_sc->bst, _sc->bsh, _reg, _val) - -struct riscv_tmr_softc { - struct resource *res[3]; - bus_space_tag_t bst; - bus_space_handle_t bsh; - bus_space_tag_t bst_timecmp; - bus_space_handle_t bsh_timecmp; +struct riscv_timer_softc { void *ih; uint32_t clkfreq; struct eventtimer et; + int intr_rid; + struct resource *intr_res; }; -static struct riscv_tmr_softc *riscv_tmr_sc = NULL; +static struct riscv_timer_softc *riscv_timer_sc = NULL; -static struct resource_spec timer_spec[] = { - { SYS_RES_MEMORY, 0, RF_ACTIVE }, - { SYS_RES_MEMORY, 1, RF_ACTIVE }, - { SYS_RES_IRQ, 0, RF_ACTIVE }, - { -1, 0 } -}; +static timecounter_get_t riscv_timer_get_timecount; -static timecounter_get_t riscv_tmr_get_timecount; - -static struct timecounter riscv_tmr_timecount = { +static struct timecounter riscv_timer_timecount = { .tc_name = "RISC-V Timecounter", - .tc_get_timecount = riscv_tmr_get_timecount, + .tc_get_timecount = riscv_timer_get_timecount, .tc_poll_pps = NULL, .tc_counter_mask = ~0u, .tc_frequency = 0, .tc_quality = 1000, }; +static inline uint64_t +get_cycles(void) +{ + uint64_t cycles; + + __asm __volatile("rdtime %0" : "=r" (cycles)); + + return (cycles); +} + static long -get_counts(struct riscv_tmr_softc *sc) +get_counts(struct riscv_timer_softc *sc) { uint64_t counts; - counts = READ8(sc, TIMER_COUNTS); + counts = get_cycles(); return (counts); } static unsigned -riscv_tmr_get_timecount(struct timecounter *tc) +riscv_timer_get_timecount(struct timecounter *tc) { - struct riscv_tmr_softc *sc; + struct riscv_timer_softc *sc; sc = tc->tc_priv; return (get_counts(sc)); } static int -riscv_tmr_start(struct eventtimer *et, sbintime_t first, sbintime_t period) +riscv_timer_start(struct eventtimer *et, sbintime_t first, sbintime_t period) { - struct riscv_tmr_softc *sc; uint64_t counts; - int cpu; - sc = (struct riscv_tmr_softc *)et->et_priv; - if (first != 0) { counts = ((uint32_t)et->et_frequency * first) >> 32; - counts += READ8(sc, TIMER_COUNTS); - cpu = PCPU_GET(cpuid); - bus_space_write_8(sc->bst_timecmp, sc->bsh_timecmp, - TIMER_MTIMECMP(cpu), counts); + sbi_set_timer(get_cycles() + counts); csr_set(sie, SIE_STIE); - sbi_set_timer(counts); return (0); } return (EINVAL); } static int -riscv_tmr_stop(struct eventtimer *et) +riscv_timer_stop(struct eventtimer *et) { - struct riscv_tmr_softc *sc; - sc = (struct riscv_tmr_softc *)et->et_priv; - /* TODO */ return (0); } static int -riscv_tmr_intr(void *arg) +riscv_timer_intr(void *arg) { - struct riscv_tmr_softc *sc; + struct riscv_timer_softc *sc; - sc = (struct riscv_tmr_softc *)arg; + sc = (struct riscv_timer_softc *)arg; csr_clear(sip, SIP_STIP); if (sc->et.et_active) sc->et.et_event_cb(&sc->et, sc->et.et_arg); return (FILTER_HANDLED); } static int -riscv_tmr_fdt_probe(device_t dev) +riscv_timer_probe(device_t dev) { - if (!ofw_bus_status_okay(dev)) - return (ENXIO); + device_set_desc(dev, "RISC-V Timer"); - if (ofw_bus_is_compatible(dev, "riscv,timer")) { - device_set_desc(dev, "RISC-V Timer"); - return (BUS_PROBE_DEFAULT); - } - - return (ENXIO); + return (BUS_PROBE_DEFAULT); } static int -riscv_tmr_attach(device_t dev) +riscv_timer_attach(device_t dev) { - struct riscv_tmr_softc *sc; - phandle_t node; - pcell_t clock; + struct riscv_timer_softc *sc; int error; sc = device_get_softc(dev); - if (riscv_tmr_sc) + if (riscv_timer_sc) return (ENXIO); - /* Get the base clock frequency */ - node = ofw_bus_get_node(dev); - if (node > 0) { - error = OF_getprop(node, "clock-frequency", &clock, - sizeof(clock)); - if (error > 0) { - sc->clkfreq = fdt32_to_cpu(clock); - } - } + if (device_get_unit(dev) != 0) + return ENXIO; - if (sc->clkfreq == 0) - sc->clkfreq = DEFAULT_FREQ; - + sc->clkfreq = DEFAULT_FREQ; if (sc->clkfreq == 0) { device_printf(dev, "No clock frequency specified\n"); return (ENXIO); } - if (bus_alloc_resources(dev, timer_spec, sc->res)) { - device_printf(dev, "could not allocate resources\n"); + riscv_timer_sc = sc; + + sc->intr_rid = 0; + sc->intr_res = bus_alloc_resource(dev, + SYS_RES_IRQ, &sc->intr_rid, IRQ_TIMER_SUPERVISOR, + IRQ_TIMER_SUPERVISOR, 1, RF_ACTIVE); + if (sc->intr_res == NULL) { + device_printf(dev, "failed to allocate irq\n"); return (ENXIO); } - /* Memory interface */ - sc->bst = rman_get_bustag(sc->res[0]); - sc->bsh = rman_get_bushandle(sc->res[0]); - sc->bst_timecmp = rman_get_bustag(sc->res[1]); - sc->bsh_timecmp = rman_get_bushandle(sc->res[1]); - - riscv_tmr_sc = sc; - /* Setup IRQs handler */ - error = bus_setup_intr(dev, sc->res[2], INTR_TYPE_CLK, - riscv_tmr_intr, NULL, sc, &sc->ih); + error = bus_setup_intr(dev, sc->intr_res, INTR_TYPE_CLK, + riscv_timer_intr, NULL, sc, &sc->ih); if (error) { device_printf(dev, "Unable to alloc int resource.\n"); return (ENXIO); } - riscv_tmr_timecount.tc_frequency = sc->clkfreq; - riscv_tmr_timecount.tc_priv = sc; - tc_init(&riscv_tmr_timecount); + riscv_timer_timecount.tc_frequency = sc->clkfreq; + riscv_timer_timecount.tc_priv = sc; + tc_init(&riscv_timer_timecount); sc->et.et_name = "RISC-V Eventtimer"; sc->et.et_flags = ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU; sc->et.et_quality = 1000; sc->et.et_frequency = sc->clkfreq; sc->et.et_min_period = (0x00000002LLU << 32) / sc->et.et_frequency; sc->et.et_max_period = (0xfffffffeLLU << 32) / sc->et.et_frequency; - sc->et.et_start = riscv_tmr_start; - sc->et.et_stop = riscv_tmr_stop; + sc->et.et_start = riscv_timer_start; + sc->et.et_stop = riscv_timer_stop; sc->et.et_priv = sc; et_register(&sc->et); return (0); } -static device_method_t riscv_tmr_fdt_methods[] = { - DEVMETHOD(device_probe, riscv_tmr_fdt_probe), - DEVMETHOD(device_attach, riscv_tmr_attach), +static device_method_t riscv_timer_methods[] = { + DEVMETHOD(device_probe, riscv_timer_probe), + DEVMETHOD(device_attach, riscv_timer_attach), { 0, 0 } }; -static driver_t riscv_tmr_fdt_driver = { +static driver_t riscv_timer_driver = { "timer", - riscv_tmr_fdt_methods, - sizeof(struct riscv_tmr_softc), + riscv_timer_methods, + sizeof(struct riscv_timer_softc), }; -static devclass_t riscv_tmr_fdt_devclass; +static devclass_t riscv_timer_devclass; -EARLY_DRIVER_MODULE(timer, simplebus, riscv_tmr_fdt_driver, riscv_tmr_fdt_devclass, +EARLY_DRIVER_MODULE(timer, nexus, riscv_timer_driver, riscv_timer_devclass, 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); -EARLY_DRIVER_MODULE(timer, ofwbus, riscv_tmr_fdt_driver, riscv_tmr_fdt_devclass, - 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); void DELAY(int usec) { int64_t counts, counts_per_usec; uint64_t first, last; /* * Check the timers are setup, if not just * use a for loop for the meantime */ - if (riscv_tmr_sc == NULL) { + if (riscv_timer_sc == NULL) { for (; usec > 0; usec--) for (counts = 200; counts > 0; counts--) /* * Prevent the compiler from optimizing * out the loop */ cpufunc_nullop(); return; } /* Get the number of times to count */ - counts_per_usec = ((riscv_tmr_timecount.tc_frequency / 1000000) + 1); + counts_per_usec = ((riscv_timer_timecount.tc_frequency / 1000000) + 1); /* * Clamp the timeout at a maximum value (about 32 seconds with * a 66MHz clock). *Nobody* should be delay()ing for anywhere * near that length of time and if they are, they should be hung * out to dry. */ if (usec >= (0x80000000U / counts_per_usec)) counts = (0x80000000U / counts_per_usec) - 1; else counts = usec * counts_per_usec; - first = get_counts(riscv_tmr_sc); + first = get_counts(riscv_timer_sc); while (counts > 0) { - last = get_counts(riscv_tmr_sc); + last = get_counts(riscv_timer_sc); counts -= (int64_t)(last - first); first = last; } } Index: head/sys/riscv/riscv/trap.c =================================================================== --- head/sys/riscv/riscv/trap.c (revision 322360) +++ head/sys/riscv/riscv/trap.c (revision 322361) @@ -1,387 +1,393 @@ /*- - * Copyright (c) 2015-2016 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #ifdef KDB #include #endif #include #include #include #include #include #include #include #include #include #include #include #ifdef KDTRACE_HOOKS #include #endif int (*dtrace_invop_jump_addr)(struct trapframe *); extern register_t fsu_intr_fault; /* Called from exception.S */ void do_trap_supervisor(struct trapframe *); void do_trap_user(struct trapframe *); static __inline void call_trapsignal(struct thread *td, int sig, int code, void *addr) { ksiginfo_t ksi; ksiginfo_init_trap(&ksi); ksi.ksi_signo = sig; ksi.ksi_code = code; ksi.ksi_addr = addr; trapsignal(td, &ksi); } int cpu_fetch_syscall_args(struct thread *td) { struct proc *p; register_t *ap; struct syscall_args *sa; int nap; nap = NARGREG; p = td->td_proc; sa = &td->td_sa; ap = &td->td_frame->tf_a[0]; sa->code = td->td_frame->tf_t[0]; if (sa->code == SYS_syscall || sa->code == SYS___syscall) { sa->code = *ap++; nap--; } if (p->p_sysent->sv_mask) sa->code &= p->p_sysent->sv_mask; if (sa->code >= p->p_sysent->sv_size) sa->callp = &p->p_sysent->sv_table[0]; else sa->callp = &p->p_sysent->sv_table[sa->code]; sa->narg = sa->callp->sy_narg; memcpy(sa->args, ap, nap * sizeof(register_t)); if (sa->narg > nap) panic("TODO: Could we have more then %d args?", NARGREG); td->td_retval[0] = 0; td->td_retval[1] = 0; return (0); } #include "../../kern/subr_syscall.c" static void dump_regs(struct trapframe *frame) { int n; int i; n = (sizeof(frame->tf_t) / sizeof(frame->tf_t[0])); for (i = 0; i < n; i++) printf("t[%d] == 0x%016lx\n", i, frame->tf_t[i]); n = (sizeof(frame->tf_s) / sizeof(frame->tf_s[0])); for (i = 0; i < n; i++) printf("s[%d] == 0x%016lx\n", i, frame->tf_s[i]); n = (sizeof(frame->tf_a) / sizeof(frame->tf_a[0])); for (i = 0; i < n; i++) printf("a[%d] == 0x%016lx\n", i, frame->tf_a[i]); printf("sepc == 0x%016lx\n", frame->tf_sepc); printf("sstatus == 0x%016lx\n", frame->tf_sstatus); } static void svc_handler(struct trapframe *frame) { struct thread *td; int error; td = curthread; td->td_frame = frame; error = syscallenter(td); syscallret(td, error); } static void data_abort(struct trapframe *frame, int lower) { struct vm_map *map; uint64_t sbadaddr; struct thread *td; struct pcb *pcb; vm_prot_t ftype; vm_offset_t va; struct proc *p; int ucode; int error; int sig; #ifdef KDB if (kdb_active) { kdb_reenter(); return; } #endif td = curthread; pcb = td->td_pcb; /* * Special case for fuswintr and suswintr. These can't sleep so * handle them early on in the trap handler. */ if (__predict_false(pcb->pcb_onfault == (vm_offset_t)&fsu_intr_fault)) { frame->tf_sepc = pcb->pcb_onfault; return; } sbadaddr = frame->tf_sbadaddr; p = td->td_proc; if (lower) map = &td->td_proc->p_vmspace->vm_map; else { /* The top bit tells us which range to use */ if ((sbadaddr >> 63) == 1) map = kernel_map; else map = &td->td_proc->p_vmspace->vm_map; } va = trunc_page(sbadaddr); - if (frame->tf_scause == EXCP_FAULT_STORE) { + if ((frame->tf_scause == EXCP_FAULT_STORE) || + (frame->tf_scause == EXCP_STORE_PAGE_FAULT)) { ftype = (VM_PROT_READ | VM_PROT_WRITE); } else { ftype = (VM_PROT_READ); } if (map != kernel_map) { /* * Keep swapout from messing with us during this * critical time. */ PROC_LOCK(p); ++p->p_lock; PROC_UNLOCK(p); /* Fault in the user page: */ error = vm_fault(map, va, ftype, VM_FAULT_NORMAL); PROC_LOCK(p); --p->p_lock; PROC_UNLOCK(p); } else { /* * Don't have to worry about process locking or stacks in the * kernel. */ error = vm_fault(map, va, ftype, VM_FAULT_NORMAL); } if (error != KERN_SUCCESS) { if (lower) { sig = SIGSEGV; if (error == KERN_PROTECTION_FAILURE) ucode = SEGV_ACCERR; else ucode = SEGV_MAPERR; call_trapsignal(td, sig, ucode, (void *)sbadaddr); } else { if (td->td_intr_nesting_level == 0 && pcb->pcb_onfault != 0) { frame->tf_a[0] = error; frame->tf_sepc = pcb->pcb_onfault; return; } dump_regs(frame); panic("vm_fault failed: %lx, va 0x%016lx", frame->tf_sepc, sbadaddr); } } if (lower) userret(td, frame); } void do_trap_supervisor(struct trapframe *frame) { uint64_t exception; uint64_t sstatus; /* Ensure we came from supervisor mode, interrupts disabled */ __asm __volatile("csrr %0, sstatus" : "=&r" (sstatus)); KASSERT((sstatus & (SSTATUS_SPP | SSTATUS_SIE)) == SSTATUS_SPP, ("We must came from S mode with interrupts disabled")); exception = (frame->tf_scause & EXCP_MASK); if (frame->tf_scause & EXCP_INTR) { /* Interrupt */ riscv_cpu_intr(frame); return; } #ifdef KDTRACE_HOOKS if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception)) return; #endif CTR3(KTR_TRAP, "do_trap_supervisor: curthread: %p, sepc: %lx, frame: %p", curthread, frame->tf_sepc, frame); switch(exception) { case EXCP_FAULT_LOAD: case EXCP_FAULT_STORE: case EXCP_FAULT_FETCH: + case EXCP_STORE_PAGE_FAULT: + case EXCP_LOAD_PAGE_FAULT: data_abort(frame, 0); break; case EXCP_BREAKPOINT: #ifdef KDTRACE_HOOKS if (dtrace_invop_jump_addr != 0) { dtrace_invop_jump_addr(frame); break; } #endif #ifdef KDB kdb_trap(exception, 0, frame); #else dump_regs(frame); panic("No debugger in kernel.\n"); #endif break; case EXCP_ILLEGAL_INSTRUCTION: dump_regs(frame); panic("Illegal instruction at 0x%016lx\n", frame->tf_sepc); break; default: dump_regs(frame); panic("Unknown kernel exception %x badaddr %lx\n", exception, frame->tf_sbadaddr); } } void do_trap_user(struct trapframe *frame) { uint64_t exception; struct thread *td; uint64_t sstatus; struct pcb *pcb; td = curthread; td->td_frame = frame; pcb = td->td_pcb; /* Ensure we came from usermode, interrupts disabled */ __asm __volatile("csrr %0, sstatus" : "=&r" (sstatus)); KASSERT((sstatus & (SSTATUS_SPP | SSTATUS_SIE)) == 0, ("We must came from U mode with interrupts disabled")); exception = (frame->tf_scause & EXCP_MASK); if (frame->tf_scause & EXCP_INTR) { /* Interrupt */ riscv_cpu_intr(frame); return; } CTR3(KTR_TRAP, "do_trap_user: curthread: %p, sepc: %lx, frame: %p", curthread, frame->tf_sepc, frame); switch(exception) { case EXCP_FAULT_LOAD: case EXCP_FAULT_STORE: case EXCP_FAULT_FETCH: + case EXCP_STORE_PAGE_FAULT: + case EXCP_LOAD_PAGE_FAULT: + case EXCP_INST_PAGE_FAULT: data_abort(frame, 1); break; case EXCP_USER_ECALL: frame->tf_sepc += 4; /* Next instruction */ svc_handler(frame); break; case EXCP_ILLEGAL_INSTRUCTION: #ifdef FPE if ((pcb->pcb_fpflags & PCB_FP_STARTED) == 0) { /* * May be a FPE trap. Enable FPE usage * for this thread and try again. */ frame->tf_sstatus |= SSTATUS_FS_INITIAL; pcb->pcb_fpflags |= PCB_FP_STARTED; break; } #endif call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_sepc); userret(td, frame); break; case EXCP_BREAKPOINT: call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_sepc); userret(td, frame); break; default: dump_regs(frame); - panic("Unknown userland exception %x badaddr %lx\n", + panic("Unknown userland exception %x, badaddr %lx\n", exception, frame->tf_sbadaddr); } } Index: head/sys/riscv/riscv/vm_machdep.c =================================================================== --- head/sys/riscv/riscv/vm_machdep.c (revision 322360) +++ head/sys/riscv/riscv/vm_machdep.c (revision 322361) @@ -1,254 +1,257 @@ /*- - * Copyright (c) 2015-2016 Ruslan Bukin + * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Finish a fork operation, with process p2 nearly set up. * Copy and update the pcb, set up the stack so that the child * ready to run and return to user mode. */ void cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) { struct pcb *pcb2; struct trapframe *tf; if ((flags & RFPROC) == 0) return; pcb2 = (struct pcb *)(td2->td_kstack + td2->td_kstack_pages * PAGE_SIZE) - 1; td2->td_pcb = pcb2; bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); td2->td_pcb->pcb_l1addr = vtophys(vmspace_pmap(td2->td_proc->p_vmspace)->pm_l1); tf = (struct trapframe *)STACKALIGN((struct trapframe *)pcb2 - 1); bcopy(td1->td_frame, tf, sizeof(*tf)); /* Clear syscall error flag */ tf->tf_t[0] = 0; /* Arguments for child */ tf->tf_a[0] = 0; tf->tf_a[1] = 0; tf->tf_sstatus |= (SSTATUS_SPIE); /* Enable interrupts. */ + tf->tf_sstatus |= (SSTATUS_SUM); /* Supervisor can access userspace. */ tf->tf_sstatus &= ~(SSTATUS_SPP); /* User mode. */ td2->td_frame = tf; /* Set the return value registers for fork() */ td2->td_pcb->pcb_s[0] = (uintptr_t)fork_return; td2->td_pcb->pcb_s[1] = (uintptr_t)td2; td2->td_pcb->pcb_ra = (uintptr_t)fork_trampoline; td2->td_pcb->pcb_sp = (uintptr_t)td2->td_frame; /* Setup to release spin count in fork_exit(). */ td2->td_md.md_spinlock_count = 1; td2->td_md.md_saved_sstatus_ie = (SSTATUS_SIE); } void cpu_reset(void) { sbi_shutdown(); while(1); } void cpu_thread_swapin(struct thread *td) { } void cpu_thread_swapout(struct thread *td) { } void cpu_set_syscall_retval(struct thread *td, int error) { struct trapframe *frame; frame = td->td_frame; switch (error) { case 0: frame->tf_a[0] = td->td_retval[0]; frame->tf_a[1] = td->td_retval[1]; frame->tf_t[0] = 0; /* syscall succeeded */ break; case ERESTART: frame->tf_sepc -= 4; /* prev instruction */ break; case EJUSTRETURN: break; default: frame->tf_a[0] = error; frame->tf_t[0] = 1; /* syscall error */ break; } } /* * Initialize machine state, mostly pcb and trap frame for a new * thread, about to return to userspace. Put enough state in the new * thread's PCB to get it to go back to the fork_return(), which * finalizes the thread state and handles peculiarities of the first * return to userspace for the new thread. */ void cpu_copy_thread(struct thread *td, struct thread *td0) { bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe)); bcopy(td0->td_pcb, td->td_pcb, sizeof(struct pcb)); td->td_pcb->pcb_s[0] = (uintptr_t)fork_return; td->td_pcb->pcb_s[1] = (uintptr_t)td; td->td_pcb->pcb_ra = (uintptr_t)fork_trampoline; td->td_pcb->pcb_sp = (uintptr_t)td->td_frame; /* Setup to release spin count in fork_exit(). */ td->td_md.md_spinlock_count = 1; td->td_md.md_saved_sstatus_ie = (SSTATUS_SIE); } /* * Set that machine state for performing an upcall that starts * the entry function with the given argument. */ void cpu_set_upcall(struct thread *td, void (*entry)(void *), void *arg, stack_t *stack) { - struct trapframe *tf = td->td_frame; + struct trapframe *tf; + + tf = td->td_frame; tf->tf_sp = STACKALIGN((uintptr_t)stack->ss_sp + stack->ss_size); tf->tf_sepc = (register_t)entry; tf->tf_a[0] = (register_t)arg; } int cpu_set_user_tls(struct thread *td, void *tls_base) { struct pcb *pcb; if ((uintptr_t)tls_base >= VM_MAXUSER_ADDRESS) return (EINVAL); pcb = td->td_pcb; pcb->pcb_tp = (register_t)tls_base; return (0); } void cpu_thread_exit(struct thread *td) { } void cpu_thread_alloc(struct thread *td) { td->td_pcb = (struct pcb *)(td->td_kstack + td->td_kstack_pages * PAGE_SIZE) - 1; td->td_frame = (struct trapframe *)STACKALIGN( (caddr_t)td->td_pcb - 8 - sizeof(struct trapframe)); } void cpu_thread_free(struct thread *td) { } void cpu_thread_clean(struct thread *td) { } /* * Intercept the return address from a freshly forked process that has NOT * been scheduled yet. * * This is needed to make kernel threads stay in kernel mode. */ void cpu_fork_kthread_handler(struct thread *td, void (*func)(void *), void *arg) { td->td_pcb->pcb_s[0] = (uintptr_t)func; td->td_pcb->pcb_s[1] = (uintptr_t)arg; td->td_pcb->pcb_ra = (uintptr_t)fork_trampoline; td->td_pcb->pcb_sp = (uintptr_t)td->td_frame; } void cpu_exit(struct thread *td) { } void swi_vm(void *v) { /* Nothing to do here - busdma bounce buffers are not implemented. */ }