diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp @@ -482,6 +482,7 @@ const bool IsX86 = getTriple().getArch() == llvm::Triple::x86; const bool IsX86_64 = getTriple().getArch() == llvm::Triple::x86_64; const bool IsMIPS64 = getTriple().isMIPS64(); + const bool IsRISCV64 = getTriple().getArch() == llvm::Triple::riscv64; SanitizerMask Res = ToolChain::getSupportedSanitizers(); Res |= SanitizerKind::Address; Res |= SanitizerKind::PointerCompare; @@ -496,7 +497,7 @@ Res |= SanitizerKind::Fuzzer; Res |= SanitizerKind::FuzzerNoLink; } - if (IsAArch64 || IsX86_64) { + if (IsAArch64 || IsX86_64 || IsRISCV64) { Res |= SanitizerKind::KernelAddress; Res |= SanitizerKind::KernelMemory; Res |= SanitizerKind::Memory; diff --git a/lib/clang/freebsd_cc_version.h b/lib/clang/freebsd_cc_version.h --- a/lib/clang/freebsd_cc_version.h +++ b/lib/clang/freebsd_cc_version.h @@ -1 +1 @@ -#define FREEBSD_CC_VERSION 1600002 +#define FREEBSD_CC_VERSION 1600003 diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -96,7 +96,8 @@ riscv/riscv/trap.c standard riscv/riscv/timer.c standard riscv/riscv/uio_machdep.c standard -riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack +riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack \ + compile-with "${NORMAL_C:N-fsanitize*}" riscv/riscv/vector.c standard riscv/riscv/vm_machdep.c standard riscv/vmm/vmm.c optional vmm diff --git a/sys/conf/kern.mk b/sys/conf/kern.mk --- a/sys/conf/kern.mk +++ b/sys/conf/kern.mk @@ -274,11 +274,13 @@ .else SAN_CFLAGS+= -fasan-shadow-offset=0xdfff208000000000 .endif -.elif ${MACHINE_CPUARCH} == "amd64" && \ - ${COMPILER_TYPE} == "clang" && ${COMPILER_VERSION} >= 180000 -# Work around https://github.com/llvm/llvm-project/issues/87923, which leads to -# an assertion failure compiling dtrace.c with asan enabled. -SAN_CFLAGS+= -mllvm -asan-use-stack-safety=0 +# TODO: similar for riscv +.elif ${MACHINE_CPUARCH} == "riscv" +.if ${COMPILER_TYPE} == "clang" +SAN_CFLAGS+= -mllvm -asan-mapping-offset=0xdfffffd000000000 +.else +SAN_CFLAGS+= -fasan-shadow-offset=0xdfffffd000000000 +.endif .endif .endif # !empty(KASAN_ENABLED) diff --git a/sys/kern/subr_asan.c b/sys/kern/subr_asan.c --- a/sys/kern/subr_asan.c +++ b/sys/kern/subr_asan.c @@ -782,32 +782,46 @@ _ASAN_ATOMIC_FUNC_STORE(name, type) \ _ASAN_ATOMIC_FUNC_STORE(rel_##name, type) +#ifndef __riscv ASAN_ATOMIC_FUNC_ADD(8, uint8_t); ASAN_ATOMIC_FUNC_ADD(16, uint16_t); +#endif ASAN_ATOMIC_FUNC_ADD(32, uint32_t); ASAN_ATOMIC_FUNC_ADD(64, uint64_t); ASAN_ATOMIC_FUNC_ADD(int, u_int); ASAN_ATOMIC_FUNC_ADD(long, u_long); ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); +#ifndef __riscv ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); +#endif ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int); ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long); ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); +#ifdef __riscv +_ASAN_ATOMIC_FUNC_SET(8, uint8_t); +_ASAN_ATOMIC_FUNC_SET(16, uint16_t); +#else ASAN_ATOMIC_FUNC_SET(8, uint8_t); ASAN_ATOMIC_FUNC_SET(16, uint16_t); +#endif ASAN_ATOMIC_FUNC_SET(32, uint32_t); ASAN_ATOMIC_FUNC_SET(64, uint64_t); ASAN_ATOMIC_FUNC_SET(int, u_int); ASAN_ATOMIC_FUNC_SET(long, u_long); ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t); +#ifdef __riscv +_ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t); +_ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t); +#else ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t); ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t); +#endif ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t); ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t); ASAN_ATOMIC_FUNC_CLEAR(int, u_int); @@ -872,11 +886,19 @@ ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); _ASAN_ATOMIC_FUNC_STORE(bool, bool); +#ifdef __riscv +_ASAN_ATOMIC_FUNC_STORE(8, uint8_t); +#else ASAN_ATOMIC_FUNC_STORE(8, uint8_t); +#endif ASAN_ATOMIC_FUNC_STORE(16, uint16_t); ASAN_ATOMIC_FUNC_STORE(32, uint32_t); ASAN_ATOMIC_FUNC_STORE(64, uint64_t); +#ifdef __riscv +_ASAN_ATOMIC_FUNC_STORE(char, u_char); +#else ASAN_ATOMIC_FUNC_STORE(char, u_char); +#endif ASAN_ATOMIC_FUNC_STORE(short, u_short); ASAN_ATOMIC_FUNC_STORE(int, u_int); ASAN_ATOMIC_FUNC_STORE(long, u_long); diff --git a/sys/riscv/include/asan.h b/sys/riscv/include/asan.h new file mode 100644 --- /dev/null +++ b/sys/riscv/include/asan.h @@ -0,0 +1,67 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2020 The FreeBSD Foundation + * + * This software was developed by Mark Johnston under sponsorship from the + * FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _MACHINE_ASAN_H_ +#define _MACHINE_ASAN_H_ + +#ifdef KASAN + +#include +#include +#include +#include + +static inline vm_offset_t +kasan_md_addr_to_shad(vm_offset_t addr) +{ + return (((addr - VM_MIN_KERNEL_ADDRESS) >> KASAN_SHADOW_SCALE_SHIFT) + + KASAN_MIN_ADDRESS); +} + +static inline bool +kasan_md_unsupported(vm_offset_t addr) +{ + return (addr < VM_MIN_KERNEL_ADDRESS || addr >= virtual_end); +} + +static inline void +kasan_md_init(void) +{ +} + +static inline void +kasan_md_init_early(vm_offset_t bootstack, size_t size) +{ + kasan_shadow_map(bootstack, size); +} + +#endif /* KASAN */ + +#endif /* !_MACHINE_ASAN_H_ */ diff --git a/sys/riscv/include/atomic.h b/sys/riscv/include/atomic.h --- a/sys/riscv/include/atomic.h +++ b/sys/riscv/include/atomic.h @@ -35,13 +35,17 @@ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ -#include - #define fence() __asm __volatile("fence" ::: "memory"); #define mb() fence() #define rmb() fence() #define wmb() fence() +#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME) +#include +#else + +#include + static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t); static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t); static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t); @@ -662,4 +666,6 @@ #define atomic_set_short atomic_set_16 #define atomic_clear_short atomic_clear_16 +#endif /* SAN_NEEDS_INTERCEPTORS && !SAN_RUNTIME */ + #endif /* _MACHINE_ATOMIC_H_ */ diff --git a/sys/riscv/include/bus.h b/sys/riscv/include/bus.h --- a/sys/riscv/include/bus.h +++ b/sys/riscv/include/bus.h @@ -253,6 +253,10 @@ bus_size_t, const u_int64_t *, bus_size_t); }; +#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME) +#include +#else + /* * Utility macros; INTERNAL USE ONLY. */ @@ -495,6 +499,8 @@ BUS_POKE_FUNC(4, uint32_t) BUS_POKE_FUNC(8, uint64_t) +#endif /* SAN_NEEDS_INTERCEPTORS && !SAN_RUNTIME */ + #include #endif /* _MACHINE_BUS_H_ */ diff --git a/sys/riscv/include/param.h b/sys/riscv/include/param.h --- a/sys/riscv/include/param.h +++ b/sys/riscv/include/param.h @@ -81,8 +81,12 @@ #define MAXPAGESIZES 3 /* maximum number of supported page sizes */ #ifndef KSTACK_PAGES +#if defined(KASAN) +#define KSTACK_PAGES 6 +#else #define KSTACK_PAGES 4 /* pages of kernel stack (with pcb) */ #endif +#endif #define KSTACK_GUARD_PAGES 1 /* pages of kstack guard; 0 disables */ #define PCPU_PAGES 1 diff --git a/sys/riscv/include/pmap.h b/sys/riscv/include/pmap.h --- a/sys/riscv/include/pmap.h +++ b/sys/riscv/include/pmap.h @@ -164,6 +164,10 @@ return (0); } +#if defined(KASAN) +void pmap_san_enter(vm_offset_t); +void pmap_bootstrap_san(void); +#endif #endif /* _KERNEL */ diff --git a/sys/riscv/include/vmparam.h b/sys/riscv/include/vmparam.h --- a/sys/riscv/include/vmparam.h +++ b/sys/riscv/include/vmparam.h @@ -129,7 +129,8 @@ * 0x0000000000000000 - 0x0000003fffffffff 256GB user map * 0x0000004000000000 - 0xffffffbfffffffff unmappable * 0xffffffc000000000 - 0xffffffc7ffffffff 32GB kernel map - * 0xffffffc800000000 - 0xffffffcfffffffff 32GB unused + * 0xffffffc800000000 - 0xffffffc8ffffffff 4GB KASAN shadow map + * 0xffffffc900000000 - 0xffffffcfffffffff 28GB unused * 0xffffffd000000000 - 0xffffffefffffffff 128GB direct map * 0xfffffff000000000 - 0xffffffffffffffff 64GB unused * @@ -138,7 +139,8 @@ * 0x0000800000000000 - 0xffff7fffffffffff unmappable * 0xffff800000000000 - 0xffffffc7ffffffff 127.75TB hole * 0xffffffc000000000 - 0xffffffc7ffffffff 32GB kernel map - * 0xffffffc800000000 - 0xffffffcfffffffff 32GB unused + * 0xffffffc800000000 - 0xffffffc8ffffffff 4GB KASAN shadow map + * 0xffffffc900000000 - 0xffffffcfffffffff 28GB unused * 0xffffffd000000000 - 0xffffffefffffffff 128GB direct map * 0xfffffff000000000 - 0xffffffffffffffff 64GB unused * @@ -161,6 +163,9 @@ #define VM_MIN_KERNEL_ADDRESS (0xffffffc000000000UL) #define VM_MAX_KERNEL_ADDRESS (0xffffffc800000000UL) +#define KASAN_MIN_ADDRESS (0xffffffc800000000UL) +#define KASAN_MAX_ADDRESS (0xffffffc900000000UL) + #define DMAP_MIN_ADDRESS (0xffffffd000000000UL) #define DMAP_MAX_ADDRESS (0xfffffff000000000UL) diff --git a/sys/riscv/riscv/bus_machdep.c b/sys/riscv/riscv/bus_machdep.c --- a/sys/riscv/riscv/bus_machdep.c +++ b/sys/riscv/riscv/bus_machdep.c @@ -33,6 +33,10 @@ * SUCH DAMAGE. */ +#if defined(KASAN) +#define SAN_RUNTIME +#endif + #include "opt_platform.h" #include diff --git a/sys/riscv/riscv/db_trace.c b/sys/riscv/riscv/db_trace.c --- a/sys/riscv/riscv/db_trace.c +++ b/sys/riscv/riscv/db_trace.c @@ -55,7 +55,7 @@ } -static void +static void __nosanitizeaddress db_stack_trace_cmd(struct thread *td, struct unwind_state *frame) { const char *name; @@ -118,7 +118,7 @@ } } -int +int __nosanitizeaddress db_trace_thread(struct thread *thr, int count) { struct unwind_state frame; @@ -133,7 +133,7 @@ return (0); } -void +void __nosanitizeaddress db_trace_self(void) { struct unwind_state frame; diff --git a/sys/riscv/riscv/locore.S b/sys/riscv/riscv/locore.S --- a/sys/riscv/riscv/locore.S +++ b/sys/riscv/riscv/locore.S @@ -260,6 +260,17 @@ sd s4, RISCV_BOOTPARAMS_DTBP_PHYS(sp) sd s3, RISCV_BOOTPARAMS_MODULEP(sp) +#ifdef KASAN + /* KASAN needs phys addr of kernel to set up the page tables */ + la t0, kernstart_pa + sd s9, 0(t0) + + /* Bootstrap a shadow map for the boot stack. */ + la a0, initstack + li a1, (PAGE_SIZE * KSTACK_PAGES) + call _C_LABEL(kasan_init_early) +#endif + mv a0, sp call _C_LABEL(initriscv) /* Off we go */ call _C_LABEL(mi_startup) @@ -310,6 +321,11 @@ /* * Static space for the bootstrap page tables. Unused after pmap_bootstrap(). */ +#ifdef KASAN + /* KASAN needs the bootstrap page table to link the early shadow map. */ + .globl bootstrap_pt_l1 +#endif + .balign PAGE_SIZE bootstrap_pt_l1: .space PAGE_SIZE @@ -390,6 +406,13 @@ li t0, 0 csrw sscratch, t0 + /* + * Initialize the per-CPU pointer before calling into C code, for the + * benefit of kernel sanitizers. + */ + la t0, bootpcpu + ld tp, 0(t0) + call init_secondary END(mpentry) #endif diff --git a/sys/riscv/riscv/machdep.c b/sys/riscv/riscv/machdep.c --- a/sys/riscv/riscv/machdep.c +++ b/sys/riscv/riscv/machdep.c @@ -39,6 +39,7 @@ #include #include +#include #include #include #include @@ -635,6 +636,15 @@ /* Bootstrap enough of pmap to enter the kernel proper */ kernlen = (lastaddr - KERNBASE); pmap_bootstrap(rvbp->kern_phys, kernlen); + + /* + * There's not always enough room for the initial shadow map after the + * kernel, so as is done on arm64, we'll end up searching for segments + * that we can safely use. + */ +#if defined(KASAN) || defined(KMSAN) + pmap_bootstrap_san(); +#endif physmem_init_kernel_globals(); @@ -659,6 +669,8 @@ kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger"); #endif + kasan_init(); + env = kern_getenv("kernelname"); if (env != NULL) strlcpy(kernelname, env, sizeof(kernelname)); diff --git a/sys/riscv/riscv/mp_machdep.c b/sys/riscv/riscv/mp_machdep.c --- a/sys/riscv/riscv/mp_machdep.c +++ b/sys/riscv/riscv/mp_machdep.c @@ -97,6 +97,9 @@ static struct mtx ap_boot_mtx; +/* Used to initialize the PCPU ahead of calling init_secondary(). */ +void *bootpcpu; + /* Stacks for AP initialization, discarded once idle threads are started. */ void *bootstack; static void *bootstacks[MAXCPU]; @@ -153,15 +156,8 @@ struct pcpu *pcpup; u_int cpuid; - /* Renumber this cpu */ - cpuid = hart; - if (cpuid < boot_hart) - cpuid += mp_maxid + 1; - cpuid -= boot_hart; - - /* Setup the pcpu pointer */ - pcpup = &__pcpu[cpuid]; - __asm __volatile("mv tp, %0" :: "r"(pcpup)); + pcpup = get_pcpu(); + cpuid = pcpup->pc_cpuid; /* Workaround: make sure wfi doesn't halt the hart */ csr_set(sie, SIE_SSIE); @@ -395,6 +391,7 @@ pcpup = &__pcpu[cpuid]; pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); pcpup->pc_hart = hart; + bootpcpu = pcpup; dpcpu[cpuid - 1] = kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); dpcpu_init(dpcpu[cpuid - 1], cpuid); diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c --- a/sys/riscv/riscv/pmap.c +++ b/sys/riscv/riscv/pmap.c @@ -117,6 +117,7 @@ #include #include +#include #include #include #include @@ -157,6 +158,7 @@ #include #include +#include #include #include #include @@ -310,6 +312,10 @@ extern cpuset_t all_harts; +#if defined(KASAN) +extern pt_entry_t bootstrap_pt_l1[]; +#endif + /* * Internal flags for pmap_enter()'s helper functions. */ @@ -888,6 +894,21 @@ pmap_store(&l1[slot], L1_PDE(pa, PTE_V)); } +#ifdef KASAN + pd_entry_t kasan_l2_pt; + /* Connect the early KASAN shadow to L1 */ + slot = pmap_l1_index(KASAN_MIN_ADDRESS); + int nkasan_l2_pt = howmany( + howmany(KASAN_MAX_ADDRESS - KASAN_MIN_ADDRESS, L2_SIZE), Ln_ENTRIES); + for (i = 0; i < nkasan_l2_pt; i++, slot++) { + /* These mappings are initially created by pmap_san_enter_early(). */ + if (((kasan_l2_pt = pmap_load(&bootstrap_pt_l1[slot])) & PTE_V) != 0) { + pa = PTE_TO_PHYS(kasan_l2_pt); + pmap_store(&l1[slot], L1_PDE(pa, PTE_V)); + } + } +#endif + /* Connect the L1 table to L0, if in use. */ if (pmap_mode == PMAP_MODE_SV48) { slot = pmap_l0_index(KERNBASE); @@ -1016,6 +1037,92 @@ EXFLAG_NOALLOC); } +#if defined(KASAN) +static void +pmap_bootstrap_allocate_san_l2(vm_paddr_t start_pa, vm_paddr_t end_pa, + vm_offset_t *vap, vm_offset_t eva) +{ + vm_paddr_t pa; + vm_offset_t va; + pd_entry_t *l2; + + va = *vap; + pa = rounddown2(end_pa - L2_SIZE, L2_SIZE); + for (; pa >= start_pa && va < eva; va += L2_SIZE, pa -= L2_SIZE) { + l2 = pmap_l2(kernel_pmap, va); + MPASS(l2 != NULL); + + /* + * KASAN stack checking results in us having already allocated + * part of our shadow map, so we can just skip those segments. + */ + if ((pmap_load(l2) & PTE_V) != 0) { + pa += L2_SIZE; + continue; + } + + bzero_early(PHYS_TO_DMAP(pa), L2_SIZE); + physmem_exclude_region(pa, L2_SIZE, EXFLAG_NOALLOC); + pmap_store(l2, L2_PTE(pa, PTE_KERN | PTE_V)); + } + *vap = va; +} + +/* + * Finish constructing the initial shadow map: + * - Count how many pages from KERNBASE to virtual_avail (scaled for + * shadow map) + * - Map that entire range using L2 superpages. + */ +void +pmap_bootstrap_san(void) +{ + vm_offset_t eva; + vm_offset_t va = KASAN_MIN_ADDRESS; + vm_paddr_t physmap[PHYS_AVAIL_ENTRIES]; + int physmap_idx, i; + + /* + * Rebuild physmap one more time, we may have excluded more regions from + * allocation since pmap_bootstrap(). + */ + physmap_idx = physmem_avail(physmap, nitems(physmap)); + + eva = kasan_md_addr_to_shad(virtual_avail); + + /* + * Find a slot in the physmap large enough for what we needed. We try to put + * the shadow map as high up as we can to avoid depleting the lower 4GB in case + * it's needed for, e.g., an xhci controller that can only do 32-bit DMA. + */ + for (i = physmap_idx - 2; i >= 0; i -= 2) { + vm_paddr_t plow, phigh; + + /* L2 mappings must be backed by memory that is L2-aligned */ + plow = roundup2(physmap[i], L2_SIZE); + phigh = physmap[i + 1]; + if (plow >= phigh) + continue; + if (phigh - plow >= L2_SIZE) { + pmap_bootstrap_allocate_san_l2(plow, phigh, &va, eva); + if (va >= eva) + break; + } + } + if (i < 0) + panic("Could not find phys region for shadow map"); + + /* + * Done. We should now have a valid shadow address mapped for all KVA + * that has been mapped so far, i.e., KERNBASE to virtual_avail. Thus, + * shadow accesses by the sanitizer runtime will succeed for this range. + * When the kernel virtual address range is later expanded, as will + * happen in vm_mem_init(), the shadow map will be grown as well. This + * is handled by pmap_san_enter(). + */ +} +#endif + /* * Initialize a vm_page's machine-dependent fields. */ @@ -1723,6 +1830,9 @@ for (i = pmap_l1_index(VM_MIN_KERNEL_ADDRESS); i < pmap_l1_index(VM_MAX_KERNEL_ADDRESS); i++) pmap->pm_top[i] = kernel_pmap->pm_top[i]; + for (i = pmap_l1_index(KASAN_MIN_ADDRESS); + i < pmap_l1_index(KASAN_MAX_ADDRESS); i++) + pmap->pm_top[i] = kernel_pmap->pm_top[i]; for (i = pmap_l1_index(DMAP_MIN_ADDRESS); i < pmap_l1_index(DMAP_MAX_ADDRESS); i++) pmap->pm_top[i] = kernel_pmap->pm_top[i]; @@ -2045,6 +2155,8 @@ addr = roundup2(addr, L2_SIZE); if (addr - 1 >= vm_map_max(kernel_map)) addr = vm_map_max(kernel_map); + if (kernel_vm_end < addr) + kasan_shadow_map(kernel_vm_end, addr - kernel_vm_end); while (kernel_vm_end < addr) { l1 = pmap_l1(kernel_pmap, kernel_vm_end); if (pmap_load(l1) == 0) { @@ -5500,6 +5612,162 @@ return (true); } +#if defined(KASAN) + +vm_paddr_t kernstart_pa; + +/* + * An L2_SIZE (2MB) shadow memory covers 16MB of actual memory, which is plenty + * for the early bootstack. + */ +#define SAN_EARLY_SHADOW_SIZE (1 * L2_SIZE) +#define SAN_EARLY_L2_PT_SIZE (1 * PAGE_SIZE) +#define SAN_EARLY_VTOPHYS(va) \ + ((vm_paddr_t)(va) - KERNBASE + kernstart_pa) +#define SAN_EARLY_PHYSTOV(pa) \ + ((vm_offset_t)(pa) - kernstart_pa + KERNBASE) + +static pt_entry_t * __nosanitizeaddress __nosanitizememory +pmap_san_alloc_table_early(void) +{ + static uint8_t bootstrap_data[SAN_EARLY_L2_PT_SIZE] __aligned(PAGE_SIZE); + static size_t offset = 0; + vm_offset_t addr; + + if (offset + PAGE_SIZE > sizeof(bootstrap_data)) { + panic("%s: out of memory for the bootstrap shadow map", + __func__); + } + + addr = SAN_EARLY_VTOPHYS(&bootstrap_data[offset]); + offset += PAGE_SIZE; + return ((pt_entry_t *)addr); +} + +static vm_paddr_t __nosanitizeaddress __nosanitizememory +pmap_san_alloc_l2_shadow_early(void) +{ + static uint8_t bootstrap_data[SAN_EARLY_SHADOW_SIZE] __aligned(L2_SIZE); + static size_t offset = 0; + vm_offset_t addr; + + if (offset + L2_SIZE > sizeof(bootstrap_data)) { + panic("%s: out of memory for the bootstrap shadow map L2 entries", + __func__); + } + + addr = SAN_EARLY_VTOPHYS(&bootstrap_data[offset]); + offset += L2_SIZE; + return (addr); +} + +/* + * Map a shadow page, before the kernel has bootstrapped its page tables. This + * is currently only used to shadow the temporary boot stack set up by locore. + */ +static void __nosanitizeaddress __nosanitizememory +pmap_san_enter_early(vm_offset_t va) +{ + pt_entry_t *l2_pt; + vm_paddr_t shadow; + int l1_slot, l2_slot; + + l1_slot = pmap_l1_index(va); + l2_slot = pmap_l2_index(va); + + /* alloc and link this L2 pt to bootstrap_pt_l1 */ + if ((pmap_load(&bootstrap_pt_l1[l1_slot]) & PTE_V) == 0) { + l2_pt = pmap_san_alloc_table_early(); + + pmap_store(&bootstrap_pt_l1[l1_slot], + L1_PDE((vm_paddr_t)l2_pt, PTE_V)); + } else { + l2_pt = (pt_entry_t *)PTE_TO_PHYS(pmap_load(&bootstrap_pt_l1[l1_slot])); + } + + /* alloc and link an L2 supperpage for the early shadow map */ + if ((pmap_load(&l2_pt[l2_slot]) & PTE_V) == 0) { + /* + * If a second VA is entered, it will currently panic in + * pmap_san_alloc_l2_shadow_early since we only have 1 * + * L2_SIZE. However, this design allows us to easily expand its + * size in the future. + */ + shadow = pmap_san_alloc_l2_shadow_early(); + + pmap_store(&l2_pt[l2_slot], + L2_PTE(shadow, PTE_KERN | PTE_V)); + } + +} + +static vm_page_t +pmap_san_enter_alloc_4k(void) +{ + vm_page_t m; + + m = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | + VM_ALLOC_ZERO); + if (m == NULL) + panic("%s: no memory to grow shadow map", __func__); + return (m); +} + +static vm_page_t +pmap_san_enter_alloc_2m(void) +{ + return (vm_page_alloc_noobj_contig(VM_ALLOC_WIRED | VM_ALLOC_ZERO, + Ln_ENTRIES, 0, ~0ul, L2_SIZE, 0, VM_MEMATTR_DEFAULT)); +} + +void __nosanitizeaddress __nosanitizememory +pmap_san_enter(vm_offset_t va) +{ + vm_page_t m; + pd_entry_t *l1, *l2; + pt_entry_t *l3; + + if (virtual_avail == 0) { + /* + * Before we enter the first C function, initriscv(), we should + * build a shadow map for its bootstack + */ + pmap_san_enter_early(va); + return; + } + + mtx_assert(&kernel_map->system_mtx, MA_OWNED); + l1 = pmap_l1(kernel_pmap, va); + MPASS(l1 != NULL); + if ((pmap_load(l1) & PTE_V) == 0) { + m = pmap_san_enter_alloc_4k(); + pmap_store(l1, L1_PDE(VM_PAGE_TO_PHYS(m), PTE_V)); + } + + l2 = pmap_l1_to_l2(l1, va); + if ((pmap_load(l2) & PTE_V) == 0) { + m = pmap_san_enter_alloc_2m(); + if (m != NULL) { + pmap_store(l2, L2_PTE(VM_PAGE_TO_PHYS(m), PTE_KERN | PTE_V)); + } else { + m = pmap_san_enter_alloc_4k(); + pmap_store(l2, L2_PDE(VM_PAGE_TO_PHYS(m), PTE_V)); + } + } + if ((pmap_load(l2) & PTE_V) != 0 && + (pmap_load(l2) & PTE_RWX) != 0) + goto out; + + l3 = pmap_l2_to_l3(l2, va); + if ((pmap_load(l3) & PTE_V) != 0) + return; + m = pmap_san_enter_alloc_4k(); + pmap_store(l3, L3_PTE(VM_PAGE_TO_PHYS(m), PTE_KERN | PTE_V)); + +out: + sfence_vma(); +} +#endif /* KASAN */ /* * Track a range of the kernel's virtual address space that is contiguous @@ -5639,6 +5907,10 @@ sbuf_printf(sb, "\nDirect map:\n"); else if (i == pmap_l1_index(VM_MIN_KERNEL_ADDRESS)) sbuf_printf(sb, "\nKernel map:\n"); +#ifdef KASAN + else if (i == pmap_l1_index(KASAN_MIN_ADDRESS)) + sbuf_printf(sb, "\nKASAN shadow map:\n"); +#endif l1 = pmap_l1(kernel_pmap, sva); l1e = pmap_load(l1); diff --git a/sys/riscv/riscv/trap.c b/sys/riscv/riscv/trap.c --- a/sys/riscv/riscv/trap.c +++ b/sys/riscv/riscv/trap.c @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -310,6 +311,8 @@ { uint64_t exception; + kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0); + /* Ensure we came from supervisor mode, interrupts disabled */ KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == SSTATUS_SPP, ("Came from S mode with interrupts enabled")); @@ -385,6 +388,8 @@ struct thread *td; struct pcb *pcb; + kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0); + td = curthread; pcb = td->td_pcb;