diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp --- a/contrib/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp +++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp @@ -482,6 +482,7 @@ const bool IsX86 = getTriple().getArch() == llvm::Triple::x86; const bool IsX86_64 = getTriple().getArch() == llvm::Triple::x86_64; const bool IsMIPS64 = getTriple().isMIPS64(); + const bool IsRISCV64 = getTriple().getArch() == llvm::Triple::riscv64; SanitizerMask Res = ToolChain::getSupportedSanitizers(); Res |= SanitizerKind::Address; Res |= SanitizerKind::PointerCompare; @@ -496,7 +497,7 @@ Res |= SanitizerKind::Fuzzer; Res |= SanitizerKind::FuzzerNoLink; } - if (IsAArch64 || IsX86_64) { + if (IsAArch64 || IsX86_64 || IsRISCV64) { Res |= SanitizerKind::KernelAddress; Res |= SanitizerKind::KernelMemory; Res |= SanitizerKind::Memory; diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -96,7 +96,8 @@ riscv/riscv/trap.c standard riscv/riscv/timer.c standard riscv/riscv/uio_machdep.c standard -riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack +riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack \ + compile-with "${NORMAL_C:N-fsanitize*}" riscv/riscv/vm_machdep.c standard riscv/vmm/vmm.c optional vmm riscv/vmm/vmm_aplic.c optional vmm diff --git a/sys/conf/kern.mk b/sys/conf/kern.mk --- a/sys/conf/kern.mk +++ b/sys/conf/kern.mk @@ -279,6 +279,13 @@ # Work around https://github.com/llvm/llvm-project/issues/87923, which leads to # an assertion failure compiling dtrace.c with asan enabled. SAN_CFLAGS+= -mllvm -asan-use-stack-safety=0 +# TODO: similar for riscv +.elif ${MACHINE_CPUARCH} == "riscv" +.if ${COMPILER_TYPE} == "clang" +SAN_CFLAGS+= -mllvm -asan-mapping-offset=0xdfffffd000000000 +.else +SAN_CFLAGS+= -fasan-shadow-offset=0xdfffffd000000000 +.endif .endif .endif # !empty(KASAN_ENABLED) diff --git a/sys/kern/subr_asan.c b/sys/kern/subr_asan.c --- a/sys/kern/subr_asan.c +++ b/sys/kern/subr_asan.c @@ -632,125 +632,134 @@ #include #include -#define _ASAN_ATOMIC_FUNC_ADD(name, type) \ - void kasan_atomic_add_##name(volatile type *ptr, type val) \ +#define DEFINE_KASAN_ATOMIC_TYPES(name, type) \ + typedef type KASAN_ATOMIC_TYPE_##name; \ + typedef type KASAN_ATOMIC_TYPE_acq_##name; \ + typedef type KASAN_ATOMIC_TYPE_rel_##name; + +DEFINE_KASAN_ATOMIC_TYPES(8, uint8_t) +DEFINE_KASAN_ATOMIC_TYPES(16, uint16_t) +DEFINE_KASAN_ATOMIC_TYPES(32, uint32_t) +DEFINE_KASAN_ATOMIC_TYPES(64, uint64_t) +DEFINE_KASAN_ATOMIC_TYPES(char, u_char) +DEFINE_KASAN_ATOMIC_TYPES(short, u_short) +DEFINE_KASAN_ATOMIC_TYPES(int, u_int) +DEFINE_KASAN_ATOMIC_TYPES(long, u_long) +DEFINE_KASAN_ATOMIC_TYPES(ptr, uintptr_t) + +typedef bool KASAN_ATOMIC_TYPE_bool; + +#undef DEFINE_KASAN_ATOMIC_TYPES + +#define ASAN_ATOMIC_FUNC_ADD(name) \ + void kasan_atomic_add_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ atomic_add_##name(ptr, val); \ } -#define ASAN_ATOMIC_FUNC_ADD(name, type) \ - _ASAN_ATOMIC_FUNC_ADD(name, type) \ - _ASAN_ATOMIC_FUNC_ADD(acq_##name, type) \ - _ASAN_ATOMIC_FUNC_ADD(rel_##name, type) +#define ASAN_ATOMIC_FUNC_ADD_ACQ(name) ASAN_ATOMIC_FUNC_ADD(acq_##name) +#define ASAN_ATOMIC_FUNC_ADD_REL(name) ASAN_ATOMIC_FUNC_ADD(rel_##name) -#define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ - void kasan_atomic_subtract_##name(volatile type *ptr, type val) \ +#define ASAN_ATOMIC_FUNC_SUBTRACT(name) \ + void kasan_atomic_subtract_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ atomic_subtract_##name(ptr, val); \ } -#define ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ - _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ - _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ - _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) +#define ASAN_ATOMIC_FUNC_SUBTRACT_ACQ(name) ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name) +#define ASAN_ATOMIC_FUNC_SUBTRACT_REL(name) ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name) -#define _ASAN_ATOMIC_FUNC_SET(name, type) \ - void kasan_atomic_set_##name(volatile type *ptr, type val) \ +#define ASAN_ATOMIC_FUNC_SET(name) \ + void kasan_atomic_set_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ atomic_set_##name(ptr, val); \ } -#define ASAN_ATOMIC_FUNC_SET(name, type) \ - _ASAN_ATOMIC_FUNC_SET(name, type) \ - _ASAN_ATOMIC_FUNC_SET(acq_##name, type) \ - _ASAN_ATOMIC_FUNC_SET(rel_##name, type) +#define ASAN_ATOMIC_FUNC_SET_ACQ(name) ASAN_ATOMIC_FUNC_SET(acq_##name) +#define ASAN_ATOMIC_FUNC_SET_REL(name) ASAN_ATOMIC_FUNC_SET(rel_##name) -#define _ASAN_ATOMIC_FUNC_CLEAR(name, type) \ - void kasan_atomic_clear_##name(volatile type *ptr, type val) \ +#define ASAN_ATOMIC_FUNC_CLEAR(name) \ + void kasan_atomic_clear_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ atomic_clear_##name(ptr, val); \ } -#define ASAN_ATOMIC_FUNC_CLEAR(name, type) \ - _ASAN_ATOMIC_FUNC_CLEAR(name, type) \ - _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ - _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type) +#define ASAN_ATOMIC_FUNC_CLEAR_ACQ(name) ASAN_ATOMIC_FUNC_CLEAR(acq_##name) +#define ASAN_ATOMIC_FUNC_CLEAR_REL(name) ASAN_ATOMIC_FUNC_CLEAR(rel_##name) -#define ASAN_ATOMIC_FUNC_FETCHADD(name, type) \ - type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \ +#define ASAN_ATOMIC_FUNC_FETCHADD(name) \ + KASAN_ATOMIC_TYPE_##name kasan_atomic_fetchadd_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ return (atomic_fetchadd_##name(ptr, val)); \ } -#define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ - type kasan_atomic_readandclear_##name(volatile type *ptr) \ +#define ASAN_ATOMIC_FUNC_READANDCLEAR(name) \ + KASAN_ATOMIC_TYPE_##name kasan_atomic_readandclear_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ return (atomic_readandclear_##name(ptr)); \ } -#define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ - int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ +#define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name) \ + int kasan_atomic_testandclear_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, u_int v) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ return (atomic_testandclear_##name(ptr, v)); \ } -#define ASAN_ATOMIC_FUNC_TESTANDSET(name, type) \ - int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \ +#define ASAN_ATOMIC_FUNC_TESTANDSET(name) \ + int kasan_atomic_testandset_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, u_int v) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ return (atomic_testandset_##name(ptr, v)); \ } -#define ASAN_ATOMIC_FUNC_SWAP(name, type) \ - type kasan_atomic_swap_##name(volatile type *ptr, type val) \ +#define ASAN_ATOMIC_FUNC_TESTANDSET_ACQ(name) ASAN_ATOMIC_FUNC_TESTANDSET(acq_##name) + +#define ASAN_ATOMIC_FUNC_SWAP(name) \ + KASAN_ATOMIC_TYPE_##name kasan_atomic_swap_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ return (atomic_swap_##name(ptr, val)); \ } -#define _ASAN_ATOMIC_FUNC_CMPSET(name, type) \ - int kasan_atomic_cmpset_##name(volatile type *ptr, type oval, \ - type nval) \ +#define ASAN_ATOMIC_FUNC_CMPSET(name) \ + int kasan_atomic_cmpset_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name oval, \ + KASAN_ATOMIC_TYPE_##name nval) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ return (atomic_cmpset_##name(ptr, oval, nval)); \ } -#define ASAN_ATOMIC_FUNC_CMPSET(name, type) \ - _ASAN_ATOMIC_FUNC_CMPSET(name, type) \ - _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ - _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type) +#define ASAN_ATOMIC_FUNC_CMPSET_ACQ(name) ASAN_ATOMIC_FUNC_CMPSET(acq_##name) +#define ASAN_ATOMIC_FUNC_CMPSET_REL(name) ASAN_ATOMIC_FUNC_CMPSET(rel_##name) -#define _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ - int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ - type nval) \ +#define ASAN_ATOMIC_FUNC_FCMPSET(name) \ + int kasan_atomic_fcmpset_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name *oval, \ + KASAN_ATOMIC_TYPE_##name nval) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ return (atomic_fcmpset_##name(ptr, oval, nval)); \ } -#define ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ - _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ - _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ - _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) +#define ASAN_ATOMIC_FUNC_FCMPSET_ACQ(name) ASAN_ATOMIC_FUNC_FCMPSET(acq_##name) +#define ASAN_ATOMIC_FUNC_FCMPSET_REL(name) ASAN_ATOMIC_FUNC_FCMPSET(rel_##name) #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name) \ void kasan_atomic_thread_fence_##name(void) \ @@ -758,129 +767,68 @@ atomic_thread_fence_##name(); \ } -#define _ASAN_ATOMIC_FUNC_LOAD(name, type) \ - type kasan_atomic_load_##name(const volatile type *ptr) \ +#define ASAN_ATOMIC_FUNC_LOAD(name) \ + KASAN_ATOMIC_TYPE_##name kasan_atomic_load_##name(const volatile KASAN_ATOMIC_TYPE_##name *ptr) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ return (atomic_load_##name(ptr)); \ } -#define ASAN_ATOMIC_FUNC_LOAD(name, type) \ - _ASAN_ATOMIC_FUNC_LOAD(name, type) \ - _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type) +#define ASAN_ATOMIC_FUNC_LOAD_ACQ(name) ASAN_ATOMIC_FUNC_LOAD(acq_##name) -#define _ASAN_ATOMIC_FUNC_STORE(name, type) \ - void kasan_atomic_store_##name(volatile type *ptr, type val) \ +#define ASAN_ATOMIC_FUNC_STORE(name) \ + void kasan_atomic_store_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \ { \ - kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ + kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \ __RET_ADDR); \ atomic_store_##name(ptr, val); \ } -#define ASAN_ATOMIC_FUNC_STORE(name, type) \ - _ASAN_ATOMIC_FUNC_STORE(name, type) \ - _ASAN_ATOMIC_FUNC_STORE(rel_##name, type) - -ASAN_ATOMIC_FUNC_ADD(8, uint8_t); -ASAN_ATOMIC_FUNC_ADD(16, uint16_t); -ASAN_ATOMIC_FUNC_ADD(32, uint32_t); -ASAN_ATOMIC_FUNC_ADD(64, uint64_t); -ASAN_ATOMIC_FUNC_ADD(int, u_int); -ASAN_ATOMIC_FUNC_ADD(long, u_long); -ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); - -ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); -ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); -ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); -ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); -ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int); -ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long); -ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); - -ASAN_ATOMIC_FUNC_SET(8, uint8_t); -ASAN_ATOMIC_FUNC_SET(16, uint16_t); -ASAN_ATOMIC_FUNC_SET(32, uint32_t); -ASAN_ATOMIC_FUNC_SET(64, uint64_t); -ASAN_ATOMIC_FUNC_SET(int, u_int); -ASAN_ATOMIC_FUNC_SET(long, u_long); -ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t); - -ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t); -ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t); -ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t); -ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t); -ASAN_ATOMIC_FUNC_CLEAR(int, u_int); -ASAN_ATOMIC_FUNC_CLEAR(long, u_long); -ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); - -ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); -ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); -ASAN_ATOMIC_FUNC_FETCHADD(int, u_int); -ASAN_ATOMIC_FUNC_FETCHADD(long, u_long); - -ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); -ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); -ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); -ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); -ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); - -ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); -ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); -ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); -ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); -ASAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t); - -ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); -ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); -ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int); -ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long); -ASAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long); -ASAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t); - -ASAN_ATOMIC_FUNC_SWAP(32, uint32_t); -ASAN_ATOMIC_FUNC_SWAP(64, uint64_t); -ASAN_ATOMIC_FUNC_SWAP(int, u_int); -ASAN_ATOMIC_FUNC_SWAP(long, u_long); -ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); - -ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t); -ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t); -ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t); -ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t); -ASAN_ATOMIC_FUNC_CMPSET(int, u_int); -ASAN_ATOMIC_FUNC_CMPSET(long, u_long); -ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); - -ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); -ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); -ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); -ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); -ASAN_ATOMIC_FUNC_FCMPSET(int, u_int); -ASAN_ATOMIC_FUNC_FCMPSET(long, u_long); -ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); - -_ASAN_ATOMIC_FUNC_LOAD(bool, bool); -ASAN_ATOMIC_FUNC_LOAD(8, uint8_t); -ASAN_ATOMIC_FUNC_LOAD(16, uint16_t); -ASAN_ATOMIC_FUNC_LOAD(32, uint32_t); -ASAN_ATOMIC_FUNC_LOAD(64, uint64_t); -ASAN_ATOMIC_FUNC_LOAD(char, u_char); -ASAN_ATOMIC_FUNC_LOAD(short, u_short); -ASAN_ATOMIC_FUNC_LOAD(int, u_int); -ASAN_ATOMIC_FUNC_LOAD(long, u_long); -ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); - -_ASAN_ATOMIC_FUNC_STORE(bool, bool); -ASAN_ATOMIC_FUNC_STORE(8, uint8_t); -ASAN_ATOMIC_FUNC_STORE(16, uint16_t); -ASAN_ATOMIC_FUNC_STORE(32, uint32_t); -ASAN_ATOMIC_FUNC_STORE(64, uint64_t); -ASAN_ATOMIC_FUNC_STORE(char, u_char); -ASAN_ATOMIC_FUNC_STORE(short, u_short); -ASAN_ATOMIC_FUNC_STORE(int, u_int); -ASAN_ATOMIC_FUNC_STORE(long, u_long); -ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); +#define ASAN_ATOMIC_FUNC_STORE_REL(name) ASAN_ATOMIC_FUNC_STORE(rel_##name) + +ARCH_SUPPORT_ATOMIC_ADD_WIDTH(ASAN_ATOMIC_FUNC_ADD) +ARCH_SUPPORT_ATOMIC_ADD_ACQ_WIDTH(ASAN_ATOMIC_FUNC_ADD_ACQ) +ARCH_SUPPORT_ATOMIC_ADD_REL_WIDTH(ASAN_ATOMIC_FUNC_ADD_REL) + +ARCH_SUPPORT_ATOMIC_SUBTRACT_WIDTH(ASAN_ATOMIC_FUNC_SUBTRACT) +ARCH_SUPPORT_ATOMIC_SUBTRACT_ACQ_WIDTH(ASAN_ATOMIC_FUNC_SUBTRACT_ACQ) +ARCH_SUPPORT_ATOMIC_SUBTRACT_REL_WIDTH(ASAN_ATOMIC_FUNC_SUBTRACT_REL) + +ARCH_SUPPORT_ATOMIC_SET_WIDTH(ASAN_ATOMIC_FUNC_SET) +ARCH_SUPPORT_ATOMIC_SET_ACQ_WIDTH(ASAN_ATOMIC_FUNC_SET_ACQ) +ARCH_SUPPORT_ATOMIC_SET_REL_WIDTH(ASAN_ATOMIC_FUNC_SET_REL) + +ARCH_SUPPORT_ATOMIC_CLEAR_WIDTH(ASAN_ATOMIC_FUNC_CLEAR) +ARCH_SUPPORT_ATOMIC_CLEAR_ACQ_WIDTH(ASAN_ATOMIC_FUNC_CLEAR_ACQ) +ARCH_SUPPORT_ATOMIC_CLEAR_REL_WIDTH(ASAN_ATOMIC_FUNC_CLEAR_REL) + +ARCH_SUPPORT_ATOMIC_FETCHADD_WIDTH(ASAN_ATOMIC_FUNC_FETCHADD) + +ARCH_SUPPORT_ATOMIC_READANDCLEAR_WIDTH(ASAN_ATOMIC_FUNC_READANDCLEAR) + +ARCH_SUPPORT_ATOMIC_TESTANDCLEAR_WIDTH(ASAN_ATOMIC_FUNC_TESTANDCLEAR) + +ARCH_SUPPORT_ATOMIC_TESTANDSET_WIDTH(ASAN_ATOMIC_FUNC_TESTANDSET) +ARCH_SUPPORT_ATOMIC_TESTANDSET_ACQ_WIDTH(ASAN_ATOMIC_FUNC_TESTANDSET_ACQ) + +ARCH_SUPPORT_ATOMIC_SWAP_WIDTH(ASAN_ATOMIC_FUNC_SWAP) + +ARCH_SUPPORT_ATOMIC_CMPSET_WIDTH(ASAN_ATOMIC_FUNC_CMPSET) +ARCH_SUPPORT_ATOMIC_CMPSET_ACQ_WIDTH(ASAN_ATOMIC_FUNC_CMPSET_ACQ) +ARCH_SUPPORT_ATOMIC_CMPSET_REL_WIDTH(ASAN_ATOMIC_FUNC_CMPSET_REL) + +ARCH_SUPPORT_ATOMIC_FCMPSET_WIDTH(ASAN_ATOMIC_FUNC_FCMPSET) +ARCH_SUPPORT_ATOMIC_FCMPSET_ACQ_WIDTH(ASAN_ATOMIC_FUNC_FCMPSET_ACQ) +ARCH_SUPPORT_ATOMIC_FCMPSET_REL_WIDTH(ASAN_ATOMIC_FUNC_FCMPSET_REL) + +ASAN_ATOMIC_FUNC_LOAD(bool); +ARCH_SUPPORT_ATOMIC_LOAD_WIDTH(ASAN_ATOMIC_FUNC_LOAD) +ARCH_SUPPORT_ATOMIC_LOAD_ACQ_WIDTH(ASAN_ATOMIC_FUNC_LOAD_ACQ) + +ASAN_ATOMIC_FUNC_STORE(bool); +ARCH_SUPPORT_ATOMIC_STORE_WIDTH(ASAN_ATOMIC_FUNC_STORE) +ARCH_SUPPORT_ATOMIC_STORE_REL_WIDTH(ASAN_ATOMIC_FUNC_STORE_REL) ASAN_ATOMIC_FUNC_THREAD_FENCE(acq); ASAN_ATOMIC_FUNC_THREAD_FENCE(rel); diff --git a/sys/riscv/include/asan.h b/sys/riscv/include/asan.h new file mode 100644 --- /dev/null +++ b/sys/riscv/include/asan.h @@ -0,0 +1,67 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2020 The FreeBSD Foundation + * + * This software was developed by Mark Johnston under sponsorship from the + * FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _MACHINE_ASAN_H_ +#define _MACHINE_ASAN_H_ + +#ifdef KASAN + +#include +#include +#include +#include + +static inline vm_offset_t +kasan_md_addr_to_shad(vm_offset_t addr) +{ + return (((addr - VM_MIN_KERNEL_ADDRESS) >> KASAN_SHADOW_SCALE_SHIFT) + + KASAN_MIN_ADDRESS); +} + +static inline bool +kasan_md_unsupported(vm_offset_t addr) +{ + return (addr < VM_MIN_KERNEL_ADDRESS || addr >= virtual_end); +} + +static inline void +kasan_md_init(void) +{ +} + +static inline void +kasan_md_init_early(vm_offset_t bootstack, size_t size) +{ + kasan_shadow_map(bootstack, size); +} + +#endif /* KASAN */ + +#endif /* !_MACHINE_ASAN_H_ */ diff --git a/sys/riscv/include/atomic.h b/sys/riscv/include/atomic.h --- a/sys/riscv/include/atomic.h +++ b/sys/riscv/include/atomic.h @@ -35,13 +35,141 @@ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ -#include - #define fence() __asm __volatile("fence" ::: "memory"); #define mb() fence() #define rmb() fence() #define wmb() fence() +#define ARCH_SUPPORT_ATOMIC_ADD_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) +#define ARCH_SUPPORT_ATOMIC_ADD_ACQ_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) +#define ARCH_SUPPORT_ATOMIC_ADD_REL_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) + +#define ARCH_SUPPORT_ATOMIC_SUBTRACT_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) +#define ARCH_SUPPORT_ATOMIC_SUBTRACT_ACQ_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) +#define ARCH_SUPPORT_ATOMIC_SUBTRACT_REL_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) + +#define ARCH_SUPPORT_ATOMIC_SET_WIDTH(X) \ + X(16) X(short) X(32) X(int) X(64) X(ptr) X(long) +#define ARCH_SUPPORT_ATOMIC_SET_ACQ_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) +#define ARCH_SUPPORT_ATOMIC_SET_REL_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) + +#define ARCH_SUPPORT_ATOMIC_CLEAR_WIDTH(X) \ + X(16) X(short) X(32) X(int) X(64) X(ptr) X(long) +#define ARCH_SUPPORT_ATOMIC_CLEAR_ACQ_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) +#define ARCH_SUPPORT_ATOMIC_CLEAR_REL_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) + +#define ARCH_SUPPORT_ATOMIC_FETCHADD_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) + +#define ARCH_SUPPORT_ATOMIC_READANDCLEAR_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) + +#define ARCH_SUPPORT_ATOMIC_SWAP_WIDTH(X) \ + X(32) X(int) X(64) X(ptr) X(long) + +#define ARCH_SUPPORT_ATOMIC_TESTANDSET_WIDTH(X) \ + X(32) X(64) X(ptr) X(long) + +#define ARCH_SUPPORT_ATOMIC_TESTANDCLEAR_WIDTH(X) \ + X(32) X(64) X(ptr) X(long) + +#define ARCH_SUPPORT_ATOMIC_LOAD_ACQ_WIDTH(X) \ + X(8) X(16) X(short) X(32) X(int) X(64) X(ptr) X(long) + +#define ARCH_SUPPORT_ATOMIC_STORE_REL_WIDTH(X) \ + X(16) X(short) X(32) X(int) X(64) X(ptr) X(long) + + +#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME) +#include + +#undef atomic_add_8 +#undef atomic_add_acq_8 +#undef atomic_add_rel_8 +#undef atomic_add_char +#undef atomic_add_acq_char +#undef atomic_add_rel_char + +#undef atomic_subtract_8 +#undef atomic_subtract_acq_8 +#undef atomic_subtract_rel_8 +#undef atomic_subtract_char +#undef atomic_subtract_acq_char +#undef atomic_subtract_rel_char + +#undef atomic_set_8 +#undef atomic_set_acq_8 +#undef atomic_set_rel_8 +#undef atomic_set_char +#undef atomic_set_acq_char +#undef atomic_set_rel_char + +#undef atomic_clear_8 +#undef atomic_clear_acq_8 +#undef atomic_clear_rel_8 +#undef atomic_clear_char +#undef atomic_clear_acq_char +#undef atomic_clear_rel_char + +#undef atomic_fetchadd_8 +#undef atomic_fetchadd_char + +#undef atomic_readandclear_8 +#undef atomic_readandclear_char + +#undef atomic_swap_8 +#undef atomic_swap_char + +#undef atomic_testandset_8 +#undef atomic_testandset_char +#undef atomic_testandclear_8 +#undef atomic_testandclear_char + +#undef atomic_add_16 +#undef atomic_add_acq_16 +#undef atomic_add_rel_16 +#undef atomic_add_short +#undef atomic_add_acq_short +#undef atomic_add_rel_short + +#undef atomic_subtract_16 +#undef atomic_subtract_acq_16 +#undef atomic_subtract_rel_16 +#undef atomic_subtract_short +#undef atomic_subtract_acq_short +#undef atomic_subtract_rel_short + +#undef atomic_fetchadd_16 +#undef atomic_fetchadd_short + +#undef atomic_readandclear_16 +#undef atomic_readandclear_short + +#undef atomic_swap_16 +#undef atomic_swap_short + +#undef atomic_testandset_16 +#undef atomic_testandset_short +#undef atomic_testandclear_16 +#undef atomic_testandclear_short + +#undef atomic_testandset_int +#undef atomic_testandclear_int + +#else + +#include + static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t); static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t); static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t); @@ -659,4 +787,6 @@ #define atomic_set_short atomic_set_16 #define atomic_clear_short atomic_clear_16 +#endif /* SAN_NEEDS_INTERCEPTORS && !SAN_RUNTIME */ + #endif /* _MACHINE_ATOMIC_H_ */ diff --git a/sys/riscv/include/bus.h b/sys/riscv/include/bus.h --- a/sys/riscv/include/bus.h +++ b/sys/riscv/include/bus.h @@ -253,6 +253,10 @@ bus_size_t, const u_int64_t *, bus_size_t); }; +#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME) +#include +#else + /* * Utility macros; INTERNAL USE ONLY. */ @@ -495,6 +499,8 @@ BUS_POKE_FUNC(4, uint32_t) BUS_POKE_FUNC(8, uint64_t) +#endif /* SAN_NEEDS_INTERCEPTORS && !SAN_RUNTIME */ + #include #endif /* _MACHINE_BUS_H_ */ diff --git a/sys/riscv/include/param.h b/sys/riscv/include/param.h --- a/sys/riscv/include/param.h +++ b/sys/riscv/include/param.h @@ -81,8 +81,12 @@ #define MAXPAGESIZES 3 /* maximum number of supported page sizes */ #ifndef KSTACK_PAGES +#if defined(KASAN) +#define KSTACK_PAGES 6 +#else #define KSTACK_PAGES 4 /* pages of kernel stack (with pcb) */ #endif +#endif #define KSTACK_GUARD_PAGES 1 /* pages of kstack guard; 0 disables */ #define PCPU_PAGES 1 diff --git a/sys/riscv/include/pmap.h b/sys/riscv/include/pmap.h --- a/sys/riscv/include/pmap.h +++ b/sys/riscv/include/pmap.h @@ -164,6 +164,10 @@ return (0); } +#if defined(KASAN) +void pmap_san_enter(vm_offset_t); +void pmap_bootstrap_san(void); +#endif #endif /* _KERNEL */ diff --git a/sys/riscv/include/vmparam.h b/sys/riscv/include/vmparam.h --- a/sys/riscv/include/vmparam.h +++ b/sys/riscv/include/vmparam.h @@ -129,7 +129,8 @@ * 0x0000000000000000 - 0x0000003fffffffff 256GB user map * 0x0000004000000000 - 0xffffffbfffffffff unmappable * 0xffffffc000000000 - 0xffffffc7ffffffff 32GB kernel map - * 0xffffffc800000000 - 0xffffffcfffffffff 32GB unused + * 0xffffffc800000000 - 0xffffffc8ffffffff 4GB KASAN shadow map + * 0xffffffc900000000 - 0xffffffcfffffffff 28GB unused * 0xffffffd000000000 - 0xffffffefffffffff 128GB direct map * 0xfffffff000000000 - 0xffffffffffffffff 64GB unused * @@ -138,7 +139,8 @@ * 0x0000800000000000 - 0xffff7fffffffffff unmappable * 0xffff800000000000 - 0xffffffc7ffffffff 127.75TB hole * 0xffffffc000000000 - 0xffffffc7ffffffff 32GB kernel map - * 0xffffffc800000000 - 0xffffffcfffffffff 32GB unused + * 0xffffffc800000000 - 0xffffffc8ffffffff 4GB KASAN shadow map + * 0xffffffc900000000 - 0xffffffcfffffffff 28GB unused * 0xffffffd000000000 - 0xffffffefffffffff 128GB direct map * 0xfffffff000000000 - 0xffffffffffffffff 64GB unused * @@ -161,6 +163,9 @@ #define VM_MIN_KERNEL_ADDRESS (0xffffffc000000000UL) #define VM_MAX_KERNEL_ADDRESS (0xffffffc800000000UL) +#define KASAN_MIN_ADDRESS (0xffffffc800000000UL) +#define KASAN_MAX_ADDRESS (0xffffffc900000000UL) + #define DMAP_MIN_ADDRESS (0xffffffd000000000UL) #define DMAP_MAX_ADDRESS (0xfffffff000000000UL) diff --git a/sys/riscv/riscv/bus_machdep.c b/sys/riscv/riscv/bus_machdep.c --- a/sys/riscv/riscv/bus_machdep.c +++ b/sys/riscv/riscv/bus_machdep.c @@ -33,6 +33,10 @@ * SUCH DAMAGE. */ +#if defined(KASAN) +#define SAN_RUNTIME +#endif + #include "opt_platform.h" #include diff --git a/sys/riscv/riscv/db_trace.c b/sys/riscv/riscv/db_trace.c --- a/sys/riscv/riscv/db_trace.c +++ b/sys/riscv/riscv/db_trace.c @@ -55,7 +55,7 @@ } -static void +static void __nosanitizeaddress db_stack_trace_cmd(struct thread *td, struct unwind_state *frame) { const char *name; @@ -118,7 +118,7 @@ } } -int +int __nosanitizeaddress db_trace_thread(struct thread *thr, int count) { struct unwind_state frame; @@ -133,7 +133,7 @@ return (0); } -void +void __nosanitizeaddress db_trace_self(void) { struct unwind_state frame; diff --git a/sys/riscv/riscv/locore.S b/sys/riscv/riscv/locore.S --- a/sys/riscv/riscv/locore.S +++ b/sys/riscv/riscv/locore.S @@ -260,6 +260,17 @@ sd s4, RISCV_BOOTPARAMS_DTBP_PHYS(sp) sd s3, RISCV_BOOTPARAMS_MODULEP(sp) +#ifdef KASAN + /* KASAN needs phys addr of kernel to set up the page tables */ + la t0, kernstart_pa + sd s9, 0(t0) + + /* Bootstrap a shadow map for the boot stack. */ + la a0, initstack + li a1, (PAGE_SIZE * KSTACK_PAGES) + call _C_LABEL(kasan_init_early) +#endif + mv a0, sp call _C_LABEL(initriscv) /* Off we go */ call _C_LABEL(mi_startup) @@ -310,6 +321,11 @@ /* * Static space for the bootstrap page tables. Unused after pmap_bootstrap(). */ +#ifdef KASAN + /* KASAN needs the bootstrap page table to link the early shadow map. */ + .globl bootstrap_pt_l1 +#endif + .balign PAGE_SIZE bootstrap_pt_l1: .space PAGE_SIZE diff --git a/sys/riscv/riscv/machdep.c b/sys/riscv/riscv/machdep.c --- a/sys/riscv/riscv/machdep.c +++ b/sys/riscv/riscv/machdep.c @@ -39,6 +39,7 @@ #include #include +#include #include #include #include @@ -634,6 +635,15 @@ /* Bootstrap enough of pmap to enter the kernel proper */ kernlen = (lastaddr - KERNBASE); pmap_bootstrap(rvbp->kern_phys, kernlen); + + /* + * There's not always enough room for the initial shadow map after the + * kernel, so as is done on arm64, we'll end up searching for segments + * that we can safely use. + */ +#if defined(KASAN) || defined(KMSAN) + pmap_bootstrap_san(); +#endif physmem_init_kernel_globals(); @@ -658,6 +668,8 @@ kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger"); #endif + kasan_init(); + env = kern_getenv("kernelname"); if (env != NULL) strlcpy(kernelname, env, sizeof(kernelname)); diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c --- a/sys/riscv/riscv/pmap.c +++ b/sys/riscv/riscv/pmap.c @@ -116,6 +116,7 @@ #include #include +#include #include #include #include @@ -152,6 +153,7 @@ #include #include +#include #include #include #include @@ -304,6 +306,10 @@ extern cpuset_t all_harts; +#if defined(KASAN) +extern pt_entry_t bootstrap_pt_l1[]; +#endif + /* * Internal flags for pmap_enter()'s helper functions. */ @@ -881,6 +887,20 @@ pmap_store(&l1[slot], L1_PDE(pa, PTE_V)); } +#ifdef KASAN + pd_entry_t kasan_l2_pt; + /* Connect the early KASAN shadow to L1 */ + slot = pmap_l1_index(KASAN_MIN_ADDRESS); + int nkasan_l2_pt = howmany( + howmany(KASAN_MAX_ADDRESS - KASAN_MIN_ADDRESS, L2_SIZE), Ln_ENTRIES); + for (i = 0; i < nkasan_l2_pt; i++, slot++) { + if (((kasan_l2_pt = pmap_load(&bootstrap_pt_l1[slot])) & PTE_V) != 0) { + pa = PTE_TO_PHYS(kasan_l2_pt); + pmap_store(&l1[slot], L1_PDE(pa, PTE_V)); + } + } +#endif + /* Connect the L1 table to L0, if in use. */ if (pmap_mode == PMAP_MODE_SV48) { slot = pmap_l0_index(KERNBASE); @@ -1009,6 +1029,64 @@ EXFLAG_NOALLOC); } +#if defined(KASAN) +static void +pmap_bootstrap_allocate_san_l2(vm_paddr_t start_pa, vm_paddr_t end_pa, + vm_offset_t *vap, vm_offset_t eva) +{ + vm_paddr_t pa; + vm_offset_t va; + pd_entry_t *l2; + + va = *vap; + pa = rounddown2(end_pa - L2_SIZE, L2_SIZE); + for (; pa >= start_pa && va < eva; va += L2_SIZE, pa -= L2_SIZE) { + l2 = pmap_l2(kernel_pmap, va); + MPASS(l2 != NULL); + + if ((pmap_load(l2) & PTE_V) != 0) { + pa += L2_SIZE; + continue; + } + + bzero_early(PHYS_TO_DMAP(pa), L2_SIZE); + physmem_exclude_region(pa, L2_SIZE, EXFLAG_NOALLOC); + pmap_store(l2, L2_PTE(pa, PTE_KERN | PTE_V)); + } + *vap = va; +} + +void +pmap_bootstrap_san(void) +{ + vm_offset_t eva; + vm_offset_t va = KASAN_MIN_ADDRESS; + vm_paddr_t physmap[PHYS_AVAIL_ENTRIES]; + int physmap_idx, i; + + physmap_idx = physmem_avail(physmap, nitems(physmap)); + + eva = kasan_md_addr_to_shad(virtual_avail); + + for (i = physmap_idx - 2; i >= 0; i -= 2) { + vm_paddr_t plow, phigh; + + /* L2 mappings must be backed by memory that is L2-aligned */ + plow = roundup2(physmap[i], L2_SIZE); + phigh = physmap[i + 1]; + if (plow >= phigh) + continue; + if (phigh - plow >= L2_SIZE) { + pmap_bootstrap_allocate_san_l2(plow, phigh, &va, eva); + if (va >= eva) + break; + } + } + if (i < 0) + panic("Could not find phys region for shadow map"); +} +#endif + /* * Initialize a vm_page's machine-dependent fields. */ @@ -1649,6 +1727,9 @@ for (i = pmap_l1_index(VM_MIN_KERNEL_ADDRESS); i < pmap_l1_index(VM_MAX_KERNEL_ADDRESS); i++) pmap->pm_top[i] = kernel_pmap->pm_top[i]; + for (i = pmap_l1_index(KASAN_MIN_ADDRESS); + i < pmap_l1_index(KASAN_MAX_ADDRESS); i++) + pmap->pm_top[i] = kernel_pmap->pm_top[i]; for (i = pmap_l1_index(DMAP_MIN_ADDRESS); i < pmap_l1_index(DMAP_MAX_ADDRESS); i++) pmap->pm_top[i] = kernel_pmap->pm_top[i]; @@ -1971,6 +2052,8 @@ addr = roundup2(addr, L2_SIZE); if (addr - 1 >= vm_map_max(kernel_map)) addr = vm_map_max(kernel_map); + if (kernel_vm_end < addr) + kasan_shadow_map(kernel_vm_end, addr - kernel_vm_end); while (kernel_vm_end < addr) { l1 = pmap_l1(kernel_pmap, kernel_vm_end); if (pmap_load(l1) == 0) { @@ -5402,6 +5485,155 @@ return (true); } +#if defined(KASAN) + +vm_paddr_t kernstart_pa; + +#define SAN_EARLY_SHADOW_SIZE (1 * L2_SIZE) +#define SAN_EARLY_L2_PT_SIZE (1 * PAGE_SIZE) +#define SAN_EARLY_VTOPHYS(va) \ + ((vm_paddr_t)(va) - KERNBASE + kernstart_pa) +#define SAN_EARLY_PHYSTOV(pa) \ + ((vm_offset_t)(pa) - kernstart_pa + KERNBASE) + +static pt_entry_t * __nosanitizeaddress __nosanitizememory +pmap_san_alloc_table_early(void) +{ + static uint8_t bootstrap_data[SAN_EARLY_L2_PT_SIZE] __aligned(PAGE_SIZE); + static size_t offset = 0; + vm_offset_t addr; + + if (offset + PAGE_SIZE > sizeof(bootstrap_data)) { + panic("%s: out of memory for the bootstrap shadow map", + __func__); + } + + addr = SAN_EARLY_VTOPHYS(&bootstrap_data[offset]); + offset += PAGE_SIZE; + return ((pt_entry_t *)addr); +} + +static vm_paddr_t __nosanitizeaddress __nosanitizememory +pmap_san_alloc_l2_shadow_early(void) +{ + static uint8_t bootstrap_data[SAN_EARLY_SHADOW_SIZE] __aligned(L2_SIZE); + static size_t offset = 0; + vm_offset_t addr; + + if (offset + L2_SIZE > sizeof(bootstrap_data)) { + panic("%s: out of memory for the bootstrap shadow map L2 entries", + __func__); + } + + addr = SAN_EARLY_VTOPHYS(&bootstrap_data[offset]); + offset += L2_SIZE; + return (addr); +} + +/* + * Map a shadow page, before the kernel has bootstrapped its page tables. This + * is currently only used to shadow the temporary boot stack set up by locore. + */ +static void __nosanitizeaddress __nosanitizememory +pmap_san_enter_early(vm_offset_t va) +{ + pt_entry_t *l2_pt; + vm_paddr_t shadow; + int l1_slot, l2_slot; + + l1_slot = pmap_l1_index(va); + l2_slot = pmap_l2_index(va); + + /* alloc and link this L2 pt to bootstrap_pt_l1 */ + if ((pmap_load(&bootstrap_pt_l1[l1_slot]) & PTE_V) == 0) { + l2_pt = pmap_san_alloc_table_early(); + + pmap_store(&bootstrap_pt_l1[l1_slot], + L1_PDE((vm_paddr_t)l2_pt, PTE_V)); + } else { + l2_pt = (pt_entry_t *)PTE_TO_PHYS(pmap_load(&bootstrap_pt_l1[l1_slot])); + } + + /* alloc and link an L2 supperpage for the early shadow map */ + if ((pmap_load(&l2_pt[l2_slot]) & PTE_V) == 0) { + /* + * If a second VA is entered, it will currently panic in + * pmap_san_alloc_l2_shadow_early since we only have 1 * + * L2_SIZE. However, this design allows us to easily expand its + * size in the future. + */ + shadow = pmap_san_alloc_l2_shadow_early(); + + pmap_store(&l2_pt[l2_slot], + L2_PTE(shadow, PTE_KERN | PTE_V)); + } + +} + +static vm_page_t +pmap_san_enter_alloc_4k(void) +{ + vm_page_t m; + + m = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | + VM_ALLOC_ZERO); + if (m == NULL) + panic("%s: no memory to grow shadow map", __func__); + return (m); +} + +static vm_page_t +pmap_san_enter_alloc_2m(void) +{ + return (vm_page_alloc_noobj_contig(VM_ALLOC_WIRED | VM_ALLOC_ZERO, + Ln_ENTRIES, 0, ~0ul, L2_SIZE, 0, VM_MEMATTR_DEFAULT)); +} + +void __nosanitizeaddress __nosanitizememory +pmap_san_enter(vm_offset_t va) +{ + pd_entry_t *l1, *l2; + pt_entry_t *l3; + vm_page_t m; + if (virtual_avail == 0) { + /* + * Before we enter the first C function, initriscv, we should + * build a shadow map for its bootstack + */ + pmap_san_enter_early(va); + return; + } + mtx_assert(&kernel_map->system_mtx, MA_OWNED); + l1 = pmap_l1(kernel_pmap, va); + MPASS(l1 != NULL); + if ((pmap_load(l1) & PTE_V) == 0) { + m = pmap_san_enter_alloc_4k(); + pmap_store(l1, L1_PDE(VM_PAGE_TO_PHYS(m), PTE_V)); + } + + l2 = pmap_l1_to_l2(l1, va); + if ((pmap_load(l2) & PTE_V) == 0) { + m = pmap_san_enter_alloc_2m(); + if (m != NULL) { + pmap_store(l2, L2_PTE(VM_PAGE_TO_PHYS(m), PTE_KERN | PTE_V)); + } else { + m = pmap_san_enter_alloc_4k(); + pmap_store(l2, L2_PDE(VM_PAGE_TO_PHYS(m), PTE_V)); + } + wmb(); + } + if ((pmap_load(l2) & PTE_V) != 0 && + (pmap_load(l2) & PTE_RWX) != 0) + return; + + l3 = pmap_l2_to_l3(l2, va); + if ((pmap_load(l3) & PTE_V) != 0) + return; + m = pmap_san_enter_alloc_4k(); + pmap_store(l3, L3_PTE(VM_PAGE_TO_PHYS(m), PTE_KERN | PTE_V)); + wmb(); +} +#endif /* * Track a range of the kernel's virtual address space that is contiguous @@ -5541,6 +5773,10 @@ sbuf_printf(sb, "\nDirect map:\n"); else if (i == pmap_l1_index(VM_MIN_KERNEL_ADDRESS)) sbuf_printf(sb, "\nKernel map:\n"); +#ifdef KASAN + else if (i == pmap_l1_index(KASAN_MIN_ADDRESS)) + sbuf_printf(sb, "\nKASAN shadow map:\n"); +#endif l1 = pmap_l1(kernel_pmap, sva); l1e = pmap_load(l1); diff --git a/sys/riscv/riscv/trap.c b/sys/riscv/riscv/trap.c --- a/sys/riscv/riscv/trap.c +++ b/sys/riscv/riscv/trap.c @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -308,6 +309,8 @@ { uint64_t exception; + kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0); + /* Ensure we came from supervisor mode, interrupts disabled */ KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == SSTATUS_SPP, ("Came from S mode with interrupts enabled")); @@ -383,6 +386,8 @@ struct thread *td; struct pcb *pcb; + kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0); + td = curthread; pcb = td->td_pcb; diff --git a/sys/sys/atomic_san.h b/sys/sys/atomic_san.h --- a/sys/sys/atomic_san.h +++ b/sys/sys/atomic_san.h @@ -406,4 +406,141 @@ #endif /* !SAN_RUNTIME */ +/* + * The following x-macros control the generation of ASAN interceptor functions + * in subr_asan.c. + * + * By default, the MI code(atomic_san.h) assumes that the underlying + * architecture supports all atomic operations across all widths. If a specific + * MD architecture(machine/atomic.h) lacks support for certain atomic widths + * (e.g., 8-bit or 16-bit), it should override these ARCH_SUPPORT_* macros in + * its machine/atomic.h and #undef the corresponding atomic_* macros to prevent + * the generation of unsupported interceptors. + */ + +#ifndef ARCH_SUPPORT_ATOMIC_ADD_WIDTH +#define ARCH_SUPPORT_ATOMIC_ADD_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_ADD_ACQ_WIDTH +#define ARCH_SUPPORT_ATOMIC_ADD_ACQ_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_ADD_REL_WIDTH +#define ARCH_SUPPORT_ATOMIC_ADD_REL_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif + +#ifndef ARCH_SUPPORT_ATOMIC_SUBTRACT_WIDTH +#define ARCH_SUPPORT_ATOMIC_SUBTRACT_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_SUBTRACT_ACQ_WIDTH +#define ARCH_SUPPORT_ATOMIC_SUBTRACT_ACQ_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_SUBTRACT_REL_WIDTH +#define ARCH_SUPPORT_ATOMIC_SUBTRACT_REL_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif + +#ifndef ARCH_SUPPORT_ATOMIC_SET_WIDTH +#define ARCH_SUPPORT_ATOMIC_SET_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_SET_ACQ_WIDTH +#define ARCH_SUPPORT_ATOMIC_SET_ACQ_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_SET_REL_WIDTH +#define ARCH_SUPPORT_ATOMIC_SET_REL_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif + +#ifndef ARCH_SUPPORT_ATOMIC_CLEAR_WIDTH +#define ARCH_SUPPORT_ATOMIC_CLEAR_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_CLEAR_ACQ_WIDTH +#define ARCH_SUPPORT_ATOMIC_CLEAR_ACQ_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_CLEAR_REL_WIDTH +#define ARCH_SUPPORT_ATOMIC_CLEAR_REL_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif + +#ifndef ARCH_SUPPORT_ATOMIC_CMPSET_WIDTH +#define ARCH_SUPPORT_ATOMIC_CMPSET_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_CMPSET_ACQ_WIDTH +#define ARCH_SUPPORT_ATOMIC_CMPSET_ACQ_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_CMPSET_REL_WIDTH +#define ARCH_SUPPORT_ATOMIC_CMPSET_REL_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif + +#ifndef ARCH_SUPPORT_ATOMIC_FCMPSET_WIDTH +#define ARCH_SUPPORT_ATOMIC_FCMPSET_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_FCMPSET_ACQ_WIDTH +#define ARCH_SUPPORT_ATOMIC_FCMPSET_ACQ_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_FCMPSET_REL_WIDTH +#define ARCH_SUPPORT_ATOMIC_FCMPSET_REL_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif + +#ifndef ARCH_SUPPORT_ATOMIC_FETCHADD_WIDTH +#define ARCH_SUPPORT_ATOMIC_FETCHADD_WIDTH(X) \ + X(32) X(int) X(64) X(long) +#endif + +#ifndef ARCH_SUPPORT_ATOMIC_READANDCLEAR_WIDTH +#define ARCH_SUPPORT_ATOMIC_READANDCLEAR_WIDTH(X) \ + X(32) X(int) X(64) X(long) X(ptr) +#endif + +#ifndef ARCH_SUPPORT_ATOMIC_SWAP_WIDTH +#define ARCH_SUPPORT_ATOMIC_SWAP_WIDTH(X) \ + X(32) X(int) X(64) X(long) X(ptr) +#endif + +#ifndef ARCH_SUPPORT_ATOMIC_TESTANDSET_WIDTH +#define ARCH_SUPPORT_ATOMIC_TESTANDSET_WIDTH(X) \ + X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_TESTANDSET_ACQ_WIDTH +#define ARCH_SUPPORT_ATOMIC_TESTANDSET_ACQ_WIDTH(X) \ + X(long) +#endif + +#ifndef ARCH_SUPPORT_ATOMIC_TESTANDCLEAR_WIDTH +#define ARCH_SUPPORT_ATOMIC_TESTANDCLEAR_WIDTH(X) \ + X(32) X(int) X(64) X(long) X(ptr) +#endif + +#ifndef ARCH_SUPPORT_ATOMIC_LOAD_WIDTH +#define ARCH_SUPPORT_ATOMIC_LOAD_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_LOAD_ACQ_WIDTH +#define ARCH_SUPPORT_ATOMIC_LOAD_ACQ_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif + +#ifndef ARCH_SUPPORT_ATOMIC_STORE_WIDTH +#define ARCH_SUPPORT_ATOMIC_STORE_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif +#ifndef ARCH_SUPPORT_ATOMIC_STORE_REL_WIDTH +#define ARCH_SUPPORT_ATOMIC_STORE_REL_WIDTH(X) \ + X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr) +#endif + #endif /* !_SYS_ATOMIC_SAN_H_ */