Page MenuHomeFreeBSD
Files F161688914

D57381.id179088.diff
No OneTemporary
Actions

D57381.id179088.diff
View Options

diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp
--- a/contrib/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp
+++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/FreeBSD.cpp
@@ -482,6 +482,7 @@
const bool IsX86 = getTriple().getArch() == llvm::Triple::x86;
const bool IsX86_64 = getTriple().getArch() == llvm::Triple::x86_64;
const bool IsMIPS64 = getTriple().isMIPS64();
+ const bool IsRISCV64 = getTriple().getArch() == llvm::Triple::riscv64;
SanitizerMask Res = ToolChain::getSupportedSanitizers();
Res |= SanitizerKind::Address;
Res |= SanitizerKind::PointerCompare;
@@ -496,7 +497,7 @@
Res |= SanitizerKind::Fuzzer;
Res |= SanitizerKind::FuzzerNoLink;
}
- if (IsAArch64 || IsX86_64) {
+ if (IsAArch64 || IsX86_64 || IsRISCV64) {
Res |= SanitizerKind::KernelAddress;
Res |= SanitizerKind::KernelMemory;
Res |= SanitizerKind::Memory;
diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv
--- a/sys/conf/files.riscv
+++ b/sys/conf/files.riscv
@@ -96,7 +96,8 @@
riscv/riscv/trap.c standard
riscv/riscv/timer.c standard
riscv/riscv/uio_machdep.c standard
-riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack
+riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack \
+ compile-with "${NORMAL_C:N-fsanitize*}"
riscv/riscv/vm_machdep.c standard
riscv/vmm/vmm.c optional vmm
riscv/vmm/vmm_aplic.c optional vmm
diff --git a/sys/conf/kern.mk b/sys/conf/kern.mk
--- a/sys/conf/kern.mk
+++ b/sys/conf/kern.mk
@@ -279,6 +279,13 @@
# Work around https://github.com/llvm/llvm-project/issues/87923, which leads to
# an assertion failure compiling dtrace.c with asan enabled.
SAN_CFLAGS+= -mllvm -asan-use-stack-safety=0
+# TODO: similar for riscv
+.elif ${MACHINE_CPUARCH} == "riscv"
+.if ${COMPILER_TYPE} == "clang"
+SAN_CFLAGS+= -mllvm -asan-mapping-offset=0xdfffffd000000000
+.else
+SAN_CFLAGS+= -fasan-shadow-offset=0xdfffffd000000000
+.endif
.endif
.endif # !empty(KASAN_ENABLED)
diff --git a/sys/kern/subr_asan.c b/sys/kern/subr_asan.c
--- a/sys/kern/subr_asan.c
+++ b/sys/kern/subr_asan.c
@@ -632,125 +632,134 @@
#include <machine/atomic.h>
#include <sys/atomic_san.h>
-#define _ASAN_ATOMIC_FUNC_ADD(name, type) \
- void kasan_atomic_add_##name(volatile type *ptr, type val) \
+#define DEFINE_KASAN_ATOMIC_TYPES(name, type) \
+ typedef type KASAN_ATOMIC_TYPE_##name; \
+ typedef type KASAN_ATOMIC_TYPE_acq_##name; \
+ typedef type KASAN_ATOMIC_TYPE_rel_##name;
+
+DEFINE_KASAN_ATOMIC_TYPES(8, uint8_t)
+DEFINE_KASAN_ATOMIC_TYPES(16, uint16_t)
+DEFINE_KASAN_ATOMIC_TYPES(32, uint32_t)
+DEFINE_KASAN_ATOMIC_TYPES(64, uint64_t)
+DEFINE_KASAN_ATOMIC_TYPES(char, u_char)
+DEFINE_KASAN_ATOMIC_TYPES(short, u_short)
+DEFINE_KASAN_ATOMIC_TYPES(int, u_int)
+DEFINE_KASAN_ATOMIC_TYPES(long, u_long)
+DEFINE_KASAN_ATOMIC_TYPES(ptr, uintptr_t)
+
+typedef bool KASAN_ATOMIC_TYPE_bool;
+
+#undef DEFINE_KASAN_ATOMIC_TYPES
+
+#define ASAN_ATOMIC_FUNC_ADD(name) \
+ void kasan_atomic_add_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
atomic_add_##name(ptr, val); \
}
-#define ASAN_ATOMIC_FUNC_ADD(name, type) \
- _ASAN_ATOMIC_FUNC_ADD(name, type) \
- _ASAN_ATOMIC_FUNC_ADD(acq_##name, type) \
- _ASAN_ATOMIC_FUNC_ADD(rel_##name, type)
+#define ASAN_ATOMIC_FUNC_ADD_ACQ(name) ASAN_ATOMIC_FUNC_ADD(acq_##name)
+#define ASAN_ATOMIC_FUNC_ADD_REL(name) ASAN_ATOMIC_FUNC_ADD(rel_##name)
-#define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
- void kasan_atomic_subtract_##name(volatile type *ptr, type val) \
+#define ASAN_ATOMIC_FUNC_SUBTRACT(name) \
+ void kasan_atomic_subtract_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
atomic_subtract_##name(ptr, val); \
}
-#define ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
- _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \
- _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
- _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
+#define ASAN_ATOMIC_FUNC_SUBTRACT_ACQ(name) ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name)
+#define ASAN_ATOMIC_FUNC_SUBTRACT_REL(name) ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name)
-#define _ASAN_ATOMIC_FUNC_SET(name, type) \
- void kasan_atomic_set_##name(volatile type *ptr, type val) \
+#define ASAN_ATOMIC_FUNC_SET(name) \
+ void kasan_atomic_set_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
atomic_set_##name(ptr, val); \
}
-#define ASAN_ATOMIC_FUNC_SET(name, type) \
- _ASAN_ATOMIC_FUNC_SET(name, type) \
- _ASAN_ATOMIC_FUNC_SET(acq_##name, type) \
- _ASAN_ATOMIC_FUNC_SET(rel_##name, type)
+#define ASAN_ATOMIC_FUNC_SET_ACQ(name) ASAN_ATOMIC_FUNC_SET(acq_##name)
+#define ASAN_ATOMIC_FUNC_SET_REL(name) ASAN_ATOMIC_FUNC_SET(rel_##name)
-#define _ASAN_ATOMIC_FUNC_CLEAR(name, type) \
- void kasan_atomic_clear_##name(volatile type *ptr, type val) \
+#define ASAN_ATOMIC_FUNC_CLEAR(name) \
+ void kasan_atomic_clear_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
atomic_clear_##name(ptr, val); \
}
-#define ASAN_ATOMIC_FUNC_CLEAR(name, type) \
- _ASAN_ATOMIC_FUNC_CLEAR(name, type) \
- _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
- _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
+#define ASAN_ATOMIC_FUNC_CLEAR_ACQ(name) ASAN_ATOMIC_FUNC_CLEAR(acq_##name)
+#define ASAN_ATOMIC_FUNC_CLEAR_REL(name) ASAN_ATOMIC_FUNC_CLEAR(rel_##name)
-#define ASAN_ATOMIC_FUNC_FETCHADD(name, type) \
- type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \
+#define ASAN_ATOMIC_FUNC_FETCHADD(name) \
+ KASAN_ATOMIC_TYPE_##name kasan_atomic_fetchadd_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
return (atomic_fetchadd_##name(ptr, val)); \
}
-#define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
- type kasan_atomic_readandclear_##name(volatile type *ptr) \
+#define ASAN_ATOMIC_FUNC_READANDCLEAR(name) \
+ KASAN_ATOMIC_TYPE_##name kasan_atomic_readandclear_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
return (atomic_readandclear_##name(ptr)); \
}
-#define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
- int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
+#define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name) \
+ int kasan_atomic_testandclear_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, u_int v) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
return (atomic_testandclear_##name(ptr, v)); \
}
-#define ASAN_ATOMIC_FUNC_TESTANDSET(name, type) \
- int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \
+#define ASAN_ATOMIC_FUNC_TESTANDSET(name) \
+ int kasan_atomic_testandset_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, u_int v) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
return (atomic_testandset_##name(ptr, v)); \
}
-#define ASAN_ATOMIC_FUNC_SWAP(name, type) \
- type kasan_atomic_swap_##name(volatile type *ptr, type val) \
+#define ASAN_ATOMIC_FUNC_TESTANDSET_ACQ(name) ASAN_ATOMIC_FUNC_TESTANDSET(acq_##name)
+
+#define ASAN_ATOMIC_FUNC_SWAP(name) \
+ KASAN_ATOMIC_TYPE_##name kasan_atomic_swap_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
return (atomic_swap_##name(ptr, val)); \
}
-#define _ASAN_ATOMIC_FUNC_CMPSET(name, type) \
- int kasan_atomic_cmpset_##name(volatile type *ptr, type oval, \
- type nval) \
+#define ASAN_ATOMIC_FUNC_CMPSET(name) \
+ int kasan_atomic_cmpset_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name oval, \
+ KASAN_ATOMIC_TYPE_##name nval) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
return (atomic_cmpset_##name(ptr, oval, nval)); \
}
-#define ASAN_ATOMIC_FUNC_CMPSET(name, type) \
- _ASAN_ATOMIC_FUNC_CMPSET(name, type) \
- _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
- _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
+#define ASAN_ATOMIC_FUNC_CMPSET_ACQ(name) ASAN_ATOMIC_FUNC_CMPSET(acq_##name)
+#define ASAN_ATOMIC_FUNC_CMPSET_REL(name) ASAN_ATOMIC_FUNC_CMPSET(rel_##name)
-#define _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
- int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
- type nval) \
+#define ASAN_ATOMIC_FUNC_FCMPSET(name) \
+ int kasan_atomic_fcmpset_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name *oval, \
+ KASAN_ATOMIC_TYPE_##name nval) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
return (atomic_fcmpset_##name(ptr, oval, nval)); \
}
-#define ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
- _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \
- _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
- _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
+#define ASAN_ATOMIC_FUNC_FCMPSET_ACQ(name) ASAN_ATOMIC_FUNC_FCMPSET(acq_##name)
+#define ASAN_ATOMIC_FUNC_FCMPSET_REL(name) ASAN_ATOMIC_FUNC_FCMPSET(rel_##name)
#define ASAN_ATOMIC_FUNC_THREAD_FENCE(name) \
void kasan_atomic_thread_fence_##name(void) \
@@ -758,129 +767,68 @@
atomic_thread_fence_##name(); \
}
-#define _ASAN_ATOMIC_FUNC_LOAD(name, type) \
- type kasan_atomic_load_##name(const volatile type *ptr) \
+#define ASAN_ATOMIC_FUNC_LOAD(name) \
+ KASAN_ATOMIC_TYPE_##name kasan_atomic_load_##name(const volatile KASAN_ATOMIC_TYPE_##name *ptr) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
return (atomic_load_##name(ptr)); \
}
-#define ASAN_ATOMIC_FUNC_LOAD(name, type) \
- _ASAN_ATOMIC_FUNC_LOAD(name, type) \
- _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type)
+#define ASAN_ATOMIC_FUNC_LOAD_ACQ(name) ASAN_ATOMIC_FUNC_LOAD(acq_##name)
-#define _ASAN_ATOMIC_FUNC_STORE(name, type) \
- void kasan_atomic_store_##name(volatile type *ptr, type val) \
+#define ASAN_ATOMIC_FUNC_STORE(name) \
+ void kasan_atomic_store_##name(volatile KASAN_ATOMIC_TYPE_##name *ptr, KASAN_ATOMIC_TYPE_##name val) \
{ \
- kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \
+ kasan_shadow_check((uintptr_t)ptr, sizeof(KASAN_ATOMIC_TYPE_##name), true, \
__RET_ADDR); \
atomic_store_##name(ptr, val); \
}
-#define ASAN_ATOMIC_FUNC_STORE(name, type) \
- _ASAN_ATOMIC_FUNC_STORE(name, type) \
- _ASAN_ATOMIC_FUNC_STORE(rel_##name, type)
-
-ASAN_ATOMIC_FUNC_ADD(8, uint8_t);
-ASAN_ATOMIC_FUNC_ADD(16, uint16_t);
-ASAN_ATOMIC_FUNC_ADD(32, uint32_t);
-ASAN_ATOMIC_FUNC_ADD(64, uint64_t);
-ASAN_ATOMIC_FUNC_ADD(int, u_int);
-ASAN_ATOMIC_FUNC_ADD(long, u_long);
-ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
-
-ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
-ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
-ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
-ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
-ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
-ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
-ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
-
-ASAN_ATOMIC_FUNC_SET(8, uint8_t);
-ASAN_ATOMIC_FUNC_SET(16, uint16_t);
-ASAN_ATOMIC_FUNC_SET(32, uint32_t);
-ASAN_ATOMIC_FUNC_SET(64, uint64_t);
-ASAN_ATOMIC_FUNC_SET(int, u_int);
-ASAN_ATOMIC_FUNC_SET(long, u_long);
-ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
-
-ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
-ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
-ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
-ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
-ASAN_ATOMIC_FUNC_CLEAR(int, u_int);
-ASAN_ATOMIC_FUNC_CLEAR(long, u_long);
-ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
-
-ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
-ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
-ASAN_ATOMIC_FUNC_FETCHADD(int, u_int);
-ASAN_ATOMIC_FUNC_FETCHADD(long, u_long);
-
-ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
-ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
-ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
-ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
-ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
-
-ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
-ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
-ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
-ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
-ASAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t);
-
-ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
-ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
-ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
-ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
-ASAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long);
-ASAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t);
-
-ASAN_ATOMIC_FUNC_SWAP(32, uint32_t);
-ASAN_ATOMIC_FUNC_SWAP(64, uint64_t);
-ASAN_ATOMIC_FUNC_SWAP(int, u_int);
-ASAN_ATOMIC_FUNC_SWAP(long, u_long);
-ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
-
-ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
-ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
-ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
-ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
-ASAN_ATOMIC_FUNC_CMPSET(int, u_int);
-ASAN_ATOMIC_FUNC_CMPSET(long, u_long);
-ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
-
-ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
-ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
-ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
-ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
-ASAN_ATOMIC_FUNC_FCMPSET(int, u_int);
-ASAN_ATOMIC_FUNC_FCMPSET(long, u_long);
-ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
-
-_ASAN_ATOMIC_FUNC_LOAD(bool, bool);
-ASAN_ATOMIC_FUNC_LOAD(8, uint8_t);
-ASAN_ATOMIC_FUNC_LOAD(16, uint16_t);
-ASAN_ATOMIC_FUNC_LOAD(32, uint32_t);
-ASAN_ATOMIC_FUNC_LOAD(64, uint64_t);
-ASAN_ATOMIC_FUNC_LOAD(char, u_char);
-ASAN_ATOMIC_FUNC_LOAD(short, u_short);
-ASAN_ATOMIC_FUNC_LOAD(int, u_int);
-ASAN_ATOMIC_FUNC_LOAD(long, u_long);
-ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
-
-_ASAN_ATOMIC_FUNC_STORE(bool, bool);
-ASAN_ATOMIC_FUNC_STORE(8, uint8_t);
-ASAN_ATOMIC_FUNC_STORE(16, uint16_t);
-ASAN_ATOMIC_FUNC_STORE(32, uint32_t);
-ASAN_ATOMIC_FUNC_STORE(64, uint64_t);
-ASAN_ATOMIC_FUNC_STORE(char, u_char);
-ASAN_ATOMIC_FUNC_STORE(short, u_short);
-ASAN_ATOMIC_FUNC_STORE(int, u_int);
-ASAN_ATOMIC_FUNC_STORE(long, u_long);
-ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
+#define ASAN_ATOMIC_FUNC_STORE_REL(name) ASAN_ATOMIC_FUNC_STORE(rel_##name)
+
+ARCH_SUPPORT_ATOMIC_ADD_WIDTH(ASAN_ATOMIC_FUNC_ADD)
+ARCH_SUPPORT_ATOMIC_ADD_ACQ_WIDTH(ASAN_ATOMIC_FUNC_ADD_ACQ)
+ARCH_SUPPORT_ATOMIC_ADD_REL_WIDTH(ASAN_ATOMIC_FUNC_ADD_REL)
+
+ARCH_SUPPORT_ATOMIC_SUBTRACT_WIDTH(ASAN_ATOMIC_FUNC_SUBTRACT)
+ARCH_SUPPORT_ATOMIC_SUBTRACT_ACQ_WIDTH(ASAN_ATOMIC_FUNC_SUBTRACT_ACQ)
+ARCH_SUPPORT_ATOMIC_SUBTRACT_REL_WIDTH(ASAN_ATOMIC_FUNC_SUBTRACT_REL)
+
+ARCH_SUPPORT_ATOMIC_SET_WIDTH(ASAN_ATOMIC_FUNC_SET)
+ARCH_SUPPORT_ATOMIC_SET_ACQ_WIDTH(ASAN_ATOMIC_FUNC_SET_ACQ)
+ARCH_SUPPORT_ATOMIC_SET_REL_WIDTH(ASAN_ATOMIC_FUNC_SET_REL)
+
+ARCH_SUPPORT_ATOMIC_CLEAR_WIDTH(ASAN_ATOMIC_FUNC_CLEAR)
+ARCH_SUPPORT_ATOMIC_CLEAR_ACQ_WIDTH(ASAN_ATOMIC_FUNC_CLEAR_ACQ)
+ARCH_SUPPORT_ATOMIC_CLEAR_REL_WIDTH(ASAN_ATOMIC_FUNC_CLEAR_REL)
+
+ARCH_SUPPORT_ATOMIC_FETCHADD_WIDTH(ASAN_ATOMIC_FUNC_FETCHADD)
+
+ARCH_SUPPORT_ATOMIC_READANDCLEAR_WIDTH(ASAN_ATOMIC_FUNC_READANDCLEAR)
+
+ARCH_SUPPORT_ATOMIC_TESTANDCLEAR_WIDTH(ASAN_ATOMIC_FUNC_TESTANDCLEAR)
+
+ARCH_SUPPORT_ATOMIC_TESTANDSET_WIDTH(ASAN_ATOMIC_FUNC_TESTANDSET)
+ARCH_SUPPORT_ATOMIC_TESTANDSET_ACQ_WIDTH(ASAN_ATOMIC_FUNC_TESTANDSET_ACQ)
+
+ARCH_SUPPORT_ATOMIC_SWAP_WIDTH(ASAN_ATOMIC_FUNC_SWAP)
+
+ARCH_SUPPORT_ATOMIC_CMPSET_WIDTH(ASAN_ATOMIC_FUNC_CMPSET)
+ARCH_SUPPORT_ATOMIC_CMPSET_ACQ_WIDTH(ASAN_ATOMIC_FUNC_CMPSET_ACQ)
+ARCH_SUPPORT_ATOMIC_CMPSET_REL_WIDTH(ASAN_ATOMIC_FUNC_CMPSET_REL)
+
+ARCH_SUPPORT_ATOMIC_FCMPSET_WIDTH(ASAN_ATOMIC_FUNC_FCMPSET)
+ARCH_SUPPORT_ATOMIC_FCMPSET_ACQ_WIDTH(ASAN_ATOMIC_FUNC_FCMPSET_ACQ)
+ARCH_SUPPORT_ATOMIC_FCMPSET_REL_WIDTH(ASAN_ATOMIC_FUNC_FCMPSET_REL)
+
+ASAN_ATOMIC_FUNC_LOAD(bool);
+ARCH_SUPPORT_ATOMIC_LOAD_WIDTH(ASAN_ATOMIC_FUNC_LOAD)
+ARCH_SUPPORT_ATOMIC_LOAD_ACQ_WIDTH(ASAN_ATOMIC_FUNC_LOAD_ACQ)
+
+ASAN_ATOMIC_FUNC_STORE(bool);
+ARCH_SUPPORT_ATOMIC_STORE_WIDTH(ASAN_ATOMIC_FUNC_STORE)
+ARCH_SUPPORT_ATOMIC_STORE_REL_WIDTH(ASAN_ATOMIC_FUNC_STORE_REL)
ASAN_ATOMIC_FUNC_THREAD_FENCE(acq);
ASAN_ATOMIC_FUNC_THREAD_FENCE(rel);
diff --git a/sys/riscv/include/asan.h b/sys/riscv/include/asan.h
new file mode 100644
--- /dev/null
+++ b/sys/riscv/include/asan.h
@@ -0,0 +1,67 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 The FreeBSD Foundation
+ *
+ * This software was developed by Mark Johnston under sponsorship from the
+ * FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_ASAN_H_
+#define _MACHINE_ASAN_H_
+
+#ifdef KASAN
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_page.h>
+#include <machine/vmparam.h>
+
+static inline vm_offset_t
+kasan_md_addr_to_shad(vm_offset_t addr)
+{
+ return (((addr - VM_MIN_KERNEL_ADDRESS) >> KASAN_SHADOW_SCALE_SHIFT) +
+ KASAN_MIN_ADDRESS);
+}
+
+static inline bool
+kasan_md_unsupported(vm_offset_t addr)
+{
+ return (addr < VM_MIN_KERNEL_ADDRESS || addr >= virtual_end);
+}
+
+static inline void
+kasan_md_init(void)
+{
+}
+
+static inline void
+kasan_md_init_early(vm_offset_t bootstack, size_t size)
+{
+ kasan_shadow_map(bootstack, size);
+}
+
+#endif /* KASAN */
+
+#endif /* !_MACHINE_ASAN_H_ */
diff --git a/sys/riscv/include/atomic.h b/sys/riscv/include/atomic.h
--- a/sys/riscv/include/atomic.h
+++ b/sys/riscv/include/atomic.h
@@ -35,13 +35,141 @@
#ifndef _MACHINE_ATOMIC_H_
#define _MACHINE_ATOMIC_H_
-#include <sys/atomic_common.h>
-
#define fence() __asm __volatile("fence" ::: "memory");
#define mb() fence()
#define rmb() fence()
#define wmb() fence()
+#define ARCH_SUPPORT_ATOMIC_ADD_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+#define ARCH_SUPPORT_ATOMIC_ADD_ACQ_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+#define ARCH_SUPPORT_ATOMIC_ADD_REL_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+
+#define ARCH_SUPPORT_ATOMIC_SUBTRACT_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+#define ARCH_SUPPORT_ATOMIC_SUBTRACT_ACQ_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+#define ARCH_SUPPORT_ATOMIC_SUBTRACT_REL_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+
+#define ARCH_SUPPORT_ATOMIC_SET_WIDTH(X) \
+ X(16) X(short) X(32) X(int) X(64) X(ptr) X(long)
+#define ARCH_SUPPORT_ATOMIC_SET_ACQ_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+#define ARCH_SUPPORT_ATOMIC_SET_REL_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+
+#define ARCH_SUPPORT_ATOMIC_CLEAR_WIDTH(X) \
+ X(16) X(short) X(32) X(int) X(64) X(ptr) X(long)
+#define ARCH_SUPPORT_ATOMIC_CLEAR_ACQ_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+#define ARCH_SUPPORT_ATOMIC_CLEAR_REL_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+
+#define ARCH_SUPPORT_ATOMIC_FETCHADD_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+
+#define ARCH_SUPPORT_ATOMIC_READANDCLEAR_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+
+#define ARCH_SUPPORT_ATOMIC_SWAP_WIDTH(X) \
+ X(32) X(int) X(64) X(ptr) X(long)
+
+#define ARCH_SUPPORT_ATOMIC_TESTANDSET_WIDTH(X) \
+ X(32) X(64) X(ptr) X(long)
+
+#define ARCH_SUPPORT_ATOMIC_TESTANDCLEAR_WIDTH(X) \
+ X(32) X(64) X(ptr) X(long)
+
+#define ARCH_SUPPORT_ATOMIC_LOAD_ACQ_WIDTH(X) \
+ X(8) X(16) X(short) X(32) X(int) X(64) X(ptr) X(long)
+
+#define ARCH_SUPPORT_ATOMIC_STORE_REL_WIDTH(X) \
+ X(16) X(short) X(32) X(int) X(64) X(ptr) X(long)
+
+
+#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME)
+#include <sys/atomic_san.h>
+
+#undef atomic_add_8
+#undef atomic_add_acq_8
+#undef atomic_add_rel_8
+#undef atomic_add_char
+#undef atomic_add_acq_char
+#undef atomic_add_rel_char
+
+#undef atomic_subtract_8
+#undef atomic_subtract_acq_8
+#undef atomic_subtract_rel_8
+#undef atomic_subtract_char
+#undef atomic_subtract_acq_char
+#undef atomic_subtract_rel_char
+
+#undef atomic_set_8
+#undef atomic_set_acq_8
+#undef atomic_set_rel_8
+#undef atomic_set_char
+#undef atomic_set_acq_char
+#undef atomic_set_rel_char
+
+#undef atomic_clear_8
+#undef atomic_clear_acq_8
+#undef atomic_clear_rel_8
+#undef atomic_clear_char
+#undef atomic_clear_acq_char
+#undef atomic_clear_rel_char
+
+#undef atomic_fetchadd_8
+#undef atomic_fetchadd_char
+
+#undef atomic_readandclear_8
+#undef atomic_readandclear_char
+
+#undef atomic_swap_8
+#undef atomic_swap_char
+
+#undef atomic_testandset_8
+#undef atomic_testandset_char
+#undef atomic_testandclear_8
+#undef atomic_testandclear_char
+
+#undef atomic_add_16
+#undef atomic_add_acq_16
+#undef atomic_add_rel_16
+#undef atomic_add_short
+#undef atomic_add_acq_short
+#undef atomic_add_rel_short
+
+#undef atomic_subtract_16
+#undef atomic_subtract_acq_16
+#undef atomic_subtract_rel_16
+#undef atomic_subtract_short
+#undef atomic_subtract_acq_short
+#undef atomic_subtract_rel_short
+
+#undef atomic_fetchadd_16
+#undef atomic_fetchadd_short
+
+#undef atomic_readandclear_16
+#undef atomic_readandclear_short
+
+#undef atomic_swap_16
+#undef atomic_swap_short
+
+#undef atomic_testandset_16
+#undef atomic_testandset_short
+#undef atomic_testandclear_16
+#undef atomic_testandclear_short
+
+#undef atomic_testandset_int
+#undef atomic_testandclear_int
+
+#else
+
+#include <sys/atomic_common.h>
+
static __inline int atomic_cmpset_8(__volatile uint8_t *, uint8_t, uint8_t);
static __inline int atomic_fcmpset_8(__volatile uint8_t *, uint8_t *, uint8_t);
static __inline int atomic_cmpset_16(__volatile uint16_t *, uint16_t, uint16_t);
@@ -659,4 +787,6 @@
#define atomic_set_short atomic_set_16
#define atomic_clear_short atomic_clear_16
+#endif /* SAN_NEEDS_INTERCEPTORS && !SAN_RUNTIME */
+
#endif /* _MACHINE_ATOMIC_H_ */
diff --git a/sys/riscv/include/bus.h b/sys/riscv/include/bus.h
--- a/sys/riscv/include/bus.h
+++ b/sys/riscv/include/bus.h
@@ -253,6 +253,10 @@
bus_size_t, const u_int64_t *, bus_size_t);
};
+#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME)
+#include <sys/bus_san.h>
+#else
+
/*
* Utility macros; INTERNAL USE ONLY.
*/
@@ -495,6 +499,8 @@
BUS_POKE_FUNC(4, uint32_t)
BUS_POKE_FUNC(8, uint64_t)
+#endif /* SAN_NEEDS_INTERCEPTORS && !SAN_RUNTIME */
+
#include <machine/bus_dma.h>
#endif /* _MACHINE_BUS_H_ */
diff --git a/sys/riscv/include/param.h b/sys/riscv/include/param.h
--- a/sys/riscv/include/param.h
+++ b/sys/riscv/include/param.h
@@ -81,8 +81,12 @@
#define MAXPAGESIZES 3 /* maximum number of supported page sizes */
#ifndef KSTACK_PAGES
+#if defined(KASAN)
+#define KSTACK_PAGES 6
+#else
#define KSTACK_PAGES 4 /* pages of kernel stack (with pcb) */
#endif
+#endif
#define KSTACK_GUARD_PAGES 1 /* pages of kstack guard; 0 disables */
#define PCPU_PAGES 1
diff --git a/sys/riscv/include/pmap.h b/sys/riscv/include/pmap.h
--- a/sys/riscv/include/pmap.h
+++ b/sys/riscv/include/pmap.h
@@ -164,6 +164,10 @@
return (0);
}
+#if defined(KASAN)
+void pmap_san_enter(vm_offset_t);
+void pmap_bootstrap_san(void);
+#endif
#endif /* _KERNEL */
diff --git a/sys/riscv/include/vmparam.h b/sys/riscv/include/vmparam.h
--- a/sys/riscv/include/vmparam.h
+++ b/sys/riscv/include/vmparam.h
@@ -129,7 +129,8 @@
* 0x0000000000000000 - 0x0000003fffffffff 256GB user map
* 0x0000004000000000 - 0xffffffbfffffffff unmappable
* 0xffffffc000000000 - 0xffffffc7ffffffff 32GB kernel map
- * 0xffffffc800000000 - 0xffffffcfffffffff 32GB unused
+ * 0xffffffc800000000 - 0xffffffc8ffffffff 4GB KASAN shadow map
+ * 0xffffffc900000000 - 0xffffffcfffffffff 28GB unused
* 0xffffffd000000000 - 0xffffffefffffffff 128GB direct map
* 0xfffffff000000000 - 0xffffffffffffffff 64GB unused
*
@@ -138,7 +139,8 @@
* 0x0000800000000000 - 0xffff7fffffffffff unmappable
* 0xffff800000000000 - 0xffffffc7ffffffff 127.75TB hole
* 0xffffffc000000000 - 0xffffffc7ffffffff 32GB kernel map
- * 0xffffffc800000000 - 0xffffffcfffffffff 32GB unused
+ * 0xffffffc800000000 - 0xffffffc8ffffffff 4GB KASAN shadow map
+ * 0xffffffc900000000 - 0xffffffcfffffffff 28GB unused
* 0xffffffd000000000 - 0xffffffefffffffff 128GB direct map
* 0xfffffff000000000 - 0xffffffffffffffff 64GB unused
*
@@ -161,6 +163,9 @@
#define VM_MIN_KERNEL_ADDRESS (0xffffffc000000000UL)
#define VM_MAX_KERNEL_ADDRESS (0xffffffc800000000UL)
+#define KASAN_MIN_ADDRESS (0xffffffc800000000UL)
+#define KASAN_MAX_ADDRESS (0xffffffc900000000UL)
+
#define DMAP_MIN_ADDRESS (0xffffffd000000000UL)
#define DMAP_MAX_ADDRESS (0xfffffff000000000UL)
diff --git a/sys/riscv/riscv/bus_machdep.c b/sys/riscv/riscv/bus_machdep.c
--- a/sys/riscv/riscv/bus_machdep.c
+++ b/sys/riscv/riscv/bus_machdep.c
@@ -33,6 +33,10 @@
* SUCH DAMAGE.
*/
+#if defined(KASAN)
+#define SAN_RUNTIME
+#endif
+
#include "opt_platform.h"
#include <sys/param.h>
diff --git a/sys/riscv/riscv/db_trace.c b/sys/riscv/riscv/db_trace.c
--- a/sys/riscv/riscv/db_trace.c
+++ b/sys/riscv/riscv/db_trace.c
@@ -55,7 +55,7 @@
}
-static void
+static void __nosanitizeaddress
db_stack_trace_cmd(struct thread *td, struct unwind_state *frame)
{
const char *name;
@@ -118,7 +118,7 @@
}
}
-int
+int __nosanitizeaddress
db_trace_thread(struct thread *thr, int count)
{
struct unwind_state frame;
@@ -133,7 +133,7 @@
return (0);
}
-void
+void __nosanitizeaddress
db_trace_self(void)
{
struct unwind_state frame;
diff --git a/sys/riscv/riscv/locore.S b/sys/riscv/riscv/locore.S
--- a/sys/riscv/riscv/locore.S
+++ b/sys/riscv/riscv/locore.S
@@ -260,6 +260,17 @@
sd s4, RISCV_BOOTPARAMS_DTBP_PHYS(sp)
sd s3, RISCV_BOOTPARAMS_MODULEP(sp)
+#ifdef KASAN
+ /* KASAN needs phys addr of kernel to set up the page tables */
+ la t0, kernstart_pa
+ sd s9, 0(t0)
+
+ /* Bootstrap a shadow map for the boot stack. */
+ la a0, initstack
+ li a1, (PAGE_SIZE * KSTACK_PAGES)
+ call _C_LABEL(kasan_init_early)
+#endif
+
mv a0, sp
call _C_LABEL(initriscv) /* Off we go */
call _C_LABEL(mi_startup)
@@ -310,6 +321,11 @@
/*
* Static space for the bootstrap page tables. Unused after pmap_bootstrap().
*/
+#ifdef KASAN
+ /* KASAN needs the bootstrap page table to link the early shadow map. */
+ .globl bootstrap_pt_l1
+#endif
+
.balign PAGE_SIZE
bootstrap_pt_l1:
.space PAGE_SIZE
diff --git a/sys/riscv/riscv/machdep.c b/sys/riscv/riscv/machdep.c
--- a/sys/riscv/riscv/machdep.c
+++ b/sys/riscv/riscv/machdep.c
@@ -39,6 +39,7 @@
#include <sys/param.h>
#include <sys/systm.h>
+#include <sys/asan.h>
#include <sys/boot.h>
#include <sys/buf.h>
#include <sys/bus.h>
@@ -634,6 +635,15 @@
/* Bootstrap enough of pmap to enter the kernel proper */
kernlen = (lastaddr - KERNBASE);
pmap_bootstrap(rvbp->kern_phys, kernlen);
+
+ /*
+ * There's not always enough room for the initial shadow map after the
+ * kernel, so as is done on arm64, we'll end up searching for segments
+ * that we can safely use.
+ */
+#if defined(KASAN) || defined(KMSAN)
+ pmap_bootstrap_san();
+#endif
physmem_init_kernel_globals();
@@ -658,6 +668,8 @@
kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger");
#endif
+ kasan_init();
+
env = kern_getenv("kernelname");
if (env != NULL)
strlcpy(kernelname, env, sizeof(kernelname));
diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c
--- a/sys/riscv/riscv/pmap.c
+++ b/sys/riscv/riscv/pmap.c
@@ -116,6 +116,7 @@
#include <sys/param.h>
#include <sys/systm.h>
+#include <sys/asan.h>
#include <sys/bitstring.h>
#include <sys/bus.h>
#include <sys/cpuset.h>
@@ -152,6 +153,7 @@
#include <vm/vm_dumpset.h>
#include <vm/uma.h>
+#include <machine/asan.h>
#include <machine/machdep.h>
#include <machine/md_var.h>
#include <machine/pcb.h>
@@ -304,6 +306,10 @@
extern cpuset_t all_harts;
+#if defined(KASAN)
+extern pt_entry_t bootstrap_pt_l1[];
+#endif
+
/*
* Internal flags for pmap_enter()'s helper functions.
*/
@@ -881,6 +887,20 @@
pmap_store(&l1[slot], L1_PDE(pa, PTE_V));
}
+#ifdef KASAN
+ pd_entry_t kasan_l2_pt;
+ /* Connect the early KASAN shadow to L1 */
+ slot = pmap_l1_index(KASAN_MIN_ADDRESS);
+ int nkasan_l2_pt = howmany(
+ howmany(KASAN_MAX_ADDRESS - KASAN_MIN_ADDRESS, L2_SIZE), Ln_ENTRIES);
+ for (i = 0; i < nkasan_l2_pt; i++, slot++) {
+ if (((kasan_l2_pt = pmap_load(&bootstrap_pt_l1[slot])) & PTE_V) != 0) {
+ pa = PTE_TO_PHYS(kasan_l2_pt);
+ pmap_store(&l1[slot], L1_PDE(pa, PTE_V));
+ }
+ }
+#endif
+
/* Connect the L1 table to L0, if in use. */
if (pmap_mode == PMAP_MODE_SV48) {
slot = pmap_l0_index(KERNBASE);
@@ -1009,6 +1029,64 @@
EXFLAG_NOALLOC);
}
+#if defined(KASAN)
+static void
+pmap_bootstrap_allocate_san_l2(vm_paddr_t start_pa, vm_paddr_t end_pa,
+ vm_offset_t *vap, vm_offset_t eva)
+{
+ vm_paddr_t pa;
+ vm_offset_t va;
+ pd_entry_t *l2;
+
+ va = *vap;
+ pa = rounddown2(end_pa - L2_SIZE, L2_SIZE);
+ for (; pa >= start_pa && va < eva; va += L2_SIZE, pa -= L2_SIZE) {
+ l2 = pmap_l2(kernel_pmap, va);
+ MPASS(l2 != NULL);
+
+ if ((pmap_load(l2) & PTE_V) != 0) {
+ pa += L2_SIZE;
+ continue;
+ }
+
+ bzero_early(PHYS_TO_DMAP(pa), L2_SIZE);
+ physmem_exclude_region(pa, L2_SIZE, EXFLAG_NOALLOC);
+ pmap_store(l2, L2_PTE(pa, PTE_KERN | PTE_V));
+ }
+ *vap = va;
+}
+
+void
+pmap_bootstrap_san(void)
+{
+ vm_offset_t eva;
+ vm_offset_t va = KASAN_MIN_ADDRESS;
+ vm_paddr_t physmap[PHYS_AVAIL_ENTRIES];
+ int physmap_idx, i;
+
+ physmap_idx = physmem_avail(physmap, nitems(physmap));
+
+ eva = kasan_md_addr_to_shad(virtual_avail);
+
+ for (i = physmap_idx - 2; i >= 0; i -= 2) {
+ vm_paddr_t plow, phigh;
+
+ /* L2 mappings must be backed by memory that is L2-aligned */
+ plow = roundup2(physmap[i], L2_SIZE);
+ phigh = physmap[i + 1];
+ if (plow >= phigh)
+ continue;
+ if (phigh - plow >= L2_SIZE) {
+ pmap_bootstrap_allocate_san_l2(plow, phigh, &va, eva);
+ if (va >= eva)
+ break;
+ }
+ }
+ if (i < 0)
+ panic("Could not find phys region for shadow map");
+}
+#endif
+
/*
* Initialize a vm_page's machine-dependent fields.
*/
@@ -1649,6 +1727,9 @@
for (i = pmap_l1_index(VM_MIN_KERNEL_ADDRESS);
i < pmap_l1_index(VM_MAX_KERNEL_ADDRESS); i++)
pmap->pm_top[i] = kernel_pmap->pm_top[i];
+ for (i = pmap_l1_index(KASAN_MIN_ADDRESS);
+ i < pmap_l1_index(KASAN_MAX_ADDRESS); i++)
+ pmap->pm_top[i] = kernel_pmap->pm_top[i];
for (i = pmap_l1_index(DMAP_MIN_ADDRESS);
i < pmap_l1_index(DMAP_MAX_ADDRESS); i++)
pmap->pm_top[i] = kernel_pmap->pm_top[i];
@@ -1971,6 +2052,8 @@
addr = roundup2(addr, L2_SIZE);
if (addr - 1 >= vm_map_max(kernel_map))
addr = vm_map_max(kernel_map);
+ if (kernel_vm_end < addr)
+ kasan_shadow_map(kernel_vm_end, addr - kernel_vm_end);
while (kernel_vm_end < addr) {
l1 = pmap_l1(kernel_pmap, kernel_vm_end);
if (pmap_load(l1) == 0) {
@@ -5402,6 +5485,155 @@
return (true);
}
+#if defined(KASAN)
+
+vm_paddr_t kernstart_pa;
+
+#define SAN_EARLY_SHADOW_SIZE (1 * L2_SIZE)
+#define SAN_EARLY_L2_PT_SIZE (1 * PAGE_SIZE)
+#define SAN_EARLY_VTOPHYS(va) \
+ ((vm_paddr_t)(va) - KERNBASE + kernstart_pa)
+#define SAN_EARLY_PHYSTOV(pa) \
+ ((vm_offset_t)(pa) - kernstart_pa + KERNBASE)
+
+static pt_entry_t * __nosanitizeaddress __nosanitizememory
+pmap_san_alloc_table_early(void)
+{
+ static uint8_t bootstrap_data[SAN_EARLY_L2_PT_SIZE] __aligned(PAGE_SIZE);
+ static size_t offset = 0;
+ vm_offset_t addr;
+
+ if (offset + PAGE_SIZE > sizeof(bootstrap_data)) {
+ panic("%s: out of memory for the bootstrap shadow map",
+ __func__);
+ }
+
+ addr = SAN_EARLY_VTOPHYS(&bootstrap_data[offset]);
+ offset += PAGE_SIZE;
+ return ((pt_entry_t *)addr);
+}
+
+static vm_paddr_t __nosanitizeaddress __nosanitizememory
+pmap_san_alloc_l2_shadow_early(void)
+{
+ static uint8_t bootstrap_data[SAN_EARLY_SHADOW_SIZE] __aligned(L2_SIZE);
+ static size_t offset = 0;
+ vm_offset_t addr;
+
+ if (offset + L2_SIZE > sizeof(bootstrap_data)) {
+ panic("%s: out of memory for the bootstrap shadow map L2 entries",
+ __func__);
+ }
+
+ addr = SAN_EARLY_VTOPHYS(&bootstrap_data[offset]);
+ offset += L2_SIZE;
+ return (addr);
+}
+
+/*
+ * Map a shadow page, before the kernel has bootstrapped its page tables. This
+ * is currently only used to shadow the temporary boot stack set up by locore.
+ */
+static void __nosanitizeaddress __nosanitizememory
+pmap_san_enter_early(vm_offset_t va)
+{
+ pt_entry_t *l2_pt;
+ vm_paddr_t shadow;
+ int l1_slot, l2_slot;
+
+ l1_slot = pmap_l1_index(va);
+ l2_slot = pmap_l2_index(va);
+
+ /* alloc and link this L2 pt to bootstrap_pt_l1 */
+ if ((pmap_load(&bootstrap_pt_l1[l1_slot]) & PTE_V) == 0) {
+ l2_pt = pmap_san_alloc_table_early();
+
+ pmap_store(&bootstrap_pt_l1[l1_slot],
+ L1_PDE((vm_paddr_t)l2_pt, PTE_V));
+ } else {
+ l2_pt = (pt_entry_t *)PTE_TO_PHYS(pmap_load(&bootstrap_pt_l1[l1_slot]));
+ }
+
+ /* alloc and link an L2 supperpage for the early shadow map */
+ if ((pmap_load(&l2_pt[l2_slot]) & PTE_V) == 0) {
+ /*
+ * If a second VA is entered, it will currently panic in
+ * pmap_san_alloc_l2_shadow_early since we only have 1 *
+ * L2_SIZE. However, this design allows us to easily expand its
+ * size in the future.
+ */
+ shadow = pmap_san_alloc_l2_shadow_early();
+
+ pmap_store(&l2_pt[l2_slot],
+ L2_PTE(shadow, PTE_KERN | PTE_V));
+ }
+
+}
+
+static vm_page_t
+pmap_san_enter_alloc_4k(void)
+{
+ vm_page_t m;
+
+ m = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED |
+ VM_ALLOC_ZERO);
+ if (m == NULL)
+ panic("%s: no memory to grow shadow map", __func__);
+ return (m);
+}
+
+static vm_page_t
+pmap_san_enter_alloc_2m(void)
+{
+ return (vm_page_alloc_noobj_contig(VM_ALLOC_WIRED | VM_ALLOC_ZERO,
+ Ln_ENTRIES, 0, ~0ul, L2_SIZE, 0, VM_MEMATTR_DEFAULT));
+}
+
+void __nosanitizeaddress __nosanitizememory
+pmap_san_enter(vm_offset_t va)
+{
+ pd_entry_t *l1, *l2;
+ pt_entry_t *l3;
+ vm_page_t m;
+ if (virtual_avail == 0) {
+ /*
+ * Before we enter the first C function, initriscv, we should
+ * build a shadow map for its bootstack
+ */
+ pmap_san_enter_early(va);
+ return;
+ }
+ mtx_assert(&kernel_map->system_mtx, MA_OWNED);
+ l1 = pmap_l1(kernel_pmap, va);
+ MPASS(l1 != NULL);
+ if ((pmap_load(l1) & PTE_V) == 0) {
+ m = pmap_san_enter_alloc_4k();
+ pmap_store(l1, L1_PDE(VM_PAGE_TO_PHYS(m), PTE_V));
+ }
+
+ l2 = pmap_l1_to_l2(l1, va);
+ if ((pmap_load(l2) & PTE_V) == 0) {
+ m = pmap_san_enter_alloc_2m();
+ if (m != NULL) {
+ pmap_store(l2, L2_PTE(VM_PAGE_TO_PHYS(m), PTE_KERN | PTE_V));
+ } else {
+ m = pmap_san_enter_alloc_4k();
+ pmap_store(l2, L2_PDE(VM_PAGE_TO_PHYS(m), PTE_V));
+ }
+ wmb();
+ }
+ if ((pmap_load(l2) & PTE_V) != 0 &&
+ (pmap_load(l2) & PTE_RWX) != 0)
+ return;
+
+ l3 = pmap_l2_to_l3(l2, va);
+ if ((pmap_load(l3) & PTE_V) != 0)
+ return;
+ m = pmap_san_enter_alloc_4k();
+ pmap_store(l3, L3_PTE(VM_PAGE_TO_PHYS(m), PTE_KERN | PTE_V));
+ wmb();
+}
+#endif
/*
* Track a range of the kernel's virtual address space that is contiguous
@@ -5541,6 +5773,10 @@
sbuf_printf(sb, "\nDirect map:\n");
else if (i == pmap_l1_index(VM_MIN_KERNEL_ADDRESS))
sbuf_printf(sb, "\nKernel map:\n");
+#ifdef KASAN
+ else if (i == pmap_l1_index(KASAN_MIN_ADDRESS))
+ sbuf_printf(sb, "\nKASAN shadow map:\n");
+#endif
l1 = pmap_l1(kernel_pmap, sva);
l1e = pmap_load(l1);
diff --git a/sys/riscv/riscv/trap.c b/sys/riscv/riscv/trap.c
--- a/sys/riscv/riscv/trap.c
+++ b/sys/riscv/riscv/trap.c
@@ -37,6 +37,7 @@
#include <sys/param.h>
#include <sys/systm.h>
+#include <sys/asan.h>
#include <sys/bus.h>
#include <sys/intr.h>
#include <sys/kernel.h>
@@ -308,6 +309,8 @@
{
uint64_t exception;
+ kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
+
/* Ensure we came from supervisor mode, interrupts disabled */
KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) ==
SSTATUS_SPP, ("Came from S mode with interrupts enabled"));
@@ -383,6 +386,8 @@
struct thread *td;
struct pcb *pcb;
+ kasan_mark(frame, sizeof(*frame), sizeof(*frame), 0);
+
td = curthread;
pcb = td->td_pcb;
diff --git a/sys/sys/atomic_san.h b/sys/sys/atomic_san.h
--- a/sys/sys/atomic_san.h
+++ b/sys/sys/atomic_san.h
@@ -406,4 +406,141 @@
#endif /* !SAN_RUNTIME */
+/*
+ * The following x-macros control the generation of ASAN interceptor functions
+ * in subr_asan.c.
+ *
+ * By default, the MI code(atomic_san.h) assumes that the underlying
+ * architecture supports all atomic operations across all widths. If a specific
+ * MD architecture(machine/atomic.h) lacks support for certain atomic widths
+ * (e.g., 8-bit or 16-bit), it should override these ARCH_SUPPORT_* macros in
+ * its machine/atomic.h and #undef the corresponding atomic_* macros to prevent
+ * the generation of unsupported interceptors.
+ */
+
+#ifndef ARCH_SUPPORT_ATOMIC_ADD_WIDTH
+#define ARCH_SUPPORT_ATOMIC_ADD_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_ADD_ACQ_WIDTH
+#define ARCH_SUPPORT_ATOMIC_ADD_ACQ_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_ADD_REL_WIDTH
+#define ARCH_SUPPORT_ATOMIC_ADD_REL_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+
+#ifndef ARCH_SUPPORT_ATOMIC_SUBTRACT_WIDTH
+#define ARCH_SUPPORT_ATOMIC_SUBTRACT_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_SUBTRACT_ACQ_WIDTH
+#define ARCH_SUPPORT_ATOMIC_SUBTRACT_ACQ_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_SUBTRACT_REL_WIDTH
+#define ARCH_SUPPORT_ATOMIC_SUBTRACT_REL_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+
+#ifndef ARCH_SUPPORT_ATOMIC_SET_WIDTH
+#define ARCH_SUPPORT_ATOMIC_SET_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_SET_ACQ_WIDTH
+#define ARCH_SUPPORT_ATOMIC_SET_ACQ_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_SET_REL_WIDTH
+#define ARCH_SUPPORT_ATOMIC_SET_REL_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+
+#ifndef ARCH_SUPPORT_ATOMIC_CLEAR_WIDTH
+#define ARCH_SUPPORT_ATOMIC_CLEAR_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_CLEAR_ACQ_WIDTH
+#define ARCH_SUPPORT_ATOMIC_CLEAR_ACQ_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_CLEAR_REL_WIDTH
+#define ARCH_SUPPORT_ATOMIC_CLEAR_REL_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+
+#ifndef ARCH_SUPPORT_ATOMIC_CMPSET_WIDTH
+#define ARCH_SUPPORT_ATOMIC_CMPSET_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_CMPSET_ACQ_WIDTH
+#define ARCH_SUPPORT_ATOMIC_CMPSET_ACQ_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_CMPSET_REL_WIDTH
+#define ARCH_SUPPORT_ATOMIC_CMPSET_REL_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+
+#ifndef ARCH_SUPPORT_ATOMIC_FCMPSET_WIDTH
+#define ARCH_SUPPORT_ATOMIC_FCMPSET_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_FCMPSET_ACQ_WIDTH
+#define ARCH_SUPPORT_ATOMIC_FCMPSET_ACQ_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_FCMPSET_REL_WIDTH
+#define ARCH_SUPPORT_ATOMIC_FCMPSET_REL_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+
+#ifndef ARCH_SUPPORT_ATOMIC_FETCHADD_WIDTH
+#define ARCH_SUPPORT_ATOMIC_FETCHADD_WIDTH(X) \
+ X(32) X(int) X(64) X(long)
+#endif
+
+#ifndef ARCH_SUPPORT_ATOMIC_READANDCLEAR_WIDTH
+#define ARCH_SUPPORT_ATOMIC_READANDCLEAR_WIDTH(X) \
+ X(32) X(int) X(64) X(long) X(ptr)
+#endif
+
+#ifndef ARCH_SUPPORT_ATOMIC_SWAP_WIDTH
+#define ARCH_SUPPORT_ATOMIC_SWAP_WIDTH(X) \
+ X(32) X(int) X(64) X(long) X(ptr)
+#endif
+
+#ifndef ARCH_SUPPORT_ATOMIC_TESTANDSET_WIDTH
+#define ARCH_SUPPORT_ATOMIC_TESTANDSET_WIDTH(X) \
+ X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_TESTANDSET_ACQ_WIDTH
+#define ARCH_SUPPORT_ATOMIC_TESTANDSET_ACQ_WIDTH(X) \
+ X(long)
+#endif
+
+#ifndef ARCH_SUPPORT_ATOMIC_TESTANDCLEAR_WIDTH
+#define ARCH_SUPPORT_ATOMIC_TESTANDCLEAR_WIDTH(X) \
+ X(32) X(int) X(64) X(long) X(ptr)
+#endif
+
+#ifndef ARCH_SUPPORT_ATOMIC_LOAD_WIDTH
+#define ARCH_SUPPORT_ATOMIC_LOAD_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_LOAD_ACQ_WIDTH
+#define ARCH_SUPPORT_ATOMIC_LOAD_ACQ_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+
+#ifndef ARCH_SUPPORT_ATOMIC_STORE_WIDTH
+#define ARCH_SUPPORT_ATOMIC_STORE_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+#ifndef ARCH_SUPPORT_ATOMIC_STORE_REL_WIDTH
+#define ARCH_SUPPORT_ATOMIC_STORE_REL_WIDTH(X) \
+ X(8) X(char) X(16) X(short) X(32) X(int) X(64) X(long) X(ptr)
+#endif
+
#endif /* !_SYS_ATOMIC_SAN_H_ */

File Metadata

Mime Type
text/plain
Expires
Mon, Jul 6, 11:26 PM (11 h, 36 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
34773115
Default Alt Text
D57381.id179088.diff (42 KB)

Event Timeline