Index: contrib/llvm/tools/clang/lib/Driver/ToolChains/FreeBSD.cpp =================================================================== --- contrib/llvm/tools/clang/lib/Driver/ToolChains/FreeBSD.cpp +++ contrib/llvm/tools/clang/lib/Driver/ToolChains/FreeBSD.cpp @@ -421,10 +421,10 @@ Res |= SanitizerKind::Address; Res |= SanitizerKind::PointerCompare; Res |= SanitizerKind::PointerSubtract; + Res |= SanitizerKind::Thread; Res |= SanitizerKind::Vptr; if (IsX86_64 || IsMIPS64) { Res |= SanitizerKind::Leak; - Res |= SanitizerKind::Thread; } if (IsX86 || IsX86_64) { Res |= SanitizerKind::Function; Index: sys/amd64/amd64/copyout.c =================================================================== --- sys/amd64/amd64/copyout.c +++ sys/amd64/amd64/copyout.c @@ -146,6 +146,10 @@ casueword_smap : casueword_nosmap); } +#undef copyinstr +#undef copyin +#undef copyout + int copyinstr_nosmap(const void *udaddr, void *kaddr, size_t len, size_t *lencopied); int copyinstr_smap(const void *udaddr, void *kaddr, size_t len, Index: sys/amd64/amd64/machdep.c =================================================================== --- sys/amd64/amd64/machdep.c +++ sys/amd64/amd64/machdep.c @@ -64,6 +64,7 @@ #include #include #include +#include #include #include #include @@ -1890,6 +1891,8 @@ cpu_probe_amdc1e(); + kcsan_cpu_init(0); + #ifdef FDT x86_init_fdt(); #endif @@ -2713,6 +2716,40 @@ void *memset_std(void *buf, int c, size_t len); void *memset_erms(void *buf, int c, size_t len); +void *memmove_std(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src, + size_t len); +void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src, + size_t len); + +#ifdef KCSAN +/* + * These fail to build as ifuncs when used with KCSAN. + */ +void * +memset(void *buf, int c, size_t len) +{ + + return memset_std(buf, c, len); +} + +void * +memmove(void * _Nonnull dst, const void * _Nonnull src, size_t len) +{ + + return memmove_std(dst, src, len); +} + +void * +memcpy(void * _Nonnull dst, const void * _Nonnull src, size_t len) +{ + + return memcpy_std(dst, src, len); +} +#else DEFINE_IFUNC(, void *, memset, (void *, int, size_t)) { @@ -2720,10 +2757,6 @@ memset_erms : memset_std); } -void *memmove_std(void * _Nonnull dst, const void * _Nonnull src, - size_t len); -void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src, - size_t len); DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull, size_t)) { @@ -2732,16 +2765,13 @@ memmove_erms : memmove_std); } -void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src, - size_t len); -void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src, - size_t len); DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull,size_t)) { return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ? memcpy_erms : memcpy_std); } +#endif void pagezero_std(void *addr); void pagezero_erms(void *addr); Index: sys/amd64/conf/GENERIC =================================================================== --- sys/amd64/conf/GENERIC +++ sys/amd64/conf/GENERIC @@ -106,6 +106,7 @@ #options KCOV # Kernel Coverage Sanitizer # Warning: KUBSAN can result in a kernel too large for loader to load #options KUBSAN # Kernel Undefined Behavior Sanitizer +options KCSAN # Kernel Concurrency Sanitizer # Kernel dump features. options EKCD # Support for encrypted kernel dumps Index: sys/amd64/include/atomic.h =================================================================== --- sys/amd64/include/atomic.h +++ sys/amd64/include/atomic.h @@ -57,6 +57,20 @@ #define wmb() __asm __volatile("sfence;" : : : "memory") #define rmb() __asm __volatile("lfence;" : : : "memory") +#ifdef _KERNEL +/* + * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). + * + * The open-coded number is used instead of the symbolic expression to + * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. + * An assertion in amd64/vm_machdep.c ensures that the value is correct. + */ +#define OFFSETOF_MONITORBUF 0x100 +#endif + +#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#include +#else #include /* @@ -345,15 +359,6 @@ #if defined(_KERNEL) -/* - * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). - * - * The open-coded number is used instead of the symbolic expression to - * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. - * An assertion in amd64/vm_machdep.c ensures that the value is correct. - */ -#define OFFSETOF_MONITORBUF 0x100 - #if defined(SMP) || defined(KLD_MODULE) static __inline void __storeload_barrier(void) @@ -679,4 +684,6 @@ #endif /* !WANT_FUNCTIONS */ +#endif /* KCSAN && !KCSAN_RUNTIME */ + #endif /* !_MACHINE_ATOMIC_H_ */ Index: sys/amd64/include/csan.h =================================================================== --- /dev/null +++ sys/amd64/include/csan.h @@ -0,0 +1,65 @@ +/* $NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $ */ + +/* + * Copyright (c) 2019 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Maxime Villard. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +static inline bool +kcsan_md_is_avail(void) +{ + return true; +} + +static inline void +kcsan_md_disable_intrs(uint64_t *state) +{ + + *state = intr_disable(); +} + +static inline void +kcsan_md_enable_intrs(uint64_t *state) +{ + + intr_restore(*state); +} + +static inline void +kcsan_md_delay(uint64_t us) +{ + DELAY(us); +} + +static void +kcsan_md_unwind(void) +{ +} Index: sys/arm64/arm64/bus_machdep.c =================================================================== --- sys/arm64/arm64/bus_machdep.c +++ sys/arm64/arm64/bus_machdep.c @@ -25,6 +25,8 @@ * */ +#define KCSAN_RUNTIME + #include "opt_platform.h" #include Index: sys/arm64/arm64/copystr.c =================================================================== --- sys/arm64/arm64/copystr.c +++ sys/arm64/arm64/copystr.c @@ -32,7 +32,7 @@ #include int -copystr(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len, +(copystr)(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len, size_t * __restrict lencopied) { const char *src; Index: sys/arm64/arm64/machdep.c =================================================================== --- sys/arm64/arm64/machdep.c +++ sys/arm64/arm64/machdep.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -1174,6 +1175,8 @@ kdb_init(); pan_enable(); + kcsan_cpu_init(0); + env = kern_getenv("kernelname"); if (env != NULL) strlcpy(kernelname, env, sizeof(kernelname)); Index: sys/arm64/arm64/mp_machdep.c =================================================================== --- sys/arm64/arm64/mp_machdep.c +++ sys/arm64/arm64/mp_machdep.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -253,6 +254,8 @@ mtx_unlock_spin(&ap_boot_mtx); + kcsan_cpu_init(cpu); + /* Enter the scheduler */ sched_throw(NULL); Index: sys/arm64/conf/GENERIC =================================================================== --- sys/arm64/conf/GENERIC +++ sys/arm64/conf/GENERIC @@ -25,7 +25,7 @@ makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support options SCHED_ULE # ULE scheduler -options NUMA # Non-Uniform Memory Architecture support +#options NUMA # Non-Uniform Memory Architecture support options PREEMPTION # Enable kernel thread preemption options VIMAGE # Subsystem virtualization, e.g. VNET options INET # InterNETworking @@ -99,6 +99,7 @@ #options KCOV # Kernel Coverage Sanitizer # Warning: KUBSAN can result in a kernel too large for loader to load #options KUBSAN # Kernel Undefined Behavior Sanitizer +options KCSAN # Kernel Concurrency Sanitizer # Kernel dump features. options EKCD # Support for encrypted kernel dumps Index: sys/arm64/include/atomic.h =================================================================== --- sys/arm64/include/atomic.h +++ sys/arm64/include/atomic.h @@ -29,8 +29,6 @@ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ -#include - #define isb() __asm __volatile("isb" : : : "memory") /* @@ -55,6 +53,12 @@ #define wmb() dmb(st) /* Full system memory barrier store */ #define rmb() dmb(ld) /* Full system memory barrier load */ +#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#include +#else + +#include + #define ATOMIC_OP(op, asm_op, bar, a, l) \ static __inline void \ atomic_##op##_##bar##8(volatile uint8_t *p, uint8_t val) \ @@ -601,5 +605,7 @@ dmb(sy); } +#endif /* KCSAN && !KCSAN_RUNTIME */ + #endif /* _MACHINE_ATOMIC_H_ */ Index: sys/arm64/include/bus.h =================================================================== --- sys/arm64/include/bus.h +++ sys/arm64/include/bus.h @@ -89,6 +89,9 @@ #define BUS_SPACE_BARRIER_READ 0x01 #define BUS_SPACE_BARRIER_WRITE 0x02 +#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#include +#else struct bus_space { /* cookie */ @@ -322,7 +325,7 @@ #define bus_space_read_stream_1(t, h, o) __bs_rs_s(1,(t), (h), (o)) #define bus_space_read_stream_2(t, h, o) __bs_rs_s(2,(t), (h), (o)) #define bus_space_read_stream_4(t, h, o) __bs_rs_s(4,(t), (h), (o)) -#define bus_space_read_stream_8(t, h, o) __bs_rs_s(8,8,(t),(h),(o)) +#define bus_space_read_stream_8(t, h, o) __bs_rs_s(8,(t), (h),(o)) /* * Bus read multiple operations. @@ -464,6 +467,8 @@ #define bus_space_copy_region_8(t, h1, o1, h2, o2, c) \ __bs_copy(8, t, h1, o1, h2, o2, c) +#endif + #include #endif /* _MACHINE_BUS_H_ */ Index: sys/arm64/include/csan.h =================================================================== --- /dev/null +++ sys/arm64/include/csan.h @@ -0,0 +1,102 @@ +/* $NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $ */ + +/* + * Copyright (c) 2019 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Maxime Villard. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +static inline bool +kcsan_md_is_avail(void) +{ + return true; +} + +static inline void +kcsan_md_disable_intrs(uint64_t *state) +{ + + *state = intr_disable(); +} + +static inline void +kcsan_md_enable_intrs(uint64_t *state) +{ + + intr_restore(*state); +} + +static inline void +kcsan_md_delay(uint64_t us) +{ + DELAY(us); +} + +static void +kcsan_md_unwind(void) +{ +#ifdef DDB + c_db_sym_t sym; + db_expr_t offset; + const char *symname; +#endif + struct unwind_state frame; + uint64_t sp; + int nsym; + + __asm __volatile("mov %0, sp" : "=&r" (sp)); + + frame.sp = sp; + frame.fp = (uint64_t)__builtin_frame_address(0); + frame.pc = (uint64_t)kcsan_md_unwind; + nsym = 0; + + while (1) { + unwind_frame(&frame); + if (!INKERNEL((vm_offset_t)frame.fp) || + !INKERNEL((vm_offset_t)frame.pc)) + break; + +#ifdef DDB + sym = db_search_symbol((vm_offset_t)frame.pc, DB_STGY_PROC, + &offset); + db_symbol_values(sym, &symname, NULL); + printf("#%d %p in %s+%#lx\n", nsym, (void *)frame.pc, + symname, offset); +#else + printf("#%d %p\n", nsym, (void *)frame.pc); +#endif + nsym++; + + if (nsym >= 15) { + break; + } + } +} Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -3799,6 +3799,8 @@ kern/subr_coverage.c optional coverage \ compile-with "${NORMAL_C:N-fsanitize*}" kern/subr_counter.c standard +kern/subr_csan.c optional kcsan \ + compile-with "${NORMAL_C:N-fsanitize*}" kern/subr_devstat.c standard kern/subr_disk.c standard kern/subr_early.c standard Index: sys/conf/files.arm64 =================================================================== --- sys/conf/files.arm64 +++ sys/conf/files.arm64 @@ -283,8 +283,10 @@ kern/subr_devmap.c standard kern/subr_intr.c optional intrng libkern/bcmp.c standard -libkern/memcmp.c standard -libkern/memset.c standard +libkern/memcmp.c standard \ + compile-with "${NORMAL_C:N-fsanitize*}" +libkern/memset.c standard \ + compile-with "${NORMAL_C:N-fsanitize*}" libkern/arm64/crc32c_armv8.S standard cddl/dev/dtrace/aarch64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/aarch64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" Index: sys/conf/kern.post.mk =================================================================== --- sys/conf/kern.post.mk +++ sys/conf/kern.post.mk @@ -38,6 +38,10 @@ MKMODULESENV+= WITH_EXTRA_TCP_STACKS="${WITH_EXTRA_TCP_STACKS}" .endif +.if defined(KCSAN_ENABLED) +MKMODULESENV+= KCSAN_ENABLED="yes" +.endif + .if defined(SAN_CFLAGS) MKMODULESENV+= SAN_CFLAGS="${SAN_CFLAGS}" .endif Index: sys/conf/kern.pre.mk =================================================================== --- sys/conf/kern.pre.mk +++ sys/conf/kern.pre.mk @@ -117,6 +117,11 @@ .endif DEFINED_PROF= ${PROF} +KCSAN_ENABLED!= grep KCSAN opt_global.h || true ; echo +.if !empty(KCSAN_ENABLED) +SAN_CFLAGS+= -fsanitize=thread +.endif + KUBSAN_ENABLED!= grep KUBSAN opt_global.h || true ; echo .if !empty(KUBSAN_ENABLED) SAN_CFLAGS+= -fsanitize=undefined Index: sys/conf/options =================================================================== --- sys/conf/options +++ sys/conf/options @@ -230,6 +230,7 @@ # Sanitizers COVERAGE opt_global.h KCOV +KCSAN opt_global.h KUBSAN opt_global.h # POSIX kernel options Index: sys/kern/subr_csan.c =================================================================== --- /dev/null +++ sys/kern/subr_csan.c @@ -0,0 +1,859 @@ +/* $NetBSD: subr_csan.c,v 1.2 2019/11/06 06:57:22 maxv Exp $ */ + +/* + * Copyright (c) 2019 The NetBSD Foundation, Inc. + * All rights reserved. + * Copyright (c) 2019 Andrew Turner + * + * This code is derived from software contributed to The NetBSD Foundation + * by Maxime Villard. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define KCSAN_RUNTIME + +#include "opt_ddb.h" + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifdef KCSAN_PANIC +#define REPORT panic +#else +#define REPORT printf +#endif + +typedef struct { + uintptr_t addr; + uint32_t size; + bool write:1; + bool atomic:1; + uintptr_t pc; + struct thread *td; +} csan_cell_t; + +typedef struct { + bool inited; + uint32_t cnt; + csan_cell_t cell; +} csan_cpu_t; + +static csan_cpu_t kcsan_cpus[MAXCPU]; +static bool kcsan_enabled __read_mostly; + +#define __RET_ADDR (uintptr_t)__builtin_return_address(0) + +#define KCSAN_NACCESSES 1024 +#define KCSAN_DELAY 10 /* 10 microseconds */ + +/* -------------------------------------------------------------------------- */ + +/* The MD code. */ +#include + +/* -------------------------------------------------------------------------- */ + +static void +kcsan_enable(void *dummy __unused) +{ + + printf("Enabling KCSCAN, expect reduced performance.\n"); + kcsan_enabled = true; +} +SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL); + +void +kcsan_cpu_init(u_int cpu) +{ + kcsan_cpus[cpu].inited = true; +} + +/* -------------------------------------------------------------------------- */ + +static inline void +kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu) +{ + const char *newsym, *oldsym; +#ifdef DDB + c_db_sym_t sym; + db_expr_t offset; + + sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset); + db_symbol_values(sym, &newsym, NULL); + + sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset); + db_symbol_values(sym, &oldsym, NULL); +#else + newsym = ""; + oldsym = ""; +#endif + REPORT("CSan: Racy Access " + "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] " + "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n", + newcpu, + (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), + (void *)new->addr, new->size, (void *)new->pc, newsym, + oldcpu, + (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), + (void *)old->addr, old->size, (void *)old->pc, oldsym); + kcsan_md_unwind(); +} + +static inline bool +kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) +{ + if (new->write && !new->atomic) + return false; + if (old->write && !old->atomic) + return false; + return true; +} + +static inline void +kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) +{ + csan_cell_t old, new; + csan_cpu_t *cpu; + struct thread *td; + uint64_t intr; + size_t i; + + if (__predict_false(!kcsan_enabled)) + return; + + td = curthread; + + new.addr = addr; + new.size = size; + new.write = write; + new.atomic = atomic; + new.pc = pc; + new.td = td; + + CPU_FOREACH(i) { + __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); + + /* Assume we can't race with ourself */ + if (old.td == td) + continue; + if (old.addr + old.size <= new.addr) + continue; + if (new.addr + new.size <= old.addr) + continue; + if (__predict_true(!old.write && !new.write)) + continue; + if (__predict_true(kcsan_access_is_atomic(&new, &old))) + continue; + + kcsan_report(&new, PCPU_GET(cpuid), &old, i); + break; + } + + if (__predict_false(!kcsan_md_is_avail())) + return; + + kcsan_md_disable_intrs(&intr); + + cpu = &kcsan_cpus[PCPU_GET(cpuid)]; + if (__predict_false(!cpu->inited)) + goto out; + cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; + if (__predict_true(cpu->cnt != 0)) + goto out; + + __builtin_memcpy(&cpu->cell, &new, sizeof(new)); + kcsan_md_delay(KCSAN_DELAY); + __builtin_memset(&cpu->cell, 0, sizeof(new)); + +out: + kcsan_md_enable_intrs(&intr); +} + +#define CSAN_READ(size) \ + void __tsan_read##size(uintptr_t); \ + void __tsan_read##size(uintptr_t addr) \ + { \ + kcsan_access(addr, size, false, false, __RET_ADDR); \ + } \ + void __tsan_unaligned_read##size(uintptr_t); \ + void __tsan_unaligned_read##size(uintptr_t addr) \ + { \ + kcsan_access(addr, size, false, false, __RET_ADDR); \ + } + +CSAN_READ(1) +CSAN_READ(2) +CSAN_READ(4) +CSAN_READ(8) +CSAN_READ(16) + +#define CSAN_WRITE(size) \ + void __tsan_write##size(uintptr_t); \ + void __tsan_write##size(uintptr_t addr) \ + { \ + kcsan_access(addr, size, true, false, __RET_ADDR); \ + } \ + void __tsan_unaligned_write##size(uintptr_t); \ + void __tsan_unaligned_write##size(uintptr_t addr) \ + { \ + kcsan_access(addr, size, true, false, __RET_ADDR); \ + } + +CSAN_WRITE(1) +CSAN_WRITE(2) +CSAN_WRITE(4) +CSAN_WRITE(8) +CSAN_WRITE(16) + +void __tsan_read_range(uintptr_t, size_t); +void __tsan_write_range(uintptr_t, size_t); + +void +__tsan_read_range(uintptr_t addr, size_t size) +{ + kcsan_access(addr, size, false, false, __RET_ADDR); +} + +void +__tsan_write_range(uintptr_t addr, size_t size) +{ + kcsan_access(addr, size, true, false, __RET_ADDR); +} + +void __tsan_init(void); +void __tsan_func_entry(void *); +void __tsan_func_exit(void); + +void +__tsan_init(void) +{ +} + +void +__tsan_func_entry(void *call_pc) +{ +} + +void +__tsan_func_exit(void) +{ +} + +/* -------------------------------------------------------------------------- */ + +void * +kcsan_memcpy(void *dst, const void *src, size_t len) +{ + kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); + kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); + return __builtin_memcpy(dst, src, len); +} + +int +kcsan_memcmp(const void *b1, const void *b2, size_t len) +{ + kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); + kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); + return __builtin_memcmp(b1, b2, len); +} + +void * +kcsan_memset(void *b, int c, size_t len) +{ + kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); + return __builtin_memset(b, c, len); +} + +void * +kcsan_memmove(void *dst, const void *src, size_t len) +{ + kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); + kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); + return __builtin_memmove(dst, src, len); +} + +char * +kcsan_strcpy(char *dst, const char *src) +{ + char *save = dst; + + while (1) { + kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); + kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); + *dst = *src; + if (*src == '\0') + break; + src++, dst++; + } + + return save; +} + +int +kcsan_strcmp(const char *s1, const char *s2) +{ + while (1) { + kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); + kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); + if (*s1 != *s2) + break; + if (*s1 == '\0') + return 0; + s1++, s2++; + } + + return (*(const unsigned char *)s1 - *(const unsigned char *)s2); +} + +size_t +kcsan_strlen(const char *str) +{ + const char *s; + + s = str; + while (1) { + kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); + if (*s == '\0') + break; + s++; + } + + return (s - str); +} + +#undef copystr +#undef copyin +#undef copyin_nofault +#undef copyinstr +#undef copyout +#undef copyout_nofault + +int +kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done) +{ + kcsan_access((uintptr_t)kdaddr, len, true, false, __RET_ADDR); + return copystr(kfaddr, kdaddr, len, done); +} + +int +kcsan_copyin(const void *uaddr, void *kaddr, size_t len) +{ + kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); + return copyin(uaddr, kaddr, len); +} + +int +kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) +{ + kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); + return copyinstr(uaddr, kaddr, len, done); +} + +int +kcsan_copyout(const void *kaddr, void *uaddr, size_t len) +{ + kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); + return copyout(kaddr, uaddr, len); +} + +/* -------------------------------------------------------------------------- */ + +#include +#include + +#define _CSAN_ATOMIC_FUNC_ADD(name, type) \ + void kcsan_atomic_add_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + atomic_add_##name(ptr, val); \ + } + +#define CSAN_ATOMIC_FUNC_ADD(name, type) \ + _CSAN_ATOMIC_FUNC_ADD(name, type) \ + _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_ADD(rel_##name, type) + +#define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ + void kcsan_atomic_clear_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + atomic_clear_##name(ptr, val); \ + } + +#define CSAN_ATOMIC_FUNC_CLEAR(name, type) \ + _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ + _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) + +#define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ + int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \ + type val2) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_cmpset_##name(ptr, val1, val2)); \ + } + +#define CSAN_ATOMIC_FUNC_CMPSET(name, type) \ + _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ + _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) + +#define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ + int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \ + type val2) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_fcmpset_##name(ptr, val1, val2)); \ + } + +#define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ + _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ + _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) + +#define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \ + type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_fetchadd_##name(ptr, val)); \ + } + +#define _CSAN_ATOMIC_FUNC_LOAD(name, type) \ + type kcsan_atomic_load_##name(volatile type *ptr) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \ + __RET_ADDR); \ + return (atomic_load_##name(ptr)); \ + } + +#define CSAN_ATOMIC_FUNC_LOAD(name, type) \ + _CSAN_ATOMIC_FUNC_LOAD(name, type) \ + _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \ + +#define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ + type kcsan_atomic_readandclear_##name(volatile type *ptr) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_readandclear_##name(ptr)); \ + } + +#define _CSAN_ATOMIC_FUNC_SET(name, type) \ + void kcsan_atomic_set_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + atomic_set_##name(ptr, val); \ + } + +#define CSAN_ATOMIC_FUNC_SET(name, type) \ + _CSAN_ATOMIC_FUNC_SET(name, type) \ + _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_SET(rel_##name, type) + +#define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ + void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + atomic_subtract_##name(ptr, val); \ + } + + +#define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ + _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ + _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ + _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) + +#define _CSAN_ATOMIC_FUNC_STORE(name, type) \ + void kcsan_atomic_store_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + atomic_store_##name(ptr, val); \ + } + +#define CSAN_ATOMIC_FUNC_STORE(name, type) \ + _CSAN_ATOMIC_FUNC_STORE(name, type) \ + _CSAN_ATOMIC_FUNC_STORE(rel_##name, type) + +#define CSAN_ATOMIC_FUNC_SWAP(name, type) \ + type kcsan_atomic_swap_##name(volatile type *ptr, type val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return(atomic_swap_##name(ptr, val)); \ + } + +#define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ + int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return(atomic_testandclear_##name(ptr, val)); \ + } + +#define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ + int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \ + { \ + kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ + __RET_ADDR); \ + return (atomic_testandset_##name(ptr, val)); \ + } + + +CSAN_ATOMIC_FUNC_ADD(8, uint8_t) +CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t) +CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t) +CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t) +_CSAN_ATOMIC_FUNC_LOAD(8, uint8_t) +CSAN_ATOMIC_FUNC_SET(8, uint8_t) +CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t) +_CSAN_ATOMIC_FUNC_STORE(8, uint8_t) +#if 0 +CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t) +CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t) +CSAN_ATOMIC_FUNC_SWAP(8, uint8_t) +CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t) +CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t) +#endif + +CSAN_ATOMIC_FUNC_ADD(16, uint16_t) +CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t) +CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t) +CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t) +#if defined(__aarch64__) +_CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) +#else +CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) +#endif +CSAN_ATOMIC_FUNC_SET(16, uint16_t) +CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t) +_CSAN_ATOMIC_FUNC_STORE(16, uint16_t) +#if 0 +CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t) +CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t) +CSAN_ATOMIC_FUNC_SWAP(16, uint16_t) +CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t) +CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t) +#endif + +CSAN_ATOMIC_FUNC_ADD(32, uint32_t) +CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t) +CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t) +CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t) +CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t) +CSAN_ATOMIC_FUNC_LOAD(32, uint32_t) +CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t) +CSAN_ATOMIC_FUNC_SET(32, uint32_t) +CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t) +CSAN_ATOMIC_FUNC_STORE(32, uint32_t) +CSAN_ATOMIC_FUNC_SWAP(32, uint32_t) +#if !defined(__aarch64__) +CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t) +CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t) +#endif + +CSAN_ATOMIC_FUNC_ADD(64, uint64_t) +CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t) +CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t) +CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t) +CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t) +CSAN_ATOMIC_FUNC_LOAD(64, uint64_t) +CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t) +CSAN_ATOMIC_FUNC_SET(64, uint64_t) +CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t) +CSAN_ATOMIC_FUNC_STORE(64, uint64_t) +CSAN_ATOMIC_FUNC_SWAP(64, uint64_t) +#if !defined(__aarch64__) +CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t) +CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t) +#endif + +CSAN_ATOMIC_FUNC_ADD(int, u_int) +CSAN_ATOMIC_FUNC_CLEAR(int, u_int) +CSAN_ATOMIC_FUNC_CMPSET(int, u_int) +CSAN_ATOMIC_FUNC_FCMPSET(int, u_int) +CSAN_ATOMIC_FUNC_FETCHADD(int, u_int) +CSAN_ATOMIC_FUNC_LOAD(int, u_int) +CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int) +CSAN_ATOMIC_FUNC_SET(int, u_int) +CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int) +CSAN_ATOMIC_FUNC_STORE(int, u_int) +CSAN_ATOMIC_FUNC_SWAP(int, u_int) +#if !defined(__aarch64__) +CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int) +CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int) +#endif + +CSAN_ATOMIC_FUNC_ADD(long, u_long) +CSAN_ATOMIC_FUNC_CLEAR(long, u_long) +CSAN_ATOMIC_FUNC_CMPSET(long, u_long) +CSAN_ATOMIC_FUNC_FCMPSET(long, u_long) +CSAN_ATOMIC_FUNC_FETCHADD(long, u_long) +CSAN_ATOMIC_FUNC_LOAD(long, u_long) +CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long) +CSAN_ATOMIC_FUNC_SET(long, u_long) +CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long) +CSAN_ATOMIC_FUNC_STORE(long, u_long) +CSAN_ATOMIC_FUNC_SWAP(long, u_long) +#if !defined(__aarch64__) +CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long) +CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long) +#endif + +CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t) +#if !defined(__amd64__) +CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t) +#endif +CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t) +#if 0 +CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t) +CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t) +#endif + +#define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ + void kcsan_atomic_thread_fence_##name(void) \ + { \ + atomic_thread_fence_##name(); \ + } + + +CSAN_ATOMIC_FUNC_THREAD_FENCE(acq) +CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel) +CSAN_ATOMIC_FUNC_THREAD_FENCE(rel) +CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst) + +/* -------------------------------------------------------------------------- */ + +#include +#include +#include + +int +kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, + int flags, bus_space_handle_t *handlep) +{ + + return (bus_space_map(tag, hnd, size, flags, handlep)); +} + +void +kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, + bus_size_t size) +{ + + bus_space_unmap(tag, hnd, size); +} + +int +kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, + bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) +{ + + return (bus_space_subregion(tag, hnd, offset, size, handlep)); +} + +#if !defined(__amd64__) +int +kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start, + bus_addr_t reg_end, bus_size_t size, bus_size_t alignment, + bus_size_t boundary, int flags, bus_addr_t *addrp, + bus_space_handle_t *handlep) +{ + + return (bus_space_alloc(tag, reg_start, reg_end, size, alignment, + boundary, flags, addrp, handlep)); +} +#endif + +void +kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, + bus_size_t size) +{ + + bus_space_free(tag, hnd, size); +} + +void +kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, + bus_size_t offset, bus_size_t size, int flags) +{ + + bus_space_barrier(tag, hnd, offset, size, flags); +} + +#define CSAN_BUS_READ_FUNC(func, width, type) \ + type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t offset) \ + { \ + return (bus_space_read##func##_##width(tag, hnd, \ + offset)); \ + } \ + +#define CSAN_BUS_READ_PTR_FUNC(func, width, type) \ + void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t size, type *buf, \ + bus_size_t count) \ + { \ + kcsan_access((uintptr_t)buf, sizeof(type) * count, \ + false, false, __RET_ADDR); \ + bus_space_read_##func##_##width(tag, hnd, size, buf, \ + count); \ + } + +CSAN_BUS_READ_FUNC(, 1, uint8_t) +CSAN_BUS_READ_FUNC(_stream, 1, uint8_t) +CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) +CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) +CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) +CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) + +CSAN_BUS_READ_FUNC(, 2, uint16_t) +CSAN_BUS_READ_FUNC(_stream, 2, uint16_t) +CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) +CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) +CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) +CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) + +CSAN_BUS_READ_FUNC(, 4, uint32_t) +CSAN_BUS_READ_FUNC(_stream, 4, uint32_t) +CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) +CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) +CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) +CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) + +CSAN_BUS_READ_FUNC(, 8, uint64_t) +#if defined(__aarch64__) +CSAN_BUS_READ_FUNC(_stream, 8, uint64_t) +CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) +CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) +CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) +CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) +#endif + +#define CSAN_BUS_WRITE_FUNC(func, width, type) \ + void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t offset, type value) \ + { \ + bus_space_write##func##_##width(tag, hnd, offset, value); \ + } \ + +#define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ + void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t size, const type *buf, \ + bus_size_t count) \ + { \ + kcsan_access((uintptr_t)buf, sizeof(type) * count, \ + true, false, __RET_ADDR); \ + bus_space_write_##func##_##width(tag, hnd, size, buf, \ + count); \ + } + +CSAN_BUS_WRITE_FUNC(, 1, uint8_t) +CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) +CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) +CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) +CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) +CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) + +CSAN_BUS_WRITE_FUNC(, 2, uint16_t) +CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) +CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) +CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) +CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) +CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) + +CSAN_BUS_WRITE_FUNC(, 4, uint32_t) +CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) +CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) +CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) +CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) +CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) + +CSAN_BUS_WRITE_FUNC(, 8, uint64_t) +#if defined(__aarch64__) +CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t) +CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t) +CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t) +CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t) +CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t) +#endif + +#define CSAN_BUS_SET_FUNC(func, width, type) \ + void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t offset, type value, \ + bus_size_t count) \ + { \ + bus_space_set_##func##_##width(tag, hnd, offset, value, \ + count); \ + } + +CSAN_BUS_SET_FUNC(multi, 1, uint8_t) +CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) +CSAN_BUS_SET_FUNC(region, 1, uint8_t) +CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) + +CSAN_BUS_SET_FUNC(multi, 2, uint16_t) +CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) +CSAN_BUS_SET_FUNC(region, 2, uint16_t) +CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) + +CSAN_BUS_SET_FUNC(multi, 4, uint32_t) +CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) +CSAN_BUS_SET_FUNC(region, 4, uint32_t) +CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) + +#if !defined(__amd64__) +CSAN_BUS_SET_FUNC(multi, 8, uint64_t) +CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t) +CSAN_BUS_SET_FUNC(region, 8, uint64_t) +CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t) +#endif + Index: sys/kern/vfs_aio.c =================================================================== --- sys/kern/vfs_aio.c +++ sys/kern/vfs_aio.c @@ -292,7 +292,7 @@ * Different ABIs provide their own operations. */ struct aiocb_ops { - int (*copyin)(struct aiocb *ujob, struct aiocb *kjob); + int (*aio_copyin)(struct aiocb *ujob, struct aiocb *kjob); long (*fetch_status)(struct aiocb *ujob); long (*fetch_error)(struct aiocb *ujob); int (*store_status)(struct aiocb *ujob, long status); @@ -1420,7 +1420,7 @@ } static struct aiocb_ops aiocb_ops = { - .copyin = aiocb_copyin, + .aio_copyin = aiocb_copyin, .fetch_status = aiocb_fetch_status, .fetch_error = aiocb_fetch_error, .store_status = aiocb_store_status, @@ -1431,7 +1431,7 @@ #ifdef COMPAT_FREEBSD6 static struct aiocb_ops aiocb_ops_osigevent = { - .copyin = aiocb_copyin_old_sigevent, + .aio_copyin = aiocb_copyin_old_sigevent, .fetch_status = aiocb_fetch_status, .fetch_error = aiocb_fetch_error, .store_status = aiocb_store_status, @@ -1478,7 +1478,7 @@ job = uma_zalloc(aiocb_zone, M_WAITOK | M_ZERO); knlist_init_mtx(&job->klist, AIO_MTX(ki)); - error = ops->copyin(ujob, &job->uaiocb); + error = ops->aio_copyin(ujob, &job->uaiocb); if (error) { ops->store_error(ujob, error); uma_zfree(aiocb_zone, job); @@ -2743,7 +2743,7 @@ } static struct aiocb_ops aiocb32_ops = { - .copyin = aiocb32_copyin, + .aio_copyin = aiocb32_copyin, .fetch_status = aiocb32_fetch_status, .fetch_error = aiocb32_fetch_error, .store_status = aiocb32_store_status, @@ -2754,7 +2754,7 @@ #ifdef COMPAT_FREEBSD6 static struct aiocb_ops aiocb32_ops_osigevent = { - .copyin = aiocb32_copyin_old_sigevent, + .aio_copyin = aiocb32_copyin_old_sigevent, .fetch_status = aiocb32_fetch_status, .fetch_error = aiocb32_fetch_error, .store_status = aiocb32_store_status, Index: sys/libkern/strcmp.c =================================================================== --- sys/libkern/strcmp.c +++ sys/libkern/strcmp.c @@ -41,7 +41,7 @@ * Compare strings. */ int -strcmp(const char *s1, const char *s2) +(strcmp)(const char *s1, const char *s2) { while (*s1 == *s2++) if (*s1++ == '\0') Index: sys/libkern/strcpy.c =================================================================== --- sys/libkern/strcpy.c +++ sys/libkern/strcpy.c @@ -35,7 +35,7 @@ #include char * -strcpy(char * __restrict to, const char * __restrict from) +(strcpy)(char * __restrict to, const char * __restrict from) { char *save = to; Index: sys/libkern/strlen.c =================================================================== --- sys/libkern/strlen.c +++ sys/libkern/strlen.c @@ -80,7 +80,7 @@ } while (0) size_t -strlen(const char *str) +(strlen)(const char *str) { const char *p; const unsigned long *lp; Index: sys/modules/Makefile =================================================================== --- sys/modules/Makefile +++ sys/modules/Makefile @@ -644,11 +644,11 @@ _chromebook_platform= chromebook_platform _cmx= cmx _coretemp= coretemp -.if ${MK_SOURCELESS_HOST} != "no" +.if ${MK_SOURCELESS_HOST} != "no" && empty(KCSAN_ENABLED) _hpt27xx= hpt27xx .endif _hptiop= hptiop -.if ${MK_SOURCELESS_HOST} != "no" +.if ${MK_SOURCELESS_HOST} != "no" && empty(KCSAN_ENABLED) _hptmv= hptmv _hptnr= hptnr _hptrr= hptrr Index: sys/sys/_cscan_atomic.h =================================================================== --- /dev/null +++ sys/sys/_cscan_atomic.h @@ -0,0 +1,307 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2019 Andrew Turner + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory (Department of Computer Science and + * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the + * DARPA SSITH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS__CSAN_ATOMIC_H_ +#define _SYS__CSAN_ATOMIC_H_ + +#ifndef _MACHINE_ATOMIC_H_ +#error do not include this header, use machine/atomic.h +#endif + +#define KCSAN_ATOMIC_FUNC_1(op, name, type) \ + void kcsan_atomic_##op##_##name(volatile type *, type); \ + void kcsan_atomic_##op##_acq_##name(volatile type *, type); \ + void kcsan_atomic_##op##_rel_##name(volatile type *, type) + +#define KCSAN_ATOMIC_CMPSET(name, type) \ + int kcsan_atomic_cmpset_##name(volatile type *, type, type); \ + int kcsan_atomic_cmpset_acq_##name(volatile type *, type, type); \ + int kcsan_atomic_cmpset_rel_##name(volatile type *, type, type) + +#define KCSAN_ATOMIC_FCMPSET(name, type) \ + int kcsan_atomic_fcmpset_##name(volatile type *, type *, type); \ + int kcsan_atomic_fcmpset_acq_##name(volatile type *, type *, type); \ + int kcsan_atomic_fcmpset_rel_##name(volatile type *, type *, type) + +#define KCSAN_ATOMIC_READ(op, name, type) \ + type kcsan_atomic_##op##_##name(volatile type *, type) + +#define KCSAN_ATOMIC_READANDCLEAR(name, type) \ + type kcsan_atomic_readandclear_##name(volatile type *) + +#define KCSAN_ATOMIC_LOAD(name, type) \ + type kcsan_atomic_load_##name(volatile type *); \ + type kcsan_atomic_load_acq_##name(volatile type *) + +#define KCSAN_ATOMIC_STORE(name, type) \ + void kcsan_atomic_store_##name(volatile type *, type); \ + void kcsan_atomic_store_rel_##name(volatile type *, type) + +#define KCSAN_ATOMIC_TEST(op, name, type) \ + int kcsan_atomic_##op##_##name(volatile type *, u_int) + +#define KCSAN_ATOMIC_FUNCS(name, type) \ + KCSAN_ATOMIC_FUNC_1(add, name, type); \ + KCSAN_ATOMIC_FUNC_1(clear, name, type); \ + KCSAN_ATOMIC_CMPSET(name, type); \ + KCSAN_ATOMIC_FCMPSET(name, type); \ + KCSAN_ATOMIC_READ(fetchadd, name, type); \ + KCSAN_ATOMIC_LOAD(name, type); \ + KCSAN_ATOMIC_READANDCLEAR(name, type); \ + KCSAN_ATOMIC_FUNC_1(set, name, type); \ + KCSAN_ATOMIC_FUNC_1(subtract, name, type); \ + KCSAN_ATOMIC_STORE(name, type); \ + KCSAN_ATOMIC_READ(swap, name, type); \ + KCSAN_ATOMIC_TEST(testandclear, name, type); \ + KCSAN_ATOMIC_TEST(testandset, name, type) + +KCSAN_ATOMIC_FUNCS(int, u_int); +KCSAN_ATOMIC_FUNCS(long, u_long); +KCSAN_ATOMIC_FUNCS(ptr, uintptr_t); +KCSAN_ATOMIC_FUNCS(8, uint8_t); +KCSAN_ATOMIC_FUNCS(16, uint16_t); +KCSAN_ATOMIC_FUNCS(32, uint32_t); +KCSAN_ATOMIC_FUNCS(64, uint64_t); + +void kcsan_atomic_thread_fence_acq(void); +void kcsan_atomic_thread_fence_acq_rel(void); +void kcsan_atomic_thread_fence_rel(void); +void kcsan_atomic_thread_fence_seq_cst(void); + +#ifndef KCSAN_RUNTIME + +#define atomic_add_int kcsan_atomic_add_int +#define atomic_add_acq_int kcsan_atomic_add_acq_int +#define atomic_add_rel_int kcsan_atomic_add_rel_int +#define atomic_clear_int kcsan_atomic_clear_int +#define atomic_clear_acq_int kcsan_atomic_clear_acq_int +#define atomic_clear_rel_int kcsan_atomic_clear_rel_int +#define atomic_cmpset_int kcsan_atomic_cmpset_int +#define atomic_cmpset_acq_int kcsan_atomic_cmpset_acq_int +#define atomic_cmpset_rel_int kcsan_atomic_cmpset_rel_int +#define atomic_fcmpset_int kcsan_atomic_fcmpset_int +#define atomic_fcmpset_acq_int kcsan_atomic_fcmpset_acq_int +#define atomic_fcmpset_rel_int kcsan_atomic_fcmpset_rel_int +#define atomic_fetchadd_int kcsan_atomic_fetchadd_int +#define atomic_load_int kcsan_atomic_load_int +#define atomic_load_acq_int kcsan_atomic_load_acq_int +#define atomic_readandclear_int kcsan_atomic_readandclear_int +#define atomic_set_int kcsan_atomic_set_int +#define atomic_set_acq_int kcsan_atomic_set_acq_int +#define atomic_set_rel_int kcsan_atomic_set_rel_int +#define atomic_subtract_int kcsan_atomic_subtract_int +#define atomic_subtract_acq_int kcsan_atomic_subtract_acq_int +#define atomic_subtract_rel_int kcsan_atomic_subtract_rel_int +#define atomic_store_int kcsan_atomic_store_int +#define atomic_store_rel_int kcsan_atomic_store_rel_int +#define atomic_swap_int kcsan_atomic_swap_int +#define atomic_testandclear_int kcsan_atomic_testandclear_int +#define atomic_testandset_int kcsan_atomic_testandset_int + +#define atomic_add_long kcsan_atomic_add_long +#define atomic_add_acq_long kcsan_atomic_add_acq_long +#define atomic_add_rel_long kcsan_atomic_add_rel_long +#define atomic_clear_long kcsan_atomic_clear_long +#define atomic_clear_acq_long kcsan_atomic_clear_acq_long +#define atomic_clear_rel_long kcsan_atomic_clear_rel_long +#define atomic_cmpset_long kcsan_atomic_cmpset_long +#define atomic_cmpset_acq_long kcsan_atomic_cmpset_acq_long +#define atomic_cmpset_rel_long kcsan_atomic_cmpset_rel_long +#define atomic_fcmpset_long kcsan_atomic_fcmpset_long +#define atomic_fcmpset_acq_long kcsan_atomic_fcmpset_acq_long +#define atomic_fcmpset_rel_long kcsan_atomic_fcmpset_rel_long +#define atomic_fetchadd_long kcsan_atomic_fetchadd_long +#define atomic_load_long kcsan_atomic_load_long +#define atomic_load_acq_long kcsan_atomic_load_acq_long +#define atomic_readandclear_long kcsan_atomic_readandclear_long +#define atomic_set_long kcsan_atomic_set_long +#define atomic_set_acq_long kcsan_atomic_set_acq_long +#define atomic_set_rel_long kcsan_atomic_set_rel_long +#define atomic_subtract_long kcsan_atomic_subtract_long +#define atomic_subtract_acq_long kcsan_atomic_subtract_acq_long +#define atomic_subtract_rel_long kcsan_atomic_subtract_rel_long +#define atomic_store_long kcsan_atomic_store_long +#define atomic_store_rel_long kcsan_atomic_store_rel_long +#define atomic_swap_long kcsan_atomic_swap_long +#define atomic_testandclear_long kcsan_atomic_testandclear_long +#define atomic_testandset_long kcsan_atomic_testandset_long + +#define atomic_add_ptr kcsan_atomic_add_ptr +#define atomic_add_acq_ptr kcsan_atomic_add_acq_ptr +#define atomic_add_rel_ptr kcsan_atomic_add_rel_ptr +#define atomic_clear_ptr kcsan_atomic_clear_ptr +#define atomic_clear_acq_ptr kcsan_atomic_clear_acq_ptr +#define atomic_clear_rel_ptr kcsan_atomic_clear_rel_ptr +#define atomic_cmpset_ptr kcsan_atomic_cmpset_ptr +#define atomic_cmpset_acq_ptr kcsan_atomic_cmpset_acq_ptr +#define atomic_cmpset_rel_ptr kcsan_atomic_cmpset_rel_ptr +#define atomic_fcmpset_ptr kcsan_atomic_fcmpset_ptr +#define atomic_fcmpset_acq_ptr kcsan_atomic_fcmpset_acq_ptr +#define atomic_fcmpset_rel_ptr kcsan_atomic_fcmpset_rel_ptr +#define atomic_fetchadd_ptr kcsan_atomic_fetchadd_ptr +#define atomic_load_ptr(x) kcsan_atomic_load_ptr((volatile uintptr_t *)(x)) +#define atomic_load_acq_ptr kcsan_atomic_load_acq_ptr +#define atomic_readandclear_ptr kcsan_atomic_readandclear_ptr +#define atomic_set_ptr kcsan_atomic_set_ptr +#define atomic_set_acq_ptr kcsan_atomic_set_acq_ptr +#define atomic_set_rel_ptr kcsan_atomic_set_rel_ptr +#define atomic_subtract_ptr kcsan_atomic_subtract_ptr +#define atomic_subtract_acq_ptr kcsan_atomic_subtract_acq_ptr +#define atomic_subtract_rel_ptr kcsan_atomic_subtract_rel_ptr +#define atomic_store_ptr kcsan_atomic_store_ptr +#define atomic_store_rel_ptr kcsan_atomic_store_rel_ptr +#define atomic_swap_ptr kcsan_atomic_swap_ptr +#define atomic_testandclear_ptr kcsan_atomic_testandclear_ptr +#define atomic_testandset_ptr kcsan_atomic_testandset_ptr + +#define atomic_add_8 kcsan_atomic_add_8 +#define atomic_add_acq_8 kcsan_atomic_add_acq_8 +#define atomic_add_rel_8 kcsan_atomic_add_rel_8 +#define atomic_clear_8 kcsan_atomic_clear_8 +#define atomic_clear_acq_8 kcsan_atomic_clear_acq_8 +#define atomic_clear_rel_8 kcsan_atomic_clear_rel_8 +#define atomic_cmpset_8 kcsan_atomic_cmpset_8 +#define atomic_cmpset_acq_8 kcsan_atomic_cmpset_acq_8 +#define atomic_cmpset_rel_8 kcsan_atomic_cmpset_rel_8 +#define atomic_fcmpset_8 kcsan_atomic_fcmpset_8 +#define atomic_fcmpset_acq_8 kcsan_atomic_fcmpset_acq_8 +#define atomic_fcmpset_rel_8 kcsan_atomic_fcmpset_rel_8 +#define atomic_fetchadd_8 kcsan_atomic_fetchadd_8 +#define atomic_load_8 kcsan_atomic_load_8 +#define atomic_load_acq_8 kcsan_atomic_load_acq_8 +#define atomic_readandclear_8 kcsan_atomic_readandclear_8 +#define atomic_set_8 kcsan_atomic_set_8 +#define atomic_set_acq_8 kcsan_atomic_set_acq_8 +#define atomic_set_rel_8 kcsan_atomic_set_rel_8 +#define atomic_subtract_8 kcsan_atomic_subtract_8 +#define atomic_subtract_acq_8 kcsan_atomic_subtract_acq_8 +#define atomic_subtract_rel_8 kcsan_atomic_subtract_rel_8 +#define atomic_store_8 kcsan_atomic_store_8 +#define atomic_store_rel_8 kcsan_atomic_store_rel_8 +#define atomic_swap_8 kcsan_atomic_swap_8 +#define atomic_testandclear_8 kcsan_atomic_testandclear_8 +#define atomic_testandset_8 kcsan_atomic_testandset_8 + +#define atomic_add_16 kcsan_atomic_add_16 +#define atomic_add_acq_16 kcsan_atomic_add_acq_16 +#define atomic_add_rel_16 kcsan_atomic_add_rel_16 +#define atomic_clear_16 kcsan_atomic_clear_16 +#define atomic_clear_acq_16 kcsan_atomic_clear_acq_16 +#define atomic_clear_rel_16 kcsan_atomic_clear_rel_16 +#define atomic_cmpset_16 kcsan_atomic_cmpset_16 +#define atomic_cmpset_acq_16 kcsan_atomic_cmpset_acq_16 +#define atomic_cmpset_rel_16 kcsan_atomic_cmpset_rel_16 +#define atomic_fcmpset_16 kcsan_atomic_fcmpset_16 +#define atomic_fcmpset_acq_16 kcsan_atomic_fcmpset_acq_16 +#define atomic_fcmpset_rel_16 kcsan_atomic_fcmpset_rel_16 +#define atomic_fetchadd_16 kcsan_atomic_fetchadd_16 +#define atomic_load_16 kcsan_atomic_load_16 +#define atomic_load_acq_16 kcsan_atomic_load_acq_16 +#define atomic_readandclear_16 kcsan_atomic_readandclear_16 +#define atomic_set_16 kcsan_atomic_set_16 +#define atomic_set_acq_16 kcsan_atomic_set_acq_16 +#define atomic_set_rel_16 kcsan_atomic_set_rel_16 +#define atomic_subtract_16 kcsan_atomic_subtract_16 +#define atomic_subtract_acq_16 kcsan_atomic_subtract_acq_16 +#define atomic_subtract_rel_16 kcsan_atomic_subtract_rel_16 +#define atomic_store_16 kcsan_atomic_store_16 +#define atomic_store_rel_16 kcsan_atomic_store_rel_16 +#define atomic_swap_16 kcsan_atomic_swap_16 +#define atomic_testandclear_16 kcsan_atomic_testandclear_16 +#define atomic_testandset_16 kcsan_atomic_testandset_16 + +#define atomic_add_32 kcsan_atomic_add_32 +#define atomic_add_acq_32 kcsan_atomic_add_acq_32 +#define atomic_add_rel_32 kcsan_atomic_add_rel_32 +#define atomic_clear_32 kcsan_atomic_clear_32 +#define atomic_clear_acq_32 kcsan_atomic_clear_acq_32 +#define atomic_clear_rel_32 kcsan_atomic_clear_rel_32 +#define atomic_cmpset_32 kcsan_atomic_cmpset_32 +#define atomic_cmpset_acq_32 kcsan_atomic_cmpset_acq_32 +#define atomic_cmpset_rel_32 kcsan_atomic_cmpset_rel_32 +#define atomic_fcmpset_32 kcsan_atomic_fcmpset_32 +#define atomic_fcmpset_acq_32 kcsan_atomic_fcmpset_acq_32 +#define atomic_fcmpset_rel_32 kcsan_atomic_fcmpset_rel_32 +#define atomic_fetchadd_32 kcsan_atomic_fetchadd_32 +#define atomic_load_32 kcsan_atomic_load_32 +#define atomic_load_acq_32 kcsan_atomic_load_acq_32 +#define atomic_readandclear_32 kcsan_atomic_readandclear_32 +#define atomic_set_32 kcsan_atomic_set_32 +#define atomic_set_acq_32 kcsan_atomic_set_acq_32 +#define atomic_set_rel_32 kcsan_atomic_set_rel_32 +#define atomic_subtract_32 kcsan_atomic_subtract_32 +#define atomic_subtract_acq_32 kcsan_atomic_subtract_acq_32 +#define atomic_subtract_rel_32 kcsan_atomic_subtract_rel_32 +#define atomic_store_32 kcsan_atomic_store_32 +#define atomic_store_rel_32 kcsan_atomic_store_rel_32 +#define atomic_swap_32 kcsan_atomic_swap_32 +#define atomic_testandclear_32 kcsan_atomic_testandclear_32 +#define atomic_testandset_32 kcsan_atomic_testandset_32 + +#define atomic_add_64 kcsan_atomic_add_64 +#define atomic_add_acq_64 kcsan_atomic_add_acq_64 +#define atomic_add_rel_64 kcsan_atomic_add_rel_64 +#define atomic_clear_64 kcsan_atomic_clear_64 +#define atomic_clear_acq_64 kcsan_atomic_clear_acq_64 +#define atomic_clear_rel_64 kcsan_atomic_clear_rel_64 +#define atomic_cmpset_64 kcsan_atomic_cmpset_64 +#define atomic_cmpset_acq_64 kcsan_atomic_cmpset_acq_64 +#define atomic_cmpset_rel_64 kcsan_atomic_cmpset_rel_64 +#define atomic_fcmpset_64 kcsan_atomic_fcmpset_64 +#define atomic_fcmpset_acq_64 kcsan_atomic_fcmpset_acq_64 +#define atomic_fcmpset_rel_64 kcsan_atomic_fcmpset_rel_64 +#define atomic_fetchadd_64 kcsan_atomic_fetchadd_64 +#define atomic_load_64 kcsan_atomic_load_64 +#define atomic_load_acq_64 kcsan_atomic_load_acq_64 +#define atomic_readandclear_64 kcsan_atomic_readandclear_64 +#define atomic_set_64 kcsan_atomic_set_64 +#define atomic_set_acq_64 kcsan_atomic_set_acq_64 +#define atomic_set_rel_64 kcsan_atomic_set_rel_64 +#define atomic_subtract_64 kcsan_atomic_subtract_64 +#define atomic_subtract_acq_64 kcsan_atomic_subtract_acq_64 +#define atomic_subtract_rel_64 kcsan_atomic_subtract_rel_64 +#define atomic_store_64 kcsan_atomic_store_64 +#define atomic_store_rel_64 kcsan_atomic_store_rel_64 +#define atomic_swap_64 kcsan_atomic_swap_64 +#define atomic_testandclear_64 kcsan_atomic_testandclear_64 +#define atomic_testandset_64 kcsan_atomic_testandset_64 + +#define atomic_thread_fence_acq kcsan_atomic_thread_fence_acq +#define atomic_thread_fence_acq_rel kcsan_atomic_thread_fence_acq_rel +#define atomic_thread_fence_rel kcsan_atomic_thread_fence_rel +#define atomic_thread_fence_seq_cst kcsan_atomic_thread_fence_seq_cst + +#endif /* !KCSAN_RUNTIME */ + +#endif /* !_SYS__CSAN_ATOMIC_H_ */ Index: sys/sys/_cscan_bus.h =================================================================== --- /dev/null +++ sys/sys/_cscan_bus.h @@ -0,0 +1,194 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2019 Andrew Turner + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory (Department of Computer Science and + * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the + * DARPA SSITH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS__CSAN_BUS_H_ +#define _SYS__CSAN_BUS_H_ + +#ifndef _SYS_BUS_H_ +//#error do not include this header, use sys/bus.h +#endif + +#define KCSAN_BS_MULTI(rw, width, type) \ + void kcsan_bus_space_##rw##_multi_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type *, bus_size_t); \ + void kcsan_bus_space_##rw##_multi_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type *, bus_size_t); \ + void kcsan_bus_space_##rw##_region_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type *, bus_size_t); \ + void kcsan_bus_space_##rw##_region_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type *, bus_size_t) + +#define KCSAN_BS_READ(width, type) \ + type kcsan_bus_space_read_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t); \ + type kcsan_bus_space_read_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t); \ + KCSAN_BS_MULTI(read, width, type) + +#define KCSAN_BS_WRITE(width, type) \ + void kcsan_bus_space_write_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type); \ + void kcsan_bus_space_write_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type); \ + KCSAN_BS_MULTI(write, width, const type) + +#define KCSAN_BS_SET(width, type) \ + void kcsan_bus_space_set_multi_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type, bus_size_t); \ + void kcsan_bus_space_set_multi_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type, bus_size_t); \ + void kcsan_bus_space_set_region_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type, bus_size_t); \ + void kcsan_bus_space_set_region_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, type, bus_size_t) + +#define KCSAN_BS_COPY(width, type) \ + void kcsan_bus_space_copy_region_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, bus_space_handle_t, \ + bus_size_t, bus_size_t); \ + void kcsan_bus_space_copy_region_stream_##width(bus_space_tag_t, \ + bus_space_handle_t, bus_size_t, bus_space_handle_t, \ + bus_size_t, bus_size_t); + +#define KCSAN_BS(width, type) \ + KCSAN_BS_READ(width, type); \ + KCSAN_BS_WRITE(width, type); \ + KCSAN_BS_SET(width, type); \ + KCSAN_BS_COPY(width, type) + +KCSAN_BS(1, uint8_t); +KCSAN_BS(2, uint16_t); +KCSAN_BS(4, uint32_t); +KCSAN_BS(8, uint64_t); + +int kcsan_bus_space_map(bus_space_tag_t, bus_addr_t, bus_size_t, int, + bus_space_handle_t *); +void kcsan_bus_space_unmap(bus_space_tag_t, bus_space_handle_t, bus_size_t); +int kcsan_bus_space_subregion(bus_space_tag_t, bus_space_handle_t, bus_size_t, + bus_size_t, bus_space_handle_t *); +int kcsan_bus_space_alloc(bus_space_tag_t, bus_addr_t, bus_addr_t, + bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *, + bus_space_handle_t *); +void kcsan_bus_space_free(bus_space_tag_t, bus_space_handle_t, bus_size_t); +void kcsan_bus_space_barrier(bus_space_tag_t, bus_space_handle_t, bus_size_t, + bus_size_t, int); + +#ifndef KCSAN_RUNTIME + +#define bus_space_map kcsan_bus_space_map +#define bus_space_unmap kcsan_bus_space_unmap +#define bus_space_subregion kcsan_bus_space_subregion +#define bus_space_alloc kcsan_bus_space_alloc +#define bus_space_free kcsan_bus_space_free +#define bus_space_barrier kcsan_bus_space_barrier + +#define bus_space_read_1 kcsan_bus_space_read_1 +#define bus_space_read_stream_1 kcsan_bus_space_read_stream_1 +#define bus_space_read_multi_1 kcsan_bus_space_read_multi_1 +#define bus_space_read_multi_stream_1 kcsan_bus_space_read_multi_stream_1 +#define bus_space_read_region_1 kcsan_bus_space_read_region_1 +#define bus_space_read_region_stream_1 kcsan_bus_space_read_region_stream_1 +#define bus_space_write_1 kcsan_bus_space_write_1 +#define bus_space_write_stream_1 kcsan_bus_space_write_stream_1 +#define bus_space_write_multi_1 kcsan_bus_space_write_multi_1 +#define bus_space_write_multi_stream_1 kcsan_bus_space_write_multi_stream_1 +#define bus_space_write_region_1 kcsan_bus_space_write_region_1 +#define bus_space_write_region_stream_1 kcsan_bus_space_write_region_stream_1 +#define bus_space_set_multi_1 kcsan_bus_space_set_multi_1 +#define bus_space_set_multi_stream_1 kcsan_bus_space_set_multi_stream_1 +#define bus_space_set_region_1 kcsan_bus_space_set_region_1 +#define bus_space_set_region_stream_1 kcsan_bus_space_set_region_stream_1 +#define bus_space_copy_multi_1 kcsan_bus_space_copy_multi_1 +#define bus_space_copy_multi_stream_1 kcsan_bus_space_copy_multi_stream_1 + +#define bus_space_read_2 kcsan_bus_space_read_2 +#define bus_space_read_stream_2 kcsan_bus_space_read_stream_2 +#define bus_space_read_multi_2 kcsan_bus_space_read_multi_2 +#define bus_space_read_multi_stream_2 kcsan_bus_space_read_multi_stream_2 +#define bus_space_read_region_2 kcsan_bus_space_read_region_2 +#define bus_space_read_region_stream_2 kcsan_bus_space_read_region_stream_2 +#define bus_space_write_2 kcsan_bus_space_write_2 +#define bus_space_write_stream_2 kcsan_bus_space_write_stream_2 +#define bus_space_write_multi_2 kcsan_bus_space_write_multi_2 +#define bus_space_write_multi_stream_2 kcsan_bus_space_write_multi_stream_2 +#define bus_space_write_region_2 kcsan_bus_space_write_region_2 +#define bus_space_write_region_stream_2 kcsan_bus_space_write_region_stream_2 +#define bus_space_set_multi_2 kcsan_bus_space_set_multi_2 +#define bus_space_set_multi_stream_2 kcsan_bus_space_set_multi_stream_2 +#define bus_space_set_region_2 kcsan_bus_space_set_region_2 +#define bus_space_set_region_stream_2 kcsan_bus_space_set_region_stream_2 +#define bus_space_copy_multi_2 kcsan_bus_space_copy_multi_2 +#define bus_space_copy_multi_stream_2 kcsan_bus_space_copy_multi_stream_2 + +#define bus_space_read_4 kcsan_bus_space_read_4 +#define bus_space_read_stream_4 kcsan_bus_space_read_stream_4 +#define bus_space_read_multi_4 kcsan_bus_space_read_multi_4 +#define bus_space_read_multi_stream_4 kcsan_bus_space_read_multi_stream_4 +#define bus_space_read_region_4 kcsan_bus_space_read_region_4 +#define bus_space_read_region_stream_4 kcsan_bus_space_read_region_stream_4 +#define bus_space_write_4 kcsan_bus_space_write_4 +#define bus_space_write_stream_4 kcsan_bus_space_write_stream_4 +#define bus_space_write_multi_4 kcsan_bus_space_write_multi_4 +#define bus_space_write_multi_stream_4 kcsan_bus_space_write_multi_stream_4 +#define bus_space_write_region_4 kcsan_bus_space_write_region_4 +#define bus_space_write_region_stream_4 kcsan_bus_space_write_region_stream_4 +#define bus_space_set_multi_4 kcsan_bus_space_set_multi_4 +#define bus_space_set_multi_stream_4 kcsan_bus_space_set_multi_stream_4 +#define bus_space_set_region_4 kcsan_bus_space_set_region_4 +#define bus_space_set_region_stream_4 kcsan_bus_space_set_region_stream_4 +#define bus_space_copy_multi_4 kcsan_bus_space_copy_multi_4 +#define bus_space_copy_multi_stream_4 kcsan_bus_space_copy_multi_stream_4 + +#define bus_space_read_8 kcsan_bus_space_read_8 +#define bus_space_read_stream_8 kcsan_bus_space_read_stream_8 +#define bus_space_read_multi_8 kcsan_bus_space_read_multi_8 +#define bus_space_read_multi_stream_8 kcsan_bus_space_read_multi_stream_8 +#define bus_space_read_region_8 kcsan_bus_space_read_region_8 +#define bus_space_read_region_stream_8 kcsan_bus_space_read_region_stream_8 +#define bus_space_write_8 kcsan_bus_space_write_8 +#define bus_space_write_stream_8 kcsan_bus_space_write_stream_8 +#define bus_space_write_multi_8 kcsan_bus_space_write_multi_8 +#define bus_space_write_multi_stream_8 kcsan_bus_space_write_multi_stream_8 +#define bus_space_write_region_8 kcsan_bus_space_write_region_8 +#define bus_space_write_region_stream_8 kcsan_bus_space_write_region_stream_8 +#define bus_space_set_multi_8 kcsan_bus_space_set_multi_8 +#define bus_space_set_multi_stream_8 kcsan_bus_space_set_multi_stream_8 +#define bus_space_set_region_8 kcsan_bus_space_set_region_8 +#define bus_space_set_region_stream_8 kcsan_bus_space_set_region_stream_8 +#define bus_space_copy_multi_8 kcsan_bus_space_copy_multi_8 +#define bus_space_copy_multi_stream_8 kcsan_bus_space_copy_multi_stream_8 + +#endif /* !KCSAN_RUNTIME */ + +#endif /* !_SYS__CSAN_BUS_H_ */ Index: sys/sys/csan.h =================================================================== --- /dev/null +++ sys/sys/csan.h @@ -0,0 +1,43 @@ +/* $NetBSD: csan.h,v 1.1 2019/11/05 20:19:18 maxv Exp $ */ + +/* + * Copyright (c) 2019 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Maxime Villard. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_CSAN_H_ +#define _SYS_CSAN_H_ + +#include + +#ifdef KCSAN +void kcsan_cpu_init(u_int); +#else +#define kcsan_cpu_init(ci) ((void)0) +#endif + +#endif /* !_SYS_CSAN_H_ */ Index: sys/sys/libkern.h =================================================================== --- sys/sys/libkern.h +++ sys/sys/libkern.h @@ -191,6 +191,15 @@ char *strstr(const char *, const char *); int strvalid(const char *, size_t); +#ifdef KCSAN +char *kcsan_strcpy(char *, const char *); +int kcsan_strcmp(const char *, const char *); +size_t kcsan_strlen(const char *); +#define strcpy(d, s) kcsan_strcpy((d), (s)) +#define strcmp(s1, s2) kcsan_strcmp((s1), (s2)) +#define strlen(s) kcsan_strlen((s)) +#endif + static __inline char * index(const char *p, int ch) { Index: sys/sys/systm.h =================================================================== --- sys/sys/systm.h +++ sys/sys/systm.h @@ -315,21 +315,36 @@ #define ovbcopy(f, t, l) bcopy((f), (t), (l)) void bcopy(const void * _Nonnull from, void * _Nonnull to, size_t len); -#define bcopy(from, to, len) __builtin_memmove((to), (from), (len)) void bzero(void * _Nonnull buf, size_t len); -#define bzero(buf, len) __builtin_memset((buf), 0, (len)) void explicit_bzero(void * _Nonnull, size_t); int bcmp(const void *b1, const void *b2, size_t len); -#define bcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len)) void *memset(void * _Nonnull buf, int c, size_t len); -#define memset(buf, c, len) __builtin_memset((buf), (c), (len)) void *memcpy(void * _Nonnull to, const void * _Nonnull from, size_t len); -#define memcpy(to, from, len) __builtin_memcpy((to), (from), (len)) void *memmove(void * _Nonnull dest, const void * _Nonnull src, size_t n); -#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n)) int memcmp(const void *b1, const void *b2, size_t len); + +#ifdef KCSAN +void *kcsan_memset(void *, int, size_t); +void *kcsan_memcpy(void *, const void *, size_t); +void *kcsan_memmove(void *, const void *, size_t); +int kcsan_memcmp(const void *, const void *, size_t); +#define bcopy(from, to, len) kcsan_memmove((to), (from), (len)) +#define bzero(buf, len) kcsan_memset((buf), 0, (len)) +#define bcmp(b1, b2, len) kcsan_memcmp((b1), (b2), (len)) +#define memset(buf, c, len) kcsan_memset((buf), (c), (len)) +#define memcpy(to, from, len) kcsan_memcpy((to), (from), (len)) +#define memmove(dest, src, n) kcsan_memmove((dest), (src), (n)) +#define memcmp(b1, b2, len) kcsan_memcmp((b1), (b2), (len)) +#else +#define bcopy(from, to, len) __builtin_memmove((to), (from), (len)) +#define bzero(buf, len) __builtin_memset((buf), 0, (len)) +#define bcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len)) +#define memset(buf, c, len) __builtin_memset((buf), (c), (len)) +#define memcpy(to, from, len) __builtin_memcpy((to), (from), (len)) +#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n)) #define memcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len)) +#endif void *memset_early(void * _Nonnull buf, int c, size_t len); #define bzero_early(buf, len) memset_early((buf), 0, (len)) @@ -352,6 +367,17 @@ int copyout_nofault(const void * _Nonnull __restrict kaddr, void * __restrict udaddr, size_t len); +#ifdef KCSAN +int kcsan_copystr(const void *, void *, size_t, size_t *); +int kcsan_copyin(const void *, void *, size_t); +int kcsan_copyinstr(const void *, void *, size_t, size_t *); +int kcsan_copyout(const void *, void *, size_t); +#define copystr(kf, k, l, lc) kcsan_copystr((kf), (k), (l), (lc)) +#define copyin(u, k, l) kcsan_copyin((u), (k), (l)) +#define copyinstr(u, k, l, lc) kcsan_copyinstr((u), (k), (l), (lc)) +#define copyout(k, u, l) kcsan_copyout((k), (u), (l)) +#endif + int fubyte(volatile const void *base); long fuword(volatile const void *base); int fuword16(volatile const void *base); Index: sys/x86/include/bus.h =================================================================== --- sys/x86/include/bus.h +++ sys/x86/include/bus.h @@ -101,6 +101,8 @@ #include #include +#include + #ifndef __GNUCLIKE_ASM #error "no assembler code for your compiler" @@ -131,6 +133,13 @@ #define BUS_SPACE_INVALID_DATA (~0) #define BUS_SPACE_UNRESTRICTED (~0) +#define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */ +#define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */ + +#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#include +#else + /* * Map a region of device bus space into CPU virtual address space. */ @@ -992,9 +1001,6 @@ * prevent reordering by the compiler; all Intel x86 processors currently * retire operations outside the CPU in program order. */ -#define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */ -#define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */ - static __inline void bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused, bus_size_t offset __unused, bus_size_t len __unused, int flags) @@ -1022,8 +1028,6 @@ #define outl(a, b) compiler_error #endif -#include - /* * Stream accesses are the same as normal accesses on x86; there are no * supported bus systems with an endianess different from the host one. @@ -1088,4 +1092,6 @@ #define bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c) \ bus_space_copy_region_4((t), (h1), (o1), (h2), (o2), (c)) +#endif /* KCSAN && !KCSAN_RUNTIME */ + #endif /* _X86_BUS_H_ */ Index: sys/x86/x86/bus_machdep.c =================================================================== --- sys/x86/x86/bus_machdep.c +++ sys/x86/x86/bus_machdep.c @@ -24,6 +24,8 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#define KCSAN_RUNTIME + #include __FBSDID("$FreeBSD$"); Index: sys/x86/x86/mp_x86.c =================================================================== --- sys/x86/x86/mp_x86.c +++ sys/x86/x86/mp_x86.c @@ -41,6 +41,7 @@ #include #include /* cngetc() */ #include +#include #ifdef GPROF #include #endif @@ -1080,6 +1081,8 @@ cpu_initclocks_ap(); #endif + kcsan_cpu_init(cpuid); + sched_throw(NULL); panic("scheduler returned us to %s", __func__);