Page MenuHomeFreeBSD
Files F142743342

D22315.id64175.diff
No OneTemporary
Actions

D22315.id64175.diff
View Options

Index: contrib/llvm/tools/clang/lib/Driver/ToolChains/FreeBSD.cpp
===================================================================
--- contrib/llvm/tools/clang/lib/Driver/ToolChains/FreeBSD.cpp
+++ contrib/llvm/tools/clang/lib/Driver/ToolChains/FreeBSD.cpp
@@ -421,10 +421,10 @@
Res |= SanitizerKind::Address;
Res |= SanitizerKind::PointerCompare;
Res |= SanitizerKind::PointerSubtract;
+ Res |= SanitizerKind::Thread;
Res |= SanitizerKind::Vptr;
if (IsX86_64 || IsMIPS64) {
Res |= SanitizerKind::Leak;
- Res |= SanitizerKind::Thread;
}
if (IsX86 || IsX86_64) {
Res |= SanitizerKind::Function;
Index: sys/amd64/amd64/copyout.c
===================================================================
--- sys/amd64/amd64/copyout.c
+++ sys/amd64/amd64/copyout.c
@@ -146,6 +146,10 @@
casueword_smap : casueword_nosmap);
}
+#undef copyinstr
+#undef copyin
+#undef copyout
+
int copyinstr_nosmap(const void *udaddr, void *kaddr, size_t len,
size_t *lencopied);
int copyinstr_smap(const void *udaddr, void *kaddr, size_t len,
Index: sys/amd64/amd64/machdep.c
===================================================================
--- sys/amd64/amd64/machdep.c
+++ sys/amd64/amd64/machdep.c
@@ -64,6 +64,7 @@
#include <sys/callout.h>
#include <sys/cons.h>
#include <sys/cpu.h>
+#include <sys/csan.h>
#include <sys/efi.h>
#include <sys/eventhandler.h>
#include <sys/exec.h>
@@ -1890,6 +1891,8 @@
cpu_probe_amdc1e();
+ kcsan_cpu_init(0);
+
#ifdef FDT
x86_init_fdt();
#endif
@@ -2713,6 +2716,40 @@
void *memset_std(void *buf, int c, size_t len);
void *memset_erms(void *buf, int c, size_t len);
+void *memmove_std(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+
+#ifdef KCSAN
+/*
+ * These fail to build as ifuncs when used with KCSAN.
+ */
+void *
+memset(void *buf, int c, size_t len)
+{
+
+ return memset_std(buf, c, len);
+}
+
+void *
+memmove(void * _Nonnull dst, const void * _Nonnull src, size_t len)
+{
+
+ return memmove_std(dst, src, len);
+}
+
+void *
+memcpy(void * _Nonnull dst, const void * _Nonnull src, size_t len)
+{
+
+ return memcpy_std(dst, src, len);
+}
+#else
DEFINE_IFUNC(, void *, memset, (void *, int, size_t))
{
@@ -2720,10 +2757,6 @@
memset_erms : memset_std);
}
-void *memmove_std(void * _Nonnull dst, const void * _Nonnull src,
- size_t len);
-void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src,
- size_t len);
DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull,
size_t))
{
@@ -2732,16 +2765,13 @@
memmove_erms : memmove_std);
}
-void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src,
- size_t len);
-void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src,
- size_t len);
DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull,size_t))
{
return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
memcpy_erms : memcpy_std);
}
+#endif
void pagezero_std(void *addr);
void pagezero_erms(void *addr);
Index: sys/amd64/conf/GENERIC
===================================================================
--- sys/amd64/conf/GENERIC
+++ sys/amd64/conf/GENERIC
@@ -106,6 +106,7 @@
#options KCOV # Kernel Coverage Sanitizer
# Warning: KUBSAN can result in a kernel too large for loader to load
#options KUBSAN # Kernel Undefined Behavior Sanitizer
+options KCSAN # Kernel Concurrency Sanitizer
# Kernel dump features.
options EKCD # Support for encrypted kernel dumps
Index: sys/amd64/include/atomic.h
===================================================================
--- sys/amd64/include/atomic.h
+++ sys/amd64/include/atomic.h
@@ -57,6 +57,20 @@
#define wmb() __asm __volatile("sfence;" : : : "memory")
#define rmb() __asm __volatile("lfence;" : : : "memory")
+#ifdef _KERNEL
+/*
+ * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
+ *
+ * The open-coded number is used instead of the symbolic expression to
+ * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
+ * An assertion in amd64/vm_machdep.c ensures that the value is correct.
+ */
+#define OFFSETOF_MONITORBUF 0x100
+#endif
+
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_atomic.h>
+#else
#include <sys/atomic_common.h>
/*
@@ -345,15 +359,6 @@
#if defined(_KERNEL)
-/*
- * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
- *
- * The open-coded number is used instead of the symbolic expression to
- * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
- * An assertion in amd64/vm_machdep.c ensures that the value is correct.
- */
-#define OFFSETOF_MONITORBUF 0x100
-
#if defined(SMP) || defined(KLD_MODULE)
static __inline void
__storeload_barrier(void)
@@ -679,4 +684,6 @@
#endif /* !WANT_FUNCTIONS */
+#endif /* KCSAN && !KCSAN_RUNTIME */
+
#endif /* !_MACHINE_ATOMIC_H_ */
Index: sys/amd64/include/csan.h
===================================================================
--- /dev/null
+++ sys/amd64/include/csan.h
@@ -0,0 +1,65 @@
+/* $NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $ */
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <machine/cpufunc.h>
+#include <machine/stack.h>
+#include <machine/vmparam.h>
+
+static inline bool
+kcsan_md_is_avail(void)
+{
+ return true;
+}
+
+static inline void
+kcsan_md_disable_intrs(uint64_t *state)
+{
+
+ *state = intr_disable();
+}
+
+static inline void
+kcsan_md_enable_intrs(uint64_t *state)
+{
+
+ intr_restore(*state);
+}
+
+static inline void
+kcsan_md_delay(uint64_t us)
+{
+ DELAY(us);
+}
+
+static void
+kcsan_md_unwind(void)
+{
+}
Index: sys/arm64/arm64/bus_machdep.c
===================================================================
--- sys/arm64/arm64/bus_machdep.c
+++ sys/arm64/arm64/bus_machdep.c
@@ -25,6 +25,8 @@
*
*/
+#define KCSAN_RUNTIME
+
#include "opt_platform.h"
#include <sys/param.h>
Index: sys/arm64/arm64/copystr.c
===================================================================
--- sys/arm64/arm64/copystr.c
+++ sys/arm64/arm64/copystr.c
@@ -32,7 +32,7 @@
#include <sys/systm.h>
int
-copystr(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len,
+(copystr)(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len,
size_t * __restrict lencopied)
{
const char *src;
Index: sys/arm64/arm64/machdep.c
===================================================================
--- sys/arm64/arm64/machdep.c
+++ sys/arm64/arm64/machdep.c
@@ -38,6 +38,7 @@
#include <sys/bus.h>
#include <sys/cons.h>
#include <sys/cpu.h>
+#include <sys/csan.h>
#include <sys/devmap.h>
#include <sys/efi.h>
#include <sys/exec.h>
@@ -1174,6 +1175,8 @@
kdb_init();
pan_enable();
+ kcsan_cpu_init(0);
+
env = kern_getenv("kernelname");
if (env != NULL)
strlcpy(kernelname, env, sizeof(kernelname));
Index: sys/arm64/arm64/mp_machdep.c
===================================================================
--- sys/arm64/arm64/mp_machdep.c
+++ sys/arm64/arm64/mp_machdep.c
@@ -40,6 +40,7 @@
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/cpu.h>
+#include <sys/csan.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/malloc.h>
@@ -253,6 +254,8 @@
mtx_unlock_spin(&ap_boot_mtx);
+ kcsan_cpu_init(cpu);
+
/* Enter the scheduler */
sched_throw(NULL);
Index: sys/arm64/conf/GENERIC
===================================================================
--- sys/arm64/conf/GENERIC
+++ sys/arm64/conf/GENERIC
@@ -25,7 +25,7 @@
makeoptions WITH_CTF=1 # Run ctfconvert(1) for DTrace support
options SCHED_ULE # ULE scheduler
-options NUMA # Non-Uniform Memory Architecture support
+#options NUMA # Non-Uniform Memory Architecture support
options PREEMPTION # Enable kernel thread preemption
options VIMAGE # Subsystem virtualization, e.g. VNET
options INET # InterNETworking
@@ -99,6 +99,7 @@
#options KCOV # Kernel Coverage Sanitizer
# Warning: KUBSAN can result in a kernel too large for loader to load
#options KUBSAN # Kernel Undefined Behavior Sanitizer
+options KCSAN # Kernel Concurrency Sanitizer
# Kernel dump features.
options EKCD # Support for encrypted kernel dumps
Index: sys/arm64/include/atomic.h
===================================================================
--- sys/arm64/include/atomic.h
+++ sys/arm64/include/atomic.h
@@ -29,8 +29,6 @@
#ifndef _MACHINE_ATOMIC_H_
#define _MACHINE_ATOMIC_H_
-#include <sys/atomic_common.h>
-
#define isb() __asm __volatile("isb" : : : "memory")
/*
@@ -55,6 +53,12 @@
#define wmb() dmb(st) /* Full system memory barrier store */
#define rmb() dmb(ld) /* Full system memory barrier load */
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_atomic.h>
+#else
+
+#include <sys/atomic_common.h>
+
#define ATOMIC_OP(op, asm_op, bar, a, l) \
static __inline void \
atomic_##op##_##bar##8(volatile uint8_t *p, uint8_t val) \
@@ -601,5 +605,7 @@
dmb(sy);
}
+#endif /* KCSAN && !KCSAN_RUNTIME */
+
#endif /* _MACHINE_ATOMIC_H_ */
Index: sys/arm64/include/bus.h
===================================================================
--- sys/arm64/include/bus.h
+++ sys/arm64/include/bus.h
@@ -89,6 +89,9 @@
#define BUS_SPACE_BARRIER_READ 0x01
#define BUS_SPACE_BARRIER_WRITE 0x02
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_bus.h>
+#else
struct bus_space {
/* cookie */
@@ -322,7 +325,7 @@
#define bus_space_read_stream_1(t, h, o) __bs_rs_s(1,(t), (h), (o))
#define bus_space_read_stream_2(t, h, o) __bs_rs_s(2,(t), (h), (o))
#define bus_space_read_stream_4(t, h, o) __bs_rs_s(4,(t), (h), (o))
-#define bus_space_read_stream_8(t, h, o) __bs_rs_s(8,8,(t),(h),(o))
+#define bus_space_read_stream_8(t, h, o) __bs_rs_s(8,(t), (h),(o))
/*
* Bus read multiple operations.
@@ -464,6 +467,8 @@
#define bus_space_copy_region_8(t, h1, o1, h2, o2, c) \
__bs_copy(8, t, h1, o1, h2, o2, c)
+#endif
+
#include <machine/bus_dma.h>
#endif /* _MACHINE_BUS_H_ */
Index: sys/arm64/include/csan.h
===================================================================
--- /dev/null
+++ sys/arm64/include/csan.h
@@ -0,0 +1,102 @@
+/* $NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $ */
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <machine/cpufunc.h>
+#include <machine/stack.h>
+#include <machine/vmparam.h>
+
+static inline bool
+kcsan_md_is_avail(void)
+{
+ return true;
+}
+
+static inline void
+kcsan_md_disable_intrs(uint64_t *state)
+{
+
+ *state = intr_disable();
+}
+
+static inline void
+kcsan_md_enable_intrs(uint64_t *state)
+{
+
+ intr_restore(*state);
+}
+
+static inline void
+kcsan_md_delay(uint64_t us)
+{
+ DELAY(us);
+}
+
+static void
+kcsan_md_unwind(void)
+{
+#ifdef DDB
+ c_db_sym_t sym;
+ db_expr_t offset;
+ const char *symname;
+#endif
+ struct unwind_state frame;
+ uint64_t sp;
+ int nsym;
+
+ __asm __volatile("mov %0, sp" : "=&r" (sp));
+
+ frame.sp = sp;
+ frame.fp = (uint64_t)__builtin_frame_address(0);
+ frame.pc = (uint64_t)kcsan_md_unwind;
+ nsym = 0;
+
+ while (1) {
+ unwind_frame(&frame);
+ if (!INKERNEL((vm_offset_t)frame.fp) ||
+ !INKERNEL((vm_offset_t)frame.pc))
+ break;
+
+#ifdef DDB
+ sym = db_search_symbol((vm_offset_t)frame.pc, DB_STGY_PROC,
+ &offset);
+ db_symbol_values(sym, &symname, NULL);
+ printf("#%d %p in %s+%#lx\n", nsym, (void *)frame.pc,
+ symname, offset);
+#else
+ printf("#%d %p\n", nsym, (void *)frame.pc);
+#endif
+ nsym++;
+
+ if (nsym >= 15) {
+ break;
+ }
+ }
+}
Index: sys/conf/files
===================================================================
--- sys/conf/files
+++ sys/conf/files
@@ -3799,6 +3799,8 @@
kern/subr_coverage.c optional coverage \
compile-with "${NORMAL_C:N-fsanitize*}"
kern/subr_counter.c standard
+kern/subr_csan.c optional kcsan \
+ compile-with "${NORMAL_C:N-fsanitize*}"
kern/subr_devstat.c standard
kern/subr_disk.c standard
kern/subr_early.c standard
Index: sys/conf/files.arm64
===================================================================
--- sys/conf/files.arm64
+++ sys/conf/files.arm64
@@ -283,8 +283,10 @@
kern/subr_devmap.c standard
kern/subr_intr.c optional intrng
libkern/bcmp.c standard
-libkern/memcmp.c standard
-libkern/memset.c standard
+libkern/memcmp.c standard \
+ compile-with "${NORMAL_C:N-fsanitize*}"
+libkern/memset.c standard \
+ compile-with "${NORMAL_C:N-fsanitize*}"
libkern/arm64/crc32c_armv8.S standard
cddl/dev/dtrace/aarch64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}"
cddl/dev/dtrace/aarch64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}"
Index: sys/conf/kern.post.mk
===================================================================
--- sys/conf/kern.post.mk
+++ sys/conf/kern.post.mk
@@ -38,6 +38,10 @@
MKMODULESENV+= WITH_EXTRA_TCP_STACKS="${WITH_EXTRA_TCP_STACKS}"
.endif
+.if defined(KCSAN_ENABLED)
+MKMODULESENV+= KCSAN_ENABLED="yes"
+.endif
+
.if defined(SAN_CFLAGS)
MKMODULESENV+= SAN_CFLAGS="${SAN_CFLAGS}"
.endif
Index: sys/conf/kern.pre.mk
===================================================================
--- sys/conf/kern.pre.mk
+++ sys/conf/kern.pre.mk
@@ -117,6 +117,11 @@
.endif
DEFINED_PROF= ${PROF}
+KCSAN_ENABLED!= grep KCSAN opt_global.h || true ; echo
+.if !empty(KCSAN_ENABLED)
+SAN_CFLAGS+= -fsanitize=thread
+.endif
+
KUBSAN_ENABLED!= grep KUBSAN opt_global.h || true ; echo
.if !empty(KUBSAN_ENABLED)
SAN_CFLAGS+= -fsanitize=undefined
Index: sys/conf/options
===================================================================
--- sys/conf/options
+++ sys/conf/options
@@ -230,6 +230,7 @@
# Sanitizers
COVERAGE opt_global.h
KCOV
+KCSAN opt_global.h
KUBSAN opt_global.h
# POSIX kernel options
Index: sys/kern/subr_csan.c
===================================================================
--- /dev/null
+++ sys/kern/subr_csan.c
@@ -0,0 +1,859 @@
+/* $NetBSD: subr_csan.c,v 1.2 2019/11/06 06:57:22 maxv Exp $ */
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ * Copyright (c) 2019 Andrew Turner
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define KCSAN_RUNTIME
+
+#include "opt_ddb.h"
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/conf.h>
+#include <sys/cpu.h>
+#include <sys/csan.h>
+#include <sys/proc.h>
+#include <sys/smp.h>
+#include <sys/systm.h>
+
+#include <ddb/ddb.h>
+#include <ddb/db_sym.h>
+
+#ifdef KCSAN_PANIC
+#define REPORT panic
+#else
+#define REPORT printf
+#endif
+
+typedef struct {
+ uintptr_t addr;
+ uint32_t size;
+ bool write:1;
+ bool atomic:1;
+ uintptr_t pc;
+ struct thread *td;
+} csan_cell_t;
+
+typedef struct {
+ bool inited;
+ uint32_t cnt;
+ csan_cell_t cell;
+} csan_cpu_t;
+
+static csan_cpu_t kcsan_cpus[MAXCPU];
+static bool kcsan_enabled __read_mostly;
+
+#define __RET_ADDR (uintptr_t)__builtin_return_address(0)
+
+#define KCSAN_NACCESSES 1024
+#define KCSAN_DELAY 10 /* 10 microseconds */
+
+/* -------------------------------------------------------------------------- */
+
+/* The MD code. */
+#include <machine/csan.h>
+
+/* -------------------------------------------------------------------------- */
+
+static void
+kcsan_enable(void *dummy __unused)
+{
+
+ printf("Enabling KCSCAN, expect reduced performance.\n");
+ kcsan_enabled = true;
+}
+SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL);
+
+void
+kcsan_cpu_init(u_int cpu)
+{
+ kcsan_cpus[cpu].inited = true;
+}
+
+/* -------------------------------------------------------------------------- */
+
+static inline void
+kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu)
+{
+ const char *newsym, *oldsym;
+#ifdef DDB
+ c_db_sym_t sym;
+ db_expr_t offset;
+
+ sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset);
+ db_symbol_values(sym, &newsym, NULL);
+
+ sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset);
+ db_symbol_values(sym, &oldsym, NULL);
+#else
+ newsym = "";
+ oldsym = "";
+#endif
+ REPORT("CSan: Racy Access "
+ "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] "
+ "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n",
+ newcpu,
+ (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
+ (void *)new->addr, new->size, (void *)new->pc, newsym,
+ oldcpu,
+ (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"),
+ (void *)old->addr, old->size, (void *)old->pc, oldsym);
+ kcsan_md_unwind();
+}
+
+static inline bool
+kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old)
+{
+ if (new->write && !new->atomic)
+ return false;
+ if (old->write && !old->atomic)
+ return false;
+ return true;
+}
+
+static inline void
+kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc)
+{
+ csan_cell_t old, new;
+ csan_cpu_t *cpu;
+ struct thread *td;
+ uint64_t intr;
+ size_t i;
+
+ if (__predict_false(!kcsan_enabled))
+ return;
+
+ td = curthread;
+
+ new.addr = addr;
+ new.size = size;
+ new.write = write;
+ new.atomic = atomic;
+ new.pc = pc;
+ new.td = td;
+
+ CPU_FOREACH(i) {
+ __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
+
+ /* Assume we can't race with ourself */
+ if (old.td == td)
+ continue;
+ if (old.addr + old.size <= new.addr)
+ continue;
+ if (new.addr + new.size <= old.addr)
+ continue;
+ if (__predict_true(!old.write && !new.write))
+ continue;
+ if (__predict_true(kcsan_access_is_atomic(&new, &old)))
+ continue;
+
+ kcsan_report(&new, PCPU_GET(cpuid), &old, i);
+ break;
+ }
+
+ if (__predict_false(!kcsan_md_is_avail()))
+ return;
+
+ kcsan_md_disable_intrs(&intr);
+
+ cpu = &kcsan_cpus[PCPU_GET(cpuid)];
+ if (__predict_false(!cpu->inited))
+ goto out;
+ cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
+ if (__predict_true(cpu->cnt != 0))
+ goto out;
+
+ __builtin_memcpy(&cpu->cell, &new, sizeof(new));
+ kcsan_md_delay(KCSAN_DELAY);
+ __builtin_memset(&cpu->cell, 0, sizeof(new));
+
+out:
+ kcsan_md_enable_intrs(&intr);
+}
+
+#define CSAN_READ(size) \
+ void __tsan_read##size(uintptr_t); \
+ void __tsan_read##size(uintptr_t addr) \
+ { \
+ kcsan_access(addr, size, false, false, __RET_ADDR); \
+ } \
+ void __tsan_unaligned_read##size(uintptr_t); \
+ void __tsan_unaligned_read##size(uintptr_t addr) \
+ { \
+ kcsan_access(addr, size, false, false, __RET_ADDR); \
+ }
+
+CSAN_READ(1)
+CSAN_READ(2)
+CSAN_READ(4)
+CSAN_READ(8)
+CSAN_READ(16)
+
+#define CSAN_WRITE(size) \
+ void __tsan_write##size(uintptr_t); \
+ void __tsan_write##size(uintptr_t addr) \
+ { \
+ kcsan_access(addr, size, true, false, __RET_ADDR); \
+ } \
+ void __tsan_unaligned_write##size(uintptr_t); \
+ void __tsan_unaligned_write##size(uintptr_t addr) \
+ { \
+ kcsan_access(addr, size, true, false, __RET_ADDR); \
+ }
+
+CSAN_WRITE(1)
+CSAN_WRITE(2)
+CSAN_WRITE(4)
+CSAN_WRITE(8)
+CSAN_WRITE(16)
+
+void __tsan_read_range(uintptr_t, size_t);
+void __tsan_write_range(uintptr_t, size_t);
+
+void
+__tsan_read_range(uintptr_t addr, size_t size)
+{
+ kcsan_access(addr, size, false, false, __RET_ADDR);
+}
+
+void
+__tsan_write_range(uintptr_t addr, size_t size)
+{
+ kcsan_access(addr, size, true, false, __RET_ADDR);
+}
+
+void __tsan_init(void);
+void __tsan_func_entry(void *);
+void __tsan_func_exit(void);
+
+void
+__tsan_init(void)
+{
+}
+
+void
+__tsan_func_entry(void *call_pc)
+{
+}
+
+void
+__tsan_func_exit(void)
+{
+}
+
+/* -------------------------------------------------------------------------- */
+
+void *
+kcsan_memcpy(void *dst, const void *src, size_t len)
+{
+ kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
+ kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
+ return __builtin_memcpy(dst, src, len);
+}
+
+int
+kcsan_memcmp(const void *b1, const void *b2, size_t len)
+{
+ kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR);
+ kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR);
+ return __builtin_memcmp(b1, b2, len);
+}
+
+void *
+kcsan_memset(void *b, int c, size_t len)
+{
+ kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR);
+ return __builtin_memset(b, c, len);
+}
+
+void *
+kcsan_memmove(void *dst, const void *src, size_t len)
+{
+ kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
+ kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
+ return __builtin_memmove(dst, src, len);
+}
+
+char *
+kcsan_strcpy(char *dst, const char *src)
+{
+ char *save = dst;
+
+ while (1) {
+ kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR);
+ kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR);
+ *dst = *src;
+ if (*src == '\0')
+ break;
+ src++, dst++;
+ }
+
+ return save;
+}
+
+int
+kcsan_strcmp(const char *s1, const char *s2)
+{
+ while (1) {
+ kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR);
+ kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR);
+ if (*s1 != *s2)
+ break;
+ if (*s1 == '\0')
+ return 0;
+ s1++, s2++;
+ }
+
+ return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
+}
+
+size_t
+kcsan_strlen(const char *str)
+{
+ const char *s;
+
+ s = str;
+ while (1) {
+ kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR);
+ if (*s == '\0')
+ break;
+ s++;
+ }
+
+ return (s - str);
+}
+
+#undef copystr
+#undef copyin
+#undef copyin_nofault
+#undef copyinstr
+#undef copyout
+#undef copyout_nofault
+
+int
+kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done)
+{
+ kcsan_access((uintptr_t)kdaddr, len, true, false, __RET_ADDR);
+ return copystr(kfaddr, kdaddr, len, done);
+}
+
+int
+kcsan_copyin(const void *uaddr, void *kaddr, size_t len)
+{
+ kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
+ return copyin(uaddr, kaddr, len);
+}
+
+int
+kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
+{
+ kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
+ return copyinstr(uaddr, kaddr, len, done);
+}
+
+int
+kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
+{
+ kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
+ return copyout(kaddr, uaddr, len);
+}
+
+/* -------------------------------------------------------------------------- */
+
+#include <machine/atomic.h>
+#include <sys/_cscan_atomic.h>
+
+#define _CSAN_ATOMIC_FUNC_ADD(name, type) \
+ void kcsan_atomic_add_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ atomic_add_##name(ptr, val); \
+ }
+
+#define CSAN_ATOMIC_FUNC_ADD(name, type) \
+ _CSAN_ATOMIC_FUNC_ADD(name, type) \
+ _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_ADD(rel_##name, type)
+
+#define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
+ void kcsan_atomic_clear_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ atomic_clear_##name(ptr, val); \
+ }
+
+#define CSAN_ATOMIC_FUNC_CLEAR(name, type) \
+ _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
+ _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
+
+#define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
+ int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \
+ type val2) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_cmpset_##name(ptr, val1, val2)); \
+ }
+
+#define CSAN_ATOMIC_FUNC_CMPSET(name, type) \
+ _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
+ _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
+
+#define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
+ int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \
+ type val2) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_fcmpset_##name(ptr, val1, val2)); \
+ }
+
+#define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
+ _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
+ _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
+
+#define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \
+ type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_fetchadd_##name(ptr, val)); \
+ }
+
+#define _CSAN_ATOMIC_FUNC_LOAD(name, type) \
+ type kcsan_atomic_load_##name(volatile type *ptr) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \
+ __RET_ADDR); \
+ return (atomic_load_##name(ptr)); \
+ }
+
+#define CSAN_ATOMIC_FUNC_LOAD(name, type) \
+ _CSAN_ATOMIC_FUNC_LOAD(name, type) \
+ _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \
+
+#define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
+ type kcsan_atomic_readandclear_##name(volatile type *ptr) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_readandclear_##name(ptr)); \
+ }
+
+#define _CSAN_ATOMIC_FUNC_SET(name, type) \
+ void kcsan_atomic_set_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ atomic_set_##name(ptr, val); \
+ }
+
+#define CSAN_ATOMIC_FUNC_SET(name, type) \
+ _CSAN_ATOMIC_FUNC_SET(name, type) \
+ _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_SET(rel_##name, type)
+
+#define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
+ void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ atomic_subtract_##name(ptr, val); \
+ }
+
+
+#define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
+ _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
+ _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
+
+#define _CSAN_ATOMIC_FUNC_STORE(name, type) \
+ void kcsan_atomic_store_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ atomic_store_##name(ptr, val); \
+ }
+
+#define CSAN_ATOMIC_FUNC_STORE(name, type) \
+ _CSAN_ATOMIC_FUNC_STORE(name, type) \
+ _CSAN_ATOMIC_FUNC_STORE(rel_##name, type)
+
+#define CSAN_ATOMIC_FUNC_SWAP(name, type) \
+ type kcsan_atomic_swap_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return(atomic_swap_##name(ptr, val)); \
+ }
+
+#define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
+ int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return(atomic_testandclear_##name(ptr, val)); \
+ }
+
+#define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \
+ int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_testandset_##name(ptr, val)); \
+ }
+
+
+CSAN_ATOMIC_FUNC_ADD(8, uint8_t)
+CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t)
+CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t)
+CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t)
+_CSAN_ATOMIC_FUNC_LOAD(8, uint8_t)
+CSAN_ATOMIC_FUNC_SET(8, uint8_t)
+CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t)
+_CSAN_ATOMIC_FUNC_STORE(8, uint8_t)
+#if 0
+CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t)
+CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t)
+CSAN_ATOMIC_FUNC_SWAP(8, uint8_t)
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t)
+CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t)
+#endif
+
+CSAN_ATOMIC_FUNC_ADD(16, uint16_t)
+CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t)
+CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t)
+CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t)
+#if defined(__aarch64__)
+_CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
+#else
+CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
+#endif
+CSAN_ATOMIC_FUNC_SET(16, uint16_t)
+CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t)
+_CSAN_ATOMIC_FUNC_STORE(16, uint16_t)
+#if 0
+CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t)
+CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t)
+CSAN_ATOMIC_FUNC_SWAP(16, uint16_t)
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t)
+CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t)
+#endif
+
+CSAN_ATOMIC_FUNC_ADD(32, uint32_t)
+CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t)
+CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t)
+CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t)
+CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t)
+CSAN_ATOMIC_FUNC_LOAD(32, uint32_t)
+CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t)
+CSAN_ATOMIC_FUNC_SET(32, uint32_t)
+CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t)
+CSAN_ATOMIC_FUNC_STORE(32, uint32_t)
+CSAN_ATOMIC_FUNC_SWAP(32, uint32_t)
+#if !defined(__aarch64__)
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t)
+CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t)
+#endif
+
+CSAN_ATOMIC_FUNC_ADD(64, uint64_t)
+CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t)
+CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t)
+CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t)
+CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t)
+CSAN_ATOMIC_FUNC_LOAD(64, uint64_t)
+CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t)
+CSAN_ATOMIC_FUNC_SET(64, uint64_t)
+CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t)
+CSAN_ATOMIC_FUNC_STORE(64, uint64_t)
+CSAN_ATOMIC_FUNC_SWAP(64, uint64_t)
+#if !defined(__aarch64__)
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t)
+CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t)
+#endif
+
+CSAN_ATOMIC_FUNC_ADD(int, u_int)
+CSAN_ATOMIC_FUNC_CLEAR(int, u_int)
+CSAN_ATOMIC_FUNC_CMPSET(int, u_int)
+CSAN_ATOMIC_FUNC_FCMPSET(int, u_int)
+CSAN_ATOMIC_FUNC_FETCHADD(int, u_int)
+CSAN_ATOMIC_FUNC_LOAD(int, u_int)
+CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int)
+CSAN_ATOMIC_FUNC_SET(int, u_int)
+CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int)
+CSAN_ATOMIC_FUNC_STORE(int, u_int)
+CSAN_ATOMIC_FUNC_SWAP(int, u_int)
+#if !defined(__aarch64__)
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int)
+CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int)
+#endif
+
+CSAN_ATOMIC_FUNC_ADD(long, u_long)
+CSAN_ATOMIC_FUNC_CLEAR(long, u_long)
+CSAN_ATOMIC_FUNC_CMPSET(long, u_long)
+CSAN_ATOMIC_FUNC_FCMPSET(long, u_long)
+CSAN_ATOMIC_FUNC_FETCHADD(long, u_long)
+CSAN_ATOMIC_FUNC_LOAD(long, u_long)
+CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long)
+CSAN_ATOMIC_FUNC_SET(long, u_long)
+CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long)
+CSAN_ATOMIC_FUNC_STORE(long, u_long)
+CSAN_ATOMIC_FUNC_SWAP(long, u_long)
+#if !defined(__aarch64__)
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long)
+CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long)
+#endif
+
+CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t)
+#if !defined(__amd64__)
+CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t)
+#endif
+CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t)
+#if 0
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t)
+#endif
+
+#define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \
+ void kcsan_atomic_thread_fence_##name(void) \
+ { \
+ atomic_thread_fence_##name(); \
+ }
+
+
+CSAN_ATOMIC_FUNC_THREAD_FENCE(acq)
+CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel)
+CSAN_ATOMIC_FUNC_THREAD_FENCE(rel)
+CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst)
+
+/* -------------------------------------------------------------------------- */
+
+#include <sys/bus.h>
+#include <machine/bus.h>
+#include <sys/_cscan_bus.h>
+
+int
+kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
+ int flags, bus_space_handle_t *handlep)
+{
+
+ return (bus_space_map(tag, hnd, size, flags, handlep));
+}
+
+void
+kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
+ bus_size_t size)
+{
+
+ bus_space_unmap(tag, hnd, size);
+}
+
+int
+kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
+ bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
+{
+
+ return (bus_space_subregion(tag, hnd, offset, size, handlep));
+}
+
+#if !defined(__amd64__)
+int
+kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start,
+ bus_addr_t reg_end, bus_size_t size, bus_size_t alignment,
+ bus_size_t boundary, int flags, bus_addr_t *addrp,
+ bus_space_handle_t *handlep)
+{
+
+ return (bus_space_alloc(tag, reg_start, reg_end, size, alignment,
+ boundary, flags, addrp, handlep));
+}
+#endif
+
+void
+kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
+ bus_size_t size)
+{
+
+ bus_space_free(tag, hnd, size);
+}
+
+void
+kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
+ bus_size_t offset, bus_size_t size, int flags)
+{
+
+ bus_space_barrier(tag, hnd, offset, size, flags);
+}
+
+#define CSAN_BUS_READ_FUNC(func, width, type) \
+ type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t offset) \
+ { \
+ return (bus_space_read##func##_##width(tag, hnd, \
+ offset)); \
+ } \
+
+#define CSAN_BUS_READ_PTR_FUNC(func, width, type) \
+ void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t size, type *buf, \
+ bus_size_t count) \
+ { \
+ kcsan_access((uintptr_t)buf, sizeof(type) * count, \
+ false, false, __RET_ADDR); \
+ bus_space_read_##func##_##width(tag, hnd, size, buf, \
+ count); \
+ }
+
+CSAN_BUS_READ_FUNC(, 1, uint8_t)
+CSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
+CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
+CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
+CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
+CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
+
+CSAN_BUS_READ_FUNC(, 2, uint16_t)
+CSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
+CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
+CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
+CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
+CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
+
+CSAN_BUS_READ_FUNC(, 4, uint32_t)
+CSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
+CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
+CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
+CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
+CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
+
+CSAN_BUS_READ_FUNC(, 8, uint64_t)
+#if defined(__aarch64__)
+CSAN_BUS_READ_FUNC(_stream, 8, uint64_t)
+CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
+CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
+CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
+CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
+#endif
+
+#define CSAN_BUS_WRITE_FUNC(func, width, type) \
+ void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t offset, type value) \
+ { \
+ bus_space_write##func##_##width(tag, hnd, offset, value); \
+ } \
+
+#define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \
+ void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t size, const type *buf, \
+ bus_size_t count) \
+ { \
+ kcsan_access((uintptr_t)buf, sizeof(type) * count, \
+ true, false, __RET_ADDR); \
+ bus_space_write_##func##_##width(tag, hnd, size, buf, \
+ count); \
+ }
+
+CSAN_BUS_WRITE_FUNC(, 1, uint8_t)
+CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
+CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
+CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
+
+CSAN_BUS_WRITE_FUNC(, 2, uint16_t)
+CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
+CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
+CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
+
+CSAN_BUS_WRITE_FUNC(, 4, uint32_t)
+CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
+CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
+CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
+
+CSAN_BUS_WRITE_FUNC(, 8, uint64_t)
+#if defined(__aarch64__)
+CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t)
+CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t)
+CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t)
+#endif
+
+#define CSAN_BUS_SET_FUNC(func, width, type) \
+ void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t offset, type value, \
+ bus_size_t count) \
+ { \
+ bus_space_set_##func##_##width(tag, hnd, offset, value, \
+ count); \
+ }
+
+CSAN_BUS_SET_FUNC(multi, 1, uint8_t)
+CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
+CSAN_BUS_SET_FUNC(region, 1, uint8_t)
+CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
+
+CSAN_BUS_SET_FUNC(multi, 2, uint16_t)
+CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
+CSAN_BUS_SET_FUNC(region, 2, uint16_t)
+CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
+
+CSAN_BUS_SET_FUNC(multi, 4, uint32_t)
+CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
+CSAN_BUS_SET_FUNC(region, 4, uint32_t)
+CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
+
+#if !defined(__amd64__)
+CSAN_BUS_SET_FUNC(multi, 8, uint64_t)
+CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t)
+CSAN_BUS_SET_FUNC(region, 8, uint64_t)
+CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t)
+#endif
+
Index: sys/kern/vfs_aio.c
===================================================================
--- sys/kern/vfs_aio.c
+++ sys/kern/vfs_aio.c
@@ -292,7 +292,7 @@
* Different ABIs provide their own operations.
*/
struct aiocb_ops {
- int (*copyin)(struct aiocb *ujob, struct aiocb *kjob);
+ int (*aio_copyin)(struct aiocb *ujob, struct aiocb *kjob);
long (*fetch_status)(struct aiocb *ujob);
long (*fetch_error)(struct aiocb *ujob);
int (*store_status)(struct aiocb *ujob, long status);
@@ -1420,7 +1420,7 @@
}
static struct aiocb_ops aiocb_ops = {
- .copyin = aiocb_copyin,
+ .aio_copyin = aiocb_copyin,
.fetch_status = aiocb_fetch_status,
.fetch_error = aiocb_fetch_error,
.store_status = aiocb_store_status,
@@ -1431,7 +1431,7 @@
#ifdef COMPAT_FREEBSD6
static struct aiocb_ops aiocb_ops_osigevent = {
- .copyin = aiocb_copyin_old_sigevent,
+ .aio_copyin = aiocb_copyin_old_sigevent,
.fetch_status = aiocb_fetch_status,
.fetch_error = aiocb_fetch_error,
.store_status = aiocb_store_status,
@@ -1478,7 +1478,7 @@
job = uma_zalloc(aiocb_zone, M_WAITOK | M_ZERO);
knlist_init_mtx(&job->klist, AIO_MTX(ki));
- error = ops->copyin(ujob, &job->uaiocb);
+ error = ops->aio_copyin(ujob, &job->uaiocb);
if (error) {
ops->store_error(ujob, error);
uma_zfree(aiocb_zone, job);
@@ -2743,7 +2743,7 @@
}
static struct aiocb_ops aiocb32_ops = {
- .copyin = aiocb32_copyin,
+ .aio_copyin = aiocb32_copyin,
.fetch_status = aiocb32_fetch_status,
.fetch_error = aiocb32_fetch_error,
.store_status = aiocb32_store_status,
@@ -2754,7 +2754,7 @@
#ifdef COMPAT_FREEBSD6
static struct aiocb_ops aiocb32_ops_osigevent = {
- .copyin = aiocb32_copyin_old_sigevent,
+ .aio_copyin = aiocb32_copyin_old_sigevent,
.fetch_status = aiocb32_fetch_status,
.fetch_error = aiocb32_fetch_error,
.store_status = aiocb32_store_status,
Index: sys/libkern/strcmp.c
===================================================================
--- sys/libkern/strcmp.c
+++ sys/libkern/strcmp.c
@@ -41,7 +41,7 @@
* Compare strings.
*/
int
-strcmp(const char *s1, const char *s2)
+(strcmp)(const char *s1, const char *s2)
{
while (*s1 == *s2++)
if (*s1++ == '\0')
Index: sys/libkern/strcpy.c
===================================================================
--- sys/libkern/strcpy.c
+++ sys/libkern/strcpy.c
@@ -35,7 +35,7 @@
#include <sys/libkern.h>
char *
-strcpy(char * __restrict to, const char * __restrict from)
+(strcpy)(char * __restrict to, const char * __restrict from)
{
char *save = to;
Index: sys/libkern/strlen.c
===================================================================
--- sys/libkern/strlen.c
+++ sys/libkern/strlen.c
@@ -80,7 +80,7 @@
} while (0)
size_t
-strlen(const char *str)
+(strlen)(const char *str)
{
const char *p;
const unsigned long *lp;
Index: sys/modules/Makefile
===================================================================
--- sys/modules/Makefile
+++ sys/modules/Makefile
@@ -644,11 +644,11 @@
_chromebook_platform= chromebook_platform
_cmx= cmx
_coretemp= coretemp
-.if ${MK_SOURCELESS_HOST} != "no"
+.if ${MK_SOURCELESS_HOST} != "no" && empty(KCSAN_ENABLED)
_hpt27xx= hpt27xx
.endif
_hptiop= hptiop
-.if ${MK_SOURCELESS_HOST} != "no"
+.if ${MK_SOURCELESS_HOST} != "no" && empty(KCSAN_ENABLED)
_hptmv= hptmv
_hptnr= hptnr
_hptrr= hptrr
Index: sys/sys/_cscan_atomic.h
===================================================================
--- /dev/null
+++ sys/sys/_cscan_atomic.h
@@ -0,0 +1,307 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 Andrew Turner
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SYS__CSAN_ATOMIC_H_
+#define _SYS__CSAN_ATOMIC_H_
+
+#ifndef _MACHINE_ATOMIC_H_
+#error do not include this header, use machine/atomic.h
+#endif
+
+#define KCSAN_ATOMIC_FUNC_1(op, name, type) \
+ void kcsan_atomic_##op##_##name(volatile type *, type); \
+ void kcsan_atomic_##op##_acq_##name(volatile type *, type); \
+ void kcsan_atomic_##op##_rel_##name(volatile type *, type)
+
+#define KCSAN_ATOMIC_CMPSET(name, type) \
+ int kcsan_atomic_cmpset_##name(volatile type *, type, type); \
+ int kcsan_atomic_cmpset_acq_##name(volatile type *, type, type); \
+ int kcsan_atomic_cmpset_rel_##name(volatile type *, type, type)
+
+#define KCSAN_ATOMIC_FCMPSET(name, type) \
+ int kcsan_atomic_fcmpset_##name(volatile type *, type *, type); \
+ int kcsan_atomic_fcmpset_acq_##name(volatile type *, type *, type); \
+ int kcsan_atomic_fcmpset_rel_##name(volatile type *, type *, type)
+
+#define KCSAN_ATOMIC_READ(op, name, type) \
+ type kcsan_atomic_##op##_##name(volatile type *, type)
+
+#define KCSAN_ATOMIC_READANDCLEAR(name, type) \
+ type kcsan_atomic_readandclear_##name(volatile type *)
+
+#define KCSAN_ATOMIC_LOAD(name, type) \
+ type kcsan_atomic_load_##name(volatile type *); \
+ type kcsan_atomic_load_acq_##name(volatile type *)
+
+#define KCSAN_ATOMIC_STORE(name, type) \
+ void kcsan_atomic_store_##name(volatile type *, type); \
+ void kcsan_atomic_store_rel_##name(volatile type *, type)
+
+#define KCSAN_ATOMIC_TEST(op, name, type) \
+ int kcsan_atomic_##op##_##name(volatile type *, u_int)
+
+#define KCSAN_ATOMIC_FUNCS(name, type) \
+ KCSAN_ATOMIC_FUNC_1(add, name, type); \
+ KCSAN_ATOMIC_FUNC_1(clear, name, type); \
+ KCSAN_ATOMIC_CMPSET(name, type); \
+ KCSAN_ATOMIC_FCMPSET(name, type); \
+ KCSAN_ATOMIC_READ(fetchadd, name, type); \
+ KCSAN_ATOMIC_LOAD(name, type); \
+ KCSAN_ATOMIC_READANDCLEAR(name, type); \
+ KCSAN_ATOMIC_FUNC_1(set, name, type); \
+ KCSAN_ATOMIC_FUNC_1(subtract, name, type); \
+ KCSAN_ATOMIC_STORE(name, type); \
+ KCSAN_ATOMIC_READ(swap, name, type); \
+ KCSAN_ATOMIC_TEST(testandclear, name, type); \
+ KCSAN_ATOMIC_TEST(testandset, name, type)
+
+KCSAN_ATOMIC_FUNCS(int, u_int);
+KCSAN_ATOMIC_FUNCS(long, u_long);
+KCSAN_ATOMIC_FUNCS(ptr, uintptr_t);
+KCSAN_ATOMIC_FUNCS(8, uint8_t);
+KCSAN_ATOMIC_FUNCS(16, uint16_t);
+KCSAN_ATOMIC_FUNCS(32, uint32_t);
+KCSAN_ATOMIC_FUNCS(64, uint64_t);
+
+void kcsan_atomic_thread_fence_acq(void);
+void kcsan_atomic_thread_fence_acq_rel(void);
+void kcsan_atomic_thread_fence_rel(void);
+void kcsan_atomic_thread_fence_seq_cst(void);
+
+#ifndef KCSAN_RUNTIME
+
+#define atomic_add_int kcsan_atomic_add_int
+#define atomic_add_acq_int kcsan_atomic_add_acq_int
+#define atomic_add_rel_int kcsan_atomic_add_rel_int
+#define atomic_clear_int kcsan_atomic_clear_int
+#define atomic_clear_acq_int kcsan_atomic_clear_acq_int
+#define atomic_clear_rel_int kcsan_atomic_clear_rel_int
+#define atomic_cmpset_int kcsan_atomic_cmpset_int
+#define atomic_cmpset_acq_int kcsan_atomic_cmpset_acq_int
+#define atomic_cmpset_rel_int kcsan_atomic_cmpset_rel_int
+#define atomic_fcmpset_int kcsan_atomic_fcmpset_int
+#define atomic_fcmpset_acq_int kcsan_atomic_fcmpset_acq_int
+#define atomic_fcmpset_rel_int kcsan_atomic_fcmpset_rel_int
+#define atomic_fetchadd_int kcsan_atomic_fetchadd_int
+#define atomic_load_int kcsan_atomic_load_int
+#define atomic_load_acq_int kcsan_atomic_load_acq_int
+#define atomic_readandclear_int kcsan_atomic_readandclear_int
+#define atomic_set_int kcsan_atomic_set_int
+#define atomic_set_acq_int kcsan_atomic_set_acq_int
+#define atomic_set_rel_int kcsan_atomic_set_rel_int
+#define atomic_subtract_int kcsan_atomic_subtract_int
+#define atomic_subtract_acq_int kcsan_atomic_subtract_acq_int
+#define atomic_subtract_rel_int kcsan_atomic_subtract_rel_int
+#define atomic_store_int kcsan_atomic_store_int
+#define atomic_store_rel_int kcsan_atomic_store_rel_int
+#define atomic_swap_int kcsan_atomic_swap_int
+#define atomic_testandclear_int kcsan_atomic_testandclear_int
+#define atomic_testandset_int kcsan_atomic_testandset_int
+
+#define atomic_add_long kcsan_atomic_add_long
+#define atomic_add_acq_long kcsan_atomic_add_acq_long
+#define atomic_add_rel_long kcsan_atomic_add_rel_long
+#define atomic_clear_long kcsan_atomic_clear_long
+#define atomic_clear_acq_long kcsan_atomic_clear_acq_long
+#define atomic_clear_rel_long kcsan_atomic_clear_rel_long
+#define atomic_cmpset_long kcsan_atomic_cmpset_long
+#define atomic_cmpset_acq_long kcsan_atomic_cmpset_acq_long
+#define atomic_cmpset_rel_long kcsan_atomic_cmpset_rel_long
+#define atomic_fcmpset_long kcsan_atomic_fcmpset_long
+#define atomic_fcmpset_acq_long kcsan_atomic_fcmpset_acq_long
+#define atomic_fcmpset_rel_long kcsan_atomic_fcmpset_rel_long
+#define atomic_fetchadd_long kcsan_atomic_fetchadd_long
+#define atomic_load_long kcsan_atomic_load_long
+#define atomic_load_acq_long kcsan_atomic_load_acq_long
+#define atomic_readandclear_long kcsan_atomic_readandclear_long
+#define atomic_set_long kcsan_atomic_set_long
+#define atomic_set_acq_long kcsan_atomic_set_acq_long
+#define atomic_set_rel_long kcsan_atomic_set_rel_long
+#define atomic_subtract_long kcsan_atomic_subtract_long
+#define atomic_subtract_acq_long kcsan_atomic_subtract_acq_long
+#define atomic_subtract_rel_long kcsan_atomic_subtract_rel_long
+#define atomic_store_long kcsan_atomic_store_long
+#define atomic_store_rel_long kcsan_atomic_store_rel_long
+#define atomic_swap_long kcsan_atomic_swap_long
+#define atomic_testandclear_long kcsan_atomic_testandclear_long
+#define atomic_testandset_long kcsan_atomic_testandset_long
+
+#define atomic_add_ptr kcsan_atomic_add_ptr
+#define atomic_add_acq_ptr kcsan_atomic_add_acq_ptr
+#define atomic_add_rel_ptr kcsan_atomic_add_rel_ptr
+#define atomic_clear_ptr kcsan_atomic_clear_ptr
+#define atomic_clear_acq_ptr kcsan_atomic_clear_acq_ptr
+#define atomic_clear_rel_ptr kcsan_atomic_clear_rel_ptr
+#define atomic_cmpset_ptr kcsan_atomic_cmpset_ptr
+#define atomic_cmpset_acq_ptr kcsan_atomic_cmpset_acq_ptr
+#define atomic_cmpset_rel_ptr kcsan_atomic_cmpset_rel_ptr
+#define atomic_fcmpset_ptr kcsan_atomic_fcmpset_ptr
+#define atomic_fcmpset_acq_ptr kcsan_atomic_fcmpset_acq_ptr
+#define atomic_fcmpset_rel_ptr kcsan_atomic_fcmpset_rel_ptr
+#define atomic_fetchadd_ptr kcsan_atomic_fetchadd_ptr
+#define atomic_load_ptr(x) kcsan_atomic_load_ptr((volatile uintptr_t *)(x))
+#define atomic_load_acq_ptr kcsan_atomic_load_acq_ptr
+#define atomic_readandclear_ptr kcsan_atomic_readandclear_ptr
+#define atomic_set_ptr kcsan_atomic_set_ptr
+#define atomic_set_acq_ptr kcsan_atomic_set_acq_ptr
+#define atomic_set_rel_ptr kcsan_atomic_set_rel_ptr
+#define atomic_subtract_ptr kcsan_atomic_subtract_ptr
+#define atomic_subtract_acq_ptr kcsan_atomic_subtract_acq_ptr
+#define atomic_subtract_rel_ptr kcsan_atomic_subtract_rel_ptr
+#define atomic_store_ptr kcsan_atomic_store_ptr
+#define atomic_store_rel_ptr kcsan_atomic_store_rel_ptr
+#define atomic_swap_ptr kcsan_atomic_swap_ptr
+#define atomic_testandclear_ptr kcsan_atomic_testandclear_ptr
+#define atomic_testandset_ptr kcsan_atomic_testandset_ptr
+
+#define atomic_add_8 kcsan_atomic_add_8
+#define atomic_add_acq_8 kcsan_atomic_add_acq_8
+#define atomic_add_rel_8 kcsan_atomic_add_rel_8
+#define atomic_clear_8 kcsan_atomic_clear_8
+#define atomic_clear_acq_8 kcsan_atomic_clear_acq_8
+#define atomic_clear_rel_8 kcsan_atomic_clear_rel_8
+#define atomic_cmpset_8 kcsan_atomic_cmpset_8
+#define atomic_cmpset_acq_8 kcsan_atomic_cmpset_acq_8
+#define atomic_cmpset_rel_8 kcsan_atomic_cmpset_rel_8
+#define atomic_fcmpset_8 kcsan_atomic_fcmpset_8
+#define atomic_fcmpset_acq_8 kcsan_atomic_fcmpset_acq_8
+#define atomic_fcmpset_rel_8 kcsan_atomic_fcmpset_rel_8
+#define atomic_fetchadd_8 kcsan_atomic_fetchadd_8
+#define atomic_load_8 kcsan_atomic_load_8
+#define atomic_load_acq_8 kcsan_atomic_load_acq_8
+#define atomic_readandclear_8 kcsan_atomic_readandclear_8
+#define atomic_set_8 kcsan_atomic_set_8
+#define atomic_set_acq_8 kcsan_atomic_set_acq_8
+#define atomic_set_rel_8 kcsan_atomic_set_rel_8
+#define atomic_subtract_8 kcsan_atomic_subtract_8
+#define atomic_subtract_acq_8 kcsan_atomic_subtract_acq_8
+#define atomic_subtract_rel_8 kcsan_atomic_subtract_rel_8
+#define atomic_store_8 kcsan_atomic_store_8
+#define atomic_store_rel_8 kcsan_atomic_store_rel_8
+#define atomic_swap_8 kcsan_atomic_swap_8
+#define atomic_testandclear_8 kcsan_atomic_testandclear_8
+#define atomic_testandset_8 kcsan_atomic_testandset_8
+
+#define atomic_add_16 kcsan_atomic_add_16
+#define atomic_add_acq_16 kcsan_atomic_add_acq_16
+#define atomic_add_rel_16 kcsan_atomic_add_rel_16
+#define atomic_clear_16 kcsan_atomic_clear_16
+#define atomic_clear_acq_16 kcsan_atomic_clear_acq_16
+#define atomic_clear_rel_16 kcsan_atomic_clear_rel_16
+#define atomic_cmpset_16 kcsan_atomic_cmpset_16
+#define atomic_cmpset_acq_16 kcsan_atomic_cmpset_acq_16
+#define atomic_cmpset_rel_16 kcsan_atomic_cmpset_rel_16
+#define atomic_fcmpset_16 kcsan_atomic_fcmpset_16
+#define atomic_fcmpset_acq_16 kcsan_atomic_fcmpset_acq_16
+#define atomic_fcmpset_rel_16 kcsan_atomic_fcmpset_rel_16
+#define atomic_fetchadd_16 kcsan_atomic_fetchadd_16
+#define atomic_load_16 kcsan_atomic_load_16
+#define atomic_load_acq_16 kcsan_atomic_load_acq_16
+#define atomic_readandclear_16 kcsan_atomic_readandclear_16
+#define atomic_set_16 kcsan_atomic_set_16
+#define atomic_set_acq_16 kcsan_atomic_set_acq_16
+#define atomic_set_rel_16 kcsan_atomic_set_rel_16
+#define atomic_subtract_16 kcsan_atomic_subtract_16
+#define atomic_subtract_acq_16 kcsan_atomic_subtract_acq_16
+#define atomic_subtract_rel_16 kcsan_atomic_subtract_rel_16
+#define atomic_store_16 kcsan_atomic_store_16
+#define atomic_store_rel_16 kcsan_atomic_store_rel_16
+#define atomic_swap_16 kcsan_atomic_swap_16
+#define atomic_testandclear_16 kcsan_atomic_testandclear_16
+#define atomic_testandset_16 kcsan_atomic_testandset_16
+
+#define atomic_add_32 kcsan_atomic_add_32
+#define atomic_add_acq_32 kcsan_atomic_add_acq_32
+#define atomic_add_rel_32 kcsan_atomic_add_rel_32
+#define atomic_clear_32 kcsan_atomic_clear_32
+#define atomic_clear_acq_32 kcsan_atomic_clear_acq_32
+#define atomic_clear_rel_32 kcsan_atomic_clear_rel_32
+#define atomic_cmpset_32 kcsan_atomic_cmpset_32
+#define atomic_cmpset_acq_32 kcsan_atomic_cmpset_acq_32
+#define atomic_cmpset_rel_32 kcsan_atomic_cmpset_rel_32
+#define atomic_fcmpset_32 kcsan_atomic_fcmpset_32
+#define atomic_fcmpset_acq_32 kcsan_atomic_fcmpset_acq_32
+#define atomic_fcmpset_rel_32 kcsan_atomic_fcmpset_rel_32
+#define atomic_fetchadd_32 kcsan_atomic_fetchadd_32
+#define atomic_load_32 kcsan_atomic_load_32
+#define atomic_load_acq_32 kcsan_atomic_load_acq_32
+#define atomic_readandclear_32 kcsan_atomic_readandclear_32
+#define atomic_set_32 kcsan_atomic_set_32
+#define atomic_set_acq_32 kcsan_atomic_set_acq_32
+#define atomic_set_rel_32 kcsan_atomic_set_rel_32
+#define atomic_subtract_32 kcsan_atomic_subtract_32
+#define atomic_subtract_acq_32 kcsan_atomic_subtract_acq_32
+#define atomic_subtract_rel_32 kcsan_atomic_subtract_rel_32
+#define atomic_store_32 kcsan_atomic_store_32
+#define atomic_store_rel_32 kcsan_atomic_store_rel_32
+#define atomic_swap_32 kcsan_atomic_swap_32
+#define atomic_testandclear_32 kcsan_atomic_testandclear_32
+#define atomic_testandset_32 kcsan_atomic_testandset_32
+
+#define atomic_add_64 kcsan_atomic_add_64
+#define atomic_add_acq_64 kcsan_atomic_add_acq_64
+#define atomic_add_rel_64 kcsan_atomic_add_rel_64
+#define atomic_clear_64 kcsan_atomic_clear_64
+#define atomic_clear_acq_64 kcsan_atomic_clear_acq_64
+#define atomic_clear_rel_64 kcsan_atomic_clear_rel_64
+#define atomic_cmpset_64 kcsan_atomic_cmpset_64
+#define atomic_cmpset_acq_64 kcsan_atomic_cmpset_acq_64
+#define atomic_cmpset_rel_64 kcsan_atomic_cmpset_rel_64
+#define atomic_fcmpset_64 kcsan_atomic_fcmpset_64
+#define atomic_fcmpset_acq_64 kcsan_atomic_fcmpset_acq_64
+#define atomic_fcmpset_rel_64 kcsan_atomic_fcmpset_rel_64
+#define atomic_fetchadd_64 kcsan_atomic_fetchadd_64
+#define atomic_load_64 kcsan_atomic_load_64
+#define atomic_load_acq_64 kcsan_atomic_load_acq_64
+#define atomic_readandclear_64 kcsan_atomic_readandclear_64
+#define atomic_set_64 kcsan_atomic_set_64
+#define atomic_set_acq_64 kcsan_atomic_set_acq_64
+#define atomic_set_rel_64 kcsan_atomic_set_rel_64
+#define atomic_subtract_64 kcsan_atomic_subtract_64
+#define atomic_subtract_acq_64 kcsan_atomic_subtract_acq_64
+#define atomic_subtract_rel_64 kcsan_atomic_subtract_rel_64
+#define atomic_store_64 kcsan_atomic_store_64
+#define atomic_store_rel_64 kcsan_atomic_store_rel_64
+#define atomic_swap_64 kcsan_atomic_swap_64
+#define atomic_testandclear_64 kcsan_atomic_testandclear_64
+#define atomic_testandset_64 kcsan_atomic_testandset_64
+
+#define atomic_thread_fence_acq kcsan_atomic_thread_fence_acq
+#define atomic_thread_fence_acq_rel kcsan_atomic_thread_fence_acq_rel
+#define atomic_thread_fence_rel kcsan_atomic_thread_fence_rel
+#define atomic_thread_fence_seq_cst kcsan_atomic_thread_fence_seq_cst
+
+#endif /* !KCSAN_RUNTIME */
+
+#endif /* !_SYS__CSAN_ATOMIC_H_ */
Index: sys/sys/_cscan_bus.h
===================================================================
--- /dev/null
+++ sys/sys/_cscan_bus.h
@@ -0,0 +1,194 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 Andrew Turner
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SYS__CSAN_BUS_H_
+#define _SYS__CSAN_BUS_H_
+
+#ifndef _SYS_BUS_H_
+//#error do not include this header, use sys/bus.h
+#endif
+
+#define KCSAN_BS_MULTI(rw, width, type) \
+ void kcsan_bus_space_##rw##_multi_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type *, bus_size_t); \
+ void kcsan_bus_space_##rw##_multi_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type *, bus_size_t); \
+ void kcsan_bus_space_##rw##_region_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type *, bus_size_t); \
+ void kcsan_bus_space_##rw##_region_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type *, bus_size_t)
+
+#define KCSAN_BS_READ(width, type) \
+ type kcsan_bus_space_read_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t); \
+ type kcsan_bus_space_read_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t); \
+ KCSAN_BS_MULTI(read, width, type)
+
+#define KCSAN_BS_WRITE(width, type) \
+ void kcsan_bus_space_write_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type); \
+ void kcsan_bus_space_write_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type); \
+ KCSAN_BS_MULTI(write, width, const type)
+
+#define KCSAN_BS_SET(width, type) \
+ void kcsan_bus_space_set_multi_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type, bus_size_t); \
+ void kcsan_bus_space_set_multi_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type, bus_size_t); \
+ void kcsan_bus_space_set_region_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type, bus_size_t); \
+ void kcsan_bus_space_set_region_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type, bus_size_t)
+
+#define KCSAN_BS_COPY(width, type) \
+ void kcsan_bus_space_copy_region_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, bus_space_handle_t, \
+ bus_size_t, bus_size_t); \
+ void kcsan_bus_space_copy_region_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, bus_space_handle_t, \
+ bus_size_t, bus_size_t);
+
+#define KCSAN_BS(width, type) \
+ KCSAN_BS_READ(width, type); \
+ KCSAN_BS_WRITE(width, type); \
+ KCSAN_BS_SET(width, type); \
+ KCSAN_BS_COPY(width, type)
+
+KCSAN_BS(1, uint8_t);
+KCSAN_BS(2, uint16_t);
+KCSAN_BS(4, uint32_t);
+KCSAN_BS(8, uint64_t);
+
+int kcsan_bus_space_map(bus_space_tag_t, bus_addr_t, bus_size_t, int,
+ bus_space_handle_t *);
+void kcsan_bus_space_unmap(bus_space_tag_t, bus_space_handle_t, bus_size_t);
+int kcsan_bus_space_subregion(bus_space_tag_t, bus_space_handle_t, bus_size_t,
+ bus_size_t, bus_space_handle_t *);
+int kcsan_bus_space_alloc(bus_space_tag_t, bus_addr_t, bus_addr_t,
+ bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *,
+ bus_space_handle_t *);
+void kcsan_bus_space_free(bus_space_tag_t, bus_space_handle_t, bus_size_t);
+void kcsan_bus_space_barrier(bus_space_tag_t, bus_space_handle_t, bus_size_t,
+ bus_size_t, int);
+
+#ifndef KCSAN_RUNTIME
+
+#define bus_space_map kcsan_bus_space_map
+#define bus_space_unmap kcsan_bus_space_unmap
+#define bus_space_subregion kcsan_bus_space_subregion
+#define bus_space_alloc kcsan_bus_space_alloc
+#define bus_space_free kcsan_bus_space_free
+#define bus_space_barrier kcsan_bus_space_barrier
+
+#define bus_space_read_1 kcsan_bus_space_read_1
+#define bus_space_read_stream_1 kcsan_bus_space_read_stream_1
+#define bus_space_read_multi_1 kcsan_bus_space_read_multi_1
+#define bus_space_read_multi_stream_1 kcsan_bus_space_read_multi_stream_1
+#define bus_space_read_region_1 kcsan_bus_space_read_region_1
+#define bus_space_read_region_stream_1 kcsan_bus_space_read_region_stream_1
+#define bus_space_write_1 kcsan_bus_space_write_1
+#define bus_space_write_stream_1 kcsan_bus_space_write_stream_1
+#define bus_space_write_multi_1 kcsan_bus_space_write_multi_1
+#define bus_space_write_multi_stream_1 kcsan_bus_space_write_multi_stream_1
+#define bus_space_write_region_1 kcsan_bus_space_write_region_1
+#define bus_space_write_region_stream_1 kcsan_bus_space_write_region_stream_1
+#define bus_space_set_multi_1 kcsan_bus_space_set_multi_1
+#define bus_space_set_multi_stream_1 kcsan_bus_space_set_multi_stream_1
+#define bus_space_set_region_1 kcsan_bus_space_set_region_1
+#define bus_space_set_region_stream_1 kcsan_bus_space_set_region_stream_1
+#define bus_space_copy_multi_1 kcsan_bus_space_copy_multi_1
+#define bus_space_copy_multi_stream_1 kcsan_bus_space_copy_multi_stream_1
+
+#define bus_space_read_2 kcsan_bus_space_read_2
+#define bus_space_read_stream_2 kcsan_bus_space_read_stream_2
+#define bus_space_read_multi_2 kcsan_bus_space_read_multi_2
+#define bus_space_read_multi_stream_2 kcsan_bus_space_read_multi_stream_2
+#define bus_space_read_region_2 kcsan_bus_space_read_region_2
+#define bus_space_read_region_stream_2 kcsan_bus_space_read_region_stream_2
+#define bus_space_write_2 kcsan_bus_space_write_2
+#define bus_space_write_stream_2 kcsan_bus_space_write_stream_2
+#define bus_space_write_multi_2 kcsan_bus_space_write_multi_2
+#define bus_space_write_multi_stream_2 kcsan_bus_space_write_multi_stream_2
+#define bus_space_write_region_2 kcsan_bus_space_write_region_2
+#define bus_space_write_region_stream_2 kcsan_bus_space_write_region_stream_2
+#define bus_space_set_multi_2 kcsan_bus_space_set_multi_2
+#define bus_space_set_multi_stream_2 kcsan_bus_space_set_multi_stream_2
+#define bus_space_set_region_2 kcsan_bus_space_set_region_2
+#define bus_space_set_region_stream_2 kcsan_bus_space_set_region_stream_2
+#define bus_space_copy_multi_2 kcsan_bus_space_copy_multi_2
+#define bus_space_copy_multi_stream_2 kcsan_bus_space_copy_multi_stream_2
+
+#define bus_space_read_4 kcsan_bus_space_read_4
+#define bus_space_read_stream_4 kcsan_bus_space_read_stream_4
+#define bus_space_read_multi_4 kcsan_bus_space_read_multi_4
+#define bus_space_read_multi_stream_4 kcsan_bus_space_read_multi_stream_4
+#define bus_space_read_region_4 kcsan_bus_space_read_region_4
+#define bus_space_read_region_stream_4 kcsan_bus_space_read_region_stream_4
+#define bus_space_write_4 kcsan_bus_space_write_4
+#define bus_space_write_stream_4 kcsan_bus_space_write_stream_4
+#define bus_space_write_multi_4 kcsan_bus_space_write_multi_4
+#define bus_space_write_multi_stream_4 kcsan_bus_space_write_multi_stream_4
+#define bus_space_write_region_4 kcsan_bus_space_write_region_4
+#define bus_space_write_region_stream_4 kcsan_bus_space_write_region_stream_4
+#define bus_space_set_multi_4 kcsan_bus_space_set_multi_4
+#define bus_space_set_multi_stream_4 kcsan_bus_space_set_multi_stream_4
+#define bus_space_set_region_4 kcsan_bus_space_set_region_4
+#define bus_space_set_region_stream_4 kcsan_bus_space_set_region_stream_4
+#define bus_space_copy_multi_4 kcsan_bus_space_copy_multi_4
+#define bus_space_copy_multi_stream_4 kcsan_bus_space_copy_multi_stream_4
+
+#define bus_space_read_8 kcsan_bus_space_read_8
+#define bus_space_read_stream_8 kcsan_bus_space_read_stream_8
+#define bus_space_read_multi_8 kcsan_bus_space_read_multi_8
+#define bus_space_read_multi_stream_8 kcsan_bus_space_read_multi_stream_8
+#define bus_space_read_region_8 kcsan_bus_space_read_region_8
+#define bus_space_read_region_stream_8 kcsan_bus_space_read_region_stream_8
+#define bus_space_write_8 kcsan_bus_space_write_8
+#define bus_space_write_stream_8 kcsan_bus_space_write_stream_8
+#define bus_space_write_multi_8 kcsan_bus_space_write_multi_8
+#define bus_space_write_multi_stream_8 kcsan_bus_space_write_multi_stream_8
+#define bus_space_write_region_8 kcsan_bus_space_write_region_8
+#define bus_space_write_region_stream_8 kcsan_bus_space_write_region_stream_8
+#define bus_space_set_multi_8 kcsan_bus_space_set_multi_8
+#define bus_space_set_multi_stream_8 kcsan_bus_space_set_multi_stream_8
+#define bus_space_set_region_8 kcsan_bus_space_set_region_8
+#define bus_space_set_region_stream_8 kcsan_bus_space_set_region_stream_8
+#define bus_space_copy_multi_8 kcsan_bus_space_copy_multi_8
+#define bus_space_copy_multi_stream_8 kcsan_bus_space_copy_multi_stream_8
+
+#endif /* !KCSAN_RUNTIME */
+
+#endif /* !_SYS__CSAN_BUS_H_ */
Index: sys/sys/csan.h
===================================================================
--- /dev/null
+++ sys/sys/csan.h
@@ -0,0 +1,43 @@
+/* $NetBSD: csan.h,v 1.1 2019/11/05 20:19:18 maxv Exp $ */
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _SYS_CSAN_H_
+#define _SYS_CSAN_H_
+
+#include <sys/types.h>
+
+#ifdef KCSAN
+void kcsan_cpu_init(u_int);
+#else
+#define kcsan_cpu_init(ci) ((void)0)
+#endif
+
+#endif /* !_SYS_CSAN_H_ */
Index: sys/sys/libkern.h
===================================================================
--- sys/sys/libkern.h
+++ sys/sys/libkern.h
@@ -191,6 +191,15 @@
char *strstr(const char *, const char *);
int strvalid(const char *, size_t);
+#ifdef KCSAN
+char *kcsan_strcpy(char *, const char *);
+int kcsan_strcmp(const char *, const char *);
+size_t kcsan_strlen(const char *);
+#define strcpy(d, s) kcsan_strcpy((d), (s))
+#define strcmp(s1, s2) kcsan_strcmp((s1), (s2))
+#define strlen(s) kcsan_strlen((s))
+#endif
+
static __inline char *
index(const char *p, int ch)
{
Index: sys/sys/systm.h
===================================================================
--- sys/sys/systm.h
+++ sys/sys/systm.h
@@ -315,21 +315,36 @@
#define ovbcopy(f, t, l) bcopy((f), (t), (l))
void bcopy(const void * _Nonnull from, void * _Nonnull to, size_t len);
-#define bcopy(from, to, len) __builtin_memmove((to), (from), (len))
void bzero(void * _Nonnull buf, size_t len);
-#define bzero(buf, len) __builtin_memset((buf), 0, (len))
void explicit_bzero(void * _Nonnull, size_t);
int bcmp(const void *b1, const void *b2, size_t len);
-#define bcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len))
void *memset(void * _Nonnull buf, int c, size_t len);
-#define memset(buf, c, len) __builtin_memset((buf), (c), (len))
void *memcpy(void * _Nonnull to, const void * _Nonnull from, size_t len);
-#define memcpy(to, from, len) __builtin_memcpy((to), (from), (len))
void *memmove(void * _Nonnull dest, const void * _Nonnull src, size_t n);
-#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
int memcmp(const void *b1, const void *b2, size_t len);
+
+#ifdef KCSAN
+void *kcsan_memset(void *, int, size_t);
+void *kcsan_memcpy(void *, const void *, size_t);
+void *kcsan_memmove(void *, const void *, size_t);
+int kcsan_memcmp(const void *, const void *, size_t);
+#define bcopy(from, to, len) kcsan_memmove((to), (from), (len))
+#define bzero(buf, len) kcsan_memset((buf), 0, (len))
+#define bcmp(b1, b2, len) kcsan_memcmp((b1), (b2), (len))
+#define memset(buf, c, len) kcsan_memset((buf), (c), (len))
+#define memcpy(to, from, len) kcsan_memcpy((to), (from), (len))
+#define memmove(dest, src, n) kcsan_memmove((dest), (src), (n))
+#define memcmp(b1, b2, len) kcsan_memcmp((b1), (b2), (len))
+#else
+#define bcopy(from, to, len) __builtin_memmove((to), (from), (len))
+#define bzero(buf, len) __builtin_memset((buf), 0, (len))
+#define bcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len))
+#define memset(buf, c, len) __builtin_memset((buf), (c), (len))
+#define memcpy(to, from, len) __builtin_memcpy((to), (from), (len))
+#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
#define memcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len))
+#endif
void *memset_early(void * _Nonnull buf, int c, size_t len);
#define bzero_early(buf, len) memset_early((buf), 0, (len))
@@ -352,6 +367,17 @@
int copyout_nofault(const void * _Nonnull __restrict kaddr,
void * __restrict udaddr, size_t len);
+#ifdef KCSAN
+int kcsan_copystr(const void *, void *, size_t, size_t *);
+int kcsan_copyin(const void *, void *, size_t);
+int kcsan_copyinstr(const void *, void *, size_t, size_t *);
+int kcsan_copyout(const void *, void *, size_t);
+#define copystr(kf, k, l, lc) kcsan_copystr((kf), (k), (l), (lc))
+#define copyin(u, k, l) kcsan_copyin((u), (k), (l))
+#define copyinstr(u, k, l, lc) kcsan_copyinstr((u), (k), (l), (lc))
+#define copyout(k, u, l) kcsan_copyout((k), (u), (l))
+#endif
+
int fubyte(volatile const void *base);
long fuword(volatile const void *base);
int fuword16(volatile const void *base);
Index: sys/x86/include/bus.h
===================================================================
--- sys/x86/include/bus.h
+++ sys/x86/include/bus.h
@@ -101,6 +101,8 @@
#include <machine/_bus.h>
#include <machine/cpufunc.h>
+#include <machine/bus_dma.h>
+
#ifndef __GNUCLIKE_ASM
#error "no assembler code for your compiler"
@@ -131,6 +133,13 @@
#define BUS_SPACE_INVALID_DATA (~0)
#define BUS_SPACE_UNRESTRICTED (~0)
+#define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */
+#define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */
+
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_bus.h>
+#else
+
/*
* Map a region of device bus space into CPU virtual address space.
*/
@@ -992,9 +1001,6 @@
* prevent reordering by the compiler; all Intel x86 processors currently
* retire operations outside the CPU in program order.
*/
-#define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */
-#define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */
-
static __inline void
bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused,
bus_size_t offset __unused, bus_size_t len __unused, int flags)
@@ -1022,8 +1028,6 @@
#define outl(a, b) compiler_error
#endif
-#include <machine/bus_dma.h>
-
/*
* Stream accesses are the same as normal accesses on x86; there are no
* supported bus systems with an endianess different from the host one.
@@ -1088,4 +1092,6 @@
#define bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c) \
bus_space_copy_region_4((t), (h1), (o1), (h2), (o2), (c))
+#endif /* KCSAN && !KCSAN_RUNTIME */
+
#endif /* _X86_BUS_H_ */
Index: sys/x86/x86/bus_machdep.c
===================================================================
--- sys/x86/x86/bus_machdep.c
+++ sys/x86/x86/bus_machdep.c
@@ -24,6 +24,8 @@
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+#define KCSAN_RUNTIME
+
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
Index: sys/x86/x86/mp_x86.c
===================================================================
--- sys/x86/x86/mp_x86.c
+++ sys/x86/x86/mp_x86.c
@@ -41,6 +41,7 @@
#include <sys/bus.h>
#include <sys/cons.h> /* cngetc() */
#include <sys/cpuset.h>
+#include <sys/csan.h>
#ifdef GPROF
#include <sys/gmon.h>
#endif
@@ -1080,6 +1081,8 @@
cpu_initclocks_ap();
#endif
+ kcsan_cpu_init(cpuid);
+
sched_throw(NULL);
panic("scheduler returned us to %s", __func__);

File Metadata

Mime Type
text/plain
Expires
Sat, Jan 24, 12:06 AM (15 h, 7 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
27891923
Default Alt Text
D22315.id64175.diff (77 KB)

Event Timeline