Page MenuHomeFreeBSD
Files F143119487

D22315.id64668.diff
No OneTemporary
Actions

D22315.id64668.diff
View Options

Index: head/sys/amd64/amd64/copyout.c
===================================================================
--- head/sys/amd64/amd64/copyout.c
+++ head/sys/amd64/amd64/copyout.c
@@ -146,6 +146,10 @@
casueword_smap : casueword_nosmap);
}
+#undef copyinstr
+#undef copyin
+#undef copyout
+
int copyinstr_nosmap(const void *udaddr, void *kaddr, size_t len,
size_t *lencopied);
int copyinstr_smap(const void *udaddr, void *kaddr, size_t len,
Index: head/sys/amd64/amd64/machdep.c
===================================================================
--- head/sys/amd64/amd64/machdep.c
+++ head/sys/amd64/amd64/machdep.c
@@ -64,6 +64,7 @@
#include <sys/callout.h>
#include <sys/cons.h>
#include <sys/cpu.h>
+#include <sys/csan.h>
#include <sys/efi.h>
#include <sys/eventhandler.h>
#include <sys/exec.h>
@@ -1899,6 +1900,8 @@
cpu_probe_amdc1e();
+ kcsan_cpu_init(0);
+
#ifdef FDT
x86_init_fdt();
#endif
@@ -2722,6 +2725,40 @@
void *memset_std(void *buf, int c, size_t len);
void *memset_erms(void *buf, int c, size_t len);
+void *memmove_std(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src,
+ size_t len);
+
+#ifdef KCSAN
+/*
+ * These fail to build as ifuncs when used with KCSAN.
+ */
+void *
+memset(void *buf, int c, size_t len)
+{
+
+ return memset_std(buf, c, len);
+}
+
+void *
+memmove(void * _Nonnull dst, const void * _Nonnull src, size_t len)
+{
+
+ return memmove_std(dst, src, len);
+}
+
+void *
+memcpy(void * _Nonnull dst, const void * _Nonnull src, size_t len)
+{
+
+ return memcpy_std(dst, src, len);
+}
+#else
DEFINE_IFUNC(, void *, memset, (void *, int, size_t))
{
@@ -2729,10 +2766,6 @@
memset_erms : memset_std);
}
-void *memmove_std(void * _Nonnull dst, const void * _Nonnull src,
- size_t len);
-void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src,
- size_t len);
DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull,
size_t))
{
@@ -2741,16 +2774,13 @@
memmove_erms : memmove_std);
}
-void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src,
- size_t len);
-void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src,
- size_t len);
DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull,size_t))
{
return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
memcpy_erms : memcpy_std);
}
+#endif
void pagezero_std(void *addr);
void pagezero_erms(void *addr);
Index: head/sys/amd64/conf/GENERIC
===================================================================
--- head/sys/amd64/conf/GENERIC
+++ head/sys/amd64/conf/GENERIC
@@ -106,6 +106,7 @@
#options KCOV # Kernel Coverage Sanitizer
# Warning: KUBSAN can result in a kernel too large for loader to load
#options KUBSAN # Kernel Undefined Behavior Sanitizer
+#options KCSAN # Kernel Concurrency Sanitizer
# Kernel dump features.
options EKCD # Support for encrypted kernel dumps
Index: head/sys/amd64/conf/GENERIC-KCSAN
===================================================================
--- head/sys/amd64/conf/GENERIC-KCSAN
+++ head/sys/amd64/conf/GENERIC-KCSAN
@@ -0,0 +1,33 @@
+#
+# GENERIC-KCSAN -- Kernel Concurrency Sanitizer kernel configuration file
+# for FreeBSD/amd64
+#
+# This configuration file removes several debugging options, including
+# WITNESS and INVARIANTS checking, which are known to have significant
+# performance impact on running systems. When benchmarking new features
+# this kernel should be used instead of the standard GENERIC.
+# This kernel configuration should never appear outside of the HEAD
+# of the FreeBSD tree.
+#
+# For more information on this file, please read the config(5) manual page,
+# and/or the handbook section on Kernel Configuration Files:
+#
+# https://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html
+#
+# The handbook is also available locally in /usr/share/doc/handbook
+# if you've installed the doc distribution, otherwise always see the
+# FreeBSD World Wide Web server (https://www.FreeBSD.org/) for the
+# latest information.
+#
+# An exhaustive list of options and more detailed explanations of the
+# device lines is also present in the ../../conf/NOTES and NOTES files.
+# If you are in doubt as to the purpose or necessity of a line, check first
+# in NOTES.
+#
+# $FreeBSD$
+
+include GENERIC
+
+ident GENERIC-KCSAN
+
+options KCSAN
Index: head/sys/amd64/include/atomic.h
===================================================================
--- head/sys/amd64/include/atomic.h
+++ head/sys/amd64/include/atomic.h
@@ -57,6 +57,20 @@
#define wmb() __asm __volatile("sfence;" : : : "memory")
#define rmb() __asm __volatile("lfence;" : : : "memory")
+#ifdef _KERNEL
+/*
+ * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
+ *
+ * The open-coded number is used instead of the symbolic expression to
+ * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
+ * An assertion in amd64/vm_machdep.c ensures that the value is correct.
+ */
+#define OFFSETOF_MONITORBUF 0x100
+#endif
+
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_atomic.h>
+#else
#include <sys/atomic_common.h>
/*
@@ -345,15 +359,6 @@
#if defined(_KERNEL)
-/*
- * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf).
- *
- * The open-coded number is used instead of the symbolic expression to
- * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers.
- * An assertion in amd64/vm_machdep.c ensures that the value is correct.
- */
-#define OFFSETOF_MONITORBUF 0x100
-
#if defined(SMP) || defined(KLD_MODULE)
static __inline void
__storeload_barrier(void)
@@ -678,5 +683,7 @@
#define atomic_readandclear_ptr atomic_readandclear_long
#endif /* !WANT_FUNCTIONS */
+
+#endif /* KCSAN && !KCSAN_RUNTIME */
#endif /* !_MACHINE_ATOMIC_H_ */
Index: head/sys/amd64/include/csan.h
===================================================================
--- head/sys/amd64/include/csan.h
+++ head/sys/amd64/include/csan.h
@@ -0,0 +1,67 @@
+/* $NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $ */
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <machine/cpufunc.h>
+#include <machine/stack.h>
+#include <machine/vmparam.h>
+
+static inline bool
+kcsan_md_is_avail(void)
+{
+ return true;
+}
+
+static inline void
+kcsan_md_disable_intrs(uint64_t *state)
+{
+
+ *state = intr_disable();
+}
+
+static inline void
+kcsan_md_enable_intrs(uint64_t *state)
+{
+
+ intr_restore(*state);
+}
+
+static inline void
+kcsan_md_delay(uint64_t us)
+{
+ DELAY(us);
+}
+
+static void
+kcsan_md_unwind(void)
+{
+}
Index: head/sys/arm64/arm64/bus_machdep.c
===================================================================
--- head/sys/arm64/arm64/bus_machdep.c
+++ head/sys/arm64/arm64/bus_machdep.c
@@ -25,6 +25,8 @@
*
*/
+#define KCSAN_RUNTIME
+
#include "opt_platform.h"
#include <sys/param.h>
Index: head/sys/arm64/arm64/copystr.c
===================================================================
--- head/sys/arm64/arm64/copystr.c
+++ head/sys/arm64/arm64/copystr.c
@@ -32,7 +32,7 @@
#include <sys/systm.h>
int
-copystr(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len,
+(copystr)(const void * __restrict kfaddr, void * __restrict kdaddr, size_t len,
size_t * __restrict lencopied)
{
const char *src;
Index: head/sys/arm64/arm64/machdep.c
===================================================================
--- head/sys/arm64/arm64/machdep.c
+++ head/sys/arm64/arm64/machdep.c
@@ -38,6 +38,7 @@
#include <sys/bus.h>
#include <sys/cons.h>
#include <sys/cpu.h>
+#include <sys/csan.h>
#include <sys/devmap.h>
#include <sys/efi.h>
#include <sys/exec.h>
@@ -1208,6 +1209,8 @@
dbg_init();
kdb_init();
pan_enable();
+
+ kcsan_cpu_init(0);
env = kern_getenv("kernelname");
if (env != NULL)
Index: head/sys/arm64/arm64/mp_machdep.c
===================================================================
--- head/sys/arm64/arm64/mp_machdep.c
+++ head/sys/arm64/arm64/mp_machdep.c
@@ -40,6 +40,7 @@
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/cpu.h>
+#include <sys/csan.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/malloc.h>
@@ -252,6 +253,8 @@
}
mtx_unlock_spin(&ap_boot_mtx);
+
+ kcsan_cpu_init(cpu);
/* Enter the scheduler */
sched_throw(NULL);
Index: head/sys/arm64/conf/GENERIC
===================================================================
--- head/sys/arm64/conf/GENERIC
+++ head/sys/arm64/conf/GENERIC
@@ -100,6 +100,7 @@
#options KCOV # Kernel Coverage Sanitizer
# Warning: KUBSAN can result in a kernel too large for loader to load
#options KUBSAN # Kernel Undefined Behavior Sanitizer
+#options KCSAN # Kernel Concurrency Sanitizer
# Kernel dump features.
options EKCD # Support for encrypted kernel dumps
Index: head/sys/arm64/include/atomic.h
===================================================================
--- head/sys/arm64/include/atomic.h
+++ head/sys/arm64/include/atomic.h
@@ -29,8 +29,6 @@
#ifndef _MACHINE_ATOMIC_H_
#define _MACHINE_ATOMIC_H_
-#include <sys/atomic_common.h>
-
#define isb() __asm __volatile("isb" : : : "memory")
/*
@@ -55,6 +53,12 @@
#define wmb() dmb(st) /* Full system memory barrier store */
#define rmb() dmb(ld) /* Full system memory barrier load */
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_atomic.h>
+#else
+
+#include <sys/atomic_common.h>
+
#define ATOMIC_OP(op, asm_op, bar, a, l) \
static __inline void \
atomic_##op##_##bar##8(volatile uint8_t *p, uint8_t val) \
@@ -600,6 +604,8 @@
dmb(sy);
}
+
+#endif /* KCSAN && !KCSAN_RUNTIME */
#endif /* _MACHINE_ATOMIC_H_ */
Index: head/sys/arm64/include/bus.h
===================================================================
--- head/sys/arm64/include/bus.h
+++ head/sys/arm64/include/bus.h
@@ -89,6 +89,9 @@
#define BUS_SPACE_BARRIER_READ 0x01
#define BUS_SPACE_BARRIER_WRITE 0x02
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_bus.h>
+#else
struct bus_space {
/* cookie */
@@ -463,6 +466,8 @@
__bs_copy(4, t, h1, o1, h2, o2, c)
#define bus_space_copy_region_8(t, h1, o1, h2, o2, c) \
__bs_copy(8, t, h1, o1, h2, o2, c)
+
+#endif
#include <machine/bus_dma.h>
Index: head/sys/arm64/include/csan.h
===================================================================
--- head/sys/arm64/include/csan.h
+++ head/sys/arm64/include/csan.h
@@ -0,0 +1,104 @@
+/* $NetBSD: csan.h,v 1.2 2019/11/06 06:57:22 maxv Exp $ */
+
+/*
+ * Copyright (c) 2019 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Maxime Villard.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <machine/cpufunc.h>
+#include <machine/stack.h>
+#include <machine/vmparam.h>
+
+static inline bool
+kcsan_md_is_avail(void)
+{
+ return true;
+}
+
+static inline void
+kcsan_md_disable_intrs(uint64_t *state)
+{
+
+ *state = intr_disable();
+}
+
+static inline void
+kcsan_md_enable_intrs(uint64_t *state)
+{
+
+ intr_restore(*state);
+}
+
+static inline void
+kcsan_md_delay(uint64_t us)
+{
+ DELAY(us);
+}
+
+static void
+kcsan_md_unwind(void)
+{
+#ifdef DDB
+ c_db_sym_t sym;
+ db_expr_t offset;
+ const char *symname;
+#endif
+ struct unwind_state frame;
+ uint64_t sp;
+ int nsym;
+
+ __asm __volatile("mov %0, sp" : "=&r" (sp));
+
+ frame.sp = sp;
+ frame.fp = (uint64_t)__builtin_frame_address(0);
+ frame.pc = (uint64_t)kcsan_md_unwind;
+ nsym = 0;
+
+ while (1) {
+ unwind_frame(&frame);
+ if (!INKERNEL((vm_offset_t)frame.fp) ||
+ !INKERNEL((vm_offset_t)frame.pc))
+ break;
+
+#ifdef DDB
+ sym = db_search_symbol((vm_offset_t)frame.pc, DB_STGY_PROC,
+ &offset);
+ db_symbol_values(sym, &symname, NULL);
+ printf("#%d %p in %s+%#lx\n", nsym, (void *)frame.pc,
+ symname, offset);
+#else
+ printf("#%d %p\n", nsym, (void *)frame.pc);
+#endif
+ nsym++;
+
+ if (nsym >= 15) {
+ break;
+ }
+ }
+}
Index: head/sys/conf/files
===================================================================
--- head/sys/conf/files
+++ head/sys/conf/files
@@ -3805,6 +3805,8 @@
kern/subr_coverage.c optional coverage \
compile-with "${NORMAL_C:N-fsanitize*}"
kern/subr_counter.c standard
+kern/subr_csan.c optional kcsan \
+ compile-with "${NORMAL_C:N-fsanitize*}"
kern/subr_devstat.c standard
kern/subr_disk.c standard
kern/subr_early.c standard
Index: head/sys/conf/files.arm64
===================================================================
--- head/sys/conf/files.arm64
+++ head/sys/conf/files.arm64
@@ -286,8 +286,10 @@
kern/subr_devmap.c standard
kern/subr_intr.c optional intrng
libkern/bcmp.c standard
-libkern/memcmp.c standard
-libkern/memset.c standard
+libkern/memcmp.c standard \
+ compile-with "${NORMAL_C:N-fsanitize*}"
+libkern/memset.c standard \
+ compile-with "${NORMAL_C:N-fsanitize*}"
libkern/arm64/crc32c_armv8.S standard
cddl/dev/dtrace/aarch64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}"
cddl/dev/dtrace/aarch64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}"
Index: head/sys/conf/kern.post.mk
===================================================================
--- head/sys/conf/kern.post.mk
+++ head/sys/conf/kern.post.mk
@@ -38,6 +38,10 @@
MKMODULESENV+= WITH_EXTRA_TCP_STACKS="${WITH_EXTRA_TCP_STACKS}"
.endif
+.if defined(KCSAN_ENABLED)
+MKMODULESENV+= KCSAN_ENABLED="yes"
+.endif
+
.if defined(SAN_CFLAGS)
MKMODULESENV+= SAN_CFLAGS="${SAN_CFLAGS}"
.endif
Index: head/sys/conf/kern.pre.mk
===================================================================
--- head/sys/conf/kern.pre.mk
+++ head/sys/conf/kern.pre.mk
@@ -117,6 +117,11 @@
.endif
DEFINED_PROF= ${PROF}
+KCSAN_ENABLED!= grep KCSAN opt_global.h || true ; echo
+.if !empty(KCSAN_ENABLED)
+SAN_CFLAGS+= -fsanitize=thread
+.endif
+
KUBSAN_ENABLED!= grep KUBSAN opt_global.h || true ; echo
.if !empty(KUBSAN_ENABLED)
SAN_CFLAGS+= -fsanitize=undefined
Index: head/sys/conf/options
===================================================================
--- head/sys/conf/options
+++ head/sys/conf/options
@@ -230,6 +230,7 @@
# Sanitizers
COVERAGE opt_global.h
KCOV
+KCSAN opt_global.h
KUBSAN opt_global.h
# POSIX kernel options
Index: head/sys/kern/subr_csan.c
===================================================================
--- head/sys/kern/subr_csan.c
+++ head/sys/kern/subr_csan.c
@@ -3,6 +3,7 @@
/*
* Copyright (c) 2019 The NetBSD Foundation, Inc.
* All rights reserved.
+ * Copyright (c) 2019 Andrew Turner
*
* This code is derived from software contributed to The NetBSD Foundation
* by Maxime Villard.
@@ -29,19 +30,26 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
+#define KCSAN_RUNTIME
+
+#include "opt_ddb.h"
+
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $");
+__FBSDID("$FreeBSD$");
#include <sys/param.h>
-#include <sys/device.h>
#include <sys/kernel.h>
-#include <sys/param.h>
+#include <sys/bus.h>
#include <sys/conf.h>
-#include <sys/systm.h>
-#include <sys/types.h>
-#include <sys/csan.h>
#include <sys/cpu.h>
+#include <sys/csan.h>
+#include <sys/proc.h>
+#include <sys/smp.h>
+#include <sys/systm.h>
+#include <ddb/ddb.h>
+#include <ddb/db_sym.h>
+
#ifdef KCSAN_PANIC
#define REPORT panic
#else
@@ -62,7 +70,7 @@
csan_cell_t cell;
} csan_cpu_t;
-static csan_cpu_t kcsan_cpus[MAXCPUS];
+static csan_cpu_t kcsan_cpus[MAXCPU];
static bool kcsan_enabled __read_mostly;
#define __RET_ADDR (uintptr_t)__builtin_return_address(0)
@@ -77,34 +85,43 @@
/* -------------------------------------------------------------------------- */
-void
-kcsan_init(void)
+static void
+kcsan_enable(void *dummy __unused)
{
+
+ printf("Enabling KCSCAN, expect reduced performance.\n");
kcsan_enabled = true;
}
+SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL);
void
-kcsan_cpu_init(struct cpu_info *ci)
+kcsan_cpu_init(u_int cpu)
{
- kcsan_cpus[cpu_index(ci)].inited = true;
+ kcsan_cpus[cpu].inited = true;
}
/* -------------------------------------------------------------------------- */
static inline void
-kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu)
+kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu)
{
const char *newsym, *oldsym;
+#ifdef DDB
+ c_db_sym_t sym;
+ db_expr_t offset;
- if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) {
- newsym = "Unknown";
- }
- if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) {
- oldsym = "Unknown";
- }
+ sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset);
+ db_symbol_values(sym, &newsym, NULL);
+
+ sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset);
+ db_symbol_values(sym, &oldsym, NULL);
+#else
+ newsym = "";
+ oldsym = "";
+#endif
REPORT("CSan: Racy Access "
- "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] "
- "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n",
+ "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] "
+ "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n",
newcpu,
(new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"),
(void *)new->addr, new->size, (void *)new->pc, newsym,
@@ -134,8 +151,6 @@
if (__predict_false(!kcsan_enabled))
return;
- if (__predict_false(kcsan_md_unsupported((vaddr_t)addr)))
- return;
new.addr = addr;
new.size = size;
@@ -143,7 +158,7 @@
new.atomic = atomic;
new.pc = pc;
- for (i = 0; i < ncpu; i++) {
+ CPU_FOREACH(i) {
__builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old));
if (old.addr + old.size <= new.addr)
@@ -155,7 +170,7 @@
if (__predict_true(kcsan_access_is_atomic(&new, &old)))
continue;
- kcsan_report(&new, cpu_number(), &old, i);
+ kcsan_report(&new, PCPU_GET(cpuid), &old, i);
break;
}
@@ -164,7 +179,7 @@
kcsan_md_disable_intrs(&intr);
- cpu = &kcsan_cpus[cpu_number()];
+ cpu = &kcsan_cpus[PCPU_GET(cpuid)];
if (__predict_false(!cpu->inited))
goto out;
cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES;
@@ -184,6 +199,11 @@
void __tsan_read##size(uintptr_t addr) \
{ \
kcsan_access(addr, size, false, false, __RET_ADDR); \
+ } \
+ void __tsan_unaligned_read##size(uintptr_t); \
+ void __tsan_unaligned_read##size(uintptr_t addr) \
+ { \
+ kcsan_access(addr, size, false, false, __RET_ADDR); \
}
CSAN_READ(1)
@@ -197,6 +217,11 @@
void __tsan_write##size(uintptr_t addr) \
{ \
kcsan_access(addr, size, true, false, __RET_ADDR); \
+ } \
+ void __tsan_unaligned_write##size(uintptr_t); \
+ void __tsan_unaligned_write##size(uintptr_t addr) \
+ { \
+ kcsan_access(addr, size, true, false, __RET_ADDR); \
}
CSAN_WRITE(1)
@@ -321,35 +346,14 @@
return (s - str);
}
-#undef kcopy
#undef copystr
-#undef copyinstr
-#undef copyoutstr
#undef copyin
+#undef copyin_nofault
+#undef copyinstr
#undef copyout
+#undef copyout_nofault
-int kcsan_kcopy(const void *, void *, size_t);
-int kcsan_copystr(const void *, void *, size_t, size_t *);
-int kcsan_copyinstr(const void *, void *, size_t, size_t *);
-int kcsan_copyoutstr(const void *, void *, size_t, size_t *);
-int kcsan_copyin(const void *, void *, size_t);
-int kcsan_copyout(const void *, void *, size_t);
-int kcopy(const void *, void *, size_t);
-int copystr(const void *, void *, size_t, size_t *);
-int copyinstr(const void *, void *, size_t, size_t *);
-int copyoutstr(const void *, void *, size_t, size_t *);
-int copyin(const void *, void *, size_t);
-int copyout(const void *, void *, size_t);
-
int
-kcsan_kcopy(const void *src, void *dst, size_t len)
-{
- kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR);
- kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR);
- return kcopy(src, dst, len);
-}
-
-int
kcsan_copystr(const void *kfaddr, void *kdaddr, size_t len, size_t *done)
{
kcsan_access((uintptr_t)kdaddr, len, true, false, __RET_ADDR);
@@ -364,13 +368,6 @@
}
int
-kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
-{
- kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
- return copyout(kaddr, uaddr, len);
-}
-
-int
kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
{
kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR);
@@ -378,377 +375,477 @@
}
int
-kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
+kcsan_copyout(const void *kaddr, void *uaddr, size_t len)
{
kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR);
- return copyoutstr(kaddr, uaddr, len, done);
+ return copyout(kaddr, uaddr, len);
}
/* -------------------------------------------------------------------------- */
-#undef atomic_add_32
-#undef atomic_add_int
-#undef atomic_add_long
-#undef atomic_add_ptr
-#undef atomic_add_64
-#undef atomic_add_32_nv
-#undef atomic_add_int_nv
-#undef atomic_add_long_nv
-#undef atomic_add_ptr_nv
-#undef atomic_add_64_nv
-#undef atomic_and_32
-#undef atomic_and_uint
-#undef atomic_and_ulong
-#undef atomic_and_64
-#undef atomic_and_32_nv
-#undef atomic_and_uint_nv
-#undef atomic_and_ulong_nv
-#undef atomic_and_64_nv
-#undef atomic_or_32
-#undef atomic_or_uint
-#undef atomic_or_ulong
-#undef atomic_or_64
-#undef atomic_or_32_nv
-#undef atomic_or_uint_nv
-#undef atomic_or_ulong_nv
-#undef atomic_or_64_nv
-#undef atomic_cas_32
-#undef atomic_cas_uint
-#undef atomic_cas_ulong
-#undef atomic_cas_ptr
-#undef atomic_cas_64
-#undef atomic_cas_32_ni
-#undef atomic_cas_uint_ni
-#undef atomic_cas_ulong_ni
-#undef atomic_cas_ptr_ni
-#undef atomic_cas_64_ni
-#undef atomic_swap_32
-#undef atomic_swap_uint
-#undef atomic_swap_ulong
-#undef atomic_swap_ptr
-#undef atomic_swap_64
-#undef atomic_dec_32
-#undef atomic_dec_uint
-#undef atomic_dec_ulong
-#undef atomic_dec_ptr
-#undef atomic_dec_64
-#undef atomic_dec_32_nv
-#undef atomic_dec_uint_nv
-#undef atomic_dec_ulong_nv
-#undef atomic_dec_ptr_nv
-#undef atomic_dec_64_nv
-#undef atomic_inc_32
-#undef atomic_inc_uint
-#undef atomic_inc_ulong
-#undef atomic_inc_ptr
-#undef atomic_inc_64
-#undef atomic_inc_32_nv
-#undef atomic_inc_uint_nv
-#undef atomic_inc_ulong_nv
-#undef atomic_inc_ptr_nv
-#undef atomic_inc_64_nv
+#include <machine/atomic.h>
+#include <sys/_cscan_atomic.h>
-#define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
- void atomic_add_##name(volatile targ1 *, targ2); \
- void kcsan_atomic_add_##name(volatile targ1 *, targ2); \
- void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- atomic_add_##name(ptr, val); \
- } \
- tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
- tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
- tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_add_##name##_nv(ptr, val); \
+#define _CSAN_ATOMIC_FUNC_ADD(name, type) \
+ void kcsan_atomic_add_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ atomic_add_##name(ptr, val); \
}
-#define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
- void atomic_and_##name(volatile targ1 *, targ2); \
- void kcsan_atomic_and_##name(volatile targ1 *, targ2); \
- void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- atomic_and_##name(ptr, val); \
- } \
- tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
- tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
- tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_and_##name##_nv(ptr, val); \
+#define CSAN_ATOMIC_FUNC_ADD(name, type) \
+ _CSAN_ATOMIC_FUNC_ADD(name, type) \
+ _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_ADD(rel_##name, type)
+
+#define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
+ void kcsan_atomic_clear_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ atomic_clear_##name(ptr, val); \
}
-#define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
- void atomic_or_##name(volatile targ1 *, targ2); \
- void kcsan_atomic_or_##name(volatile targ1 *, targ2); \
- void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- atomic_or_##name(ptr, val); \
- } \
- tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
- tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
- tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_or_##name##_nv(ptr, val); \
+#define CSAN_ATOMIC_FUNC_CLEAR(name, type) \
+ _CSAN_ATOMIC_FUNC_CLEAR(name, type) \
+ _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
+
+#define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
+ int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \
+ type val2) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_cmpset_##name(ptr, val1, val2)); \
}
-#define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
- tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
- tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
- tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_cas_##name(ptr, exp, new); \
- } \
- tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
- tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
- tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_cas_##name##_ni(ptr, exp, new); \
+#define CSAN_ATOMIC_FUNC_CMPSET(name, type) \
+ _CSAN_ATOMIC_FUNC_CMPSET(name, type) \
+ _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
+
+#define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
+ int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \
+ type val2) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_fcmpset_##name(ptr, val1, val2)); \
}
-#define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
- tret atomic_swap_##name(volatile targ1 *, targ2); \
- tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \
- tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_swap_##name(ptr, val); \
+#define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
+ _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \
+ _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
+
+#define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \
+ type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_fetchadd_##name(ptr, val)); \
}
-#define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
- void atomic_dec_##name(volatile targ1 *); \
- void kcsan_atomic_dec_##name(volatile targ1 *); \
- void kcsan_atomic_dec_##name(volatile targ1 *ptr) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- atomic_dec_##name(ptr); \
- } \
- tret atomic_dec_##name##_nv(volatile targ1 *); \
- tret kcsan_atomic_dec_##name##_nv(volatile targ1 *); \
- tret kcsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_dec_##name##_nv(ptr); \
+#define _CSAN_ATOMIC_FUNC_LOAD(name, type) \
+ type kcsan_atomic_load_##name(volatile type *ptr) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \
+ __RET_ADDR); \
+ return (atomic_load_##name(ptr)); \
}
-#define CSAN_ATOMIC_FUNC_INC(name, tret, targ1) \
- void atomic_inc_##name(volatile targ1 *); \
- void kcsan_atomic_inc_##name(volatile targ1 *); \
- void kcsan_atomic_inc_##name(volatile targ1 *ptr) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- atomic_inc_##name(ptr); \
- } \
- tret atomic_inc_##name##_nv(volatile targ1 *); \
- tret kcsan_atomic_inc_##name##_nv(volatile targ1 *); \
- tret kcsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \
- { \
- kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \
- __RET_ADDR); \
- return atomic_inc_##name##_nv(ptr); \
+#define CSAN_ATOMIC_FUNC_LOAD(name, type) \
+ _CSAN_ATOMIC_FUNC_LOAD(name, type) \
+ _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \
+
+#define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
+ type kcsan_atomic_readandclear_##name(volatile type *ptr) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_readandclear_##name(ptr)); \
}
-CSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t);
-CSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t);
-CSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int);
-CSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long);
-CSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t);
+#define _CSAN_ATOMIC_FUNC_SET(name, type) \
+ void kcsan_atomic_set_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ atomic_set_##name(ptr, val); \
+ }
-CSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t);
-CSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t);
-CSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int);
-CSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long);
+#define CSAN_ATOMIC_FUNC_SET(name, type) \
+ _CSAN_ATOMIC_FUNC_SET(name, type) \
+ _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_SET(rel_##name, type)
-CSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t);
-CSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t);
-CSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int);
-CSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long);
+#define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
+ void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ atomic_subtract_##name(ptr, val); \
+ }
-CSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t);
-CSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t);
-CSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int);
-CSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long);
-CSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *);
-CSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t);
-CSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t);
-CSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int);
-CSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long);
-CSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *);
+#define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
+ _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
+ _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
+ _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
-CSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t)
-CSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t)
-CSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int);
-CSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long);
-CSAN_ATOMIC_FUNC_DEC(ptr, void *, void);
+#define _CSAN_ATOMIC_FUNC_STORE(name, type) \
+ void kcsan_atomic_store_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ atomic_store_##name(ptr, val); \
+ }
-CSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t)
-CSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t)
-CSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int);
-CSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
-CSAN_ATOMIC_FUNC_INC(ptr, void *, void);
+#define CSAN_ATOMIC_FUNC_STORE(name, type) \
+ _CSAN_ATOMIC_FUNC_STORE(name, type) \
+ _CSAN_ATOMIC_FUNC_STORE(rel_##name, type)
+#define CSAN_ATOMIC_FUNC_SWAP(name, type) \
+ type kcsan_atomic_swap_##name(volatile type *ptr, type val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return(atomic_swap_##name(ptr, val)); \
+ }
+
+#define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
+ int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return(atomic_testandclear_##name(ptr, val)); \
+ }
+
+#define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \
+ int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \
+ { \
+ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \
+ __RET_ADDR); \
+ return (atomic_testandset_##name(ptr, val)); \
+ }
+
+
+CSAN_ATOMIC_FUNC_ADD(8, uint8_t)
+CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t)
+CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t)
+CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t)
+_CSAN_ATOMIC_FUNC_LOAD(8, uint8_t)
+CSAN_ATOMIC_FUNC_SET(8, uint8_t)
+CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t)
+_CSAN_ATOMIC_FUNC_STORE(8, uint8_t)
+#if 0
+CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t)
+CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t)
+CSAN_ATOMIC_FUNC_SWAP(8, uint8_t)
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t)
+CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t)
+#endif
+
+CSAN_ATOMIC_FUNC_ADD(16, uint16_t)
+CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t)
+CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t)
+CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t)
+#if defined(__aarch64__)
+_CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
+#else
+CSAN_ATOMIC_FUNC_LOAD(16, uint16_t)
+#endif
+CSAN_ATOMIC_FUNC_SET(16, uint16_t)
+CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t)
+_CSAN_ATOMIC_FUNC_STORE(16, uint16_t)
+#if 0
+CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t)
+CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t)
+CSAN_ATOMIC_FUNC_SWAP(16, uint16_t)
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t)
+CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t)
+#endif
+
+CSAN_ATOMIC_FUNC_ADD(32, uint32_t)
+CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t)
+CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t)
+CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t)
+CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t)
+CSAN_ATOMIC_FUNC_LOAD(32, uint32_t)
+CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t)
+CSAN_ATOMIC_FUNC_SET(32, uint32_t)
+CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t)
+CSAN_ATOMIC_FUNC_STORE(32, uint32_t)
+CSAN_ATOMIC_FUNC_SWAP(32, uint32_t)
+#if !defined(__aarch64__)
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t)
+CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t)
+#endif
+
+CSAN_ATOMIC_FUNC_ADD(64, uint64_t)
+CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t)
+CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t)
+CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t)
+CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t)
+CSAN_ATOMIC_FUNC_LOAD(64, uint64_t)
+CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t)
+CSAN_ATOMIC_FUNC_SET(64, uint64_t)
+CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t)
+CSAN_ATOMIC_FUNC_STORE(64, uint64_t)
+CSAN_ATOMIC_FUNC_SWAP(64, uint64_t)
+#if !defined(__aarch64__)
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t)
+CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t)
+#endif
+
+CSAN_ATOMIC_FUNC_ADD(int, u_int)
+CSAN_ATOMIC_FUNC_CLEAR(int, u_int)
+CSAN_ATOMIC_FUNC_CMPSET(int, u_int)
+CSAN_ATOMIC_FUNC_FCMPSET(int, u_int)
+CSAN_ATOMIC_FUNC_FETCHADD(int, u_int)
+CSAN_ATOMIC_FUNC_LOAD(int, u_int)
+CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int)
+CSAN_ATOMIC_FUNC_SET(int, u_int)
+CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int)
+CSAN_ATOMIC_FUNC_STORE(int, u_int)
+CSAN_ATOMIC_FUNC_SWAP(int, u_int)
+#if !defined(__aarch64__)
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int)
+CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int)
+#endif
+
+CSAN_ATOMIC_FUNC_ADD(long, u_long)
+CSAN_ATOMIC_FUNC_CLEAR(long, u_long)
+CSAN_ATOMIC_FUNC_CMPSET(long, u_long)
+CSAN_ATOMIC_FUNC_FCMPSET(long, u_long)
+CSAN_ATOMIC_FUNC_FETCHADD(long, u_long)
+CSAN_ATOMIC_FUNC_LOAD(long, u_long)
+CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long)
+CSAN_ATOMIC_FUNC_SET(long, u_long)
+CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long)
+CSAN_ATOMIC_FUNC_STORE(long, u_long)
+CSAN_ATOMIC_FUNC_SWAP(long, u_long)
+#if !defined(__aarch64__)
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long)
+CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long)
+#endif
+
+CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t)
+#if !defined(__amd64__)
+CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t)
+#endif
+CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t)
+#if 0
+CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t)
+CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t)
+#endif
+
+#define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \
+ void kcsan_atomic_thread_fence_##name(void) \
+ { \
+ atomic_thread_fence_##name(); \
+ }
+
+
+CSAN_ATOMIC_FUNC_THREAD_FENCE(acq)
+CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel)
+CSAN_ATOMIC_FUNC_THREAD_FENCE(rel)
+CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst)
+
/* -------------------------------------------------------------------------- */
#include <sys/bus.h>
+#include <machine/bus.h>
+#include <sys/_cscan_bus.h>
-#undef bus_space_read_multi_1
-#undef bus_space_read_multi_2
-#undef bus_space_read_multi_4
-#undef bus_space_read_multi_8
-#undef bus_space_read_multi_stream_1
-#undef bus_space_read_multi_stream_2
-#undef bus_space_read_multi_stream_4
-#undef bus_space_read_multi_stream_8
-#undef bus_space_read_region_1
-#undef bus_space_read_region_2
-#undef bus_space_read_region_4
-#undef bus_space_read_region_8
-#undef bus_space_read_region_stream_1
-#undef bus_space_read_region_stream_2
-#undef bus_space_read_region_stream_4
-#undef bus_space_read_region_stream_8
-#undef bus_space_write_multi_1
-#undef bus_space_write_multi_2
-#undef bus_space_write_multi_4
-#undef bus_space_write_multi_8
-#undef bus_space_write_multi_stream_1
-#undef bus_space_write_multi_stream_2
-#undef bus_space_write_multi_stream_4
-#undef bus_space_write_multi_stream_8
-#undef bus_space_write_region_1
-#undef bus_space_write_region_2
-#undef bus_space_write_region_4
-#undef bus_space_write_region_8
-#undef bus_space_write_region_stream_1
-#undef bus_space_write_region_stream_2
-#undef bus_space_write_region_stream_4
-#undef bus_space_write_region_stream_8
+int
+kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
+ int flags, bus_space_handle_t *handlep)
+{
-#define CSAN_BUS_READ_FUNC(bytes, bits) \
- void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
- bus_size_t, uint##bits##_t *, bus_size_t); \
- void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t, \
- bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
- void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \
- bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
- bus_size_t count) \
- { \
- kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
- false, false, __RET_ADDR); \
- bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \
- } \
- void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
- bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
- void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
- bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
- void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \
- bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
- bus_size_t count) \
- { \
- kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
- false, false, __RET_ADDR); \
- bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\
- } \
- void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \
- bus_size_t, uint##bits##_t *, bus_size_t); \
- void kcsan_bus_space_read_region_##bytes(bus_space_tag_t, \
- bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
- void kcsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \
- bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
- bus_size_t count) \
- { \
- kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
- false, false, __RET_ADDR); \
- bus_space_read_region_##bytes(tag, hnd, size, buf, count); \
- } \
- void bus_space_read_region_stream_##bytes(bus_space_tag_t, \
- bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
- void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \
- bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
- void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \
- bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
- bus_size_t count) \
- { \
- kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
- false, false, __RET_ADDR); \
- bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\
+ return (bus_space_map(tag, hnd, size, flags, handlep));
+}
+
+void
+kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
+ bus_size_t size)
+{
+
+ bus_space_unmap(tag, hnd, size);
+}
+
+int
+kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
+ bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
+{
+
+ return (bus_space_subregion(tag, hnd, offset, size, handlep));
+}
+
+#if !defined(__amd64__)
+int
+kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start,
+ bus_addr_t reg_end, bus_size_t size, bus_size_t alignment,
+ bus_size_t boundary, int flags, bus_addr_t *addrp,
+ bus_space_handle_t *handlep)
+{
+
+ return (bus_space_alloc(tag, reg_start, reg_end, size, alignment,
+ boundary, flags, addrp, handlep));
+}
+#endif
+
+void
+kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
+ bus_size_t size)
+{
+
+ bus_space_free(tag, hnd, size);
+}
+
+void
+kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
+ bus_size_t offset, bus_size_t size, int flags)
+{
+
+ bus_space_barrier(tag, hnd, offset, size, flags);
+}
+
+#define CSAN_BUS_READ_FUNC(func, width, type) \
+ type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t offset) \
+ { \
+ return (bus_space_read##func##_##width(tag, hnd, \
+ offset)); \
+ } \
+
+#define CSAN_BUS_READ_PTR_FUNC(func, width, type) \
+ void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t size, type *buf, \
+ bus_size_t count) \
+ { \
+ kcsan_access((uintptr_t)buf, sizeof(type) * count, \
+ false, false, __RET_ADDR); \
+ bus_space_read_##func##_##width(tag, hnd, size, buf, \
+ count); \
}
-#define CSAN_BUS_WRITE_FUNC(bytes, bits) \
- void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
- bus_size_t, const uint##bits##_t *, bus_size_t); \
- void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t, \
- bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
- void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \
- bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
- bus_size_t count) \
- { \
- kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
- true, false, __RET_ADDR); \
- bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \
- } \
- void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
- bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
- void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
- bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
- void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \
- bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
- bus_size_t count) \
- { \
- kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
- true, false, __RET_ADDR); \
- bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\
- } \
- void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\
- bus_size_t, const uint##bits##_t *, bus_size_t); \
- void kcsan_bus_space_write_region_##bytes(bus_space_tag_t, \
- bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
- void kcsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \
- bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
- bus_size_t count) \
- { \
- kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
- true, false, __RET_ADDR); \
- bus_space_write_region_##bytes(tag, hnd, size, buf, count); \
- } \
- void bus_space_write_region_stream_##bytes(bus_space_tag_t, \
- bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
- void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \
- bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
- void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \
- bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
- bus_size_t count) \
- { \
- kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \
- true, false, __RET_ADDR); \
- bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\
+CSAN_BUS_READ_FUNC(, 1, uint8_t)
+CSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
+CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
+CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
+CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
+CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
+
+CSAN_BUS_READ_FUNC(, 2, uint16_t)
+CSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
+CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
+CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
+CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
+CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
+
+CSAN_BUS_READ_FUNC(, 4, uint32_t)
+CSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
+CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
+CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
+CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
+CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
+
+CSAN_BUS_READ_FUNC(, 8, uint64_t)
+#if defined(__aarch64__)
+CSAN_BUS_READ_FUNC(_stream, 8, uint64_t)
+CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
+CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
+CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
+CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
+#endif
+
+#define CSAN_BUS_WRITE_FUNC(func, width, type) \
+ void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t offset, type value) \
+ { \
+ bus_space_write##func##_##width(tag, hnd, offset, value); \
+ } \
+
+#define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \
+ void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t size, const type *buf, \
+ bus_size_t count) \
+ { \
+ kcsan_access((uintptr_t)buf, sizeof(type) * count, \
+ true, false, __RET_ADDR); \
+ bus_space_write_##func##_##width(tag, hnd, size, buf, \
+ count); \
}
-CSAN_BUS_READ_FUNC(1, 8)
-CSAN_BUS_READ_FUNC(2, 16)
-CSAN_BUS_READ_FUNC(4, 32)
-CSAN_BUS_READ_FUNC(8, 64)
+CSAN_BUS_WRITE_FUNC(, 1, uint8_t)
+CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
+CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
+CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
-CSAN_BUS_WRITE_FUNC(1, 8)
-CSAN_BUS_WRITE_FUNC(2, 16)
-CSAN_BUS_WRITE_FUNC(4, 32)
-CSAN_BUS_WRITE_FUNC(8, 64)
+CSAN_BUS_WRITE_FUNC(, 2, uint16_t)
+CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
+CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
+CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
+
+CSAN_BUS_WRITE_FUNC(, 4, uint32_t)
+CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
+CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
+CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
+
+CSAN_BUS_WRITE_FUNC(, 8, uint64_t)
+#if defined(__aarch64__)
+CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t)
+CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t)
+CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t)
+CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t)
+#endif
+
+#define CSAN_BUS_SET_FUNC(func, width, type) \
+ void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
+ bus_space_handle_t hnd, bus_size_t offset, type value, \
+ bus_size_t count) \
+ { \
+ bus_space_set_##func##_##width(tag, hnd, offset, value, \
+ count); \
+ }
+
+CSAN_BUS_SET_FUNC(multi, 1, uint8_t)
+CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
+CSAN_BUS_SET_FUNC(region, 1, uint8_t)
+CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
+
+CSAN_BUS_SET_FUNC(multi, 2, uint16_t)
+CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
+CSAN_BUS_SET_FUNC(region, 2, uint16_t)
+CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
+
+CSAN_BUS_SET_FUNC(multi, 4, uint32_t)
+CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
+CSAN_BUS_SET_FUNC(region, 4, uint32_t)
+CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
+
+#if !defined(__amd64__)
+CSAN_BUS_SET_FUNC(multi, 8, uint64_t)
+CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t)
+CSAN_BUS_SET_FUNC(region, 8, uint64_t)
+CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t)
+#endif
+
Index: head/sys/kern/vfs_aio.c
===================================================================
--- head/sys/kern/vfs_aio.c
+++ head/sys/kern/vfs_aio.c
@@ -292,7 +292,7 @@
* Different ABIs provide their own operations.
*/
struct aiocb_ops {
- int (*copyin)(struct aiocb *ujob, struct aiocb *kjob);
+ int (*aio_copyin)(struct aiocb *ujob, struct aiocb *kjob);
long (*fetch_status)(struct aiocb *ujob);
long (*fetch_error)(struct aiocb *ujob);
int (*store_status)(struct aiocb *ujob, long status);
@@ -1420,7 +1420,7 @@
}
static struct aiocb_ops aiocb_ops = {
- .copyin = aiocb_copyin,
+ .aio_copyin = aiocb_copyin,
.fetch_status = aiocb_fetch_status,
.fetch_error = aiocb_fetch_error,
.store_status = aiocb_store_status,
@@ -1431,7 +1431,7 @@
#ifdef COMPAT_FREEBSD6
static struct aiocb_ops aiocb_ops_osigevent = {
- .copyin = aiocb_copyin_old_sigevent,
+ .aio_copyin = aiocb_copyin_old_sigevent,
.fetch_status = aiocb_fetch_status,
.fetch_error = aiocb_fetch_error,
.store_status = aiocb_store_status,
@@ -1478,7 +1478,7 @@
job = uma_zalloc(aiocb_zone, M_WAITOK | M_ZERO);
knlist_init_mtx(&job->klist, AIO_MTX(ki));
- error = ops->copyin(ujob, &job->uaiocb);
+ error = ops->aio_copyin(ujob, &job->uaiocb);
if (error) {
ops->store_error(ujob, error);
uma_zfree(aiocb_zone, job);
@@ -2743,7 +2743,7 @@
}
static struct aiocb_ops aiocb32_ops = {
- .copyin = aiocb32_copyin,
+ .aio_copyin = aiocb32_copyin,
.fetch_status = aiocb32_fetch_status,
.fetch_error = aiocb32_fetch_error,
.store_status = aiocb32_store_status,
@@ -2754,7 +2754,7 @@
#ifdef COMPAT_FREEBSD6
static struct aiocb_ops aiocb32_ops_osigevent = {
- .copyin = aiocb32_copyin_old_sigevent,
+ .aio_copyin = aiocb32_copyin_old_sigevent,
.fetch_status = aiocb32_fetch_status,
.fetch_error = aiocb32_fetch_error,
.store_status = aiocb32_store_status,
Index: head/sys/libkern/strcmp.c
===================================================================
--- head/sys/libkern/strcmp.c
+++ head/sys/libkern/strcmp.c
@@ -41,7 +41,7 @@
* Compare strings.
*/
int
-strcmp(const char *s1, const char *s2)
+(strcmp)(const char *s1, const char *s2)
{
while (*s1 == *s2++)
if (*s1++ == '\0')
Index: head/sys/libkern/strcpy.c
===================================================================
--- head/sys/libkern/strcpy.c
+++ head/sys/libkern/strcpy.c
@@ -35,7 +35,7 @@
#include <sys/libkern.h>
char *
-strcpy(char * __restrict to, const char * __restrict from)
+(strcpy)(char * __restrict to, const char * __restrict from)
{
char *save = to;
Index: head/sys/libkern/strlen.c
===================================================================
--- head/sys/libkern/strlen.c
+++ head/sys/libkern/strlen.c
@@ -80,7 +80,7 @@
} while (0)
size_t
-strlen(const char *str)
+(strlen)(const char *str)
{
const char *p;
const unsigned long *lp;
Index: head/sys/modules/Makefile
===================================================================
--- head/sys/modules/Makefile
+++ head/sys/modules/Makefile
@@ -644,11 +644,11 @@
_chromebook_platform= chromebook_platform
_cmx= cmx
_coretemp= coretemp
-.if ${MK_SOURCELESS_HOST} != "no"
+.if ${MK_SOURCELESS_HOST} != "no" && empty(KCSAN_ENABLED)
_hpt27xx= hpt27xx
.endif
_hptiop= hptiop
-.if ${MK_SOURCELESS_HOST} != "no"
+.if ${MK_SOURCELESS_HOST} != "no" && empty(KCSAN_ENABLED)
_hptmv= hptmv
_hptnr= hptnr
_hptrr= hptrr
Index: head/sys/sys/_cscan_atomic.h
===================================================================
--- head/sys/sys/_cscan_atomic.h
+++ head/sys/sys/_cscan_atomic.h
@@ -0,0 +1,307 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 Andrew Turner
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SYS__CSAN_ATOMIC_H_
+#define _SYS__CSAN_ATOMIC_H_
+
+#ifndef _MACHINE_ATOMIC_H_
+#error do not include this header, use machine/atomic.h
+#endif
+
+#define KCSAN_ATOMIC_FUNC_1(op, name, type) \
+ void kcsan_atomic_##op##_##name(volatile type *, type); \
+ void kcsan_atomic_##op##_acq_##name(volatile type *, type); \
+ void kcsan_atomic_##op##_rel_##name(volatile type *, type)
+
+#define KCSAN_ATOMIC_CMPSET(name, type) \
+ int kcsan_atomic_cmpset_##name(volatile type *, type, type); \
+ int kcsan_atomic_cmpset_acq_##name(volatile type *, type, type); \
+ int kcsan_atomic_cmpset_rel_##name(volatile type *, type, type)
+
+#define KCSAN_ATOMIC_FCMPSET(name, type) \
+ int kcsan_atomic_fcmpset_##name(volatile type *, type *, type); \
+ int kcsan_atomic_fcmpset_acq_##name(volatile type *, type *, type); \
+ int kcsan_atomic_fcmpset_rel_##name(volatile type *, type *, type)
+
+#define KCSAN_ATOMIC_READ(op, name, type) \
+ type kcsan_atomic_##op##_##name(volatile type *, type)
+
+#define KCSAN_ATOMIC_READANDCLEAR(name, type) \
+ type kcsan_atomic_readandclear_##name(volatile type *)
+
+#define KCSAN_ATOMIC_LOAD(name, type) \
+ type kcsan_atomic_load_##name(volatile type *); \
+ type kcsan_atomic_load_acq_##name(volatile type *)
+
+#define KCSAN_ATOMIC_STORE(name, type) \
+ void kcsan_atomic_store_##name(volatile type *, type); \
+ void kcsan_atomic_store_rel_##name(volatile type *, type)
+
+#define KCSAN_ATOMIC_TEST(op, name, type) \
+ int kcsan_atomic_##op##_##name(volatile type *, u_int)
+
+#define KCSAN_ATOMIC_FUNCS(name, type) \
+ KCSAN_ATOMIC_FUNC_1(add, name, type); \
+ KCSAN_ATOMIC_FUNC_1(clear, name, type); \
+ KCSAN_ATOMIC_CMPSET(name, type); \
+ KCSAN_ATOMIC_FCMPSET(name, type); \
+ KCSAN_ATOMIC_READ(fetchadd, name, type); \
+ KCSAN_ATOMIC_LOAD(name, type); \
+ KCSAN_ATOMIC_READANDCLEAR(name, type); \
+ KCSAN_ATOMIC_FUNC_1(set, name, type); \
+ KCSAN_ATOMIC_FUNC_1(subtract, name, type); \
+ KCSAN_ATOMIC_STORE(name, type); \
+ KCSAN_ATOMIC_READ(swap, name, type); \
+ KCSAN_ATOMIC_TEST(testandclear, name, type); \
+ KCSAN_ATOMIC_TEST(testandset, name, type)
+
+KCSAN_ATOMIC_FUNCS(int, u_int);
+KCSAN_ATOMIC_FUNCS(long, u_long);
+KCSAN_ATOMIC_FUNCS(ptr, uintptr_t);
+KCSAN_ATOMIC_FUNCS(8, uint8_t);
+KCSAN_ATOMIC_FUNCS(16, uint16_t);
+KCSAN_ATOMIC_FUNCS(32, uint32_t);
+KCSAN_ATOMIC_FUNCS(64, uint64_t);
+
+void kcsan_atomic_thread_fence_acq(void);
+void kcsan_atomic_thread_fence_acq_rel(void);
+void kcsan_atomic_thread_fence_rel(void);
+void kcsan_atomic_thread_fence_seq_cst(void);
+
+#ifndef KCSAN_RUNTIME
+
+#define atomic_add_int kcsan_atomic_add_int
+#define atomic_add_acq_int kcsan_atomic_add_acq_int
+#define atomic_add_rel_int kcsan_atomic_add_rel_int
+#define atomic_clear_int kcsan_atomic_clear_int
+#define atomic_clear_acq_int kcsan_atomic_clear_acq_int
+#define atomic_clear_rel_int kcsan_atomic_clear_rel_int
+#define atomic_cmpset_int kcsan_atomic_cmpset_int
+#define atomic_cmpset_acq_int kcsan_atomic_cmpset_acq_int
+#define atomic_cmpset_rel_int kcsan_atomic_cmpset_rel_int
+#define atomic_fcmpset_int kcsan_atomic_fcmpset_int
+#define atomic_fcmpset_acq_int kcsan_atomic_fcmpset_acq_int
+#define atomic_fcmpset_rel_int kcsan_atomic_fcmpset_rel_int
+#define atomic_fetchadd_int kcsan_atomic_fetchadd_int
+#define atomic_load_int kcsan_atomic_load_int
+#define atomic_load_acq_int kcsan_atomic_load_acq_int
+#define atomic_readandclear_int kcsan_atomic_readandclear_int
+#define atomic_set_int kcsan_atomic_set_int
+#define atomic_set_acq_int kcsan_atomic_set_acq_int
+#define atomic_set_rel_int kcsan_atomic_set_rel_int
+#define atomic_subtract_int kcsan_atomic_subtract_int
+#define atomic_subtract_acq_int kcsan_atomic_subtract_acq_int
+#define atomic_subtract_rel_int kcsan_atomic_subtract_rel_int
+#define atomic_store_int kcsan_atomic_store_int
+#define atomic_store_rel_int kcsan_atomic_store_rel_int
+#define atomic_swap_int kcsan_atomic_swap_int
+#define atomic_testandclear_int kcsan_atomic_testandclear_int
+#define atomic_testandset_int kcsan_atomic_testandset_int
+
+#define atomic_add_long kcsan_atomic_add_long
+#define atomic_add_acq_long kcsan_atomic_add_acq_long
+#define atomic_add_rel_long kcsan_atomic_add_rel_long
+#define atomic_clear_long kcsan_atomic_clear_long
+#define atomic_clear_acq_long kcsan_atomic_clear_acq_long
+#define atomic_clear_rel_long kcsan_atomic_clear_rel_long
+#define atomic_cmpset_long kcsan_atomic_cmpset_long
+#define atomic_cmpset_acq_long kcsan_atomic_cmpset_acq_long
+#define atomic_cmpset_rel_long kcsan_atomic_cmpset_rel_long
+#define atomic_fcmpset_long kcsan_atomic_fcmpset_long
+#define atomic_fcmpset_acq_long kcsan_atomic_fcmpset_acq_long
+#define atomic_fcmpset_rel_long kcsan_atomic_fcmpset_rel_long
+#define atomic_fetchadd_long kcsan_atomic_fetchadd_long
+#define atomic_load_long kcsan_atomic_load_long
+#define atomic_load_acq_long kcsan_atomic_load_acq_long
+#define atomic_readandclear_long kcsan_atomic_readandclear_long
+#define atomic_set_long kcsan_atomic_set_long
+#define atomic_set_acq_long kcsan_atomic_set_acq_long
+#define atomic_set_rel_long kcsan_atomic_set_rel_long
+#define atomic_subtract_long kcsan_atomic_subtract_long
+#define atomic_subtract_acq_long kcsan_atomic_subtract_acq_long
+#define atomic_subtract_rel_long kcsan_atomic_subtract_rel_long
+#define atomic_store_long kcsan_atomic_store_long
+#define atomic_store_rel_long kcsan_atomic_store_rel_long
+#define atomic_swap_long kcsan_atomic_swap_long
+#define atomic_testandclear_long kcsan_atomic_testandclear_long
+#define atomic_testandset_long kcsan_atomic_testandset_long
+
+#define atomic_add_ptr kcsan_atomic_add_ptr
+#define atomic_add_acq_ptr kcsan_atomic_add_acq_ptr
+#define atomic_add_rel_ptr kcsan_atomic_add_rel_ptr
+#define atomic_clear_ptr kcsan_atomic_clear_ptr
+#define atomic_clear_acq_ptr kcsan_atomic_clear_acq_ptr
+#define atomic_clear_rel_ptr kcsan_atomic_clear_rel_ptr
+#define atomic_cmpset_ptr kcsan_atomic_cmpset_ptr
+#define atomic_cmpset_acq_ptr kcsan_atomic_cmpset_acq_ptr
+#define atomic_cmpset_rel_ptr kcsan_atomic_cmpset_rel_ptr
+#define atomic_fcmpset_ptr kcsan_atomic_fcmpset_ptr
+#define atomic_fcmpset_acq_ptr kcsan_atomic_fcmpset_acq_ptr
+#define atomic_fcmpset_rel_ptr kcsan_atomic_fcmpset_rel_ptr
+#define atomic_fetchadd_ptr kcsan_atomic_fetchadd_ptr
+#define atomic_load_ptr(x) kcsan_atomic_load_ptr((volatile uintptr_t *)(x))
+#define atomic_load_acq_ptr kcsan_atomic_load_acq_ptr
+#define atomic_readandclear_ptr kcsan_atomic_readandclear_ptr
+#define atomic_set_ptr kcsan_atomic_set_ptr
+#define atomic_set_acq_ptr kcsan_atomic_set_acq_ptr
+#define atomic_set_rel_ptr kcsan_atomic_set_rel_ptr
+#define atomic_subtract_ptr kcsan_atomic_subtract_ptr
+#define atomic_subtract_acq_ptr kcsan_atomic_subtract_acq_ptr
+#define atomic_subtract_rel_ptr kcsan_atomic_subtract_rel_ptr
+#define atomic_store_ptr kcsan_atomic_store_ptr
+#define atomic_store_rel_ptr kcsan_atomic_store_rel_ptr
+#define atomic_swap_ptr kcsan_atomic_swap_ptr
+#define atomic_testandclear_ptr kcsan_atomic_testandclear_ptr
+#define atomic_testandset_ptr kcsan_atomic_testandset_ptr
+
+#define atomic_add_8 kcsan_atomic_add_8
+#define atomic_add_acq_8 kcsan_atomic_add_acq_8
+#define atomic_add_rel_8 kcsan_atomic_add_rel_8
+#define atomic_clear_8 kcsan_atomic_clear_8
+#define atomic_clear_acq_8 kcsan_atomic_clear_acq_8
+#define atomic_clear_rel_8 kcsan_atomic_clear_rel_8
+#define atomic_cmpset_8 kcsan_atomic_cmpset_8
+#define atomic_cmpset_acq_8 kcsan_atomic_cmpset_acq_8
+#define atomic_cmpset_rel_8 kcsan_atomic_cmpset_rel_8
+#define atomic_fcmpset_8 kcsan_atomic_fcmpset_8
+#define atomic_fcmpset_acq_8 kcsan_atomic_fcmpset_acq_8
+#define atomic_fcmpset_rel_8 kcsan_atomic_fcmpset_rel_8
+#define atomic_fetchadd_8 kcsan_atomic_fetchadd_8
+#define atomic_load_8 kcsan_atomic_load_8
+#define atomic_load_acq_8 kcsan_atomic_load_acq_8
+#define atomic_readandclear_8 kcsan_atomic_readandclear_8
+#define atomic_set_8 kcsan_atomic_set_8
+#define atomic_set_acq_8 kcsan_atomic_set_acq_8
+#define atomic_set_rel_8 kcsan_atomic_set_rel_8
+#define atomic_subtract_8 kcsan_atomic_subtract_8
+#define atomic_subtract_acq_8 kcsan_atomic_subtract_acq_8
+#define atomic_subtract_rel_8 kcsan_atomic_subtract_rel_8
+#define atomic_store_8 kcsan_atomic_store_8
+#define atomic_store_rel_8 kcsan_atomic_store_rel_8
+#define atomic_swap_8 kcsan_atomic_swap_8
+#define atomic_testandclear_8 kcsan_atomic_testandclear_8
+#define atomic_testandset_8 kcsan_atomic_testandset_8
+
+#define atomic_add_16 kcsan_atomic_add_16
+#define atomic_add_acq_16 kcsan_atomic_add_acq_16
+#define atomic_add_rel_16 kcsan_atomic_add_rel_16
+#define atomic_clear_16 kcsan_atomic_clear_16
+#define atomic_clear_acq_16 kcsan_atomic_clear_acq_16
+#define atomic_clear_rel_16 kcsan_atomic_clear_rel_16
+#define atomic_cmpset_16 kcsan_atomic_cmpset_16
+#define atomic_cmpset_acq_16 kcsan_atomic_cmpset_acq_16
+#define atomic_cmpset_rel_16 kcsan_atomic_cmpset_rel_16
+#define atomic_fcmpset_16 kcsan_atomic_fcmpset_16
+#define atomic_fcmpset_acq_16 kcsan_atomic_fcmpset_acq_16
+#define atomic_fcmpset_rel_16 kcsan_atomic_fcmpset_rel_16
+#define atomic_fetchadd_16 kcsan_atomic_fetchadd_16
+#define atomic_load_16 kcsan_atomic_load_16
+#define atomic_load_acq_16 kcsan_atomic_load_acq_16
+#define atomic_readandclear_16 kcsan_atomic_readandclear_16
+#define atomic_set_16 kcsan_atomic_set_16
+#define atomic_set_acq_16 kcsan_atomic_set_acq_16
+#define atomic_set_rel_16 kcsan_atomic_set_rel_16
+#define atomic_subtract_16 kcsan_atomic_subtract_16
+#define atomic_subtract_acq_16 kcsan_atomic_subtract_acq_16
+#define atomic_subtract_rel_16 kcsan_atomic_subtract_rel_16
+#define atomic_store_16 kcsan_atomic_store_16
+#define atomic_store_rel_16 kcsan_atomic_store_rel_16
+#define atomic_swap_16 kcsan_atomic_swap_16
+#define atomic_testandclear_16 kcsan_atomic_testandclear_16
+#define atomic_testandset_16 kcsan_atomic_testandset_16
+
+#define atomic_add_32 kcsan_atomic_add_32
+#define atomic_add_acq_32 kcsan_atomic_add_acq_32
+#define atomic_add_rel_32 kcsan_atomic_add_rel_32
+#define atomic_clear_32 kcsan_atomic_clear_32
+#define atomic_clear_acq_32 kcsan_atomic_clear_acq_32
+#define atomic_clear_rel_32 kcsan_atomic_clear_rel_32
+#define atomic_cmpset_32 kcsan_atomic_cmpset_32
+#define atomic_cmpset_acq_32 kcsan_atomic_cmpset_acq_32
+#define atomic_cmpset_rel_32 kcsan_atomic_cmpset_rel_32
+#define atomic_fcmpset_32 kcsan_atomic_fcmpset_32
+#define atomic_fcmpset_acq_32 kcsan_atomic_fcmpset_acq_32
+#define atomic_fcmpset_rel_32 kcsan_atomic_fcmpset_rel_32
+#define atomic_fetchadd_32 kcsan_atomic_fetchadd_32
+#define atomic_load_32 kcsan_atomic_load_32
+#define atomic_load_acq_32 kcsan_atomic_load_acq_32
+#define atomic_readandclear_32 kcsan_atomic_readandclear_32
+#define atomic_set_32 kcsan_atomic_set_32
+#define atomic_set_acq_32 kcsan_atomic_set_acq_32
+#define atomic_set_rel_32 kcsan_atomic_set_rel_32
+#define atomic_subtract_32 kcsan_atomic_subtract_32
+#define atomic_subtract_acq_32 kcsan_atomic_subtract_acq_32
+#define atomic_subtract_rel_32 kcsan_atomic_subtract_rel_32
+#define atomic_store_32 kcsan_atomic_store_32
+#define atomic_store_rel_32 kcsan_atomic_store_rel_32
+#define atomic_swap_32 kcsan_atomic_swap_32
+#define atomic_testandclear_32 kcsan_atomic_testandclear_32
+#define atomic_testandset_32 kcsan_atomic_testandset_32
+
+#define atomic_add_64 kcsan_atomic_add_64
+#define atomic_add_acq_64 kcsan_atomic_add_acq_64
+#define atomic_add_rel_64 kcsan_atomic_add_rel_64
+#define atomic_clear_64 kcsan_atomic_clear_64
+#define atomic_clear_acq_64 kcsan_atomic_clear_acq_64
+#define atomic_clear_rel_64 kcsan_atomic_clear_rel_64
+#define atomic_cmpset_64 kcsan_atomic_cmpset_64
+#define atomic_cmpset_acq_64 kcsan_atomic_cmpset_acq_64
+#define atomic_cmpset_rel_64 kcsan_atomic_cmpset_rel_64
+#define atomic_fcmpset_64 kcsan_atomic_fcmpset_64
+#define atomic_fcmpset_acq_64 kcsan_atomic_fcmpset_acq_64
+#define atomic_fcmpset_rel_64 kcsan_atomic_fcmpset_rel_64
+#define atomic_fetchadd_64 kcsan_atomic_fetchadd_64
+#define atomic_load_64 kcsan_atomic_load_64
+#define atomic_load_acq_64 kcsan_atomic_load_acq_64
+#define atomic_readandclear_64 kcsan_atomic_readandclear_64
+#define atomic_set_64 kcsan_atomic_set_64
+#define atomic_set_acq_64 kcsan_atomic_set_acq_64
+#define atomic_set_rel_64 kcsan_atomic_set_rel_64
+#define atomic_subtract_64 kcsan_atomic_subtract_64
+#define atomic_subtract_acq_64 kcsan_atomic_subtract_acq_64
+#define atomic_subtract_rel_64 kcsan_atomic_subtract_rel_64
+#define atomic_store_64 kcsan_atomic_store_64
+#define atomic_store_rel_64 kcsan_atomic_store_rel_64
+#define atomic_swap_64 kcsan_atomic_swap_64
+#define atomic_testandclear_64 kcsan_atomic_testandclear_64
+#define atomic_testandset_64 kcsan_atomic_testandset_64
+
+#define atomic_thread_fence_acq kcsan_atomic_thread_fence_acq
+#define atomic_thread_fence_acq_rel kcsan_atomic_thread_fence_acq_rel
+#define atomic_thread_fence_rel kcsan_atomic_thread_fence_rel
+#define atomic_thread_fence_seq_cst kcsan_atomic_thread_fence_seq_cst
+
+#endif /* !KCSAN_RUNTIME */
+
+#endif /* !_SYS__CSAN_ATOMIC_H_ */
Index: head/sys/sys/_cscan_bus.h
===================================================================
--- head/sys/sys/_cscan_bus.h
+++ head/sys/sys/_cscan_bus.h
@@ -0,0 +1,190 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 Andrew Turner
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory (Department of Computer Science and
+ * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
+ * DARPA SSITH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SYS__CSAN_BUS_H_
+#define _SYS__CSAN_BUS_H_
+
+#define KCSAN_BS_MULTI(rw, width, type) \
+ void kcsan_bus_space_##rw##_multi_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type *, bus_size_t); \
+ void kcsan_bus_space_##rw##_multi_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type *, bus_size_t); \
+ void kcsan_bus_space_##rw##_region_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type *, bus_size_t); \
+ void kcsan_bus_space_##rw##_region_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type *, bus_size_t)
+
+#define KCSAN_BS_READ(width, type) \
+ type kcsan_bus_space_read_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t); \
+ type kcsan_bus_space_read_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t); \
+ KCSAN_BS_MULTI(read, width, type)
+
+#define KCSAN_BS_WRITE(width, type) \
+ void kcsan_bus_space_write_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type); \
+ void kcsan_bus_space_write_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type); \
+ KCSAN_BS_MULTI(write, width, const type)
+
+#define KCSAN_BS_SET(width, type) \
+ void kcsan_bus_space_set_multi_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type, bus_size_t); \
+ void kcsan_bus_space_set_multi_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type, bus_size_t); \
+ void kcsan_bus_space_set_region_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type, bus_size_t); \
+ void kcsan_bus_space_set_region_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, type, bus_size_t)
+
+#define KCSAN_BS_COPY(width, type) \
+ void kcsan_bus_space_copy_region_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, bus_space_handle_t, \
+ bus_size_t, bus_size_t); \
+ void kcsan_bus_space_copy_region_stream_##width(bus_space_tag_t, \
+ bus_space_handle_t, bus_size_t, bus_space_handle_t, \
+ bus_size_t, bus_size_t);
+
+#define KCSAN_BS(width, type) \
+ KCSAN_BS_READ(width, type); \
+ KCSAN_BS_WRITE(width, type); \
+ KCSAN_BS_SET(width, type); \
+ KCSAN_BS_COPY(width, type)
+
+KCSAN_BS(1, uint8_t);
+KCSAN_BS(2, uint16_t);
+KCSAN_BS(4, uint32_t);
+KCSAN_BS(8, uint64_t);
+
+int kcsan_bus_space_map(bus_space_tag_t, bus_addr_t, bus_size_t, int,
+ bus_space_handle_t *);
+void kcsan_bus_space_unmap(bus_space_tag_t, bus_space_handle_t, bus_size_t);
+int kcsan_bus_space_subregion(bus_space_tag_t, bus_space_handle_t, bus_size_t,
+ bus_size_t, bus_space_handle_t *);
+int kcsan_bus_space_alloc(bus_space_tag_t, bus_addr_t, bus_addr_t,
+ bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *,
+ bus_space_handle_t *);
+void kcsan_bus_space_free(bus_space_tag_t, bus_space_handle_t, bus_size_t);
+void kcsan_bus_space_barrier(bus_space_tag_t, bus_space_handle_t, bus_size_t,
+ bus_size_t, int);
+
+#ifndef KCSAN_RUNTIME
+
+#define bus_space_map kcsan_bus_space_map
+#define bus_space_unmap kcsan_bus_space_unmap
+#define bus_space_subregion kcsan_bus_space_subregion
+#define bus_space_alloc kcsan_bus_space_alloc
+#define bus_space_free kcsan_bus_space_free
+#define bus_space_barrier kcsan_bus_space_barrier
+
+#define bus_space_read_1 kcsan_bus_space_read_1
+#define bus_space_read_stream_1 kcsan_bus_space_read_stream_1
+#define bus_space_read_multi_1 kcsan_bus_space_read_multi_1
+#define bus_space_read_multi_stream_1 kcsan_bus_space_read_multi_stream_1
+#define bus_space_read_region_1 kcsan_bus_space_read_region_1
+#define bus_space_read_region_stream_1 kcsan_bus_space_read_region_stream_1
+#define bus_space_write_1 kcsan_bus_space_write_1
+#define bus_space_write_stream_1 kcsan_bus_space_write_stream_1
+#define bus_space_write_multi_1 kcsan_bus_space_write_multi_1
+#define bus_space_write_multi_stream_1 kcsan_bus_space_write_multi_stream_1
+#define bus_space_write_region_1 kcsan_bus_space_write_region_1
+#define bus_space_write_region_stream_1 kcsan_bus_space_write_region_stream_1
+#define bus_space_set_multi_1 kcsan_bus_space_set_multi_1
+#define bus_space_set_multi_stream_1 kcsan_bus_space_set_multi_stream_1
+#define bus_space_set_region_1 kcsan_bus_space_set_region_1
+#define bus_space_set_region_stream_1 kcsan_bus_space_set_region_stream_1
+#define bus_space_copy_multi_1 kcsan_bus_space_copy_multi_1
+#define bus_space_copy_multi_stream_1 kcsan_bus_space_copy_multi_stream_1
+
+#define bus_space_read_2 kcsan_bus_space_read_2
+#define bus_space_read_stream_2 kcsan_bus_space_read_stream_2
+#define bus_space_read_multi_2 kcsan_bus_space_read_multi_2
+#define bus_space_read_multi_stream_2 kcsan_bus_space_read_multi_stream_2
+#define bus_space_read_region_2 kcsan_bus_space_read_region_2
+#define bus_space_read_region_stream_2 kcsan_bus_space_read_region_stream_2
+#define bus_space_write_2 kcsan_bus_space_write_2
+#define bus_space_write_stream_2 kcsan_bus_space_write_stream_2
+#define bus_space_write_multi_2 kcsan_bus_space_write_multi_2
+#define bus_space_write_multi_stream_2 kcsan_bus_space_write_multi_stream_2
+#define bus_space_write_region_2 kcsan_bus_space_write_region_2
+#define bus_space_write_region_stream_2 kcsan_bus_space_write_region_stream_2
+#define bus_space_set_multi_2 kcsan_bus_space_set_multi_2
+#define bus_space_set_multi_stream_2 kcsan_bus_space_set_multi_stream_2
+#define bus_space_set_region_2 kcsan_bus_space_set_region_2
+#define bus_space_set_region_stream_2 kcsan_bus_space_set_region_stream_2
+#define bus_space_copy_multi_2 kcsan_bus_space_copy_multi_2
+#define bus_space_copy_multi_stream_2 kcsan_bus_space_copy_multi_stream_2
+
+#define bus_space_read_4 kcsan_bus_space_read_4
+#define bus_space_read_stream_4 kcsan_bus_space_read_stream_4
+#define bus_space_read_multi_4 kcsan_bus_space_read_multi_4
+#define bus_space_read_multi_stream_4 kcsan_bus_space_read_multi_stream_4
+#define bus_space_read_region_4 kcsan_bus_space_read_region_4
+#define bus_space_read_region_stream_4 kcsan_bus_space_read_region_stream_4
+#define bus_space_write_4 kcsan_bus_space_write_4
+#define bus_space_write_stream_4 kcsan_bus_space_write_stream_4
+#define bus_space_write_multi_4 kcsan_bus_space_write_multi_4
+#define bus_space_write_multi_stream_4 kcsan_bus_space_write_multi_stream_4
+#define bus_space_write_region_4 kcsan_bus_space_write_region_4
+#define bus_space_write_region_stream_4 kcsan_bus_space_write_region_stream_4
+#define bus_space_set_multi_4 kcsan_bus_space_set_multi_4
+#define bus_space_set_multi_stream_4 kcsan_bus_space_set_multi_stream_4
+#define bus_space_set_region_4 kcsan_bus_space_set_region_4
+#define bus_space_set_region_stream_4 kcsan_bus_space_set_region_stream_4
+#define bus_space_copy_multi_4 kcsan_bus_space_copy_multi_4
+#define bus_space_copy_multi_stream_4 kcsan_bus_space_copy_multi_stream_4
+
+#define bus_space_read_8 kcsan_bus_space_read_8
+#define bus_space_read_stream_8 kcsan_bus_space_read_stream_8
+#define bus_space_read_multi_8 kcsan_bus_space_read_multi_8
+#define bus_space_read_multi_stream_8 kcsan_bus_space_read_multi_stream_8
+#define bus_space_read_region_8 kcsan_bus_space_read_region_8
+#define bus_space_read_region_stream_8 kcsan_bus_space_read_region_stream_8
+#define bus_space_write_8 kcsan_bus_space_write_8
+#define bus_space_write_stream_8 kcsan_bus_space_write_stream_8
+#define bus_space_write_multi_8 kcsan_bus_space_write_multi_8
+#define bus_space_write_multi_stream_8 kcsan_bus_space_write_multi_stream_8
+#define bus_space_write_region_8 kcsan_bus_space_write_region_8
+#define bus_space_write_region_stream_8 kcsan_bus_space_write_region_stream_8
+#define bus_space_set_multi_8 kcsan_bus_space_set_multi_8
+#define bus_space_set_multi_stream_8 kcsan_bus_space_set_multi_stream_8
+#define bus_space_set_region_8 kcsan_bus_space_set_region_8
+#define bus_space_set_region_stream_8 kcsan_bus_space_set_region_stream_8
+#define bus_space_copy_multi_8 kcsan_bus_space_copy_multi_8
+#define bus_space_copy_multi_stream_8 kcsan_bus_space_copy_multi_stream_8
+
+#endif /* !KCSAN_RUNTIME */
+
+#endif /* !_SYS__CSAN_BUS_H_ */
Index: head/sys/sys/csan.h
===================================================================
--- head/sys/sys/csan.h
+++ head/sys/sys/csan.h
@@ -27,23 +27,19 @@
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
*/
#ifndef _SYS_CSAN_H_
#define _SYS_CSAN_H_
-#ifdef _KERNEL_OPT
-#include "opt_kcsan.h"
-#endif
-
#include <sys/types.h>
#ifdef KCSAN
-void kcsan_init(void);
-void kcsan_cpu_init(struct cpu_info *);
+void kcsan_cpu_init(u_int);
#else
-#define kcsan_init() __nothing
-#define kcsan_cpu_init(ci) __nothing
+#define kcsan_cpu_init(ci) ((void)0)
#endif
#endif /* !_SYS_CSAN_H_ */
Index: head/sys/sys/libkern.h
===================================================================
--- head/sys/sys/libkern.h
+++ head/sys/sys/libkern.h
@@ -191,6 +191,15 @@
char *strstr(const char *, const char *);
int strvalid(const char *, size_t);
+#ifdef KCSAN
+char *kcsan_strcpy(char *, const char *);
+int kcsan_strcmp(const char *, const char *);
+size_t kcsan_strlen(const char *);
+#define strcpy(d, s) kcsan_strcpy((d), (s))
+#define strcmp(s1, s2) kcsan_strcmp((s1), (s2))
+#define strlen(s) kcsan_strlen((s))
+#endif
+
static __inline char *
index(const char *p, int ch)
{
Index: head/sys/sys/systm.h
===================================================================
--- head/sys/sys/systm.h
+++ head/sys/sys/systm.h
@@ -315,21 +315,36 @@
#define ovbcopy(f, t, l) bcopy((f), (t), (l))
void bcopy(const void * _Nonnull from, void * _Nonnull to, size_t len);
-#define bcopy(from, to, len) __builtin_memmove((to), (from), (len))
void bzero(void * _Nonnull buf, size_t len);
-#define bzero(buf, len) __builtin_memset((buf), 0, (len))
void explicit_bzero(void * _Nonnull, size_t);
int bcmp(const void *b1, const void *b2, size_t len);
-#define bcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len))
void *memset(void * _Nonnull buf, int c, size_t len);
-#define memset(buf, c, len) __builtin_memset((buf), (c), (len))
void *memcpy(void * _Nonnull to, const void * _Nonnull from, size_t len);
-#define memcpy(to, from, len) __builtin_memcpy((to), (from), (len))
void *memmove(void * _Nonnull dest, const void * _Nonnull src, size_t n);
-#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
int memcmp(const void *b1, const void *b2, size_t len);
+
+#ifdef KCSAN
+void *kcsan_memset(void *, int, size_t);
+void *kcsan_memcpy(void *, const void *, size_t);
+void *kcsan_memmove(void *, const void *, size_t);
+int kcsan_memcmp(const void *, const void *, size_t);
+#define bcopy(from, to, len) kcsan_memmove((to), (from), (len))
+#define bzero(buf, len) kcsan_memset((buf), 0, (len))
+#define bcmp(b1, b2, len) kcsan_memcmp((b1), (b2), (len))
+#define memset(buf, c, len) kcsan_memset((buf), (c), (len))
+#define memcpy(to, from, len) kcsan_memcpy((to), (from), (len))
+#define memmove(dest, src, n) kcsan_memmove((dest), (src), (n))
+#define memcmp(b1, b2, len) kcsan_memcmp((b1), (b2), (len))
+#else
+#define bcopy(from, to, len) __builtin_memmove((to), (from), (len))
+#define bzero(buf, len) __builtin_memset((buf), 0, (len))
+#define bcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len))
+#define memset(buf, c, len) __builtin_memset((buf), (c), (len))
+#define memcpy(to, from, len) __builtin_memcpy((to), (from), (len))
+#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
#define memcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len))
+#endif
void *memset_early(void * _Nonnull buf, int c, size_t len);
#define bzero_early(buf, len) memset_early((buf), 0, (len))
@@ -351,6 +366,17 @@
void * __restrict udaddr, size_t len);
int copyout_nofault(const void * _Nonnull __restrict kaddr,
void * __restrict udaddr, size_t len);
+
+#ifdef KCSAN
+int kcsan_copystr(const void *, void *, size_t, size_t *);
+int kcsan_copyin(const void *, void *, size_t);
+int kcsan_copyinstr(const void *, void *, size_t, size_t *);
+int kcsan_copyout(const void *, void *, size_t);
+#define copystr(kf, k, l, lc) kcsan_copystr((kf), (k), (l), (lc))
+#define copyin(u, k, l) kcsan_copyin((u), (k), (l))
+#define copyinstr(u, k, l, lc) kcsan_copyinstr((u), (k), (l), (lc))
+#define copyout(k, u, l) kcsan_copyout((k), (u), (l))
+#endif
int fubyte(volatile const void *base);
long fuword(volatile const void *base);
Index: head/sys/x86/include/bus.h
===================================================================
--- head/sys/x86/include/bus.h
+++ head/sys/x86/include/bus.h
@@ -101,6 +101,7 @@
#include <machine/_bus.h>
#include <machine/cpufunc.h>
+#include <machine/bus_dma.h>
#ifndef __GNUCLIKE_ASM
#error "no assembler code for your compiler"
@@ -131,6 +132,13 @@
#define BUS_SPACE_INVALID_DATA (~0)
#define BUS_SPACE_UNRESTRICTED (~0)
+#define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */
+#define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */
+
+#if defined(KCSAN) && !defined(KCSAN_RUNTIME)
+#include <sys/_cscan_bus.h>
+#else
+
/*
* Map a region of device bus space into CPU virtual address space.
*/
@@ -992,9 +1000,6 @@
* prevent reordering by the compiler; all Intel x86 processors currently
* retire operations outside the CPU in program order.
*/
-#define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */
-#define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */
-
static __inline void
bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused,
bus_size_t offset __unused, bus_size_t len __unused, int flags)
@@ -1022,8 +1027,6 @@
#define outl(a, b) compiler_error
#endif
-#include <machine/bus_dma.h>
-
/*
* Stream accesses are the same as normal accesses on x86; there are no
* supported bus systems with an endianess different from the host one.
@@ -1087,5 +1090,7 @@
bus_space_copy_region_2((t), (h1), (o1), (h2), (o2), (c))
#define bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c) \
bus_space_copy_region_4((t), (h1), (o1), (h2), (o2), (c))
+
+#endif /* KCSAN && !KCSAN_RUNTIME */
#endif /* _X86_BUS_H_ */
Index: head/sys/x86/x86/bus_machdep.c
===================================================================
--- head/sys/x86/x86/bus_machdep.c
+++ head/sys/x86/x86/bus_machdep.c
@@ -24,6 +24,8 @@
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+#define KCSAN_RUNTIME
+
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
Index: head/sys/x86/x86/mp_x86.c
===================================================================
--- head/sys/x86/x86/mp_x86.c
+++ head/sys/x86/x86/mp_x86.c
@@ -41,6 +41,7 @@
#include <sys/bus.h>
#include <sys/cons.h> /* cngetc() */
#include <sys/cpuset.h>
+#include <sys/csan.h>
#ifdef GPROF
#include <sys/gmon.h>
#endif
@@ -1079,6 +1080,8 @@
/* Start per-CPU event timers. */
cpu_initclocks_ap();
#endif
+
+ kcsan_cpu_init(cpuid);
sched_throw(NULL);

File Metadata

Mime Type
text/plain
Expires
Tue, Jan 27, 5:35 AM (8 h, 30 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
28051080
Default Alt Text
D22315.id64668.diff (87 KB)

Event Timeline