diff --git a/sys/amd64/amd64/copyout.c b/sys/amd64/amd64/copyout.c index a954411cdd0c..d5f94dd39276 100644 --- a/sys/amd64/amd64/copyout.c +++ b/sys/amd64/amd64/copyout.c @@ -1,199 +1,199 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2018 The FreeBSD Foundation * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); +#ifdef SAN_NEEDS_INTERCEPTORS +#define SAN_RUNTIME +#endif + #include #include #include #include #include int fubyte_nosmap(volatile const void *base); int fubyte_smap(volatile const void *base); DEFINE_IFUNC(, int, fubyte, (volatile const void *)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? fubyte_smap : fubyte_nosmap); } int fuword16_nosmap(volatile const void *base); int fuword16_smap(volatile const void *base); DEFINE_IFUNC(, int, fuword16, (volatile const void *)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? fuword16_smap : fuword16_nosmap); } int fueword_nosmap(volatile const void *base, long *val); int fueword_smap(volatile const void *base, long *val); DEFINE_IFUNC(, int, fueword, (volatile const void *, long *)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? fueword_smap : fueword_nosmap); } DEFINE_IFUNC(, int, fueword64, (volatile const void *, int64_t *)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? fueword_smap : fueword_nosmap); } int fueword32_nosmap(volatile const void *base, int32_t *val); int fueword32_smap(volatile const void *base, int32_t *val); DEFINE_IFUNC(, int, fueword32, (volatile const void *, int32_t *)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? fueword32_smap : fueword32_nosmap); } int subyte_nosmap(volatile void *base, int byte); int subyte_smap(volatile void *base, int byte); DEFINE_IFUNC(, int, subyte, (volatile void *, int)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? subyte_smap : subyte_nosmap); } int suword16_nosmap(volatile void *base, int word); int suword16_smap(volatile void *base, int word); DEFINE_IFUNC(, int, suword16, (volatile void *, int)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? suword16_smap : suword16_nosmap); } int suword32_nosmap(volatile void *base, int32_t word); int suword32_smap(volatile void *base, int32_t word); DEFINE_IFUNC(, int, suword32, (volatile void *, int32_t)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? suword32_smap : suword32_nosmap); } int suword_nosmap(volatile void *base, long word); int suword_smap(volatile void *base, long word); DEFINE_IFUNC(, int, suword, (volatile void *, long)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? suword_smap : suword_nosmap); } DEFINE_IFUNC(, int, suword64, (volatile void *, int64_t)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? suword_smap : suword_nosmap); } int casueword32_nosmap(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, uint32_t newval); int casueword32_smap(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, uint32_t newval); DEFINE_IFUNC(, int, casueword32, (volatile uint32_t *, uint32_t, uint32_t *, uint32_t)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? casueword32_smap : casueword32_nosmap); } int casueword_nosmap(volatile u_long *p, u_long oldval, u_long *oldvalp, u_long newval); int casueword_smap(volatile u_long *p, u_long oldval, u_long *oldvalp, u_long newval); DEFINE_IFUNC(, int, casueword, (volatile u_long *, u_long, u_long *, u_long)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? casueword_smap : casueword_nosmap); } -#undef copyinstr -#undef copyin -#undef copyout - int copyinstr_nosmap(const void *udaddr, void *kaddr, size_t len, size_t *lencopied); int copyinstr_smap(const void *udaddr, void *kaddr, size_t len, size_t *lencopied); DEFINE_IFUNC(, int, copyinstr, (const void *, void *, size_t, size_t *)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? copyinstr_smap : copyinstr_nosmap); } int copyin_nosmap_std(const void *udaddr, void *kaddr, size_t len); int copyin_smap_std(const void *udaddr, void *kaddr, size_t len); int copyin_nosmap_erms(const void *udaddr, void *kaddr, size_t len); int copyin_smap_erms(const void *udaddr, void *kaddr, size_t len); DEFINE_IFUNC(, int, copyin, (const void *, void *, size_t)) { switch (cpu_stdext_feature & (CPUID_STDEXT_SMAP | CPUID_STDEXT_ERMS)) { case CPUID_STDEXT_SMAP: return (copyin_smap_std); case CPUID_STDEXT_ERMS: return (copyin_nosmap_erms); case CPUID_STDEXT_SMAP | CPUID_STDEXT_ERMS: return (copyin_smap_erms); default: return (copyin_nosmap_std); } } int copyout_nosmap_std(const void *kaddr, void *udaddr, size_t len); int copyout_smap_std(const void *kaddr, void *udaddr, size_t len); int copyout_nosmap_erms(const void *kaddr, void *udaddr, size_t len); int copyout_smap_erms(const void *kaddr, void *udaddr, size_t len); DEFINE_IFUNC(, int, copyout, (const void *, void *, size_t)) { switch (cpu_stdext_feature & (CPUID_STDEXT_SMAP | CPUID_STDEXT_ERMS)) { case CPUID_STDEXT_SMAP: return (copyout_smap_std); case CPUID_STDEXT_ERMS: return (copyout_nosmap_erms); case CPUID_STDEXT_SMAP | CPUID_STDEXT_ERMS: return (copyout_smap_erms); default: return (copyout_nosmap_std); } } diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h index 0cb65c768fd8..d61fb359e261 100644 --- a/sys/amd64/include/atomic.h +++ b/sys/amd64/include/atomic.h @@ -1,649 +1,641 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif /* * To express interprocessor (as opposed to processor and device) memory * ordering constraints, use the atomic_*() functions with acquire and release * semantics rather than the *mb() functions. An architecture's memory * ordering (or memory consistency) model governs the order in which a * program's accesses to different locations may be performed by an * implementation of that architecture. In general, for memory regions * defined as writeback cacheable, the memory ordering implemented by amd64 * processors preserves the program ordering of a load followed by a load, a * load followed by a store, and a store followed by a store. Only a store * followed by a load to a different memory location may be reordered. * Therefore, except for special cases, like non-temporal memory accesses or * memory regions defined as write combining, the memory ordering effects * provided by the sfence instruction in the wmb() function and the lfence * instruction in the rmb() function are redundant. In contrast, the * atomic_*() functions with acquire and release semantics do not perform * redundant instructions for ordinary cases of interprocessor memory * ordering on any architecture. */ #define mb() __asm __volatile("mfence;" : : : "memory") #define wmb() __asm __volatile("sfence;" : : : "memory") #define rmb() __asm __volatile("lfence;" : : : "memory") #ifdef _KERNEL /* * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). * * The open-coded number is used instead of the symbolic expression to * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in amd64/vm_machdep.c ensures that the value is correct. */ #define OFFSETOF_MONITORBUF 0x100 #endif -#ifndef SAN_RUNTIME -#if defined(KASAN) -#define ATOMIC_SAN_PREFIX kasan -#elif defined(KCSAN) -#define ATOMIC_SAN_PREFIX kcsan -#endif -#endif - -#ifdef ATOMIC_SAN_PREFIX +#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME) #include #else #include /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. * * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) * atomic_add_char(P, V) (*(u_char *)(P) += (V)) * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) * * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) * atomic_add_short(P, V) (*(u_short *)(P) += (V)) * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) * * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ #if !defined(__GNUCLIKE_ASM) #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_char(volatile u_char *dst, u_char expect, u_char src); int atomic_cmpset_short(volatile u_short *dst, u_short expect, u_short src); int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src); int atomic_fcmpset_char(volatile u_char *dst, u_char *expect, u_char src); int atomic_fcmpset_short(volatile u_short *dst, u_short *expect, u_short src); int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src); int atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); u_long atomic_fetchadd_long(volatile u_long *p, u_long v); int atomic_testandset_int(volatile u_int *p, u_int v); int atomic_testandset_long(volatile u_long *p, u_int v); int atomic_testandclear_int(volatile u_int *p, u_int v); int atomic_testandclear_long(volatile u_long *p, u_int v); void atomic_thread_fence_acq(void); void atomic_thread_fence_acq_rel(void); void atomic_thread_fence_rel(void); void atomic_thread_fence_seq_cst(void); #define ATOMIC_LOAD(TYPE) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) #define ATOMIC_STORE(TYPE) \ void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) #else /* !__GNUCLIKE_ASM */ /* * Always use lock prefixes. The result is slighly less optimal for * UP systems, but it matters less now, and sometimes UP is emulated * over SMP. * * The assembly is volatilized to avoid code chunk removal by the compiler. * GCC aggressively reorders operations and memory clobbering is necessary * in order to avoid that for memory barriers. */ #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile("lock; " OP \ : "+m" (*p) \ : CONS (V) \ : "cc"); \ } \ \ static __inline void \ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile("lock; " OP \ : "+m" (*p) \ : CONS (V) \ : "memory", "cc"); \ } \ struct __hack /* * Atomic compare and set, used by the mutex functions. * * cmpset: * if (*dst == expect) * *dst = src * * fcmpset: * if (*dst == *expect) * *dst = src * else * *expect = *dst * * Returns 0 on failure, non-zero on success. */ #define ATOMIC_CMPSET(TYPE) \ static __inline int \ atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " lock; cmpxchg %3,%1 ; " \ "# atomic_cmpset_" #TYPE " " \ : "=@cce" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (expect) /* 2 */ \ : "r" (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } \ \ static __inline int \ atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " lock; cmpxchg %3,%1 ; " \ "# atomic_fcmpset_" #TYPE " " \ : "=@cce" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (*expect) /* 2 */ \ : "r" (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } ATOMIC_CMPSET(char); ATOMIC_CMPSET(short); ATOMIC_CMPSET(int); ATOMIC_CMPSET(long); /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { __asm __volatile( " lock; xaddl %0,%1 ; " "# atomic_fetchadd_int" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } /* * Atomically add the value of v to the long integer pointed to by p and return * the previous value of *p. */ static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { __asm __volatile( " lock; xaddq %0,%1 ; " "# atomic_fetchadd_long" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " lock; btsl %2,%1 ; " "# atomic_testandset_int" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { u_char res; __asm __volatile( " lock; btsq %2,%1 ; " "# atomic_testandset_long" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Jr" ((u_long)(v & 0x3f)) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " lock; btrl %2,%1 ; " "# atomic_testandclear_int" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { u_char res; __asm __volatile( " lock; btrq %2,%1 ; " "# atomic_testandclear_long" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Jr" ((u_long)(v & 0x3f)) /* 2 */ : "cc"); return (res); } /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. * * However, a load may pass a store if they are performed on distinct * addresses, so we need a Store/Load barrier for sequentially * consistent fences in SMP kernels. We use "lock addl $0,mem" for a * Store/Load barrier, as recommended by the AMD Software Optimization * Guide, and not mfence. To avoid false data dependencies, we use a * special address for "mem". In the kernel, we use a private per-cpu * cache line. In user space, we use a word in the stack's red zone * (-8(%rsp)). */ static __inline void __storeload_barrier(void) { #if defined(_KERNEL) __asm __volatile("lock; addl $0,%%gs:%0" : "+m" (*(u_int *)OFFSETOF_MONITORBUF) : : "memory", "cc"); #else /* !_KERNEL */ __asm __volatile("lock; addl $0,-8(%%rsp)" : : : "memory", "cc"); #endif /* _KERNEL*/ } #define ATOMIC_LOAD(TYPE) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ u_##TYPE res; \ \ res = *p; \ __compiler_membar(); \ return (res); \ } \ struct __hack #define ATOMIC_STORE(TYPE) \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ __compiler_membar(); \ *p = v; \ } \ struct __hack static __inline void atomic_thread_fence_acq(void) { __compiler_membar(); } static __inline void atomic_thread_fence_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_acq_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_seq_cst(void) { __storeload_barrier(); } #endif /* !__GNUCLIKE_ASM */ ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); ATOMIC_ASM(set, int, "orl %1,%0", "ir", v); ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); ATOMIC_ASM(set, long, "orq %1,%0", "er", v); ATOMIC_ASM(clear, long, "andq %1,%0", "er", ~v); ATOMIC_ASM(add, long, "addq %1,%0", "er", v); ATOMIC_ASM(subtract, long, "subq %1,%0", "er", v); #define ATOMIC_LOADSTORE(TYPE) \ ATOMIC_LOAD(TYPE); \ ATOMIC_STORE(TYPE) ATOMIC_LOADSTORE(char); ATOMIC_LOADSTORE(short); ATOMIC_LOADSTORE(int); ATOMIC_LOADSTORE(long); #undef ATOMIC_ASM #undef ATOMIC_LOAD #undef ATOMIC_STORE #undef ATOMIC_LOADSTORE #ifndef WANT_FUNCTIONS /* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int atomic_swap_int(volatile u_int *p, u_int v) { __asm __volatile( " xchgl %1,%0 ; " "# atomic_swap_int" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } static __inline u_long atomic_swap_long(volatile u_long *p, u_long v) { __asm __volatile( " xchgq %1,%0 ; " "# atomic_swap_long" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } #else /* !__GNUCLIKE_ASM */ u_int atomic_swap_int(volatile u_int *p, u_int v); u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* __GNUCLIKE_ASM */ #define atomic_set_acq_char atomic_set_barr_char #define atomic_set_rel_char atomic_set_barr_char #define atomic_clear_acq_char atomic_clear_barr_char #define atomic_clear_rel_char atomic_clear_barr_char #define atomic_add_acq_char atomic_add_barr_char #define atomic_add_rel_char atomic_add_barr_char #define atomic_subtract_acq_char atomic_subtract_barr_char #define atomic_subtract_rel_char atomic_subtract_barr_char #define atomic_cmpset_acq_char atomic_cmpset_char #define atomic_cmpset_rel_char atomic_cmpset_char #define atomic_fcmpset_acq_char atomic_fcmpset_char #define atomic_fcmpset_rel_char atomic_fcmpset_char #define atomic_set_acq_short atomic_set_barr_short #define atomic_set_rel_short atomic_set_barr_short #define atomic_clear_acq_short atomic_clear_barr_short #define atomic_clear_rel_short atomic_clear_barr_short #define atomic_add_acq_short atomic_add_barr_short #define atomic_add_rel_short atomic_add_barr_short #define atomic_subtract_acq_short atomic_subtract_barr_short #define atomic_subtract_rel_short atomic_subtract_barr_short #define atomic_cmpset_acq_short atomic_cmpset_short #define atomic_cmpset_rel_short atomic_cmpset_short #define atomic_fcmpset_acq_short atomic_fcmpset_short #define atomic_fcmpset_rel_short atomic_fcmpset_short #define atomic_set_acq_int atomic_set_barr_int #define atomic_set_rel_int atomic_set_barr_int #define atomic_clear_acq_int atomic_clear_barr_int #define atomic_clear_rel_int atomic_clear_barr_int #define atomic_add_acq_int atomic_add_barr_int #define atomic_add_rel_int atomic_add_barr_int #define atomic_subtract_acq_int atomic_subtract_barr_int #define atomic_subtract_rel_int atomic_subtract_barr_int #define atomic_cmpset_acq_int atomic_cmpset_int #define atomic_cmpset_rel_int atomic_cmpset_int #define atomic_fcmpset_acq_int atomic_fcmpset_int #define atomic_fcmpset_rel_int atomic_fcmpset_int #define atomic_set_acq_long atomic_set_barr_long #define atomic_set_rel_long atomic_set_barr_long #define atomic_clear_acq_long atomic_clear_barr_long #define atomic_clear_rel_long atomic_clear_barr_long #define atomic_add_acq_long atomic_add_barr_long #define atomic_add_rel_long atomic_add_barr_long #define atomic_subtract_acq_long atomic_subtract_barr_long #define atomic_subtract_rel_long atomic_subtract_barr_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long #define atomic_fcmpset_acq_long atomic_fcmpset_long #define atomic_fcmpset_rel_long atomic_fcmpset_long #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) #define atomic_testandset_acq_long atomic_testandset_long /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char #define atomic_set_rel_8 atomic_set_rel_char #define atomic_clear_8 atomic_clear_char #define atomic_clear_acq_8 atomic_clear_acq_char #define atomic_clear_rel_8 atomic_clear_rel_char #define atomic_add_8 atomic_add_char #define atomic_add_acq_8 atomic_add_acq_char #define atomic_add_rel_8 atomic_add_rel_char #define atomic_subtract_8 atomic_subtract_char #define atomic_subtract_acq_8 atomic_subtract_acq_char #define atomic_subtract_rel_8 atomic_subtract_rel_char #define atomic_load_acq_8 atomic_load_acq_char #define atomic_store_rel_8 atomic_store_rel_char #define atomic_cmpset_8 atomic_cmpset_char #define atomic_cmpset_acq_8 atomic_cmpset_acq_char #define atomic_cmpset_rel_8 atomic_cmpset_rel_char #define atomic_fcmpset_8 atomic_fcmpset_char #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char /* Operations on 16-bit words. */ #define atomic_set_16 atomic_set_short #define atomic_set_acq_16 atomic_set_acq_short #define atomic_set_rel_16 atomic_set_rel_short #define atomic_clear_16 atomic_clear_short #define atomic_clear_acq_16 atomic_clear_acq_short #define atomic_clear_rel_16 atomic_clear_rel_short #define atomic_add_16 atomic_add_short #define atomic_add_acq_16 atomic_add_acq_short #define atomic_add_rel_16 atomic_add_rel_short #define atomic_subtract_16 atomic_subtract_short #define atomic_subtract_acq_16 atomic_subtract_acq_short #define atomic_subtract_rel_16 atomic_subtract_rel_short #define atomic_load_acq_16 atomic_load_acq_short #define atomic_store_rel_16 atomic_store_rel_short #define atomic_cmpset_16 atomic_cmpset_short #define atomic_cmpset_acq_16 atomic_cmpset_acq_short #define atomic_cmpset_rel_16 atomic_cmpset_rel_short #define atomic_fcmpset_16 atomic_fcmpset_short #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short /* Operations on 32-bit double words. */ #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_testandset_32 atomic_testandset_int #define atomic_testandclear_32 atomic_testandclear_int /* Operations on 64-bit quad words. */ #define atomic_set_64 atomic_set_long #define atomic_set_acq_64 atomic_set_acq_long #define atomic_set_rel_64 atomic_set_rel_long #define atomic_clear_64 atomic_clear_long #define atomic_clear_acq_64 atomic_clear_acq_long #define atomic_clear_rel_64 atomic_clear_rel_long #define atomic_add_64 atomic_add_long #define atomic_add_acq_64 atomic_add_acq_long #define atomic_add_rel_64 atomic_add_rel_long #define atomic_subtract_64 atomic_subtract_long #define atomic_subtract_acq_64 atomic_subtract_acq_long #define atomic_subtract_rel_64 atomic_subtract_rel_long #define atomic_load_acq_64 atomic_load_acq_long #define atomic_store_rel_64 atomic_store_rel_long #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long #define atomic_fcmpset_64 atomic_fcmpset_long #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long #define atomic_swap_64 atomic_swap_long #define atomic_readandclear_64 atomic_readandclear_long #define atomic_fetchadd_64 atomic_fetchadd_long #define atomic_testandset_64 atomic_testandset_long #define atomic_testandclear_64 atomic_testandclear_long /* Operations on pointers. */ #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long #define atomic_clear_ptr atomic_clear_long #define atomic_clear_acq_ptr atomic_clear_acq_long #define atomic_clear_rel_ptr atomic_clear_rel_long #define atomic_add_ptr atomic_add_long #define atomic_add_acq_ptr atomic_add_acq_long #define atomic_add_rel_ptr atomic_add_rel_long #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long #define atomic_load_acq_ptr atomic_load_acq_long #define atomic_store_rel_ptr atomic_store_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long #define atomic_fcmpset_ptr atomic_fcmpset_long #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #define atomic_swap_ptr atomic_swap_long #define atomic_readandclear_ptr atomic_readandclear_long #endif /* !WANT_FUNCTIONS */ -#endif /* !ATOMIC_SAN_PREFIX */ +#endif /* !SAN_NEEDS_INTERCEPTORS || SAN_RUNTIME */ #endif /* !_MACHINE_ATOMIC_H_ */ diff --git a/sys/arm64/include/bus.h b/sys/arm64/include/bus.h index fc768fc3e709..f7879a67d8e8 100644 --- a/sys/arm64/include/bus.h +++ b/sys/arm64/include/bus.h @@ -1,530 +1,522 @@ /* $NetBSD: bus.h,v 1.11 2003/07/28 17:35:54 thorpej Exp $ */ /*- * Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, * NASA Ames Research Center. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1996 Charles M. Hannum. All rights reserved. * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Christopher G. Demetriou * for the NetBSD Project. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * From: sys/arm/include/bus.h * * $FreeBSD$ */ #ifndef _MACHINE_BUS_H_ #define _MACHINE_BUS_H_ #include #define BUS_SPACE_ALIGNED_POINTER(p, t) ALIGNED_POINTER(p, t) #define BUS_SPACE_MAXADDR_24BIT 0xFFFFFFUL #define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFFUL #define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFUL #define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFFUL #define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFFUL #define BUS_SPACE_MAXSIZE_40BIT 0xFFFFFFFFFFUL #define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFUL #define BUS_SPACE_MAXSIZE 0xFFFFFFFFFFFFFFFFUL #define BUS_SPACE_MAP_CACHEABLE 0x01 #define BUS_SPACE_MAP_LINEAR 0x02 #define BUS_SPACE_MAP_PREFETCHABLE 0x04 #define BUS_SPACE_UNRESTRICTED (~0) #define BUS_SPACE_BARRIER_READ 0x01 #define BUS_SPACE_BARRIER_WRITE 0x02 -#ifndef SAN_RUNTIME -#if defined(KASAN) -#define BUS_SAN_PREFIX kasan -#elif defined(KCSAN) -#define BUS_SAN_PREFIX kcsan -#endif -#endif - struct bus_space { /* cookie */ void *bs_cookie; /* mapping/unmapping */ int (*bs_map) (void *, bus_addr_t, bus_size_t, int, bus_space_handle_t *); void (*bs_unmap) (void *, bus_space_handle_t, bus_size_t); int (*bs_subregion) (void *, bus_space_handle_t, bus_size_t, bus_size_t, bus_space_handle_t *); /* allocation/deallocation */ int (*bs_alloc) (void *, bus_addr_t, bus_addr_t, bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *, bus_space_handle_t *); void (*bs_free) (void *, bus_space_handle_t, bus_size_t); /* get kernel virtual address */ /* barrier */ void (*bs_barrier) (void *, bus_space_handle_t, bus_size_t, bus_size_t, int); /* read single */ u_int8_t (*bs_r_1) (void *, bus_space_handle_t, bus_size_t); u_int16_t (*bs_r_2) (void *, bus_space_handle_t, bus_size_t); u_int32_t (*bs_r_4) (void *, bus_space_handle_t, bus_size_t); u_int64_t (*bs_r_8) (void *, bus_space_handle_t, bus_size_t); /* read multiple */ void (*bs_rm_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t *, bus_size_t); void (*bs_rm_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t *, bus_size_t); void (*bs_rm_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t *, bus_size_t); void (*bs_rm_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t *, bus_size_t); /* read region */ void (*bs_rr_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t *, bus_size_t); void (*bs_rr_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t *, bus_size_t); void (*bs_rr_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t *, bus_size_t); void (*bs_rr_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t *, bus_size_t); /* write single */ void (*bs_w_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t); void (*bs_w_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t); void (*bs_w_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t); void (*bs_w_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t); /* write multiple */ void (*bs_wm_1) (void *, bus_space_handle_t, bus_size_t, const u_int8_t *, bus_size_t); void (*bs_wm_2) (void *, bus_space_handle_t, bus_size_t, const u_int16_t *, bus_size_t); void (*bs_wm_4) (void *, bus_space_handle_t, bus_size_t, const u_int32_t *, bus_size_t); void (*bs_wm_8) (void *, bus_space_handle_t, bus_size_t, const u_int64_t *, bus_size_t); /* write region */ void (*bs_wr_1) (void *, bus_space_handle_t, bus_size_t, const u_int8_t *, bus_size_t); void (*bs_wr_2) (void *, bus_space_handle_t, bus_size_t, const u_int16_t *, bus_size_t); void (*bs_wr_4) (void *, bus_space_handle_t, bus_size_t, const u_int32_t *, bus_size_t); void (*bs_wr_8) (void *, bus_space_handle_t, bus_size_t, const u_int64_t *, bus_size_t); /* set multiple */ void (*bs_sm_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t, bus_size_t); void (*bs_sm_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t, bus_size_t); void (*bs_sm_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t, bus_size_t); void (*bs_sm_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t, bus_size_t); /* set region */ void (*bs_sr_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t, bus_size_t); void (*bs_sr_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t, bus_size_t); void (*bs_sr_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t, bus_size_t); void (*bs_sr_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t, bus_size_t); /* copy */ void (*bs_c_1) (void *, bus_space_handle_t, bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t); void (*bs_c_2) (void *, bus_space_handle_t, bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t); void (*bs_c_4) (void *, bus_space_handle_t, bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t); void (*bs_c_8) (void *, bus_space_handle_t, bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t); /* read single stream */ u_int8_t (*bs_r_1_s) (void *, bus_space_handle_t, bus_size_t); u_int16_t (*bs_r_2_s) (void *, bus_space_handle_t, bus_size_t); u_int32_t (*bs_r_4_s) (void *, bus_space_handle_t, bus_size_t); u_int64_t (*bs_r_8_s) (void *, bus_space_handle_t, bus_size_t); /* read multiple stream */ void (*bs_rm_1_s) (void *, bus_space_handle_t, bus_size_t, u_int8_t *, bus_size_t); void (*bs_rm_2_s) (void *, bus_space_handle_t, bus_size_t, u_int16_t *, bus_size_t); void (*bs_rm_4_s) (void *, bus_space_handle_t, bus_size_t, u_int32_t *, bus_size_t); void (*bs_rm_8_s) (void *, bus_space_handle_t, bus_size_t, u_int64_t *, bus_size_t); /* read region stream */ void (*bs_rr_1_s) (void *, bus_space_handle_t, bus_size_t, u_int8_t *, bus_size_t); void (*bs_rr_2_s) (void *, bus_space_handle_t, bus_size_t, u_int16_t *, bus_size_t); void (*bs_rr_4_s) (void *, bus_space_handle_t, bus_size_t, u_int32_t *, bus_size_t); void (*bs_rr_8_s) (void *, bus_space_handle_t, bus_size_t, u_int64_t *, bus_size_t); /* write single stream */ void (*bs_w_1_s) (void *, bus_space_handle_t, bus_size_t, u_int8_t); void (*bs_w_2_s) (void *, bus_space_handle_t, bus_size_t, u_int16_t); void (*bs_w_4_s) (void *, bus_space_handle_t, bus_size_t, u_int32_t); void (*bs_w_8_s) (void *, bus_space_handle_t, bus_size_t, u_int64_t); /* write multiple stream */ void (*bs_wm_1_s) (void *, bus_space_handle_t, bus_size_t, const u_int8_t *, bus_size_t); void (*bs_wm_2_s) (void *, bus_space_handle_t, bus_size_t, const u_int16_t *, bus_size_t); void (*bs_wm_4_s) (void *, bus_space_handle_t, bus_size_t, const u_int32_t *, bus_size_t); void (*bs_wm_8_s) (void *, bus_space_handle_t, bus_size_t, const u_int64_t *, bus_size_t); /* write region stream */ void (*bs_wr_1_s) (void *, bus_space_handle_t, bus_size_t, const u_int8_t *, bus_size_t); void (*bs_wr_2_s) (void *, bus_space_handle_t, bus_size_t, const u_int16_t *, bus_size_t); void (*bs_wr_4_s) (void *, bus_space_handle_t, bus_size_t, const u_int32_t *, bus_size_t); void (*bs_wr_8_s) (void *, bus_space_handle_t, bus_size_t, const u_int64_t *, bus_size_t); /* peek */ int (*bs_peek_1)(void *, bus_space_handle_t, bus_size_t , uint8_t *); int (*bs_peek_2)(void *, bus_space_handle_t, bus_size_t , uint16_t *); int (*bs_peek_4)(void *, bus_space_handle_t, bus_size_t , uint32_t *); int (*bs_peek_8)(void *, bus_space_handle_t, bus_size_t , uint64_t *); /* poke */ int (*bs_poke_1)(void *, bus_space_handle_t, bus_size_t, uint8_t); int (*bs_poke_2)(void *, bus_space_handle_t, bus_size_t, uint16_t); int (*bs_poke_4)(void *, bus_space_handle_t, bus_size_t, uint32_t); int (*bs_poke_8)(void *, bus_space_handle_t, bus_size_t, uint64_t); }; -#ifdef BUS_SAN_PREFIX +#ifdef SAN_NEEDS_INTERCEPTORS #include #else /* * Utility macros; INTERNAL USE ONLY. */ #define __bs_c(a,b) __CONCAT(a,b) #define __bs_opname(op,size) __bs_c(__bs_c(__bs_c(bs_,op),_),size) #define __bs_rs(sz, t, h, o) \ (*(t)->__bs_opname(r,sz))((t)->bs_cookie, h, o) #define __bs_ws(sz, t, h, o, v) \ (*(t)->__bs_opname(w,sz))((t)->bs_cookie, h, o, v) #define __bs_nonsingle(type, sz, t, h, o, a, c) \ (*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, a, c) #define __bs_set(type, sz, t, h, o, v, c) \ (*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, v, c) #define __bs_copy(sz, t, h1, o1, h2, o2, cnt) \ (*(t)->__bs_opname(c,sz))((t)->bs_cookie, h1, o1, h2, o2, cnt) #define __bs_opname_s(op,size) __bs_c(__bs_c(__bs_c(__bs_c(bs_,op),_),size),_s) #define __bs_rs_s(sz, t, h, o) \ (*(t)->__bs_opname_s(r,sz))((t)->bs_cookie, h, o) #define __bs_ws_s(sz, t, h, o, v) \ (*(t)->__bs_opname_s(w,sz))((t)->bs_cookie, h, o, v) #define __bs_peek(sz, t, h, o, vp) \ (*(t)->__bs_opname(peek, sz))((t)->bs_cookie, h, o, vp) #define __bs_poke(sz, t, h, o, v) \ (*(t)->__bs_opname(poke, sz))((t)->bs_cookie, h, o, v) #define __bs_nonsingle_s(type, sz, t, h, o, a, c) \ (*(t)->__bs_opname_s(type,sz))((t)->bs_cookie, h, o, a, c) /* * Mapping and unmapping operations. */ #define bus_space_map(t, a, s, c, hp) \ (*(t)->bs_map)((t)->bs_cookie, (a), (s), (c), (hp)) #define bus_space_unmap(t, h, s) \ (*(t)->bs_unmap)((t)->bs_cookie, (h), (s)) #define bus_space_subregion(t, h, o, s, hp) \ (*(t)->bs_subregion)((t)->bs_cookie, (h), (o), (s), (hp)) /* * Allocation and deallocation operations. */ #define bus_space_alloc(t, rs, re, s, a, b, c, ap, hp) \ (*(t)->bs_alloc)((t)->bs_cookie, (rs), (re), (s), (a), (b), \ (c), (ap), (hp)) #define bus_space_free(t, h, s) \ (*(t)->bs_free)((t)->bs_cookie, (h), (s)) /* * Bus barrier operations. */ #define bus_space_barrier(t, h, o, l, f) \ (*(t)->bs_barrier)((t)->bs_cookie, (h), (o), (l), (f)) /* * Bus read (single) operations. */ #define bus_space_read_1(t, h, o) __bs_rs(1,(t),(h),(o)) #define bus_space_read_2(t, h, o) __bs_rs(2,(t),(h),(o)) #define bus_space_read_4(t, h, o) __bs_rs(4,(t),(h),(o)) #define bus_space_read_8(t, h, o) __bs_rs(8,(t),(h),(o)) #define bus_space_read_stream_1(t, h, o) __bs_rs_s(1,(t), (h), (o)) #define bus_space_read_stream_2(t, h, o) __bs_rs_s(2,(t), (h), (o)) #define bus_space_read_stream_4(t, h, o) __bs_rs_s(4,(t), (h), (o)) #define bus_space_read_stream_8(t, h, o) __bs_rs_s(8,(t), (h), (o)) /* * Bus read multiple operations. */ #define bus_space_read_multi_1(t, h, o, a, c) \ __bs_nonsingle(rm,1,(t),(h),(o),(a),(c)) #define bus_space_read_multi_2(t, h, o, a, c) \ __bs_nonsingle(rm,2,(t),(h),(o),(a),(c)) #define bus_space_read_multi_4(t, h, o, a, c) \ __bs_nonsingle(rm,4,(t),(h),(o),(a),(c)) #define bus_space_read_multi_8(t, h, o, a, c) \ __bs_nonsingle(rm,8,(t),(h),(o),(a),(c)) #define bus_space_read_multi_stream_1(t, h, o, a, c) \ __bs_nonsingle_s(rm,1,(t),(h),(o),(a),(c)) #define bus_space_read_multi_stream_2(t, h, o, a, c) \ __bs_nonsingle_s(rm,2,(t),(h),(o),(a),(c)) #define bus_space_read_multi_stream_4(t, h, o, a, c) \ __bs_nonsingle_s(rm,4,(t),(h),(o),(a),(c)) #define bus_space_read_multi_stream_8(t, h, o, a, c) \ __bs_nonsingle_s(rm,8,(t),(h),(o),(a),(c)) /* * Bus read region operations. */ #define bus_space_read_region_1(t, h, o, a, c) \ __bs_nonsingle(rr,1,(t),(h),(o),(a),(c)) #define bus_space_read_region_2(t, h, o, a, c) \ __bs_nonsingle(rr,2,(t),(h),(o),(a),(c)) #define bus_space_read_region_4(t, h, o, a, c) \ __bs_nonsingle(rr,4,(t),(h),(o),(a),(c)) #define bus_space_read_region_8(t, h, o, a, c) \ __bs_nonsingle(rr,8,(t),(h),(o),(a),(c)) #define bus_space_read_region_stream_1(t, h, o, a, c) \ __bs_nonsingle_s(rr,1,(t),(h),(o),(a),(c)) #define bus_space_read_region_stream_2(t, h, o, a, c) \ __bs_nonsingle_s(rr,2,(t),(h),(o),(a),(c)) #define bus_space_read_region_stream_4(t, h, o, a, c) \ __bs_nonsingle_s(rr,4,(t),(h),(o),(a),(c)) #define bus_space_read_region_stream_8(t, h, o, a, c) \ __bs_nonsingle_s(rr,8,(t),(h),(o),(a),(c)) /* * Bus write (single) operations. */ #define bus_space_write_1(t, h, o, v) __bs_ws(1,(t),(h),(o),(v)) #define bus_space_write_2(t, h, o, v) __bs_ws(2,(t),(h),(o),(v)) #define bus_space_write_4(t, h, o, v) __bs_ws(4,(t),(h),(o),(v)) #define bus_space_write_8(t, h, o, v) __bs_ws(8,(t),(h),(o),(v)) #define bus_space_write_stream_1(t, h, o, v) __bs_ws_s(1,(t),(h),(o),(v)) #define bus_space_write_stream_2(t, h, o, v) __bs_ws_s(2,(t),(h),(o),(v)) #define bus_space_write_stream_4(t, h, o, v) __bs_ws_s(4,(t),(h),(o),(v)) #define bus_space_write_stream_8(t, h, o, v) __bs_ws_s(8,(t),(h),(o),(v)) /* * Bus write multiple operations. */ #define bus_space_write_multi_1(t, h, o, a, c) \ __bs_nonsingle(wm,1,(t),(h),(o),(a),(c)) #define bus_space_write_multi_2(t, h, o, a, c) \ __bs_nonsingle(wm,2,(t),(h),(o),(a),(c)) #define bus_space_write_multi_4(t, h, o, a, c) \ __bs_nonsingle(wm,4,(t),(h),(o),(a),(c)) #define bus_space_write_multi_8(t, h, o, a, c) \ __bs_nonsingle(wm,8,(t),(h),(o),(a),(c)) #define bus_space_write_multi_stream_1(t, h, o, a, c) \ __bs_nonsingle_s(wm,1,(t),(h),(o),(a),(c)) #define bus_space_write_multi_stream_2(t, h, o, a, c) \ __bs_nonsingle_s(wm,2,(t),(h),(o),(a),(c)) #define bus_space_write_multi_stream_4(t, h, o, a, c) \ __bs_nonsingle_s(wm,4,(t),(h),(o),(a),(c)) #define bus_space_write_multi_stream_8(t, h, o, a, c) \ __bs_nonsingle_s(wm,8,(t),(h),(o),(a),(c)) /* * Bus write region operations. */ #define bus_space_write_region_1(t, h, o, a, c) \ __bs_nonsingle(wr,1,(t),(h),(o),(a),(c)) #define bus_space_write_region_2(t, h, o, a, c) \ __bs_nonsingle(wr,2,(t),(h),(o),(a),(c)) #define bus_space_write_region_4(t, h, o, a, c) \ __bs_nonsingle(wr,4,(t),(h),(o),(a),(c)) #define bus_space_write_region_8(t, h, o, a, c) \ __bs_nonsingle(wr,8,(t),(h),(o),(a),(c)) #define bus_space_write_region_stream_1(t, h, o, a, c) \ __bs_nonsingle_s(wr,1,(t),(h),(o),(a),(c)) #define bus_space_write_region_stream_2(t, h, o, a, c) \ __bs_nonsingle_s(wr,2,(t),(h),(o),(a),(c)) #define bus_space_write_region_stream_4(t, h, o, a, c) \ __bs_nonsingle_s(wr,4,(t),(h),(o),(a),(c)) #define bus_space_write_region_stream_8(t, h, o, a, c) \ __bs_nonsingle_s(wr,8,(t),(h),(o),(a),(c)) /* * Set multiple operations. */ #define bus_space_set_multi_1(t, h, o, v, c) \ __bs_set(sm,1,(t),(h),(o),(v),(c)) #define bus_space_set_multi_2(t, h, o, v, c) \ __bs_set(sm,2,(t),(h),(o),(v),(c)) #define bus_space_set_multi_4(t, h, o, v, c) \ __bs_set(sm,4,(t),(h),(o),(v),(c)) #define bus_space_set_multi_8(t, h, o, v, c) \ __bs_set(sm,8,(t),(h),(o),(v),(c)) #define bus_space_set_multi_stream_1(t, h, o, v, c) \ bus_space_set_multi_1((t), (h), (o), (v), (c)) #define bus_space_set_multi_stream_2(t, h, o, v, c) \ bus_space_set_multi_2((t), (h), (o), (v), (c)) #define bus_space_set_multi_stream_4(t, h, o, v, c) \ bus_space_set_multi_4((t), (h), (o), (v), (c)) #define bus_space_set_multi_stream_8(t, h, o, v, c) \ bus_space_set_multi_8((t), (h), (o), (v), (c)) /* * Set region operations. */ #define bus_space_set_region_1(t, h, o, v, c) \ __bs_set(sr,1,(t),(h),(o),(v),(c)) #define bus_space_set_region_2(t, h, o, v, c) \ __bs_set(sr,2,(t),(h),(o),(v),(c)) #define bus_space_set_region_4(t, h, o, v, c) \ __bs_set(sr,4,(t),(h),(o),(v),(c)) #define bus_space_set_region_8(t, h, o, v, c) \ __bs_set(sr,8,(t),(h),(o),(v),(c)) #define bus_space_set_region_stream_1(t, h, o, v, c) \ bus_space_set_region_1((t), (h), (o), (v), (c)) #define bus_space_set_region_stream_2(t, h, o, v, c) \ bus_space_set_region_2((t), (h), (o), (v), (c)) #define bus_space_set_region_stream_4(t, h, o, v, c) \ bus_space_set_region_4((t), (h), (o), (v), (c)) #define bus_space_set_region_stream_8(t, h, o, v, c) \ bus_space_set_region_8((t), (h), (o), (v), (c)) /* * Copy operations. */ #define bus_space_copy_region_1(t, h1, o1, h2, o2, c) \ __bs_copy(1, t, h1, o1, h2, o2, c) #define bus_space_copy_region_2(t, h1, o1, h2, o2, c) \ __bs_copy(2, t, h1, o1, h2, o2, c) #define bus_space_copy_region_4(t, h1, o1, h2, o2, c) \ __bs_copy(4, t, h1, o1, h2, o2, c) #define bus_space_copy_region_8(t, h1, o1, h2, o2, c) \ __bs_copy(8, t, h1, o1, h2, o2, c) /* * Poke (checked write) operations. */ #define bus_space_poke_1(t, h, o, v) __bs_poke(1, (t), (h), (o), (v)) #define bus_space_poke_2(t, h, o, v) __bs_poke(2, (t), (h), (o), (v)) #define bus_space_poke_4(t, h, o, v) __bs_poke(4, (t), (h), (o), (v)) #define bus_space_poke_8(t, h, o, v) __bs_poke(8, (t), (h), (o), (v)) /* * Peek (checked read) operations. */ #define bus_space_peek_1(t, h, o, vp) __bs_peek(1, (t), (h), (o), (vp)) #define bus_space_peek_2(t, h, o, vp) __bs_peek(2, (t), (h), (o), (vp)) #define bus_space_peek_4(t, h, o, vp) __bs_peek(4, (t), (h), (o), (vp)) #define bus_space_peek_8(t, h, o, vp) __bs_peek(8, (t), (h), (o), (vp)) -#endif +#endif /* !SAN_NEEDS_INTERCEPTORS */ #include #endif /* _MACHINE_BUS_H_ */ diff --git a/sys/conf/kern.pre.mk b/sys/conf/kern.pre.mk index f2f69342c865..645c27627124 100644 --- a/sys/conf/kern.pre.mk +++ b/sys/conf/kern.pre.mk @@ -1,388 +1,390 @@ # $FreeBSD$ # Part of a unified Makefile for building kernels. This part contains all # of the definitions that need to be before %BEFORE_DEPEND. # Allow user to configure things that only effect src tree builds. # Note: This is duplicated from src.sys.mk to ensure that we include # /etc/src.conf when building the kernel. Kernels can be built without # the rest of /usr/src, but they still always process SRCCONF even though # the normal mechanisms to prevent that (compiling out of tree) won't # work. To ensure they do work, we have to duplicate thee few lines here. SRCCONF?= /etc/src.conf .if (exists(${SRCCONF}) || ${SRCCONF} != "/etc/src.conf") && !target(_srcconf_included_) .include "${SRCCONF}" _srcconf_included_: .endif .include .include .include "kern.opts.mk" # The kernel build always occurs in the object directory which is .CURDIR. .if ${.MAKE.MODE:Unormal:Mmeta} .MAKE.MODE+= curdirOk=yes .endif # The kernel build always expects .OBJDIR=.CURDIR. .OBJDIR: ${.CURDIR} .if defined(NO_OBJWALK) || ${MK_AUTO_OBJ} == "yes" NO_OBJWALK= t NO_MODULES_OBJ= t .endif .if !defined(NO_OBJWALK) _obj= obj .endif # Can be overridden by makeoptions or /etc/make.conf KERNEL_KO?= kernel KERNEL?= kernel KODIR?= /boot/${KERNEL} LDSCRIPT_NAME?= ldscript.$M LDSCRIPT?= $S/conf/${LDSCRIPT_NAME} M= ${MACHINE} AWK?= awk CP?= cp ELFDUMP?= elfdump NM?= nm OBJCOPY?= objcopy SIZE?= size .if defined(DEBUG) CTFFLAGS+= -g .endif .if ${MACHINE_CPUARCH} == "amd64" && ${COMPILER_TYPE} != "clang" _COPTFLAGS_EXTRA=-frename-registers .else _COPTFLAGS_EXTRA= .endif COPTFLAGS?=-O2 -pipe ${_COPTFLAGS_EXTRA} .if !empty(COPTFLAGS:M-O[23s]) && empty(COPTFLAGS:M-fno-strict-aliasing) COPTFLAGS+= -fno-strict-aliasing .endif .if !defined(NO_CPU_COPTFLAGS) COPTFLAGS+= ${_CPUCFLAGS} .endif NOSTDINC= -nostdinc INCLUDES= ${NOSTDINC} ${INCLMAGIC} -I. -I$S -I$S/contrib/ck/include CFLAGS= ${COPTFLAGS} ${DEBUG} CFLAGS+= ${INCLUDES} -D_KERNEL -DHAVE_KERNEL_OPTION_HEADERS -include opt_global.h CFLAGS_PARAM_INLINE_UNIT_GROWTH?=100 CFLAGS_PARAM_LARGE_FUNCTION_GROWTH?=1000 .if ${MACHINE_CPUARCH} == "mips" CFLAGS_ARCH_PARAMS?=--param max-inline-insns-single=1000 -DMACHINE_ARCH='"${MACHINE_ARCH}"' .endif CFLAGS.gcc+= -fms-extensions -finline-limit=${INLINE_LIMIT} CFLAGS.gcc+= --param inline-unit-growth=${CFLAGS_PARAM_INLINE_UNIT_GROWTH} CFLAGS.gcc+= --param large-function-growth=${CFLAGS_PARAM_LARGE_FUNCTION_GROWTH} CFLAGS.gcc+= -fms-extensions .if defined(CFLAGS_ARCH_PARAMS) CFLAGS.gcc+=${CFLAGS_ARCH_PARAMS} .endif WERROR?= -Werror # The following should be removed no earlier than LLVM11 being imported into the # tree, to ensure we don't regress the build. LLVM11 and GCC10 will switch the # default over to -fno-common, making this redundant. CFLAGS+= -fno-common # XXX LOCORE means "don't declare C stuff" not "for locore.s". ASM_CFLAGS= -x assembler-with-cpp -DLOCORE ${CFLAGS} ${ASM_CFLAGS.${.IMPSRC:T}} .if defined(PROFLEVEL) && ${PROFLEVEL} >= 1 CFLAGS+= -DGPROF CFLAGS.gcc+= -falign-functions=16 .if ${PROFLEVEL} >= 2 CFLAGS+= -DGPROF4 -DGUPROF PROF= -pg .if ${COMPILER_TYPE} == "gcc" PROF+= -mprofiler-epilogue .endif .else PROF= -pg .endif .endif DEFINED_PROF= ${PROF} COMPAT_FREEBSD32_ENABLED!= grep COMPAT_FREEBSD32 opt_global.h || true ; echo KASAN_ENABLED!= grep KASAN opt_global.h || true ; echo .if !empty(KASAN_ENABLED) -SAN_CFLAGS+= -fsanitize=kernel-address \ +SAN_CFLAGS+= -DSAN_NEEDS_INTERCEPTORS -DSAN_INTERCEPTOR_PREFIX=kasan \ + -fsanitize=kernel-address \ -mllvm -asan-stack=true \ -mllvm -asan-instrument-dynamic-allocas=true \ -mllvm -asan-globals=true \ -mllvm -asan-use-after-scope=true \ -mllvm -asan-instrumentation-with-call-threshold=0 \ -mllvm -asan-instrument-byval=false .endif KCSAN_ENABLED!= grep KCSAN opt_global.h || true ; echo .if !empty(KCSAN_ENABLED) -SAN_CFLAGS+= -fsanitize=thread +SAN_CFLAGS+= -DSAN_NEEDS_INTERCEPTORS -DSAN_INTERCEPTOR_PREFIX=kcsan \ + -fsanitize=thread .endif KMSAN_ENABLED!= grep KMSAN opt_global.h || true ; echo .if !empty(KMSAN_ENABLED) SAN_CFLAGS+= -DSAN_NEEDS_INTERCEPTORS -DSAN_INTERCEPTOR_PREFIX=kmsan \ -fsanitize=kernel-memory .endif KUBSAN_ENABLED!= grep KUBSAN opt_global.h || true ; echo .if !empty(KUBSAN_ENABLED) SAN_CFLAGS+= -fsanitize=undefined .endif COVERAGE_ENABLED!= grep COVERAGE opt_global.h || true ; echo .if !empty(COVERAGE_ENABLED) .if ${COMPILER_TYPE} == "clang" || \ (${COMPILER_TYPE} == "gcc" && ${COMPILER_VERSION} >= 80100) SAN_CFLAGS+= -fsanitize-coverage=trace-pc,trace-cmp .else SAN_CFLAGS+= -fsanitize-coverage=trace-pc .endif .endif CFLAGS+= ${SAN_CFLAGS} GCOV_ENABLED!= grep GCOV opt_global.h || true ; echo .if !empty(GCOV_ENABLED) .if ${COMPILER_TYPE} == "gcc" GCOV_CFLAGS+= -fprofile-arcs -ftest-coverage .endif .endif CFLAGS+= ${GCOV_CFLAGS} # Put configuration-specific C flags last (except for ${PROF}) so that they # can override the others. CFLAGS+= ${CONF_CFLAGS} .if defined(LINKER_FEATURES) && ${LINKER_FEATURES:Mbuild-id} LDFLAGS+= --build-id=sha1 .endif .if (${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64" || \ ${MACHINE_CPUARCH} == "i386" || ${MACHINE} == "powerpc") && \ defined(LINKER_FEATURES) && ${LINKER_FEATURES:Mifunc} == "" && \ !make(install) .error amd64/arm64/i386/ppc* kernel requires linker ifunc support .endif .if ${MACHINE_CPUARCH} == "amd64" LDFLAGS+= -z max-page-size=2097152 .if ${LINKER_TYPE} != "lld" LDFLAGS+= -z common-page-size=4096 .else .if defined(LINKER_FEATURES) && !${LINKER_FEATURES:Mifunc-noplt} .warning "Linker ${LD} does not support -z ifunc-noplt -> ifunc calls are unoptimized." .else LDFLAGS+= -z notext -z ifunc-noplt .endif .endif .endif # ${MACHINE_CPUARCH} == "amd64" .if ${MACHINE_CPUARCH} == "riscv" # Hack: Work around undefined weak symbols being out of range when linking with # LLD (address is a PC-relative calculation, and BFD works around this by # rewriting the instructions to generate an absolute address of 0); -fPIE # avoids this since it uses the GOT for all extern symbols, which is overly # inefficient for us. Drop once undefined weak symbols work with medany. .if ${LINKER_TYPE} == "lld" CFLAGS+= -fPIE .endif .endif NORMAL_C= ${CC} -c ${CFLAGS} ${WERROR} ${PROF} ${.IMPSRC} NORMAL_S= ${CC:N${CCACHE_BIN}} -c ${ASM_CFLAGS} ${WERROR} ${.IMPSRC} PROFILE_C= ${CC} -c ${CFLAGS} ${WERROR} ${.IMPSRC} NORMAL_C_NOWERROR= ${CC} -c ${CFLAGS} ${PROF} ${.IMPSRC} NORMAL_M= ${AWK} -f $S/tools/makeobjops.awk ${.IMPSRC} -c ; \ ${CC} -c ${CFLAGS} ${WERROR} ${PROF} ${.PREFIX}.c NORMAL_FW= uudecode -o ${.TARGET} ${.ALLSRC} NORMAL_FWO= ${CC:N${CCACHE_BIN}} -c ${ASM_CFLAGS} ${WERROR} -o ${.TARGET} \ $S/kern/firmw.S -DFIRMW_FILE="${.ALLSRC:M*.fw}" \ -DFIRMW_SYMBOL="${.ALLSRC:M*.fw:C/[-.\/]/_/g}" # for ZSTD in the kernel (include zstd/lib/freebsd before other CFLAGS) ZSTD_C= ${CC} -c -DZSTD_HEAPMODE=1 -I$S/contrib/zstd/lib/freebsd ${CFLAGS} \ -I$S/contrib/zstd/lib -I$S/contrib/zstd/lib/common ${WERROR} \ -Wno-missing-prototypes ${PROF} -U__BMI__ \ -DZSTD_NO_INTRINSICS \ ${.IMPSRC} # https://github.com/facebook/zstd/commit/812e8f2a [zstd 1.4.1] # "Note that [GCC] autovectorization still does not do a good job on the # optimized version, so it's turned off via attribute and flag. I found # that neither attribute nor command-line flag were entirely successful in # turning off vectorization, which is why there were both." .if ${COMPILER_TYPE} == "gcc" ZSTD_DECOMPRESS_BLOCK_FLAGS= -fno-tree-vectorize .endif ZINCDIR=$S/contrib/openzfs/include # Common for dtrace / zfs CDDL_CFLAGS= \ -DFREEBSD_NAMECACHE \ -D_SYS_VMEM_H_ \ -D__KERNEL \ -D__KERNEL__ \ -nostdinc \ -include $S/modules/zfs/static_ccompile.h \ -I${ZINCDIR} \ -I${ZINCDIR}/os/freebsd \ -I${ZINCDIR}/os/freebsd/spl \ -I${ZINCDIR}/os/freebsd/zfs \ -I$S/modules/zfs \ -I$S/contrib/openzfs/module/zstd/include \ ${CFLAGS} \ -Wno-cast-qual \ -Wno-duplicate-decl-specifier \ -Wno-missing-braces \ -Wno-missing-prototypes \ -Wno-nested-externs \ -Wno-parentheses \ -Wno-pointer-arith \ -Wno-redundant-decls \ -Wno-strict-prototypes \ -Wno-switch \ -Wno-undef \ -Wno-uninitialized \ -Wno-unknown-pragmas \ -Wno-unused \ -include ${ZINCDIR}/os/freebsd/spl/sys/ccompile.h \ -I$S/cddl/contrib/opensolaris/uts/common \ -I$S -I$S/cddl/compat/opensolaris CDDL_C= ${CC} -c ${CDDL_CFLAGS} ${WERROR} ${PROF} ${.IMPSRC} # Special flags for managing the compat compiles for ZFS ZFS_CFLAGS+= ${CDDL_CFLAGS} -DBUILDING_ZFS -DHAVE_UIO_ZEROCOPY \ -DWITH_NETDUMP -D__KERNEL__ -D_SYS_CONDVAR_H_ -DSMP \ -DIN_FREEBSD_BASE .if ${MACHINE_ARCH} == "amd64" ZFS_CFLAGS+= -DHAVE_AVX2 -DHAVE_AVX -D__x86_64 -DHAVE_SSE2 -DHAVE_AVX512F \ -DHAVE_SSSE3 -DHAVE_AVX512BW .endif .if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "powerpc" || \ ${MACHINE_ARCH} == "powerpcspe" || ${MACHINE_ARCH} == "arm" ZFS_CFLAGS+= -DBITS_PER_LONG=32 .else ZFS_CFLAGS+= -DBITS_PER_LONG=64 .endif ZFS_ASM_CFLAGS= -x assembler-with-cpp -DLOCORE ${ZFS_CFLAGS} ZFS_C= ${CC} -c ${ZFS_CFLAGS} ${WERROR} ${PROF} ${.IMPSRC} ZFS_RPC_C= ${CC} -c ${ZFS_CFLAGS} -DHAVE_RPC_TYPES ${WERROR} ${PROF} ${.IMPSRC} ZFS_S= ${CC} -c ${ZFS_ASM_CFLAGS} ${WERROR} ${.IMPSRC} # Special flags for managing the compat compiles for DTrace DTRACE_CFLAGS= -DBUILDING_DTRACE ${CDDL_CFLAGS} -I$S/cddl/dev/dtrace -I$S/cddl/dev/dtrace/${MACHINE_CPUARCH} .if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386" DTRACE_CFLAGS+= -I$S/cddl/contrib/opensolaris/uts/intel -I$S/cddl/dev/dtrace/x86 .endif DTRACE_CFLAGS+= -I$S/cddl/contrib/opensolaris/common/util -I$S -DDIS_MEM -DSMP -I$S/cddl/compat/opensolaris DTRACE_CFLAGS+= -I$S/cddl/contrib/opensolaris/uts/common DTRACE_ASM_CFLAGS= -x assembler-with-cpp -DLOCORE ${DTRACE_CFLAGS} DTRACE_C= ${CC} -c ${DTRACE_CFLAGS} ${WERROR} ${PROF} ${.IMPSRC} DTRACE_S= ${CC} -c ${DTRACE_ASM_CFLAGS} ${WERROR} ${.IMPSRC} # Special flags for managing the compat compiles for DTrace/FBT FBT_CFLAGS= -DBUILDING_DTRACE -nostdinc -I$S/cddl/dev/fbt/${MACHINE_CPUARCH} -I$S/cddl/dev/fbt ${CDDL_CFLAGS} -I$S/cddl/compat/opensolaris -I$S/cddl/contrib/opensolaris/uts/common .if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386" FBT_CFLAGS+= -I$S/cddl/dev/fbt/x86 .endif FBT_C= ${CC} -c ${FBT_CFLAGS} ${WERROR} ${PROF} ${.IMPSRC} .if ${MK_CTF} != "no" NORMAL_CTFCONVERT= ${CTFCONVERT} ${CTFFLAGS} ${.TARGET} .elif ${MAKE_VERSION} >= 5201111300 NORMAL_CTFCONVERT= .else NORMAL_CTFCONVERT= @: .endif # Linux Kernel Programming Interface C-flags LINUXKPI_INCLUDES= -I$S/compat/linuxkpi/common/include \ -I$S/compat/linuxkpi/dummy/include LINUXKPI_C= ${NORMAL_C} ${LINUXKPI_INCLUDES} # Infiniband C flags. Correct include paths and omit errors that linux # does not honor. OFEDINCLUDES= -I$S/ofed/include -I$S/ofed/include/uapi ${LINUXKPI_INCLUDES} OFEDNOERR= -Wno-cast-qual -Wno-pointer-arith -Wno-redundant-decls OFEDCFLAGS= ${CFLAGS:N-I*} -DCONFIG_INFINIBAND_USER_MEM \ ${OFEDINCLUDES} ${CFLAGS:M-I*} ${OFEDNOERR} OFED_C_NOIMP= ${CC} -c -o ${.TARGET} ${OFEDCFLAGS} ${WERROR} ${PROF} OFED_C= ${OFED_C_NOIMP} ${.IMPSRC} # mlxfw C flags. MLXFW_C= ${OFED_C_NOIMP} \ -I${SRCTOP}/sys/contrib/xz-embedded/freebsd \ -I${SRCTOP}/sys/contrib/xz-embedded/linux/lib/xz \ ${.IMPSRC} GEN_CFILES= $S/$M/$M/genassym.c ${MFILES:T:S/.m$/.c/} SYSTEM_CFILES= config.c env.c hints.c vnode_if.c SYSTEM_DEP= Makefile ${SYSTEM_OBJS} SYSTEM_OBJS= locore.o ${MDOBJS} ${OBJS} SYSTEM_OBJS+= ${SYSTEM_CFILES:.c=.o} SYSTEM_OBJS+= hack.pico KEYMAP=kbdcontrol -P ${SRCTOP}/share/vt/keymaps -P ${SRCTOP}/share/syscons/keymaps KEYMAP_FIX=sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' MD_ROOT_SIZE_CONFIGURED!= grep MD_ROOT_SIZE opt_md.h || true ; echo .if ${MFS_IMAGE:Uno} != "no" .if empty(MD_ROOT_SIZE_CONFIGURED) SYSTEM_OBJS+= embedfs_${MFS_IMAGE:T:R}.o .endif .endif SYSTEM_LD_BASECMD= \ ${LD} -m ${LD_EMULATION} -Bdynamic -T ${LDSCRIPT} ${_LDFLAGS} \ --no-warn-mismatch --warn-common --export-dynamic \ --dynamic-linker /red/herring -X SYSTEM_LD= @${SYSTEM_LD_BASECMD} -o ${.TARGET} ${SYSTEM_OBJS} vers.o SYSTEM_LD_TAIL= @${OBJCOPY} --strip-symbol gcc2_compiled. ${.TARGET} ; \ ${SIZE} ${.TARGET} ; chmod 755 ${.TARGET} SYSTEM_DEP+= ${LDSCRIPT} # Calculate path for .m files early, if needed. .if !defined(NO_MODULES) && !defined(__MPATH) && !make(install) && \ (empty(.MAKEFLAGS:M-V) || defined(NO_SKIP_MPATH)) __MPATH!=find ${S:tA}/ -name \*_if.m .endif # MKMODULESENV is set here so that port makefiles can augment # them. MKMODULESENV+= MAKEOBJDIRPREFIX=${.OBJDIR}/modules KMODDIR=${KODIR} MKMODULESENV+= MACHINE_CPUARCH=${MACHINE_CPUARCH} MKMODULESENV+= MACHINE=${MACHINE} MACHINE_ARCH=${MACHINE_ARCH} MKMODULESENV+= MODULES_EXTRA="${MODULES_EXTRA}" WITHOUT_MODULES="${WITHOUT_MODULES}" MKMODULESENV+= ARCH_FLAGS="${ARCH_FLAGS}" .if (${KERN_IDENT} == LINT) MKMODULESENV+= ALL_MODULES=LINT .endif .if defined(MODULES_OVERRIDE) MKMODULESENV+= MODULES_OVERRIDE="${MODULES_OVERRIDE}" .endif .if defined(DEBUG) MKMODULESENV+= DEBUG_FLAGS="${DEBUG}" .endif .if !defined(NO_MODULES) MKMODULESENV+= __MPATH="${__MPATH}" .endif # Detect kernel config options that force stack frames to be turned on. DDB_ENABLED!= grep DDB opt_ddb.h || true ; echo DTR_ENABLED!= grep KDTRACE_FRAME opt_kdtrace.h || true ; echo HWPMC_ENABLED!= grep HWPMC opt_hwpmc_hooks.h || true ; echo diff --git a/sys/kern/subr_asan.c b/sys/kern/subr_asan.c index bdb7fc3da368..30b05c54cd16 100644 --- a/sys/kern/subr_asan.c +++ b/sys/kern/subr_asan.c @@ -1,1129 +1,1127 @@ /* $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $ */ /* * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net * All rights reserved. * * This code is part of the KASAN subsystem of the NetBSD kernel. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #define SAN_RUNTIME #include __FBSDID("$FreeBSD$"); #if 0 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $"); #endif #include #include #include #include #include #include #include #include /* ASAN constants. Part of the compiler ABI. */ #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE - 1) #define KASAN_ALLOCA_SCALE_SIZE 32 /* ASAN ABI version. */ #if defined(__clang__) && (__clang_major__ - 0 >= 6) #define ASAN_ABI_VERSION 8 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__) #define ASAN_ABI_VERSION 8 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__) #define ASAN_ABI_VERSION 6 #else #error "Unsupported compiler version" #endif #define __RET_ADDR (unsigned long)__builtin_return_address(0) /* Global variable descriptor. Part of the compiler ABI. */ struct __asan_global_source_location { const char *filename; int line_no; int column_no; }; struct __asan_global { const void *beg; /* address of the global variable */ size_t size; /* size of the global variable */ size_t size_with_redzone; /* size with the redzone */ const void *name; /* name of the variable */ const void *module_name; /* name of the module where the var is declared */ unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */ struct __asan_global_source_location *location; #if ASAN_ABI_VERSION >= 7 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */ #endif }; FEATURE(kasan, "Kernel address sanitizer"); static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "KASAN options"); static int panic_on_violation = 1; SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN, &panic_on_violation, 0, "Panic if an invalid access is detected"); static bool kasan_enabled __read_mostly = false; /* -------------------------------------------------------------------------- */ void kasan_shadow_map(vm_offset_t addr, size_t size) { size_t sz, npages, i; vm_offset_t sva, eva; KASSERT(addr % KASAN_SHADOW_SCALE == 0, ("%s: invalid address %#lx", __func__, addr)); sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE; sva = kasan_md_addr_to_shad(addr); eva = kasan_md_addr_to_shad(addr) + sz; sva = rounddown(sva, PAGE_SIZE); eva = roundup(eva, PAGE_SIZE); npages = (eva - sva) / PAGE_SIZE; KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS, ("%s: invalid address range %#lx-%#lx", __func__, sva, eva)); for (i = 0; i < npages; i++) pmap_kasan_enter(sva + ptoa(i)); } void kasan_init(void) { int disabled; disabled = 0; TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled); if (disabled) return; /* MD initialization. */ kasan_md_init(); /* Now officially enabled. */ kasan_enabled = true; } static inline const char * kasan_code_name(uint8_t code) { switch (code) { case KASAN_GENERIC_REDZONE: return "GenericRedZone"; case KASAN_MALLOC_REDZONE: return "MallocRedZone"; case KASAN_KMEM_REDZONE: return "KmemRedZone"; case KASAN_UMA_FREED: return "UMAUseAfterFree"; case KASAN_KSTACK_FREED: return "KernelStack"; case KASAN_EXEC_ARGS_FREED: return "ExecKVA"; case 1 ... 7: return "RedZonePartial"; case KASAN_STACK_LEFT: return "StackLeft"; case KASAN_STACK_MID: return "StackMiddle"; case KASAN_STACK_RIGHT: return "StackRight"; case KASAN_USE_AFTER_RET: return "UseAfterRet"; case KASAN_USE_AFTER_SCOPE: return "UseAfterScope"; default: return "Unknown"; } } #define REPORT(f, ...) do { \ if (panic_on_violation) { \ kasan_enabled = false; \ panic(f, __VA_ARGS__); \ } else { \ struct stack st; \ \ stack_save(&st); \ printf(f "\n", __VA_ARGS__); \ stack_print_ddb(&st); \ } \ } while (0) static void kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc, uint8_t code) { REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)", size, (write ? "write" : "read"), addr, kasan_code_name(code), code); } static __always_inline void kasan_shadow_1byte_markvalid(unsigned long addr) { int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr); int8_t last = (addr & KASAN_SHADOW_MASK) + 1; *byte = last; } static __always_inline void kasan_shadow_Nbyte_markvalid(const void *addr, size_t size) { size_t i; for (i = 0; i < size; i++) { kasan_shadow_1byte_markvalid((unsigned long)addr + i); } } static __always_inline void kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code) { void *shad; if (__predict_false(size == 0)) return; if (__predict_false(kasan_md_unsupported((vm_offset_t)addr))) return; KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0, ("%s: invalid address %p", __func__, addr)); KASSERT(size % KASAN_SHADOW_SCALE == 0, ("%s: invalid size %zu", __func__, size)); shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr); size = size >> KASAN_SHADOW_SCALE_SHIFT; __builtin_memset(shad, code, size); } /* * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid, * and the rest as invalid. There are generally two use cases: * * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks * the redzone at the end of the buffer as invalid. If the entire is to be * marked invalid, origsize will be 0. * * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid. */ void kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code) { size_t i, n, redz; int8_t *shad; if (__predict_false(!kasan_enabled)) return; if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS && (vm_offset_t)addr < DMAP_MAX_ADDRESS) return; KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS && (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS, ("%s: invalid address %p", __func__, addr)); KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0, ("%s: invalid address %p", __func__, addr)); redz = redzsize - roundup(size, KASAN_SHADOW_SCALE); KASSERT(redz % KASAN_SHADOW_SCALE == 0, ("%s: invalid size %zu", __func__, redz)); shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr); /* Chunks of 8 bytes, valid. */ n = size / KASAN_SHADOW_SCALE; for (i = 0; i < n; i++) { *shad++ = 0; } /* Possibly one chunk, mid. */ if ((size & KASAN_SHADOW_MASK) != 0) { *shad++ = (size & KASAN_SHADOW_MASK); } /* Chunks of 8 bytes, invalid. */ n = redz / KASAN_SHADOW_SCALE; for (i = 0; i < n; i++) { *shad++ = code; } } /* -------------------------------------------------------------------------- */ #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \ (addr >> KASAN_SHADOW_SCALE_SHIFT) != \ ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT) static __always_inline bool kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code) { int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr); int8_t last = (addr & KASAN_SHADOW_MASK) + 1; if (__predict_true(*byte == 0 || last <= *byte)) { return (true); } *code = *byte; return (false); } static __always_inline bool kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code) { int8_t *byte, last; if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) { return (kasan_shadow_1byte_isvalid(addr, code) && kasan_shadow_1byte_isvalid(addr+1, code)); } byte = (int8_t *)kasan_md_addr_to_shad(addr); last = ((addr + 1) & KASAN_SHADOW_MASK) + 1; if (__predict_true(*byte == 0 || last <= *byte)) { return (true); } *code = *byte; return (false); } static __always_inline bool kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code) { int8_t *byte, last; if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) { return (kasan_shadow_2byte_isvalid(addr, code) && kasan_shadow_2byte_isvalid(addr+2, code)); } byte = (int8_t *)kasan_md_addr_to_shad(addr); last = ((addr + 3) & KASAN_SHADOW_MASK) + 1; if (__predict_true(*byte == 0 || last <= *byte)) { return (true); } *code = *byte; return (false); } static __always_inline bool kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code) { int8_t *byte, last; if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) { return (kasan_shadow_4byte_isvalid(addr, code) && kasan_shadow_4byte_isvalid(addr+4, code)); } byte = (int8_t *)kasan_md_addr_to_shad(addr); last = ((addr + 7) & KASAN_SHADOW_MASK) + 1; if (__predict_true(*byte == 0 || last <= *byte)) { return (true); } *code = *byte; return (false); } static __always_inline bool kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code) { size_t i; for (i = 0; i < size; i++) { if (!kasan_shadow_1byte_isvalid(addr+i, code)) return (false); } return (true); } static __always_inline void kasan_shadow_check(unsigned long addr, size_t size, bool write, unsigned long retaddr) { uint8_t code; bool valid; if (__predict_false(!kasan_enabled)) return; if (__predict_false(size == 0)) return; if (__predict_false(kasan_md_unsupported(addr))) return; if (KERNEL_PANICKED()) return; if (__builtin_constant_p(size)) { switch (size) { case 1: valid = kasan_shadow_1byte_isvalid(addr, &code); break; case 2: valid = kasan_shadow_2byte_isvalid(addr, &code); break; case 4: valid = kasan_shadow_4byte_isvalid(addr, &code); break; case 8: valid = kasan_shadow_8byte_isvalid(addr, &code); break; default: valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); break; } } else { valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); } if (__predict_false(!valid)) { kasan_report(addr, size, write, retaddr, code); } } /* -------------------------------------------------------------------------- */ void * kasan_memcpy(void *dst, const void *src, size_t len) { kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); return (__builtin_memcpy(dst, src, len)); } int kasan_memcmp(const void *b1, const void *b2, size_t len) { kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR); kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR); return (__builtin_memcmp(b1, b2, len)); } void * kasan_memset(void *b, int c, size_t len) { kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR); return (__builtin_memset(b, c, len)); } void * kasan_memmove(void *dst, const void *src, size_t len) { kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); return (__builtin_memmove(dst, src, len)); } size_t kasan_strlen(const char *str) { const char *s; s = str; while (1) { kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR); if (*s == '\0') break; s++; } return (s - str); } char * kasan_strcpy(char *dst, const char *src) { char *save = dst; while (1) { kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR); kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR); *dst = *src; if (*src == '\0') break; src++, dst++; } return save; } int kasan_strcmp(const char *s1, const char *s2) { while (1) { kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR); kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR); if (*s1 != *s2) break; if (*s1 == '\0') return 0; s1++, s2++; } return (*(const unsigned char *)s1 - *(const unsigned char *)s2); } int kasan_copyin(const void *uaddr, void *kaddr, size_t len) { kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); return (copyin(uaddr, kaddr, len)); } int kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) { kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); return (copyinstr(uaddr, kaddr, len, done)); } int kasan_copyout(const void *kaddr, void *uaddr, size_t len) { kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR); return (copyout(kaddr, uaddr, len)); } /* -------------------------------------------------------------------------- */ #include -#define ATOMIC_SAN_PREFIX kasan #include #define _ASAN_ATOMIC_FUNC_ADD(name, type) \ void kasan_atomic_add_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_add_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_ADD(name, type) \ _ASAN_ATOMIC_FUNC_ADD(name, type) \ _ASAN_ATOMIC_FUNC_ADD(acq_##name, type) \ _ASAN_ATOMIC_FUNC_ADD(rel_##name, type) #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ void kasan_atomic_subtract_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_subtract_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) #define _ASAN_ATOMIC_FUNC_SET(name, type) \ void kasan_atomic_set_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_set_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_SET(name, type) \ _ASAN_ATOMIC_FUNC_SET(name, type) \ _ASAN_ATOMIC_FUNC_SET(acq_##name, type) \ _ASAN_ATOMIC_FUNC_SET(rel_##name, type) #define _ASAN_ATOMIC_FUNC_CLEAR(name, type) \ void kasan_atomic_clear_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_clear_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_CLEAR(name, type) \ _ASAN_ATOMIC_FUNC_CLEAR(name, type) \ _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type) #define ASAN_ATOMIC_FUNC_FETCHADD(name, type) \ type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_fetchadd_##name(ptr, val)); \ } #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ type kasan_atomic_readandclear_##name(volatile type *ptr) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_readandclear_##name(ptr)); \ } #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_testandclear_##name(ptr, v)); \ } #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type) \ int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_testandset_##name(ptr, v)); \ } #define ASAN_ATOMIC_FUNC_SWAP(name, type) \ type kasan_atomic_swap_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_swap_##name(ptr, val)); \ } #define _ASAN_ATOMIC_FUNC_CMPSET(name, type) \ int kasan_atomic_cmpset_##name(volatile type *ptr, type oval, \ type nval) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_cmpset_##name(ptr, oval, nval)); \ } #define ASAN_ATOMIC_FUNC_CMPSET(name, type) \ _ASAN_ATOMIC_FUNC_CMPSET(name, type) \ _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type) #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ type nval) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_fcmpset_##name(ptr, oval, nval)); \ } #define ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name) \ void kasan_atomic_thread_fence_##name(void) \ { \ atomic_thread_fence_##name(); \ } #define _ASAN_ATOMIC_FUNC_LOAD(name, type) \ type kasan_atomic_load_##name(volatile type *ptr) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_load_##name(ptr)); \ } #define ASAN_ATOMIC_FUNC_LOAD(name, type) \ _ASAN_ATOMIC_FUNC_LOAD(name, type) \ _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type) #define _ASAN_ATOMIC_FUNC_STORE(name, type) \ void kasan_atomic_store_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_store_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_STORE(name, type) \ _ASAN_ATOMIC_FUNC_STORE(name, type) \ _ASAN_ATOMIC_FUNC_STORE(rel_##name, type) ASAN_ATOMIC_FUNC_ADD(8, uint8_t); ASAN_ATOMIC_FUNC_ADD(16, uint16_t); ASAN_ATOMIC_FUNC_ADD(32, uint32_t); ASAN_ATOMIC_FUNC_ADD(64, uint64_t); ASAN_ATOMIC_FUNC_ADD(int, u_int); ASAN_ATOMIC_FUNC_ADD(long, u_long); ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int); ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long); ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); ASAN_ATOMIC_FUNC_SET(8, uint8_t); ASAN_ATOMIC_FUNC_SET(16, uint16_t); ASAN_ATOMIC_FUNC_SET(32, uint32_t); ASAN_ATOMIC_FUNC_SET(64, uint64_t); ASAN_ATOMIC_FUNC_SET(int, u_int); ASAN_ATOMIC_FUNC_SET(long, u_long); ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t); ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t); ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t); ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t); ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t); ASAN_ATOMIC_FUNC_CLEAR(int, u_int); ASAN_ATOMIC_FUNC_CLEAR(long, u_long); ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); ASAN_ATOMIC_FUNC_FETCHADD(int, u_int); ASAN_ATOMIC_FUNC_FETCHADD(long, u_long); ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int); ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long); ASAN_ATOMIC_FUNC_SWAP(32, uint32_t); ASAN_ATOMIC_FUNC_SWAP(64, uint64_t); ASAN_ATOMIC_FUNC_SWAP(int, u_int); ASAN_ATOMIC_FUNC_SWAP(long, u_long); ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t); ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t); ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t); ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t); ASAN_ATOMIC_FUNC_CMPSET(int, u_int); ASAN_ATOMIC_FUNC_CMPSET(long, u_long); ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); ASAN_ATOMIC_FUNC_FCMPSET(int, u_int); ASAN_ATOMIC_FUNC_FCMPSET(long, u_long); ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); ASAN_ATOMIC_FUNC_LOAD(8, uint8_t); ASAN_ATOMIC_FUNC_LOAD(16, uint16_t); ASAN_ATOMIC_FUNC_LOAD(32, uint32_t); ASAN_ATOMIC_FUNC_LOAD(64, uint64_t); ASAN_ATOMIC_FUNC_LOAD(char, u_char); ASAN_ATOMIC_FUNC_LOAD(short, u_short); ASAN_ATOMIC_FUNC_LOAD(int, u_int); ASAN_ATOMIC_FUNC_LOAD(long, u_long); ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); ASAN_ATOMIC_FUNC_STORE(8, uint8_t); ASAN_ATOMIC_FUNC_STORE(16, uint16_t); ASAN_ATOMIC_FUNC_STORE(32, uint32_t); ASAN_ATOMIC_FUNC_STORE(64, uint64_t); ASAN_ATOMIC_FUNC_STORE(char, u_char); ASAN_ATOMIC_FUNC_STORE(short, u_short); ASAN_ATOMIC_FUNC_STORE(int, u_int); ASAN_ATOMIC_FUNC_STORE(long, u_long); ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); ASAN_ATOMIC_FUNC_THREAD_FENCE(acq); ASAN_ATOMIC_FUNC_THREAD_FENCE(rel); ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); void kasan_atomic_interrupt_fence(void) { } /* -------------------------------------------------------------------------- */ #include #include -#define BUS_SAN_PREFIX kasan #include int kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, int flags, bus_space_handle_t *handlep) { return (bus_space_map(tag, hnd, size, flags, handlep)); } void kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_unmap(tag, hnd, size); } int kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) { return (bus_space_subregion(tag, hnd, offset, size, handlep)); } void kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_free(tag, hnd, size); } void kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, int flags) { bus_space_barrier(tag, hnd, offset, size, flags); } #define ASAN_BUS_READ_FUNC(func, width, type) \ type kasan_bus_space_read##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset) \ { \ return (bus_space_read##func##_##width(tag, hnd, \ offset)); \ } \ #define ASAN_BUS_READ_PTR_FUNC(func, width, type) \ void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, type *buf, \ bus_size_t count) \ { \ kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ false, __RET_ADDR); \ bus_space_read_##func##_##width(tag, hnd, size, buf, \ count); \ } ASAN_BUS_READ_FUNC(, 1, uint8_t) ASAN_BUS_READ_FUNC(_stream, 1, uint8_t) ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) ASAN_BUS_READ_FUNC(, 2, uint16_t) ASAN_BUS_READ_FUNC(_stream, 2, uint16_t) ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) ASAN_BUS_READ_FUNC(, 4, uint32_t) ASAN_BUS_READ_FUNC(_stream, 4, uint32_t) ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) ASAN_BUS_READ_FUNC(, 8, uint64_t) #define ASAN_BUS_WRITE_FUNC(func, width, type) \ void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ bus_space_write##func##_##width(tag, hnd, offset, value);\ } \ #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type) \ void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ bus_space_handle_t hnd, bus_size_t size, const type *buf, \ bus_size_t count) \ { \ kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ true, __RET_ADDR); \ bus_space_write_##func##_##width(tag, hnd, size, buf, \ count); \ } ASAN_BUS_WRITE_FUNC(, 1, uint8_t) ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) ASAN_BUS_WRITE_FUNC(, 2, uint16_t) ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) ASAN_BUS_WRITE_FUNC(, 4, uint32_t) ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) ASAN_BUS_WRITE_FUNC(, 8, uint64_t) #define ASAN_BUS_SET_FUNC(func, width, type) \ void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value, \ bus_size_t count) \ { \ bus_space_set_##func##_##width(tag, hnd, offset, value, \ count); \ } ASAN_BUS_SET_FUNC(multi, 1, uint8_t) ASAN_BUS_SET_FUNC(region, 1, uint8_t) ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t) ASAN_BUS_SET_FUNC(multi, 2, uint16_t) ASAN_BUS_SET_FUNC(region, 2, uint16_t) ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t) ASAN_BUS_SET_FUNC(multi, 4, uint32_t) ASAN_BUS_SET_FUNC(region, 4, uint32_t) ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t) #define ASAN_BUS_PEEK_FUNC(width, type) \ int kasan_bus_space_peek_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type *valuep) \ { \ return (bus_space_peek_##width(tag, hnd, offset, \ valuep)); \ } ASAN_BUS_PEEK_FUNC(1, uint8_t) ASAN_BUS_PEEK_FUNC(2, uint16_t) ASAN_BUS_PEEK_FUNC(4, uint32_t) ASAN_BUS_PEEK_FUNC(8, uint64_t) #define ASAN_BUS_POKE_FUNC(width, type) \ int kasan_bus_space_poke_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ return (bus_space_poke_##width(tag, hnd, offset, \ value)); \ } ASAN_BUS_POKE_FUNC(1, uint8_t) ASAN_BUS_POKE_FUNC(2, uint16_t) ASAN_BUS_POKE_FUNC(4, uint32_t) ASAN_BUS_POKE_FUNC(8, uint64_t) /* -------------------------------------------------------------------------- */ void __asan_register_globals(struct __asan_global *, size_t); void __asan_unregister_globals(struct __asan_global *, size_t); void __asan_register_globals(struct __asan_global *globals, size_t n) { size_t i; for (i = 0; i < n; i++) { kasan_mark(globals[i].beg, globals[i].size, globals[i].size_with_redzone, KASAN_GENERIC_REDZONE); } } void __asan_unregister_globals(struct __asan_global *globals, size_t n) { size_t i; for (i = 0; i < n; i++) { kasan_mark(globals[i].beg, globals[i].size_with_redzone, globals[i].size_with_redzone, 0); } } #define ASAN_LOAD_STORE(size) \ void __asan_load##size(unsigned long); \ void __asan_load##size(unsigned long addr) \ { \ kasan_shadow_check(addr, size, false, __RET_ADDR);\ } \ void __asan_load##size##_noabort(unsigned long); \ void __asan_load##size##_noabort(unsigned long addr) \ { \ kasan_shadow_check(addr, size, false, __RET_ADDR);\ } \ void __asan_store##size(unsigned long); \ void __asan_store##size(unsigned long addr) \ { \ kasan_shadow_check(addr, size, true, __RET_ADDR);\ } \ void __asan_store##size##_noabort(unsigned long); \ void __asan_store##size##_noabort(unsigned long addr) \ { \ kasan_shadow_check(addr, size, true, __RET_ADDR);\ } ASAN_LOAD_STORE(1); ASAN_LOAD_STORE(2); ASAN_LOAD_STORE(4); ASAN_LOAD_STORE(8); ASAN_LOAD_STORE(16); void __asan_loadN(unsigned long, size_t); void __asan_loadN_noabort(unsigned long, size_t); void __asan_storeN(unsigned long, size_t); void __asan_storeN_noabort(unsigned long, size_t); void __asan_handle_no_return(void); void __asan_loadN(unsigned long addr, size_t size) { kasan_shadow_check(addr, size, false, __RET_ADDR); } void __asan_loadN_noabort(unsigned long addr, size_t size) { kasan_shadow_check(addr, size, false, __RET_ADDR); } void __asan_storeN(unsigned long addr, size_t size) { kasan_shadow_check(addr, size, true, __RET_ADDR); } void __asan_storeN_noabort(unsigned long addr, size_t size) { kasan_shadow_check(addr, size, true, __RET_ADDR); } void __asan_handle_no_return(void) { /* nothing */ } #define ASAN_SET_SHADOW(byte) \ void __asan_set_shadow_##byte(void *, size_t); \ void __asan_set_shadow_##byte(void *addr, size_t size) \ { \ __builtin_memset((void *)addr, 0x##byte, size); \ } ASAN_SET_SHADOW(00); ASAN_SET_SHADOW(f1); ASAN_SET_SHADOW(f2); ASAN_SET_SHADOW(f3); ASAN_SET_SHADOW(f5); ASAN_SET_SHADOW(f8); void __asan_poison_stack_memory(const void *, size_t); void __asan_unpoison_stack_memory(const void *, size_t); void __asan_poison_stack_memory(const void *addr, size_t size) { size = roundup(size, KASAN_SHADOW_SCALE); kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE); } void __asan_unpoison_stack_memory(const void *addr, size_t size) { kasan_shadow_Nbyte_markvalid(addr, size); } void __asan_alloca_poison(const void *, size_t); void __asan_allocas_unpoison(const void *, const void *); void __asan_alloca_poison(const void *addr, size_t size) { const void *l, *r; KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0, ("%s: invalid address %p", __func__, addr)); l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE; r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE); kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT); kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE), KASAN_STACK_MID); kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT); } void __asan_allocas_unpoison(const void *stkbegin, const void *stkend) { size_t size; if (__predict_false(!stkbegin)) return; if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend)) return; size = (uintptr_t)stkend - (uintptr_t)stkbegin; kasan_shadow_Nbyte_fill(stkbegin, size, 0); } void __asan_poison_memory_region(const void *addr, size_t size); void __asan_unpoison_memory_region(const void *addr, size_t size); void __asan_poison_memory_region(const void *addr, size_t size) { } void __asan_unpoison_memory_region(const void *addr, size_t size) { } diff --git a/sys/kern/subr_csan.c b/sys/kern/subr_csan.c index 56d2e59ff12c..7bed25076fa9 100644 --- a/sys/kern/subr_csan.c +++ b/sys/kern/subr_csan.c @@ -1,912 +1,910 @@ /* $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $ */ /* * Copyright (c) 2019 The NetBSD Foundation, Inc. * All rights reserved. * Copyright (c) 2019 Andrew Turner * * This code is derived from software contributed to The NetBSD Foundation * by Maxime Villard. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #define SAN_RUNTIME #include "opt_ddb.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #ifdef KCSAN_PANIC #define REPORT panic #else #define REPORT printf #endif typedef struct { uintptr_t addr; uint32_t size; bool write:1; bool atomic:1; uintptr_t pc; } csan_cell_t; typedef struct { bool inited; uint32_t cnt; csan_cell_t cell; } csan_cpu_t; static csan_cpu_t kcsan_cpus[MAXCPU]; static bool kcsan_enabled __read_mostly; #define __RET_ADDR (uintptr_t)__builtin_return_address(0) #define KCSAN_NACCESSES 1024 #define KCSAN_DELAY 10 /* 10 microseconds */ /* -------------------------------------------------------------------------- */ /* The MD code. */ #include /* -------------------------------------------------------------------------- */ static void kcsan_enable(void *dummy __unused) { printf("Enabling KCSCAN, expect reduced performance.\n"); kcsan_enabled = true; } SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL); void kcsan_cpu_init(u_int cpu) { kcsan_cpus[cpu].inited = true; } /* -------------------------------------------------------------------------- */ static inline void kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu) { const char *newsym, *oldsym; #ifdef DDB c_db_sym_t sym; db_expr_t offset; sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset); db_symbol_values(sym, &newsym, NULL); sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset); db_symbol_values(sym, &oldsym, NULL); #else newsym = ""; oldsym = ""; #endif REPORT("CSan: Racy Access " "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] " "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n", newcpu, (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), (void *)new->addr, new->size, (void *)new->pc, newsym, oldcpu, (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), (void *)old->addr, old->size, (void *)old->pc, oldsym); kcsan_md_unwind(); } static inline bool kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) { if (new->write && !new->atomic) return false; if (old->write && !old->atomic) return false; return true; } static inline void kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) { csan_cell_t old, new; csan_cpu_t *cpu; uint64_t intr; size_t i; if (__predict_false(!kcsan_enabled)) return; if (__predict_false(kcsan_md_unsupported((vm_offset_t)addr))) return; if (KERNEL_PANICKED()) return; new.addr = addr; new.size = size; new.write = write; new.atomic = atomic; new.pc = pc; CPU_FOREACH(i) { __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); if (old.addr + old.size <= new.addr) continue; if (new.addr + new.size <= old.addr) continue; if (__predict_true(!old.write && !new.write)) continue; if (__predict_true(kcsan_access_is_atomic(&new, &old))) continue; kcsan_report(&new, PCPU_GET(cpuid), &old, i); break; } if (__predict_false(!kcsan_md_is_avail())) return; kcsan_md_disable_intrs(&intr); cpu = &kcsan_cpus[PCPU_GET(cpuid)]; if (__predict_false(!cpu->inited)) goto out; cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; if (__predict_true(cpu->cnt != 0)) goto out; __builtin_memcpy(&cpu->cell, &new, sizeof(new)); kcsan_md_delay(KCSAN_DELAY); __builtin_memset(&cpu->cell, 0, sizeof(new)); out: kcsan_md_enable_intrs(&intr); } #define CSAN_READ(size) \ void __tsan_read##size(uintptr_t); \ void __tsan_read##size(uintptr_t addr) \ { \ kcsan_access(addr, size, false, false, __RET_ADDR); \ } \ void __tsan_unaligned_read##size(uintptr_t); \ void __tsan_unaligned_read##size(uintptr_t addr) \ { \ kcsan_access(addr, size, false, false, __RET_ADDR); \ } CSAN_READ(1) CSAN_READ(2) CSAN_READ(4) CSAN_READ(8) CSAN_READ(16) #define CSAN_WRITE(size) \ void __tsan_write##size(uintptr_t); \ void __tsan_write##size(uintptr_t addr) \ { \ kcsan_access(addr, size, true, false, __RET_ADDR); \ } \ void __tsan_unaligned_write##size(uintptr_t); \ void __tsan_unaligned_write##size(uintptr_t addr) \ { \ kcsan_access(addr, size, true, false, __RET_ADDR); \ } CSAN_WRITE(1) CSAN_WRITE(2) CSAN_WRITE(4) CSAN_WRITE(8) CSAN_WRITE(16) void __tsan_read_range(uintptr_t, size_t); void __tsan_write_range(uintptr_t, size_t); void __tsan_read_range(uintptr_t addr, size_t size) { kcsan_access(addr, size, false, false, __RET_ADDR); } void __tsan_write_range(uintptr_t addr, size_t size) { kcsan_access(addr, size, true, false, __RET_ADDR); } void __tsan_init(void); void __tsan_func_entry(void *); void __tsan_func_exit(void); void __tsan_init(void) { } void __tsan_func_entry(void *call_pc) { } void __tsan_func_exit(void) { } /* -------------------------------------------------------------------------- */ void * kcsan_memcpy(void *dst, const void *src, size_t len) { kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); return __builtin_memcpy(dst, src, len); } int kcsan_memcmp(const void *b1, const void *b2, size_t len) { kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); return __builtin_memcmp(b1, b2, len); } void * kcsan_memset(void *b, int c, size_t len) { kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); return __builtin_memset(b, c, len); } void * kcsan_memmove(void *dst, const void *src, size_t len) { kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); return __builtin_memmove(dst, src, len); } char * kcsan_strcpy(char *dst, const char *src) { char *save = dst; while (1) { kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); *dst = *src; if (*src == '\0') break; src++, dst++; } return save; } int kcsan_strcmp(const char *s1, const char *s2) { while (1) { kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); if (*s1 != *s2) break; if (*s1 == '\0') return 0; s1++, s2++; } return (*(const unsigned char *)s1 - *(const unsigned char *)s2); } size_t kcsan_strlen(const char *str) { const char *s; s = str; while (1) { kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); if (*s == '\0') break; s++; } return (s - str); } int kcsan_copyin(const void *uaddr, void *kaddr, size_t len) { kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); return copyin(uaddr, kaddr, len); } int kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) { kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); return copyinstr(uaddr, kaddr, len, done); } int kcsan_copyout(const void *kaddr, void *uaddr, size_t len) { kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); return copyout(kaddr, uaddr, len); } /* -------------------------------------------------------------------------- */ #include -#define ATOMIC_SAN_PREFIX kcsan #include #define _CSAN_ATOMIC_FUNC_ADD(name, type) \ void kcsan_atomic_add_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_add_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_ADD(name, type) \ _CSAN_ATOMIC_FUNC_ADD(name, type) \ _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ _CSAN_ATOMIC_FUNC_ADD(rel_##name, type) #define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ void kcsan_atomic_clear_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_clear_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_CLEAR(name, type) \ _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) #define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \ type val2) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_cmpset_##name(ptr, val1, val2)); \ } #define CSAN_ATOMIC_FUNC_CMPSET(name, type) \ _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) #define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \ type val2) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_fcmpset_##name(ptr, val1, val2)); \ } #define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) #define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \ type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_fetchadd_##name(ptr, val)); \ } #define _CSAN_ATOMIC_FUNC_LOAD(name, type) \ type kcsan_atomic_load_##name(volatile type *ptr) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \ __RET_ADDR); \ return (atomic_load_##name(ptr)); \ } #define CSAN_ATOMIC_FUNC_LOAD(name, type) \ _CSAN_ATOMIC_FUNC_LOAD(name, type) \ _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \ #define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ type kcsan_atomic_readandclear_##name(volatile type *ptr) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_readandclear_##name(ptr)); \ } #define _CSAN_ATOMIC_FUNC_SET(name, type) \ void kcsan_atomic_set_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_set_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_SET(name, type) \ _CSAN_ATOMIC_FUNC_SET(name, type) \ _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \ _CSAN_ATOMIC_FUNC_SET(rel_##name, type) #define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_subtract_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) #define _CSAN_ATOMIC_FUNC_STORE(name, type) \ void kcsan_atomic_store_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_store_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_STORE(name, type) \ _CSAN_ATOMIC_FUNC_STORE(name, type) \ _CSAN_ATOMIC_FUNC_STORE(rel_##name, type) #define CSAN_ATOMIC_FUNC_SWAP(name, type) \ type kcsan_atomic_swap_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return(atomic_swap_##name(ptr, val)); \ } #define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return(atomic_testandclear_##name(ptr, val)); \ } #define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_testandset_##name(ptr, val)); \ } CSAN_ATOMIC_FUNC_ADD(8, uint8_t) CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t) CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t) CSAN_ATOMIC_FUNC_LOAD(8, uint8_t) CSAN_ATOMIC_FUNC_SET(8, uint8_t) CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t) _CSAN_ATOMIC_FUNC_STORE(8, uint8_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t) CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_SWAP(8, uint8_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t) #endif CSAN_ATOMIC_FUNC_ADD(16, uint16_t) CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t) CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t) CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) CSAN_ATOMIC_FUNC_SET(16, uint16_t) CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t) _CSAN_ATOMIC_FUNC_STORE(16, uint16_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t) CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_SWAP(16, uint16_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t) #endif CSAN_ATOMIC_FUNC_ADD(32, uint32_t) CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t) CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t) CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t) CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t) CSAN_ATOMIC_FUNC_LOAD(32, uint32_t) CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t) CSAN_ATOMIC_FUNC_SET(32, uint32_t) CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t) CSAN_ATOMIC_FUNC_STORE(32, uint32_t) CSAN_ATOMIC_FUNC_SWAP(32, uint32_t) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t) CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t) #endif CSAN_ATOMIC_FUNC_ADD(64, uint64_t) CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t) CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t) CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t) CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t) CSAN_ATOMIC_FUNC_LOAD(64, uint64_t) CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t) CSAN_ATOMIC_FUNC_SET(64, uint64_t) CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t) CSAN_ATOMIC_FUNC_STORE(64, uint64_t) CSAN_ATOMIC_FUNC_SWAP(64, uint64_t) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t) CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t) #endif CSAN_ATOMIC_FUNC_ADD(char, uint8_t) CSAN_ATOMIC_FUNC_CLEAR(char, uint8_t) CSAN_ATOMIC_FUNC_CMPSET(char, uint8_t) CSAN_ATOMIC_FUNC_FCMPSET(char, uint8_t) CSAN_ATOMIC_FUNC_LOAD(char, uint8_t) CSAN_ATOMIC_FUNC_SET(char, uint8_t) CSAN_ATOMIC_FUNC_SUBTRACT(char, uint8_t) _CSAN_ATOMIC_FUNC_STORE(char, uint8_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(char, uint8_t) CSAN_ATOMIC_FUNC_READANDCLEAR(char, uint8_t) CSAN_ATOMIC_FUNC_SWAP(char, uint8_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(char, uint8_t) CSAN_ATOMIC_FUNC_TESTANDSET(char, uint8_t) #endif CSAN_ATOMIC_FUNC_ADD(short, uint16_t) CSAN_ATOMIC_FUNC_CLEAR(short, uint16_t) CSAN_ATOMIC_FUNC_CMPSET(short, uint16_t) CSAN_ATOMIC_FUNC_FCMPSET(short, uint16_t) CSAN_ATOMIC_FUNC_LOAD(short, uint16_t) CSAN_ATOMIC_FUNC_SET(short, uint16_t) CSAN_ATOMIC_FUNC_SUBTRACT(short, uint16_t) _CSAN_ATOMIC_FUNC_STORE(short, uint16_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(short, uint16_t) CSAN_ATOMIC_FUNC_READANDCLEAR(short, uint16_t) CSAN_ATOMIC_FUNC_SWAP(short, uint16_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(short, uint16_t) CSAN_ATOMIC_FUNC_TESTANDSET(short, uint16_t) #endif CSAN_ATOMIC_FUNC_ADD(int, u_int) CSAN_ATOMIC_FUNC_CLEAR(int, u_int) CSAN_ATOMIC_FUNC_CMPSET(int, u_int) CSAN_ATOMIC_FUNC_FCMPSET(int, u_int) CSAN_ATOMIC_FUNC_FETCHADD(int, u_int) CSAN_ATOMIC_FUNC_LOAD(int, u_int) CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int) CSAN_ATOMIC_FUNC_SET(int, u_int) CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int) CSAN_ATOMIC_FUNC_STORE(int, u_int) CSAN_ATOMIC_FUNC_SWAP(int, u_int) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int) CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int) #endif CSAN_ATOMIC_FUNC_ADD(long, u_long) CSAN_ATOMIC_FUNC_CLEAR(long, u_long) CSAN_ATOMIC_FUNC_CMPSET(long, u_long) CSAN_ATOMIC_FUNC_FCMPSET(long, u_long) CSAN_ATOMIC_FUNC_FETCHADD(long, u_long) CSAN_ATOMIC_FUNC_LOAD(long, u_long) CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long) CSAN_ATOMIC_FUNC_SET(long, u_long) CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long) CSAN_ATOMIC_FUNC_STORE(long, u_long) CSAN_ATOMIC_FUNC_SWAP(long, u_long) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long) CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long) CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long) #endif CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t) CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t) CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t) CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t) #if !defined(__amd64__) CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t) #endif CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t) CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t) CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t) CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t) CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t) CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t) #if 0 CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t) CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t) #endif #define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ void kcsan_atomic_thread_fence_##name(void) \ { \ atomic_thread_fence_##name(); \ } CSAN_ATOMIC_FUNC_THREAD_FENCE(acq) CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel) CSAN_ATOMIC_FUNC_THREAD_FENCE(rel) CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst) void kcsan_atomic_interrupt_fence(void) { atomic_interrupt_fence(); } /* -------------------------------------------------------------------------- */ #include #include -#define BUS_SAN_PREFIX kcsan #include int kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, int flags, bus_space_handle_t *handlep) { return (bus_space_map(tag, hnd, size, flags, handlep)); } void kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_unmap(tag, hnd, size); } int kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) { return (bus_space_subregion(tag, hnd, offset, size, handlep)); } #if !defined(__amd64__) int kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start, bus_addr_t reg_end, bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags, bus_addr_t *addrp, bus_space_handle_t *handlep) { return (bus_space_alloc(tag, reg_start, reg_end, size, alignment, boundary, flags, addrp, handlep)); } #endif void kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_free(tag, hnd, size); } void kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, int flags) { bus_space_barrier(tag, hnd, offset, size, flags); } #define CSAN_BUS_READ_FUNC(func, width, type) \ type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset) \ { \ return (bus_space_read##func##_##width(tag, hnd, \ offset)); \ } \ #define CSAN_BUS_READ_PTR_FUNC(func, width, type) \ void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, type *buf, \ bus_size_t count) \ { \ kcsan_access((uintptr_t)buf, sizeof(type) * count, \ false, false, __RET_ADDR); \ bus_space_read_##func##_##width(tag, hnd, size, buf, \ count); \ } CSAN_BUS_READ_FUNC(, 1, uint8_t) CSAN_BUS_READ_FUNC(_stream, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) CSAN_BUS_READ_FUNC(, 2, uint16_t) CSAN_BUS_READ_FUNC(_stream, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) CSAN_BUS_READ_FUNC(, 4, uint32_t) CSAN_BUS_READ_FUNC(_stream, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) CSAN_BUS_READ_FUNC(, 8, uint64_t) #if defined(__aarch64__) CSAN_BUS_READ_FUNC(_stream, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) #endif #define CSAN_BUS_WRITE_FUNC(func, width, type) \ void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ bus_space_write##func##_##width(tag, hnd, offset, value); \ } \ #define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, const type *buf, \ bus_size_t count) \ { \ kcsan_access((uintptr_t)buf, sizeof(type) * count, \ true, false, __RET_ADDR); \ bus_space_write_##func##_##width(tag, hnd, size, buf, \ count); \ } CSAN_BUS_WRITE_FUNC(, 1, uint8_t) CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) CSAN_BUS_WRITE_FUNC(, 2, uint16_t) CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) CSAN_BUS_WRITE_FUNC(, 4, uint32_t) CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) CSAN_BUS_WRITE_FUNC(, 8, uint64_t) #if defined(__aarch64__) CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t) #endif #define CSAN_BUS_SET_FUNC(func, width, type) \ void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value, \ bus_size_t count) \ { \ bus_space_set_##func##_##width(tag, hnd, offset, value, \ count); \ } CSAN_BUS_SET_FUNC(multi, 1, uint8_t) CSAN_BUS_SET_FUNC(region, 1, uint8_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) #endif CSAN_BUS_SET_FUNC(multi, 2, uint16_t) CSAN_BUS_SET_FUNC(region, 2, uint16_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) #endif CSAN_BUS_SET_FUNC(multi, 4, uint32_t) CSAN_BUS_SET_FUNC(region, 4, uint32_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) #endif #if !defined(__amd64__) CSAN_BUS_SET_FUNC(multi, 8, uint64_t) CSAN_BUS_SET_FUNC(region, 8, uint64_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t) CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t) #endif #endif #define CSAN_BUS_PEEK_FUNC(width, type) \ int kcsan_bus_space_peek_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type *value) \ { \ kcsan_access((uintptr_t)value, sizeof(type), true, false, \ __RET_ADDR); \ return (bus_space_peek_##width(tag, hnd, offset, value)); \ } CSAN_BUS_PEEK_FUNC(1, uint8_t) CSAN_BUS_PEEK_FUNC(2, uint16_t) CSAN_BUS_PEEK_FUNC(4, uint32_t) #if !defined(__i386__) CSAN_BUS_PEEK_FUNC(8, uint64_t) #endif #define CSAN_BUS_POKE_FUNC(width, type) \ int kcsan_bus_space_poke_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ return (bus_space_poke_##width(tag, hnd, offset, value)); \ } CSAN_BUS_POKE_FUNC(1, uint8_t) CSAN_BUS_POKE_FUNC(2, uint16_t) CSAN_BUS_POKE_FUNC(4, uint32_t) #if !defined(__i386__) CSAN_BUS_POKE_FUNC(8, uint64_t) #endif diff --git a/sys/sys/atomic_san.h b/sys/sys/atomic_san.h index 5d10f58f7565..30bd01c97cf2 100644 --- a/sys/sys/atomic_san.h +++ b/sys/sys/atomic_san.h @@ -1,402 +1,398 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Andrew Turner * Copyright (c) 2021 The FreeBSD Foundation * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory (Department of Computer Science and * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the * DARPA SSITH research programme. * * Portions of this software were written by Mark Johnston under sponsorship * by the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_ATOMIC_SAN_H_ #define _SYS_ATOMIC_SAN_H_ #ifndef _MACHINE_ATOMIC_H_ #error do not include this header, use machine/atomic.h #endif -#ifndef ATOMIC_SAN_PREFIX -#error No sanitizer prefix available -#endif - #define ATOMIC_SAN_FUNC_1(sp, op, name, type) \ void sp##_atomic_##op##_##name(volatile type *, type); \ void sp##_atomic_##op##_acq_##name(volatile type *, type); \ void sp##_atomic_##op##_rel_##name(volatile type *, type) #define ATOMIC_SAN_CMPSET(sp, name, type) \ int sp##_atomic_cmpset_##name(volatile type *, type, type); \ int sp##_atomic_cmpset_acq_##name(volatile type *, type, type); \ int sp##_atomic_cmpset_rel_##name(volatile type *, type, type) #define ATOMIC_SAN_FCMPSET(sp, name, type) \ int sp##_atomic_fcmpset_##name(volatile type *, type *, type); \ int sp##_atomic_fcmpset_acq_##name(volatile type *, type *, type); \ int sp##_atomic_fcmpset_rel_##name(volatile type *, type *, type) #define ATOMIC_SAN_READ(sp, op, name, type) \ type sp##_atomic_##op##_##name(volatile type *, type) #define ATOMIC_SAN_READANDCLEAR(sp, name, type) \ type sp##_atomic_readandclear_##name(volatile type *) #define ATOMIC_SAN_LOAD(sp, name, type) \ type sp##_atomic_load_##name(volatile type *); \ type sp##_atomic_load_acq_##name(volatile type *) #define ATOMIC_SAN_STORE(sp, name, type) \ void sp##_atomic_store_##name(volatile type *, type); \ void sp##_atomic_store_rel_##name(volatile type *, type) #define ATOMIC_SAN_TEST(sp, op, name, type) \ int sp##_atomic_##op##_##name(volatile type *, u_int); \ int sp##_atomic_##op##_acq_##name(volatile type *, u_int) #define ATOMIC_SAN_THREAD_FENCE(sp) \ void sp##_atomic_thread_fence_acq(void); \ void sp##_atomic_thread_fence_rel(void); \ void sp##_atomic_thread_fence_acq_rel(void); \ void sp##_atomic_thread_fence_seq_cst(void); \ void sp##_atomic_interrupt_fence(void) #define _ATOMIC_SAN_FUNCS(sp, name, type) \ ATOMIC_SAN_FUNC_1(sp, add, name, type); \ ATOMIC_SAN_FUNC_1(sp, clear, name, type); \ ATOMIC_SAN_CMPSET(sp, name, type); \ ATOMIC_SAN_FCMPSET(sp, name, type); \ ATOMIC_SAN_READ(sp, fetchadd, name, type); \ ATOMIC_SAN_LOAD(sp, name, type); \ ATOMIC_SAN_READANDCLEAR(sp, name, type); \ ATOMIC_SAN_FUNC_1(sp, set, name, type); \ ATOMIC_SAN_FUNC_1(sp, subtract, name, type); \ ATOMIC_SAN_STORE(sp, name, type); \ ATOMIC_SAN_READ(sp, swap, name, type); \ ATOMIC_SAN_TEST(sp, testandclear, name, type); \ ATOMIC_SAN_TEST(sp, testandset, name, type); \ ATOMIC_SAN_THREAD_FENCE(sp); #define ATOMIC_SAN_FUNCS(name, type) \ - _ATOMIC_SAN_FUNCS(ATOMIC_SAN_PREFIX, name, type) + _ATOMIC_SAN_FUNCS(SAN_INTERCEPTOR_PREFIX, name, type) ATOMIC_SAN_FUNCS(char, uint8_t); ATOMIC_SAN_FUNCS(short, uint16_t); ATOMIC_SAN_FUNCS(int, u_int); ATOMIC_SAN_FUNCS(long, u_long); ATOMIC_SAN_FUNCS(ptr, uintptr_t); ATOMIC_SAN_FUNCS(8, uint8_t); ATOMIC_SAN_FUNCS(16, uint16_t); ATOMIC_SAN_FUNCS(32, uint32_t); ATOMIC_SAN_FUNCS(64, uint64_t); #ifndef SAN_RUNTIME /* * Redirect uses of an atomic(9) function to the sanitizer's interceptor. * For instance, KASAN callers of atomic_add_char() will be redirected to * kasan_atomic_add_char(). */ -#define ATOMIC_SAN(func) \ - __CONCAT(ATOMIC_SAN_PREFIX, __CONCAT(_atomic_, func)) +#define ATOMIC_SAN(func) \ + __CONCAT(SAN_INTERCEPTOR_PREFIX, __CONCAT(_atomic_, func)) #define atomic_add_char ATOMIC_SAN(add_char) #define atomic_add_acq_char ATOMIC_SAN(add_acq_char) #define atomic_add_rel_char ATOMIC_SAN(add_rel_char) #define atomic_clear_char ATOMIC_SAN(clear_char) #define atomic_clear_acq_char ATOMIC_SAN(clear_acq_char) #define atomic_clear_rel_char ATOMIC_SAN(clear_rel_char) #define atomic_cmpset_char ATOMIC_SAN(cmpset_char) #define atomic_cmpset_acq_char ATOMIC_SAN(cmpset_acq_char) #define atomic_cmpset_rel_char ATOMIC_SAN(cmpset_rel_char) #define atomic_fcmpset_char ATOMIC_SAN(fcmpset_char) #define atomic_fcmpset_acq_char ATOMIC_SAN(fcmpset_acq_char) #define atomic_fcmpset_rel_char ATOMIC_SAN(fcmpset_rel_char) #define atomic_fetchadd_char ATOMIC_SAN(fetchadd_char) #define atomic_load_char ATOMIC_SAN(load_char) #define atomic_load_acq_char ATOMIC_SAN(load_acq_char) #define atomic_readandclear_char ATOMIC_SAN(readandclear_char) #define atomic_set_char ATOMIC_SAN(set_char) #define atomic_set_acq_char ATOMIC_SAN(set_acq_char) #define atomic_set_rel_char ATOMIC_SAN(set_rel_char) #define atomic_subtract_char ATOMIC_SAN(subtract_char) #define atomic_subtract_acq_char ATOMIC_SAN(subtract_acq_char) #define atomic_subtract_rel_char ATOMIC_SAN(subtract_rel_char) #define atomic_store_char ATOMIC_SAN(store_char) #define atomic_store_rel_char ATOMIC_SAN(store_rel_char) #define atomic_swap_char ATOMIC_SAN(swap_char) #define atomic_testandclear_char ATOMIC_SAN(testandclear_char) #define atomic_testandset_char ATOMIC_SAN(testandset_char) #define atomic_add_short ATOMIC_SAN(add_short) #define atomic_add_acq_short ATOMIC_SAN(add_acq_short) #define atomic_add_rel_short ATOMIC_SAN(add_rel_short) #define atomic_clear_short ATOMIC_SAN(clear_short) #define atomic_clear_acq_short ATOMIC_SAN(clear_acq_short) #define atomic_clear_rel_short ATOMIC_SAN(clear_rel_short) #define atomic_cmpset_short ATOMIC_SAN(cmpset_short) #define atomic_cmpset_acq_short ATOMIC_SAN(cmpset_acq_short) #define atomic_cmpset_rel_short ATOMIC_SAN(cmpset_rel_short) #define atomic_fcmpset_short ATOMIC_SAN(fcmpset_short) #define atomic_fcmpset_acq_short ATOMIC_SAN(fcmpset_acq_short) #define atomic_fcmpset_rel_short ATOMIC_SAN(fcmpset_rel_short) #define atomic_fetchadd_short ATOMIC_SAN(fetchadd_short) #define atomic_load_short ATOMIC_SAN(load_short) #define atomic_load_acq_short ATOMIC_SAN(load_acq_short) #define atomic_readandclear_short ATOMIC_SAN(readandclear_short) #define atomic_set_short ATOMIC_SAN(set_short) #define atomic_set_acq_short ATOMIC_SAN(set_acq_short) #define atomic_set_rel_short ATOMIC_SAN(set_rel_short) #define atomic_subtract_short ATOMIC_SAN(subtract_short) #define atomic_subtract_acq_short ATOMIC_SAN(subtract_acq_short) #define atomic_subtract_rel_short ATOMIC_SAN(subtract_rel_short) #define atomic_store_short ATOMIC_SAN(store_short) #define atomic_store_rel_short ATOMIC_SAN(store_rel_short) #define atomic_swap_short ATOMIC_SAN(swap_short) #define atomic_testandclear_short ATOMIC_SAN(testandclear_short) #define atomic_testandset_short ATOMIC_SAN(testandset_short) #define atomic_add_int ATOMIC_SAN(add_int) #define atomic_add_acq_int ATOMIC_SAN(add_acq_int) #define atomic_add_rel_int ATOMIC_SAN(add_rel_int) #define atomic_clear_int ATOMIC_SAN(clear_int) #define atomic_clear_acq_int ATOMIC_SAN(clear_acq_int) #define atomic_clear_rel_int ATOMIC_SAN(clear_rel_int) #define atomic_cmpset_int ATOMIC_SAN(cmpset_int) #define atomic_cmpset_acq_int ATOMIC_SAN(cmpset_acq_int) #define atomic_cmpset_rel_int ATOMIC_SAN(cmpset_rel_int) #define atomic_fcmpset_int ATOMIC_SAN(fcmpset_int) #define atomic_fcmpset_acq_int ATOMIC_SAN(fcmpset_acq_int) #define atomic_fcmpset_rel_int ATOMIC_SAN(fcmpset_rel_int) #define atomic_fetchadd_int ATOMIC_SAN(fetchadd_int) #define atomic_load_int ATOMIC_SAN(load_int) #define atomic_load_acq_int ATOMIC_SAN(load_acq_int) #define atomic_readandclear_int ATOMIC_SAN(readandclear_int) #define atomic_set_int ATOMIC_SAN(set_int) #define atomic_set_acq_int ATOMIC_SAN(set_acq_int) #define atomic_set_rel_int ATOMIC_SAN(set_rel_int) #define atomic_subtract_int ATOMIC_SAN(subtract_int) #define atomic_subtract_acq_int ATOMIC_SAN(subtract_acq_int) #define atomic_subtract_rel_int ATOMIC_SAN(subtract_rel_int) #define atomic_store_int ATOMIC_SAN(store_int) #define atomic_store_rel_int ATOMIC_SAN(store_rel_int) #define atomic_swap_int ATOMIC_SAN(swap_int) #define atomic_testandclear_int ATOMIC_SAN(testandclear_int) #define atomic_testandset_int ATOMIC_SAN(testandset_int) #define atomic_add_long ATOMIC_SAN(add_long) #define atomic_add_acq_long ATOMIC_SAN(add_acq_long) #define atomic_add_rel_long ATOMIC_SAN(add_rel_long) #define atomic_clear_long ATOMIC_SAN(clear_long) #define atomic_clear_acq_long ATOMIC_SAN(clear_acq_long) #define atomic_clear_rel_long ATOMIC_SAN(clear_rel_long) #define atomic_cmpset_long ATOMIC_SAN(cmpset_long) #define atomic_cmpset_acq_long ATOMIC_SAN(cmpset_acq_long) #define atomic_cmpset_rel_long ATOMIC_SAN(cmpset_rel_long) #define atomic_fcmpset_long ATOMIC_SAN(fcmpset_long) #define atomic_fcmpset_acq_long ATOMIC_SAN(fcmpset_acq_long) #define atomic_fcmpset_rel_long ATOMIC_SAN(fcmpset_rel_long) #define atomic_fetchadd_long ATOMIC_SAN(fetchadd_long) #define atomic_load_long ATOMIC_SAN(load_long) #define atomic_load_acq_long ATOMIC_SAN(load_acq_long) #define atomic_readandclear_long ATOMIC_SAN(readandclear_long) #define atomic_set_long ATOMIC_SAN(set_long) #define atomic_set_acq_long ATOMIC_SAN(set_acq_long) #define atomic_set_rel_long ATOMIC_SAN(set_rel_long) #define atomic_subtract_long ATOMIC_SAN(subtract_long) #define atomic_subtract_acq_long ATOMIC_SAN(subtract_acq_long) #define atomic_subtract_rel_long ATOMIC_SAN(subtract_rel_long) #define atomic_store_long ATOMIC_SAN(store_long) #define atomic_store_rel_long ATOMIC_SAN(store_rel_long) #define atomic_swap_long ATOMIC_SAN(swap_long) #define atomic_testandclear_long ATOMIC_SAN(testandclear_long) #define atomic_testandset_long ATOMIC_SAN(testandset_long) #define atomic_testandset_acq_long ATOMIC_SAN(testandset_acq_long) #define atomic_add_ptr ATOMIC_SAN(add_ptr) #define atomic_add_acq_ptr ATOMIC_SAN(add_acq_ptr) #define atomic_add_rel_ptr ATOMIC_SAN(add_rel_ptr) #define atomic_clear_ptr ATOMIC_SAN(clear_ptr) #define atomic_clear_acq_ptr ATOMIC_SAN(clear_acq_ptr) #define atomic_clear_rel_ptr ATOMIC_SAN(clear_rel_ptr) #define atomic_cmpset_ptr ATOMIC_SAN(cmpset_ptr) #define atomic_cmpset_acq_ptr ATOMIC_SAN(cmpset_acq_ptr) #define atomic_cmpset_rel_ptr ATOMIC_SAN(cmpset_rel_ptr) #define atomic_fcmpset_ptr ATOMIC_SAN(fcmpset_ptr) #define atomic_fcmpset_acq_ptr ATOMIC_SAN(fcmpset_acq_ptr) #define atomic_fcmpset_rel_ptr ATOMIC_SAN(fcmpset_rel_ptr) #define atomic_fetchadd_ptr ATOMIC_SAN(fetchadd_ptr) #define atomic_load_ptr(x) ({ \ __typeof(*x) __retptr; \ __retptr = (void *)ATOMIC_SAN(load_ptr)((volatile uintptr_t *)(x)); \ __retptr; \ }) #define atomic_load_acq_ptr ATOMIC_SAN(load_acq_ptr) #define atomic_load_consume_ptr(x) ({ \ __typeof(*x) __retptr; \ __retptr = (void *)atomic_load_acq_ptr((volatile uintptr_t *)(x));\ __retptr; \ }) #define atomic_readandclear_ptr ATOMIC_SAN(readandclear_ptr) #define atomic_set_ptr ATOMIC_SAN(set_ptr) #define atomic_set_acq_ptr ATOMIC_SAN(set_acq_ptr) #define atomic_set_rel_ptr ATOMIC_SAN(set_rel_ptr) #define atomic_subtract_ptr ATOMIC_SAN(subtract_ptr) #define atomic_subtract_acq_ptr ATOMIC_SAN(subtract_acq_ptr) #define atomic_subtract_rel_ptr ATOMIC_SAN(subtract_rel_ptr) #define atomic_store_ptr(x, v) ({ \ __typeof(*x) __value = (v); \ ATOMIC_SAN(store_ptr)((volatile uintptr_t *)(x), (uintptr_t)(__value));\ }) #define atomic_store_rel_ptr ATOMIC_SAN(store_rel_ptr) #define atomic_swap_ptr ATOMIC_SAN(swap_ptr) #define atomic_testandclear_ptr ATOMIC_SAN(testandclear_ptr) #define atomic_testandset_ptr ATOMIC_SAN(testandset_ptr) #define atomic_add_8 ATOMIC_SAN(add_8) #define atomic_add_acq_8 ATOMIC_SAN(add_acq_8) #define atomic_add_rel_8 ATOMIC_SAN(add_rel_8) #define atomic_clear_8 ATOMIC_SAN(clear_8) #define atomic_clear_acq_8 ATOMIC_SAN(clear_acq_8) #define atomic_clear_rel_8 ATOMIC_SAN(clear_rel_8) #define atomic_cmpset_8 ATOMIC_SAN(cmpset_8) #define atomic_cmpset_acq_8 ATOMIC_SAN(cmpset_acq_8) #define atomic_cmpset_rel_8 ATOMIC_SAN(cmpset_rel_8) #define atomic_fcmpset_8 ATOMIC_SAN(fcmpset_8) #define atomic_fcmpset_acq_8 ATOMIC_SAN(fcmpset_acq_8) #define atomic_fcmpset_rel_8 ATOMIC_SAN(fcmpset_rel_8) #define atomic_fetchadd_8 ATOMIC_SAN(fetchadd_8) #define atomic_load_8 ATOMIC_SAN(load_8) #define atomic_load_acq_8 ATOMIC_SAN(load_acq_8) #define atomic_readandclear_8 ATOMIC_SAN(readandclear_8) #define atomic_set_8 ATOMIC_SAN(set_8) #define atomic_set_acq_8 ATOMIC_SAN(set_acq_8) #define atomic_set_rel_8 ATOMIC_SAN(set_rel_8) #define atomic_subtract_8 ATOMIC_SAN(subtract_8) #define atomic_subtract_acq_8 ATOMIC_SAN(subtract_acq_8) #define atomic_subtract_rel_8 ATOMIC_SAN(subtract_rel_8) #define atomic_store_8 ATOMIC_SAN(store_8) #define atomic_store_rel_8 ATOMIC_SAN(store_rel_8) #define atomic_swap_8 ATOMIC_SAN(swap_8) #define atomic_testandclear_8 ATOMIC_SAN(testandclear_8) #define atomic_testandset_8 ATOMIC_SAN(testandset_8) #define atomic_add_16 ATOMIC_SAN(add_16) #define atomic_add_acq_16 ATOMIC_SAN(add_acq_16) #define atomic_add_rel_16 ATOMIC_SAN(add_rel_16) #define atomic_clear_16 ATOMIC_SAN(clear_16) #define atomic_clear_acq_16 ATOMIC_SAN(clear_acq_16) #define atomic_clear_rel_16 ATOMIC_SAN(clear_rel_16) #define atomic_cmpset_16 ATOMIC_SAN(cmpset_16) #define atomic_cmpset_acq_16 ATOMIC_SAN(cmpset_acq_16) #define atomic_cmpset_rel_16 ATOMIC_SAN(cmpset_rel_16) #define atomic_fcmpset_16 ATOMIC_SAN(fcmpset_16) #define atomic_fcmpset_acq_16 ATOMIC_SAN(fcmpset_acq_16) #define atomic_fcmpset_rel_16 ATOMIC_SAN(fcmpset_rel_16) #define atomic_fetchadd_16 ATOMIC_SAN(fetchadd_16) #define atomic_load_16 ATOMIC_SAN(load_16) #define atomic_load_acq_16 ATOMIC_SAN(load_acq_16) #define atomic_readandclear_16 ATOMIC_SAN(readandclear_16) #define atomic_set_16 ATOMIC_SAN(set_16) #define atomic_set_acq_16 ATOMIC_SAN(set_acq_16) #define atomic_set_rel_16 ATOMIC_SAN(set_rel_16) #define atomic_subtract_16 ATOMIC_SAN(subtract_16) #define atomic_subtract_acq_16 ATOMIC_SAN(subtract_acq_16) #define atomic_subtract_rel_16 ATOMIC_SAN(subtract_rel_16) #define atomic_store_16 ATOMIC_SAN(store_16) #define atomic_store_rel_16 ATOMIC_SAN(store_rel_16) #define atomic_swap_16 ATOMIC_SAN(swap_16) #define atomic_testandclear_16 ATOMIC_SAN(testandclear_16) #define atomic_testandset_16 ATOMIC_SAN(testandset_16) #define atomic_add_32 ATOMIC_SAN(add_32) #define atomic_add_acq_32 ATOMIC_SAN(add_acq_32) #define atomic_add_rel_32 ATOMIC_SAN(add_rel_32) #define atomic_clear_32 ATOMIC_SAN(clear_32) #define atomic_clear_acq_32 ATOMIC_SAN(clear_acq_32) #define atomic_clear_rel_32 ATOMIC_SAN(clear_rel_32) #define atomic_cmpset_32 ATOMIC_SAN(cmpset_32) #define atomic_cmpset_acq_32 ATOMIC_SAN(cmpset_acq_32) #define atomic_cmpset_rel_32 ATOMIC_SAN(cmpset_rel_32) #define atomic_fcmpset_32 ATOMIC_SAN(fcmpset_32) #define atomic_fcmpset_acq_32 ATOMIC_SAN(fcmpset_acq_32) #define atomic_fcmpset_rel_32 ATOMIC_SAN(fcmpset_rel_32) #define atomic_fetchadd_32 ATOMIC_SAN(fetchadd_32) #define atomic_load_32 ATOMIC_SAN(load_32) #define atomic_load_acq_32 ATOMIC_SAN(load_acq_32) #define atomic_readandclear_32 ATOMIC_SAN(readandclear_32) #define atomic_set_32 ATOMIC_SAN(set_32) #define atomic_set_acq_32 ATOMIC_SAN(set_acq_32) #define atomic_set_rel_32 ATOMIC_SAN(set_rel_32) #define atomic_subtract_32 ATOMIC_SAN(subtract_32) #define atomic_subtract_acq_32 ATOMIC_SAN(subtract_acq_32) #define atomic_subtract_rel_32 ATOMIC_SAN(subtract_rel_32) #define atomic_store_32 ATOMIC_SAN(store_32) #define atomic_store_rel_32 ATOMIC_SAN(store_rel_32) #define atomic_swap_32 ATOMIC_SAN(swap_32) #define atomic_testandclear_32 ATOMIC_SAN(testandclear_32) #define atomic_testandset_32 ATOMIC_SAN(testandset_32) #define atomic_add_64 ATOMIC_SAN(add_64) #define atomic_add_acq_64 ATOMIC_SAN(add_acq_64) #define atomic_add_rel_64 ATOMIC_SAN(add_rel_64) #define atomic_clear_64 ATOMIC_SAN(clear_64) #define atomic_clear_acq_64 ATOMIC_SAN(clear_acq_64) #define atomic_clear_rel_64 ATOMIC_SAN(clear_rel_64) #define atomic_cmpset_64 ATOMIC_SAN(cmpset_64) #define atomic_cmpset_acq_64 ATOMIC_SAN(cmpset_acq_64) #define atomic_cmpset_rel_64 ATOMIC_SAN(cmpset_rel_64) #define atomic_fcmpset_64 ATOMIC_SAN(fcmpset_64) #define atomic_fcmpset_acq_64 ATOMIC_SAN(fcmpset_acq_64) #define atomic_fcmpset_rel_64 ATOMIC_SAN(fcmpset_rel_64) #define atomic_fetchadd_64 ATOMIC_SAN(fetchadd_64) #define atomic_load_64 ATOMIC_SAN(load_64) #define atomic_load_acq_64 ATOMIC_SAN(load_acq_64) #define atomic_readandclear_64 ATOMIC_SAN(readandclear_64) #define atomic_set_64 ATOMIC_SAN(set_64) #define atomic_set_acq_64 ATOMIC_SAN(set_acq_64) #define atomic_set_rel_64 ATOMIC_SAN(set_rel_64) #define atomic_subtract_64 ATOMIC_SAN(subtract_64) #define atomic_subtract_acq_64 ATOMIC_SAN(subtract_acq_64) #define atomic_subtract_rel_64 ATOMIC_SAN(subtract_rel_64) #define atomic_store_64 ATOMIC_SAN(store_64) #define atomic_store_rel_64 ATOMIC_SAN(store_rel_64) #define atomic_swap_64 ATOMIC_SAN(swap_64) #define atomic_testandclear_64 ATOMIC_SAN(testandclear_64) #define atomic_testandset_64 ATOMIC_SAN(testandset_64) #define atomic_thread_fence_acq ATOMIC_SAN(thread_fence_acq) #define atomic_thread_fence_acq_rel ATOMIC_SAN(thread_fence_acq_rel) #define atomic_thread_fence_rel ATOMIC_SAN(thread_fence_rel) #define atomic_thread_fence_seq_cst ATOMIC_SAN(thread_fence_seq_cst) #define atomic_interrupt_fence ATOMIC_SAN(interrupt_fence) #endif /* !SAN_RUNTIME */ #endif /* !_SYS_ATOMIC_SAN_H_ */ diff --git a/sys/sys/bus_san.h b/sys/sys/bus_san.h index 05d5ecd4b844..fd3c4c803675 100644 --- a/sys/sys/bus_san.h +++ b/sys/sys/bus_san.h @@ -1,229 +1,226 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Andrew Turner * Copyright (c) 2021 The FreeBSD Foundation * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory (Department of Computer Science and * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the * DARPA SSITH research programme. * * Portions of this software were written by Mark Johnston under sponsorship by * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_BUS_SAN_H_ #define _SYS_BUS_SAN_H_ #ifndef _MACHINE_BUS_H_ #error do not include this header, use machine/bus.h #endif -#ifndef BUS_SAN_PREFIX -#error No sanitizer prefix defined -#endif - #define BUS_SAN_MULTI(sp, rw, width, type) \ void sp##_bus_space_##rw##_multi_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *, bus_size_t); \ void sp##_bus_space_##rw##_multi_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *, bus_size_t); \ void sp##_bus_space_##rw##_region_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *, bus_size_t); \ void sp##_bus_space_##rw##_region_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *, bus_size_t) #define BUS_SAN_READ(sp, width, type) \ type sp##_bus_space_read_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t); \ type sp##_bus_space_read_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t); \ BUS_SAN_MULTI(sp, read, width, type) #define BUS_SAN_WRITE(sp, width, type) \ void sp##_bus_space_write_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type); \ void sp##_bus_space_write_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type); \ BUS_SAN_MULTI(sp, write, width, const type) #define BUS_SAN_SET(sp, width, type) \ void sp##_bus_space_set_multi_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type, bus_size_t); \ void sp##_bus_space_set_multi_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type, bus_size_t); \ void sp##_bus_space_set_region_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type, bus_size_t); \ void sp##_bus_space_set_region_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type, bus_size_t) #define BUS_SAN_COPY(sp, width, type) \ void sp##_bus_space_copy_region_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, bus_space_handle_t, \ bus_size_t, bus_size_t); \ void sp##_bus_space_copy_region_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, bus_space_handle_t, \ bus_size_t, bus_size_t); #define BUS_SAN_PEEK(sp, width, type) \ int sp##_bus_space_peek_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *); #define BUS_SAN_POKE(sp, width, type) \ int sp##_bus_space_poke_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type); #define BUS_SAN_MISC(sp) \ int sp##_bus_space_map(bus_space_tag_t, bus_addr_t, bus_size_t, \ int, bus_space_handle_t *); \ void sp##_bus_space_unmap(bus_space_tag_t, bus_space_handle_t, \ bus_size_t); \ int sp##_bus_space_subregion(bus_space_tag_t, bus_space_handle_t,\ bus_size_t, bus_size_t, bus_space_handle_t *); \ int sp##_bus_space_alloc(bus_space_tag_t, bus_addr_t, bus_addr_t,\ bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *, \ bus_space_handle_t *); \ void sp##_bus_space_free(bus_space_tag_t, bus_space_handle_t, \ bus_size_t); \ void sp##_bus_space_barrier(bus_space_tag_t, bus_space_handle_t,\ bus_size_t, bus_size_t, int); #define _BUS_SAN_FUNCS(sp, width, type) \ BUS_SAN_READ(sp, width, type); \ BUS_SAN_WRITE(sp, width, type); \ BUS_SAN_SET(sp, width, type); \ BUS_SAN_COPY(sp, width, type) \ BUS_SAN_PEEK(sp, width, type); \ BUS_SAN_POKE(sp, width, type); \ BUS_SAN_MISC(sp); #define BUS_SAN_FUNCS(width, type) \ - _BUS_SAN_FUNCS(BUS_SAN_PREFIX, width, type) + _BUS_SAN_FUNCS(SAN_INTERCEPTOR_PREFIX, width, type) BUS_SAN_FUNCS(1, uint8_t); BUS_SAN_FUNCS(2, uint16_t); BUS_SAN_FUNCS(4, uint32_t); BUS_SAN_FUNCS(8, uint64_t); #ifndef SAN_RUNTIME -#define BUS_SAN(func) __CONCAT(BUS_SAN_PREFIX, __CONCAT(_bus_space_, func)) +#define BUS_SAN(func) \ + __CONCAT(SAN_INTERCEPTOR_PREFIX, __CONCAT(_bus_space_, func)) #define bus_space_map BUS_SAN(map) #define bus_space_unmap BUS_SAN(unmap) #define bus_space_subregion BUS_SAN(subregion) #define bus_space_alloc BUS_SAN(alloc) #define bus_space_free BUS_SAN(free) #define bus_space_barrier BUS_SAN(barrier) #define bus_space_read_1 BUS_SAN(read_1) #define bus_space_read_stream_1 BUS_SAN(read_stream_1) #define bus_space_read_multi_1 BUS_SAN(read_multi_1) #define bus_space_read_multi_stream_1 BUS_SAN(read_multi_stream_1) #define bus_space_read_region_1 BUS_SAN(read_region_1) #define bus_space_read_region_stream_1 BUS_SAN(read_region_stream_1) #define bus_space_write_1 BUS_SAN(write_1) #define bus_space_write_stream_1 BUS_SAN(write_stream_1) #define bus_space_write_multi_1 BUS_SAN(write_multi_1) #define bus_space_write_multi_stream_1 BUS_SAN(write_multi_stream_1) #define bus_space_write_region_1 BUS_SAN(write_region_1) #define bus_space_write_region_stream_1 BUS_SAN(write_region_stream_1) #define bus_space_set_multi_1 BUS_SAN(set_multi_1) #define bus_space_set_multi_stream_1 BUS_SAN(set_multi_stream_1) #define bus_space_set_region_1 BUS_SAN(set_region_1) #define bus_space_set_region_stream_1 BUS_SAN(set_region_stream_1) #define bus_space_copy_multi_1 BUS_SAN(copy_multi_1) #define bus_space_copy_multi_stream_1 BUS_SAN(copy_multi_stream_1) #define bus_space_poke_1 BUS_SAN(poke_1) #define bus_space_peek_1 BUS_SAN(peek_1) #define bus_space_read_2 BUS_SAN(read_2) #define bus_space_read_stream_2 BUS_SAN(read_stream_2) #define bus_space_read_multi_2 BUS_SAN(read_multi_2) #define bus_space_read_multi_stream_2 BUS_SAN(read_multi_stream_2) #define bus_space_read_region_2 BUS_SAN(read_region_2) #define bus_space_read_region_stream_2 BUS_SAN(read_region_stream_2) #define bus_space_write_2 BUS_SAN(write_2) #define bus_space_write_stream_2 BUS_SAN(write_stream_2) #define bus_space_write_multi_2 BUS_SAN(write_multi_2) #define bus_space_write_multi_stream_2 BUS_SAN(write_multi_stream_2) #define bus_space_write_region_2 BUS_SAN(write_region_2) #define bus_space_write_region_stream_2 BUS_SAN(write_region_stream_2) #define bus_space_set_multi_2 BUS_SAN(set_multi_2) #define bus_space_set_multi_stream_2 BUS_SAN(set_multi_stream_2) #define bus_space_set_region_2 BUS_SAN(set_region_2) #define bus_space_set_region_stream_2 BUS_SAN(set_region_stream_2) #define bus_space_copy_multi_2 BUS_SAN(copy_multi_2) #define bus_space_copy_multi_stream_2 BUS_SAN(copy_multi_stream_2) #define bus_space_poke_2 BUS_SAN(poke_2) #define bus_space_peek_2 BUS_SAN(peek_2) #define bus_space_read_4 BUS_SAN(read_4) #define bus_space_read_stream_4 BUS_SAN(read_stream_4) #define bus_space_read_multi_4 BUS_SAN(read_multi_4) #define bus_space_read_multi_stream_4 BUS_SAN(read_multi_stream_4) #define bus_space_read_region_4 BUS_SAN(read_region_4) #define bus_space_read_region_stream_4 BUS_SAN(read_region_stream_4) #define bus_space_write_4 BUS_SAN(write_4) #define bus_space_write_stream_4 BUS_SAN(write_stream_4) #define bus_space_write_multi_4 BUS_SAN(write_multi_4) #define bus_space_write_multi_stream_4 BUS_SAN(write_multi_stream_4) #define bus_space_write_region_4 BUS_SAN(write_region_4) #define bus_space_write_region_stream_4 BUS_SAN(write_region_stream_4) #define bus_space_set_multi_4 BUS_SAN(set_multi_4) #define bus_space_set_multi_stream_4 BUS_SAN(set_multi_stream_4) #define bus_space_set_region_4 BUS_SAN(set_region_4) #define bus_space_set_region_stream_4 BUS_SAN(set_region_stream_4) #define bus_space_copy_multi_4 BUS_SAN(copy_multi_4) #define bus_space_copy_multi_stream_4 BUS_SAN(copy_multi_stream_4) #define bus_space_poke_4 BUS_SAN(poke_4) #define bus_space_peek_4 BUS_SAN(peek_4) #define bus_space_read_8 BUS_SAN(read_8) #define bus_space_read_stream_8 BUS_SAN(read_stream_8) #define bus_space_read_multi_8 BUS_SAN(read_multi_8) #define bus_space_read_multi_stream_8 BUS_SAN(read_multi_stream_8) #define bus_space_read_region_8 BUS_SAN(read_region_8) #define bus_space_read_region_stream_8 BUS_SAN(read_region_stream_8) #define bus_space_write_8 BUS_SAN(write_8) #define bus_space_write_stream_8 BUS_SAN(write_stream_8) #define bus_space_write_multi_8 BUS_SAN(write_multi_8) #define bus_space_write_multi_stream_8 BUS_SAN(write_multi_stream_8) #define bus_space_write_region_8 BUS_SAN(write_region_8) #define bus_space_write_region_stream_8 BUS_SAN(write_region_stream_8) #define bus_space_set_multi_8 BUS_SAN(set_multi_8) #define bus_space_set_multi_stream_8 BUS_SAN(set_multi_stream_8) #define bus_space_set_region_8 BUS_SAN(set_region_8) #define bus_space_set_region_stream_8 BUS_SAN(set_region_stream_8) #define bus_space_copy_multi_8 BUS_SAN(copy_multi_8) #define bus_space_copy_multi_stream_8 BUS_SAN(copy_multi_stream_8) #define bus_space_poke_8 BUS_SAN(poke_8) #define bus_space_peek_8 BUS_SAN(peek_8) -#endif /* !KCSAN_RUNTIME */ +#endif /* !SAN_RUNTIME */ #endif /* !_SYS_BUS_SAN_H_ */ diff --git a/sys/sys/libkern.h b/sys/sys/libkern.h index d8d3dce1b705..147eba9bd1bd 100644 --- a/sys/sys/libkern.h +++ b/sys/sys/libkern.h @@ -1,237 +1,241 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1992, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)libkern.h 8.1 (Berkeley) 6/10/93 * $FreeBSD$ */ #ifndef _SYS_LIBKERN_H_ #define _SYS_LIBKERN_H_ #include #include #ifdef _KERNEL #include #endif #ifndef LIBKERN_INLINE #define LIBKERN_INLINE static __inline #define LIBKERN_BODY #endif /* BCD conversions. */ extern u_char const bcd2bin_data[]; extern u_char const bin2bcd_data[]; extern char const hex2ascii_data[]; #define LIBKERN_LEN_BCD2BIN 154 #define LIBKERN_LEN_BIN2BCD 100 #define LIBKERN_LEN_HEX2ASCII 36 static inline u_char bcd2bin(int bcd) { KASSERT(bcd >= 0 && bcd < LIBKERN_LEN_BCD2BIN, ("invalid bcd %d", bcd)); return (bcd2bin_data[bcd]); } static inline u_char bin2bcd(int bin) { KASSERT(bin >= 0 && bin < LIBKERN_LEN_BIN2BCD, ("invalid bin %d", bin)); return (bin2bcd_data[bin]); } static inline char hex2ascii(int hex) { KASSERT(hex >= 0 && hex < LIBKERN_LEN_HEX2ASCII, ("invalid hex %d", hex)); return (hex2ascii_data[hex]); } static inline bool validbcd(int bcd) { return (bcd == 0 || (bcd > 0 && bcd <= 0x99 && bcd2bin_data[bcd] != 0)); } static __inline int imax(int a, int b) { return (a > b ? a : b); } static __inline int imin(int a, int b) { return (a < b ? a : b); } static __inline long lmax(long a, long b) { return (a > b ? a : b); } static __inline long lmin(long a, long b) { return (a < b ? a : b); } static __inline u_int max(u_int a, u_int b) { return (a > b ? a : b); } static __inline u_int min(u_int a, u_int b) { return (a < b ? a : b); } static __inline quad_t qmax(quad_t a, quad_t b) { return (a > b ? a : b); } static __inline quad_t qmin(quad_t a, quad_t b) { return (a < b ? a : b); } static __inline u_quad_t uqmax(u_quad_t a, u_quad_t b) { return (a > b ? a : b); } static __inline u_quad_t uqmin(u_quad_t a, u_quad_t b) { return (a < b ? a : b); } static __inline u_long ulmax(u_long a, u_long b) { return (a > b ? a : b); } static __inline u_long ulmin(u_long a, u_long b) { return (a < b ? a : b); } static __inline __uintmax_t ummax(__uintmax_t a, __uintmax_t b) { return (a > b ? a : b); } static __inline __uintmax_t ummin(__uintmax_t a, __uintmax_t b) { return (a < b ? a : b); } static __inline off_t omax(off_t a, off_t b) { return (a > b ? a : b); } static __inline off_t omin(off_t a, off_t b) { return (a < b ? a : b); } static __inline int abs(int a) { return (a < 0 ? -a : a); } static __inline long labs(long a) { return (a < 0 ? -a : a); } static __inline quad_t qabs(quad_t a) { return (a < 0 ? -a : a); } #ifndef RANDOM_FENESTRASX #define ARC4_ENTR_NONE 0 /* Don't have entropy yet. */ #define ARC4_ENTR_HAVE 1 /* Have entropy. */ #define ARC4_ENTR_SEED 2 /* Reseeding. */ extern int arc4rand_iniseed_state; #endif /* Prototypes for non-quad routines. */ struct malloc_type; uint32_t arc4random(void); void arc4random_buf(void *, size_t); uint32_t arc4random_uniform(uint32_t); void arc4rand(void *, u_int, int); int timingsafe_bcmp(const void *, const void *, size_t); void *bsearch(const void *, const void *, size_t, size_t, int (*)(const void *, const void *)); #ifndef HAVE_INLINE_FFS int ffs(int); #endif #ifndef HAVE_INLINE_FFSL int ffsl(long); #endif #ifndef HAVE_INLINE_FFSLL int ffsll(long long); #endif #ifndef HAVE_INLINE_FLS int fls(int); #endif #ifndef HAVE_INLINE_FLSL int flsl(long); #endif #ifndef HAVE_INLINE_FLSLL int flsll(long long); #endif #define bitcount64(x) __bitcount64((uint64_t)(x)) #define bitcount32(x) __bitcount32((uint32_t)(x)) #define bitcount16(x) __bitcount16((uint16_t)(x)) #define bitcountl(x) __bitcountl((u_long)(x)) #define bitcount(x) __bitcount((u_int)(x)) int fnmatch(const char *, const char *, int); int locc(int, char *, u_int); void *memchr(const void *s, int c, size_t n); void *memcchr(const void *s, int c, size_t n); void *memmem(const void *l, size_t l_len, const void *s, size_t s_len); void qsort(void *base, size_t nmemb, size_t size, int (*compar)(const void *, const void *)); void qsort_r(void *base, size_t nmemb, size_t size, void *thunk, int (*compar)(void *, const void *, const void *)); u_long random(void); int scanc(u_int, const u_char *, const u_char *, int); int strcasecmp(const char *, const char *); char *strcasestr(const char *, const char *); char *strcat(char * __restrict, const char * __restrict); char *strchr(const char *, int); char *strchrnul(const char *, int); int strcmp(const char *, const char *); char *strcpy(char * __restrict, const char * __restrict); char *strdup_flags(const char *__restrict, struct malloc_type *, int); size_t strcspn(const char *, const char *) __pure; char *strdup(const char *__restrict, struct malloc_type *); char *strncat(char *, const char *, size_t); char *strndup(const char *__restrict, size_t, struct malloc_type *); size_t strlcat(char *, const char *, size_t); size_t strlcpy(char *, const char *, size_t); size_t strlen(const char *); int strncasecmp(const char *, const char *, size_t); int strncmp(const char *, const char *, size_t); char *strncpy(char * __restrict, const char * __restrict, size_t); size_t strnlen(const char *, size_t); char *strnstr(const char *, const char *, size_t); char *strrchr(const char *, int); char *strsep(char **, const char *delim); size_t strspn(const char *, const char *); char *strstr(const char *, const char *); int strvalid(const char *, size_t); -#ifdef SAN_PREFIX +#ifdef SAN_NEEDS_INTERCEPTORS +#ifndef SAN_INTERCEPTOR +#define SAN_INTERCEPTOR(func) \ + __CONCAT(SAN_INTERCEPTOR_PREFIX, __CONCAT(_, func)) +#endif char *SAN_INTERCEPTOR(strcpy)(char *, const char *); int SAN_INTERCEPTOR(strcmp)(const char *, const char *); size_t SAN_INTERCEPTOR(strlen)(const char *); #ifndef SAN_RUNTIME #define strcpy(d, s) SAN_INTERCEPTOR(strcpy)((d), (s)) #define strcmp(s1, s2) SAN_INTERCEPTOR(strcmp)((s1), (s2)) #define strlen(s) SAN_INTERCEPTOR(strlen)(s) #endif /* !SAN_RUNTIME */ -#else -#define strcpy(d, s) __builtin_strcpy((d), (s)) -#define strcmp(s1, s2) __builtin_strcmp((s1), (s2)) -#define strlen(s) __builtin_strlen((s)) -#endif /* SAN_PREFIX */ +#else /* !SAN_NEEDS_INTERCEPTORS */ +#define strcpy(d, s) __builtin_strcpy((d), (s)) +#define strcmp(s1, s2) __builtin_strcmp((s1), (s2)) +#define strlen(s) __builtin_strlen((s)) +#endif /* SAN_NEEDS_INTERCEPTORS */ static __inline char * index(const char *p, int ch) { return (strchr(p, ch)); } static __inline char * rindex(const char *p, int ch) { return (strrchr(p, ch)); } /* fnmatch() return values. */ #define FNM_NOMATCH 1 /* Match failed. */ /* fnmatch() flags. */ #define FNM_NOESCAPE 0x01 /* Disable backslash escaping. */ #define FNM_PATHNAME 0x02 /* Slash must be matched by slash. */ #define FNM_PERIOD 0x04 /* Period must be matched by period. */ #define FNM_LEADING_DIR 0x08 /* Ignore / after Imatch. */ #define FNM_CASEFOLD 0x10 /* Case insensitive search. */ #define FNM_IGNORECASE FNM_CASEFOLD #define FNM_FILE_NAME FNM_PATHNAME #endif /* !_SYS_LIBKERN_H_ */ diff --git a/sys/sys/systm.h b/sys/sys/systm.h index bccf88573991..828297e5b948 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -1,547 +1,541 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1982, 1988, 1991, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)systm.h 8.7 (Berkeley) 3/29/95 * $FreeBSD$ */ #ifndef _SYS_SYSTM_H_ #define _SYS_SYSTM_H_ #include #include #include #include #include #include #include /* for people using printf mainly */ __NULLABILITY_PRAGMA_PUSH #ifdef _KERNEL extern int cold; /* nonzero if we are doing a cold boot */ extern int suspend_blocked; /* block suspend due to pending shutdown */ extern int rebooting; /* kern_reboot() has been called. */ extern char version[]; /* system version */ extern char compiler_version[]; /* compiler version */ extern char copyright[]; /* system copyright */ extern int kstack_pages; /* number of kernel stack pages */ extern u_long pagesizes[]; /* supported page sizes */ extern long physmem; /* physical memory */ extern long realmem; /* 'real' memory */ extern char *rootdevnames[2]; /* names of possible root devices */ extern int boothowto; /* reboot flags, from console subsystem */ extern int bootverbose; /* nonzero to print verbose messages */ extern int maxusers; /* system tune hint */ extern int ngroups_max; /* max # of supplemental groups */ extern int vm_guest; /* Running as virtual machine guest? */ extern u_long maxphys; /* max raw I/O transfer size */ /* * Detected virtual machine guest types. The intention is to expand * and/or add to the VM_GUEST_VM type if specific VM functionality is * ever implemented (e.g. vendor-specific paravirtualization features). * Keep in sync with vm_guest_sysctl_names[]. */ enum VM_GUEST { VM_GUEST_NO = 0, VM_GUEST_VM, VM_GUEST_XEN, VM_GUEST_HV, VM_GUEST_VMWARE, VM_GUEST_KVM, VM_GUEST_BHYVE, VM_GUEST_VBOX, VM_GUEST_PARALLELS, VM_LAST }; #endif /* KERNEL */ /* * Align variables. */ #define __read_mostly __section(".data.read_mostly") #define __read_frequently __section(".data.read_frequently") #define __exclusive_cache_line __aligned(CACHE_LINE_SIZE) \ __section(".data.exclusive_cache_line") #if defined(_STANDALONE) struct ucred; #endif #ifdef _KERNEL #include /* MAXCPU */ #include /* curthread */ #include extern int osreldate; extern const void *zero_region; /* address space maps to a zeroed page */ extern int unmapped_buf_allowed; #ifdef __LP64__ #define IOSIZE_MAX iosize_max() #define DEVFS_IOSIZE_MAX devfs_iosize_max() #else #define IOSIZE_MAX SSIZE_MAX #define DEVFS_IOSIZE_MAX SSIZE_MAX #endif /* * General function declarations. */ struct inpcb; struct lock_object; struct malloc_type; struct mtx; struct proc; struct socket; struct thread; struct tty; struct ucred; struct uio; struct _jmp_buf; struct trapframe; struct eventtimer; int setjmp(struct _jmp_buf *) __returns_twice; void longjmp(struct _jmp_buf *, int) __dead2; int dumpstatus(vm_offset_t addr, off_t count); int nullop(void); int eopnotsupp(void); int ureadc(int, struct uio *); void hashdestroy(void *, struct malloc_type *, u_long); void *hashinit(int count, struct malloc_type *type, u_long *hashmask); void *hashinit_flags(int count, struct malloc_type *type, u_long *hashmask, int flags); #define HASH_NOWAIT 0x00000001 #define HASH_WAITOK 0x00000002 void *phashinit(int count, struct malloc_type *type, u_long *nentries); void *phashinit_flags(int count, struct malloc_type *type, u_long *nentries, int flags); void g_waitidle(void); void cpu_flush_dcache(void *, size_t); void cpu_rootconf(void); void critical_enter_KBI(void); void critical_exit_KBI(void); void critical_exit_preempt(void); void init_param1(void); void init_param2(long physpages); void init_static_kenv(char *, size_t); void tablefull(const char *); /* * Allocate per-thread "current" state in the linuxkpi */ extern int (*lkpi_alloc_current)(struct thread *, int); int linux_alloc_current_noop(struct thread *, int); #if defined(KLD_MODULE) || defined(KTR_CRITICAL) || !defined(_KERNEL) || defined(GENOFFSET) #define critical_enter() critical_enter_KBI() #define critical_exit() critical_exit_KBI() #else static __inline void critical_enter(void) { struct thread_lite *td; td = (struct thread_lite *)curthread; td->td_critnest++; atomic_interrupt_fence(); } static __inline void critical_exit(void) { struct thread_lite *td; td = (struct thread_lite *)curthread; KASSERT(td->td_critnest != 0, ("critical_exit: td_critnest == 0")); atomic_interrupt_fence(); td->td_critnest--; atomic_interrupt_fence(); if (__predict_false(td->td_owepreempt)) critical_exit_preempt(); } #endif #ifdef EARLY_PRINTF typedef void early_putc_t(int ch); extern early_putc_t *early_putc; #endif int kvprintf(char const *, void (*)(int, void*), void *, int, __va_list) __printflike(1, 0); void log(int, const char *, ...) __printflike(2, 3); void log_console(struct uio *); void vlog(int, const char *, __va_list) __printflike(2, 0); int asprintf(char **ret, struct malloc_type *mtp, const char *format, ...) __printflike(3, 4); int printf(const char *, ...) __printflike(1, 2); int snprintf(char *, size_t, const char *, ...) __printflike(3, 4); int sprintf(char *buf, const char *, ...) __printflike(2, 3); int uprintf(const char *, ...) __printflike(1, 2); int vprintf(const char *, __va_list) __printflike(1, 0); int vasprintf(char **ret, struct malloc_type *mtp, const char *format, __va_list ap) __printflike(3, 0); int vsnprintf(char *, size_t, const char *, __va_list) __printflike(3, 0); int vsnrprintf(char *, size_t, int, const char *, __va_list) __printflike(4, 0); int vsprintf(char *buf, const char *, __va_list) __printflike(2, 0); int sscanf(const char *, char const * _Nonnull, ...) __scanflike(2, 3); int vsscanf(const char * _Nonnull, char const * _Nonnull, __va_list) __scanflike(2, 0); long strtol(const char *, char **, int); u_long strtoul(const char *, char **, int); quad_t strtoq(const char *, char **, int); u_quad_t strtouq(const char *, char **, int); void tprintf(struct proc *p, int pri, const char *, ...) __printflike(3, 4); void vtprintf(struct proc *, int, const char *, __va_list) __printflike(3, 0); void hexdump(const void *ptr, int length, const char *hdr, int flags); #define HD_COLUMN_MASK 0xff #define HD_DELIM_MASK 0xff00 #define HD_OMIT_COUNT (1 << 16) #define HD_OMIT_HEX (1 << 17) #define HD_OMIT_CHARS (1 << 18) #define ovbcopy(f, t, l) bcopy((f), (t), (l)) void bcopy(const void * _Nonnull from, void * _Nonnull to, size_t len); void bzero(void * _Nonnull buf, size_t len); void explicit_bzero(void * _Nonnull, size_t); int bcmp(const void *b1, const void *b2, size_t len); void *memset(void * _Nonnull buf, int c, size_t len); void *memcpy(void * _Nonnull to, const void * _Nonnull from, size_t len); void *memmove(void * _Nonnull dest, const void * _Nonnull src, size_t n); int memcmp(const void *b1, const void *b2, size_t len); -#if defined(KASAN) -#define SAN_PREFIX kasan_ -#elif defined(KCSAN) -#define SAN_PREFIX kcsan_ -#endif - -#ifdef SAN_PREFIX -#define SAN_INTERCEPTOR(func) __CONCAT(SAN_PREFIX, func) - +#ifdef SAN_NEEDS_INTERCEPTORS +#define SAN_INTERCEPTOR(func) \ + __CONCAT(SAN_INTERCEPTOR_PREFIX, __CONCAT(_, func)) void *SAN_INTERCEPTOR(memset)(void *, int, size_t); void *SAN_INTERCEPTOR(memcpy)(void *, const void *, size_t); void *SAN_INTERCEPTOR(memmove)(void *, const void *, size_t); int SAN_INTERCEPTOR(memcmp)(const void *, const void *, size_t); #ifndef SAN_RUNTIME #define bcopy(from, to, len) SAN_INTERCEPTOR(memmove)((to), (from), (len)) #define bzero(buf, len) SAN_INTERCEPTOR(memset)((buf), 0, (len)) #define bcmp(b1, b2, len) SAN_INTERCEPTOR(memcmp)((b1), (b2), (len)) #define memset(buf, c, len) SAN_INTERCEPTOR(memset)((buf), (c), (len)) #define memcpy(to, from, len) SAN_INTERCEPTOR(memcpy)((to), (from), (len)) #define memmove(dest, src, n) SAN_INTERCEPTOR(memmove)((dest), (src), (n)) #define memcmp(b1, b2, len) SAN_INTERCEPTOR(memcmp)((b1), (b2), (len)) #endif /* !SAN_RUNTIME */ -#else -#define bcopy(from, to, len) __builtin_memmove((to), (from), (len)) -#define bzero(buf, len) __builtin_memset((buf), 0, (len)) -#define bcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len)) -#define memset(buf, c, len) __builtin_memset((buf), (c), (len)) -#define memcpy(to, from, len) __builtin_memcpy((to), (from), (len)) -#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n)) -#define memcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len)) -#endif /* !SAN_PREFIX */ +#else /* !SAN_NEEDS_INTERCEPTORS */ +#define bcopy(from, to, len) __builtin_memmove((to), (from), (len)) +#define bzero(buf, len) __builtin_memset((buf), 0, (len)) +#define bcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len)) +#define memset(buf, c, len) __builtin_memset((buf), (c), (len)) +#define memcpy(to, from, len) __builtin_memcpy((to), (from), (len)) +#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n)) +#define memcmp(b1, b2, len) __builtin_memcmp((b1), (b2), (len)) +#endif /* SAN_NEEDS_INTERCEPTORS */ void *memset_early(void * _Nonnull buf, int c, size_t len); #define bzero_early(buf, len) memset_early((buf), 0, (len)) void *memcpy_early(void * _Nonnull to, const void * _Nonnull from, size_t len); void *memmove_early(void * _Nonnull dest, const void * _Nonnull src, size_t n); #define bcopy_early(from, to, len) memmove_early((to), (from), (len)) #define copystr(src, dst, len, outlen) ({ \ size_t __r, __len, *__outlen; \ \ __len = (len); \ __outlen = (outlen); \ __r = strlcpy((dst), (src), __len); \ if (__outlen != NULL) \ *__outlen = ((__r >= __len) ? __len : __r + 1); \ ((__r >= __len) ? ENAMETOOLONG : 0); \ }) int copyinstr(const void * __restrict udaddr, void * _Nonnull __restrict kaddr, size_t len, size_t * __restrict lencopied); int copyin(const void * __restrict udaddr, void * _Nonnull __restrict kaddr, size_t len); int copyin_nofault(const void * __restrict udaddr, void * _Nonnull __restrict kaddr, size_t len); int copyout(const void * _Nonnull __restrict kaddr, void * __restrict udaddr, size_t len); int copyout_nofault(const void * _Nonnull __restrict kaddr, void * __restrict udaddr, size_t len); -#ifdef SAN_PREFIX +#ifdef SAN_NEEDS_INTERCEPTORS int SAN_INTERCEPTOR(copyin)(const void *, void *, size_t); int SAN_INTERCEPTOR(copyinstr)(const void *, void *, size_t, size_t *); int SAN_INTERCEPTOR(copyout)(const void *, void *, size_t); #ifndef SAN_RUNTIME #define copyin(u, k, l) SAN_INTERCEPTOR(copyin)((u), (k), (l)) #define copyinstr(u, k, l, lc) SAN_INTERCEPTOR(copyinstr)((u), (k), (l), (lc)) #define copyout(k, u, l) SAN_INTERCEPTOR(copyout)((k), (u), (l)) #endif /* !SAN_RUNTIME */ -#endif /* SAN_PREFIX */ +#endif /* SAN_NEEDS_INTERCEPTORS */ int fubyte(volatile const void *base); long fuword(volatile const void *base); int fuword16(volatile const void *base); int32_t fuword32(volatile const void *base); int64_t fuword64(volatile const void *base); int fueword(volatile const void *base, long *val); int fueword32(volatile const void *base, int32_t *val); int fueword64(volatile const void *base, int64_t *val); int subyte(volatile void *base, int byte); int suword(volatile void *base, long word); int suword16(volatile void *base, int word); int suword32(volatile void *base, int32_t word); int suword64(volatile void *base, int64_t word); uint32_t casuword32(volatile uint32_t *base, uint32_t oldval, uint32_t newval); u_long casuword(volatile u_long *p, u_long oldval, u_long newval); int casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, uint32_t newval); int casueword(volatile u_long *p, u_long oldval, u_long *oldvalp, u_long newval); int sysbeep(int hertz, sbintime_t duration); void hardclock(int cnt, int usermode); void hardclock_sync(int cpu); void softclock(void *); void statclock(int cnt, int usermode); void profclock(int cnt, int usermode, uintfptr_t pc); int hardclockintr(void); void startprofclock(struct proc *); void stopprofclock(struct proc *); void cpu_startprofclock(void); void cpu_stopprofclock(void); void suspendclock(void); void resumeclock(void); sbintime_t cpu_idleclock(void); void cpu_activeclock(void); void cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt); void cpu_et_frequency(struct eventtimer *et, uint64_t newfreq); extern int cpu_disable_c2_sleep; extern int cpu_disable_c3_sleep; char *kern_getenv(const char *name); void freeenv(char *env); int getenv_int(const char *name, int *data); int getenv_uint(const char *name, unsigned int *data); int getenv_long(const char *name, long *data); int getenv_ulong(const char *name, unsigned long *data); int getenv_string(const char *name, char *data, int size); int getenv_int64(const char *name, int64_t *data); int getenv_uint64(const char *name, uint64_t *data); int getenv_quad(const char *name, quad_t *data); int getenv_bool(const char *name, bool *data); bool getenv_is_true(const char *name); bool getenv_is_false(const char *name); int kern_setenv(const char *name, const char *value); int kern_unsetenv(const char *name); int testenv(const char *name); int getenv_array(const char *name, void *data, int size, int *psize, int type_size, bool allow_signed); #define GETENV_UNSIGNED false /* negative numbers not allowed */ #define GETENV_SIGNED true /* negative numbers allowed */ typedef uint64_t (cpu_tick_f)(void); void set_cputicker(cpu_tick_f *func, uint64_t freq, bool isvariable); extern cpu_tick_f *cpu_ticks; uint64_t cpu_tickrate(void); uint64_t cputick2usec(uint64_t tick); #include /* Initialize the world */ void consinit(void); void cpu_initclocks(void); void cpu_initclocks_bsp(void); void cpu_initclocks_ap(void); void usrinfoinit(void); /* Finalize the world */ void kern_reboot(int) __dead2; void shutdown_nice(int); /* Stubs for obsolete functions that used to be for interrupt management */ static __inline intrmask_t splhigh(void) { return 0; } static __inline intrmask_t splimp(void) { return 0; } static __inline intrmask_t splnet(void) { return 0; } static __inline intrmask_t spltty(void) { return 0; } static __inline void splx(intrmask_t ipl __unused) { return; } /* * Common `proc' functions are declared here so that proc.h can be included * less often. */ int _sleep(const void * _Nonnull chan, struct lock_object *lock, int pri, const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags); #define msleep(chan, mtx, pri, wmesg, timo) \ _sleep((chan), &(mtx)->lock_object, (pri), (wmesg), \ tick_sbt * (timo), 0, C_HARDCLOCK) #define msleep_sbt(chan, mtx, pri, wmesg, bt, pr, flags) \ _sleep((chan), &(mtx)->lock_object, (pri), (wmesg), (bt), (pr), \ (flags)) int msleep_spin_sbt(const void * _Nonnull chan, struct mtx *mtx, const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags); #define msleep_spin(chan, mtx, wmesg, timo) \ msleep_spin_sbt((chan), (mtx), (wmesg), tick_sbt * (timo), \ 0, C_HARDCLOCK) int pause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags); static __inline int pause(const char *wmesg, int timo) { return (pause_sbt(wmesg, tick_sbt * timo, 0, C_HARDCLOCK)); } #define pause_sig(wmesg, timo) \ pause_sbt((wmesg), tick_sbt * (timo), 0, C_HARDCLOCK | C_CATCH) #define tsleep(chan, pri, wmesg, timo) \ _sleep((chan), NULL, (pri), (wmesg), tick_sbt * (timo), \ 0, C_HARDCLOCK) #define tsleep_sbt(chan, pri, wmesg, bt, pr, flags) \ _sleep((chan), NULL, (pri), (wmesg), (bt), (pr), (flags)) void wakeup(const void *chan); void wakeup_one(const void *chan); void wakeup_any(const void *chan); /* * Common `struct cdev *' stuff are declared here to avoid #include poisoning */ struct cdev; dev_t dev2udev(struct cdev *x); const char *devtoname(struct cdev *cdev); #ifdef __LP64__ size_t devfs_iosize_max(void); size_t iosize_max(void); #endif int poll_no_poll(int events); /* XXX: Should be void nanodelay(u_int nsec); */ void DELAY(int usec); /* Root mount holdback API */ struct root_hold_token { int flags; const char *who; TAILQ_ENTRY(root_hold_token) list; }; struct root_hold_token *root_mount_hold(const char *identifier); void root_mount_hold_token(const char *identifier, struct root_hold_token *h); void root_mount_rel(struct root_hold_token *h); int root_mounted(void); /* * Unit number allocation API. (kern/subr_unit.c) */ struct unrhdr; #define UNR_NO_MTX ((void *)(uintptr_t)-1) struct unrhdr *new_unrhdr(int low, int high, struct mtx *mutex); void init_unrhdr(struct unrhdr *uh, int low, int high, struct mtx *mutex); void delete_unrhdr(struct unrhdr *uh); void clear_unrhdr(struct unrhdr *uh); void clean_unrhdr(struct unrhdr *uh); void clean_unrhdrl(struct unrhdr *uh); int alloc_unr(struct unrhdr *uh); int alloc_unr_specific(struct unrhdr *uh, u_int item); int alloc_unrl(struct unrhdr *uh); void free_unr(struct unrhdr *uh, u_int item); #ifndef __LP64__ #define UNR64_LOCKED #endif struct unrhdr64 { uint64_t counter; }; static __inline void new_unrhdr64(struct unrhdr64 *unr64, uint64_t low) { unr64->counter = low; } #ifdef UNR64_LOCKED uint64_t alloc_unr64(struct unrhdr64 *); #else static __inline uint64_t alloc_unr64(struct unrhdr64 *unr64) { return (atomic_fetchadd_64(&unr64->counter, 1)); } #endif void intr_prof_stack_use(struct thread *td, struct trapframe *frame); void counted_warning(unsigned *counter, const char *msg); /* * APIs to manage deprecation and obsolescence. */ void _gone_in(int major, const char *msg); void _gone_in_dev(device_t dev, int major, const char *msg); #ifdef NO_OBSOLETE_CODE #define __gone_ok(m, msg) \ _Static_assert(m < P_OSREL_MAJOR(__FreeBSD_version)), \ "Obsolete code: " msg); #else #define __gone_ok(m, msg) #endif #define gone_in(major, msg) __gone_ok(major, msg) _gone_in(major, msg) #define gone_in_dev(dev, major, msg) __gone_ok(major, msg) _gone_in_dev(dev, major, msg) #if defined(INVARIANTS) || defined(WITNESS) #define __diagused #else #define __diagused __unused #endif #endif /* _KERNEL */ __NULLABILITY_PRAGMA_POP #endif /* !_SYS_SYSTM_H_ */ diff --git a/sys/x86/include/bus.h b/sys/x86/include/bus.h index 0ba68e250b3d..9522e5db7c78 100644 --- a/sys/x86/include/bus.h +++ b/sys/x86/include/bus.h @@ -1,1134 +1,1126 @@ /*- * SPDX-License-Identifier: BSD-3-Clause AND BSD-2-Clause-NetBSDE * * Copyright (c) KATO Takenori, 1999. * * All rights reserved. Unpublished rights reserved under the copyright * laws of Japan. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer as * the first lines of this file unmodified. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ /* $NetBSD: bus.h,v 1.12 1997/10/01 08:25:15 fvdl Exp $ */ /*- * Copyright (c) 1996, 1997 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, * NASA Ames Research Center. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1996 Charles M. Hannum. All rights reserved. * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Christopher G. Demetriou * for the NetBSD Project. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _MACHINE_BUS_H_ #define _MACHINE_BUS_H_ #include #include #include #ifndef __GNUCLIKE_ASM #error "no assembler code for your compiler" #endif /* * Values for the x86 bus space tag, not to be used directly by MI code. */ #define X86_BUS_SPACE_IO 0 /* space is i/o space */ #define X86_BUS_SPACE_MEM 1 /* space is mem space */ #define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF #define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF #if defined(__amd64__) #define BUS_SPACE_MAXSIZE 0xFFFFFFFFFFFFFFFFULL #else #define BUS_SPACE_MAXSIZE 0xFFFFFFFF #endif #define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF #define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF #if defined(__amd64__) || defined(PAE) #define BUS_SPACE_MAXADDR_48BIT 0xFFFFFFFFFFFFULL #define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL #else #define BUS_SPACE_MAXADDR 0xFFFFFFFF #endif #define BUS_SPACE_INVALID_DATA (~0) #define BUS_SPACE_UNRESTRICTED (~0) #define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */ #define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */ -#ifndef SAN_RUNTIME -#if defined(KASAN) -#define BUS_SAN_PREFIX kasan -#elif defined(KCSAN) -#define BUS_SAN_PREFIX kcsan -#endif -#endif - -#ifdef BUS_SAN_PREFIX +#if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME) #include #else /* * Map a region of device bus space into CPU virtual address space. */ int bus_space_map(bus_space_tag_t tag, bus_addr_t addr, bus_size_t size, int flags, bus_space_handle_t *bshp); /* * Unmap a region of device bus space. */ void bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t size); /* * Get a new handle for a subregion of an already-mapped area of bus space. */ static __inline int bus_space_subregion(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t offset, bus_size_t size, bus_space_handle_t *nbshp); static __inline int bus_space_subregion(bus_space_tag_t t __unused, bus_space_handle_t bsh, bus_size_t offset, bus_size_t size __unused, bus_space_handle_t *nbshp) { *nbshp = bsh + offset; return (0); } /* * Allocate a region of memory that is accessible to devices in bus space. */ int bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart, bus_addr_t rend, bus_size_t size, bus_size_t align, bus_size_t boundary, int flags, bus_addr_t *addrp, bus_space_handle_t *bshp); /* * Free a region of bus space accessible memory. */ static __inline void bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size); static __inline void bus_space_free(bus_space_tag_t t __unused, bus_space_handle_t bsh __unused, bus_size_t size __unused) { } /* * Read a 1, 2, 4, or 8 byte quantity from bus space * described by tag/handle/offset. */ static __inline u_int8_t bus_space_read_1(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset); static __inline u_int16_t bus_space_read_2(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset); static __inline u_int32_t bus_space_read_4(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset); #ifdef __amd64__ static __inline uint64_t bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset); #endif static __inline u_int8_t bus_space_read_1(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset) { if (tag == X86_BUS_SPACE_IO) return (inb(handle + offset)); return (*(volatile u_int8_t *)(handle + offset)); } static __inline u_int16_t bus_space_read_2(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset) { if (tag == X86_BUS_SPACE_IO) return (inw(handle + offset)); return (*(volatile u_int16_t *)(handle + offset)); } static __inline u_int32_t bus_space_read_4(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset) { if (tag == X86_BUS_SPACE_IO) return (inl(handle + offset)); return (*(volatile u_int32_t *)(handle + offset)); } #ifdef __amd64__ static __inline uint64_t bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset) { if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */ return (BUS_SPACE_INVALID_DATA); return (*(volatile uint64_t *)(handle + offset)); } #endif /* * Read `count' 1, 2, 4, or 8 byte quantities from bus space * described by tag/handle/offset and copy into buffer provided. */ static __inline void bus_space_read_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t *addr, size_t count); static __inline void bus_space_read_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t *addr, size_t count); static __inline void bus_space_read_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t *addr, size_t count); static __inline void bus_space_read_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) insb(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: movb (%2),%%al \n\ stosb \n\ loop 1b" : "=D" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory"); #endif } } static __inline void bus_space_read_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) insw(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: movw (%2),%%ax \n\ stosw \n\ loop 1b" : "=D" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory"); #endif } } static __inline void bus_space_read_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) insl(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: movl (%2),%%eax \n\ stosl \n\ loop 1b" : "=D" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory"); #endif } } #if 0 /* Cause a link error for bus_space_read_multi_8 */ #define bus_space_read_multi_8 !!! bus_space_read_multi_8 unimplemented !!! #endif /* * Read `count' 1, 2, 4, or 8 byte quantities from bus space * described by tag/handle and starting at `offset' and copy into * buffer provided. */ static __inline void bus_space_read_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t *addr, size_t count); static __inline void bus_space_read_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t *addr, size_t count); static __inline void bus_space_read_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t *addr, size_t count); static __inline void bus_space_read_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: inb %w2,%%al \n\ stosb \n\ incl %2 \n\ loop 1b" : "=D" (addr), "=c" (count), "=d" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsb" : "=D" (addr), "=c" (count), "=S" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "memory", "cc"); #endif } } static __inline void bus_space_read_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: inw %w2,%%ax \n\ stosw \n\ addl $2,%2 \n\ loop 1b" : "=D" (addr), "=c" (count), "=d" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsw" : "=D" (addr), "=c" (count), "=S" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "memory", "cc"); #endif } } static __inline void bus_space_read_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: inl %w2,%%eax \n\ stosl \n\ addl $4,%2 \n\ loop 1b" : "=D" (addr), "=c" (count), "=d" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsl" : "=D" (addr), "=c" (count), "=S" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "memory", "cc"); #endif } } #if 0 /* Cause a link error for bus_space_read_region_8 */ #define bus_space_read_region_8 !!! bus_space_read_region_8 unimplemented !!! #endif /* * Write the 1, 2, 4, or 8 byte value `value' to bus space * described by tag/handle/offset. */ static __inline void bus_space_write_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value); static __inline void bus_space_write_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value); static __inline void bus_space_write_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value); #ifdef __amd64__ static __inline void bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, uint64_t value); #endif static __inline void bus_space_write_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value) { if (tag == X86_BUS_SPACE_IO) outb(bsh + offset, value); else *(volatile u_int8_t *)(bsh + offset) = value; } static __inline void bus_space_write_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value) { if (tag == X86_BUS_SPACE_IO) outw(bsh + offset, value); else *(volatile u_int16_t *)(bsh + offset) = value; } static __inline void bus_space_write_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value) { if (tag == X86_BUS_SPACE_IO) outl(bsh + offset, value); else *(volatile u_int32_t *)(bsh + offset) = value; } #ifdef __amd64__ static __inline void bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, uint64_t value) { if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */ return; else *(volatile uint64_t *)(bsh + offset) = value; } #endif /* * Write `count' 1, 2, 4, or 8 byte quantities from the buffer * provided to bus space described by tag/handle/offset. */ static __inline void bus_space_write_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int8_t *addr, size_t count); static __inline void bus_space_write_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int16_t *addr, size_t count); static __inline void bus_space_write_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int32_t *addr, size_t count); static __inline void bus_space_write_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int8_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) outsb(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsb \n\ movb %%al,(%2) \n\ loop 1b" : "=S" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory", "cc"); #endif } } static __inline void bus_space_write_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int16_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) outsw(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsw \n\ movw %%ax,(%2) \n\ loop 1b" : "=S" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory", "cc"); #endif } } static __inline void bus_space_write_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int32_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) outsl(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsl \n\ movl %%eax,(%2) \n\ loop 1b" : "=S" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory", "cc"); #endif } } #if 0 /* Cause a link error for bus_space_write_multi_8 */ #define bus_space_write_multi_8(t, h, o, a, c) \ !!! bus_space_write_multi_8 unimplemented !!! #endif /* * Write `count' 1, 2, 4, or 8 byte quantities from the buffer provided * to bus space described by tag/handle starting at `offset'. */ static __inline void bus_space_write_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int8_t *addr, size_t count); static __inline void bus_space_write_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int16_t *addr, size_t count); static __inline void bus_space_write_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int32_t *addr, size_t count); static __inline void bus_space_write_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int8_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsb \n\ outb %%al,%w0 \n\ incl %0 \n\ loop 1b" : "=d" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsb" : "=D" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "memory", "cc"); #endif } } static __inline void bus_space_write_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int16_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsw \n\ outw %%ax,%w0 \n\ addl $2,%0 \n\ loop 1b" : "=d" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsw" : "=D" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "memory", "cc"); #endif } } static __inline void bus_space_write_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int32_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsl \n\ outl %%eax,%w0 \n\ addl $4,%0 \n\ loop 1b" : "=d" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsl" : "=D" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "memory", "cc"); #endif } } #if 0 /* Cause a link error for bus_space_write_region_8 */ #define bus_space_write_region_8 \ !!! bus_space_write_region_8 unimplemented !!! #endif /* * Write the 1, 2, 4, or 8 byte value `val' to bus space described * by tag/handle/offset `count' times. */ static __inline void bus_space_set_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value, size_t count); static __inline void bus_space_set_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value, size_t count); static __inline void bus_space_set_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value, size_t count); static __inline void bus_space_set_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) while (count--) outb(addr, value); else while (count--) *(volatile u_int8_t *)(addr) = value; } static __inline void bus_space_set_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) while (count--) outw(addr, value); else while (count--) *(volatile u_int16_t *)(addr) = value; } static __inline void bus_space_set_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) while (count--) outl(addr, value); else while (count--) *(volatile u_int32_t *)(addr) = value; } #if 0 /* Cause a link error for bus_space_set_multi_8 */ #define bus_space_set_multi_8 !!! bus_space_set_multi_8 unimplemented !!! #endif /* * Write `count' 1, 2, 4, or 8 byte value `val' to bus space described * by tag/handle starting at `offset'. */ static __inline void bus_space_set_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value, size_t count); static __inline void bus_space_set_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value, size_t count); static __inline void bus_space_set_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value, size_t count); static __inline void bus_space_set_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) for (; count != 0; count--, addr++) outb(addr, value); else for (; count != 0; count--, addr++) *(volatile u_int8_t *)(addr) = value; } static __inline void bus_space_set_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) for (; count != 0; count--, addr += 2) outw(addr, value); else for (; count != 0; count--, addr += 2) *(volatile u_int16_t *)(addr) = value; } static __inline void bus_space_set_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) for (; count != 0; count--, addr += 4) outl(addr, value); else for (; count != 0; count--, addr += 4) *(volatile u_int32_t *)(addr) = value; } #if 0 /* Cause a link error for bus_space_set_region_8 */ #define bus_space_set_region_8 !!! bus_space_set_region_8 unimplemented !!! #endif /* * Copy `count' 1, 2, 4, or 8 byte values from bus space starting * at tag/bsh1/off1 to bus space starting at tag/bsh2/off2. */ static __inline void bus_space_copy_region_1(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count); static __inline void bus_space_copy_region_2(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count); static __inline void bus_space_copy_region_4(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count); static __inline void bus_space_copy_region_1(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count) { bus_space_handle_t addr1 = bsh1 + off1; bus_space_handle_t addr2 = bsh2 + off2; if (tag == X86_BUS_SPACE_IO) { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1++, addr2++) outb(addr2, inb(addr1)); } else { /* dest after src: copy backwards */ for (addr1 += (count - 1), addr2 += (count - 1); count != 0; count--, addr1--, addr2--) outb(addr2, inb(addr1)); } } else { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1++, addr2++) *(volatile u_int8_t *)(addr2) = *(volatile u_int8_t *)(addr1); } else { /* dest after src: copy backwards */ for (addr1 += (count - 1), addr2 += (count - 1); count != 0; count--, addr1--, addr2--) *(volatile u_int8_t *)(addr2) = *(volatile u_int8_t *)(addr1); } } } static __inline void bus_space_copy_region_2(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count) { bus_space_handle_t addr1 = bsh1 + off1; bus_space_handle_t addr2 = bsh2 + off2; if (tag == X86_BUS_SPACE_IO) { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1 += 2, addr2 += 2) outw(addr2, inw(addr1)); } else { /* dest after src: copy backwards */ for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1); count != 0; count--, addr1 -= 2, addr2 -= 2) outw(addr2, inw(addr1)); } } else { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1 += 2, addr2 += 2) *(volatile u_int16_t *)(addr2) = *(volatile u_int16_t *)(addr1); } else { /* dest after src: copy backwards */ for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1); count != 0; count--, addr1 -= 2, addr2 -= 2) *(volatile u_int16_t *)(addr2) = *(volatile u_int16_t *)(addr1); } } } static __inline void bus_space_copy_region_4(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count) { bus_space_handle_t addr1 = bsh1 + off1; bus_space_handle_t addr2 = bsh2 + off2; if (tag == X86_BUS_SPACE_IO) { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1 += 4, addr2 += 4) outl(addr2, inl(addr1)); } else { /* dest after src: copy backwards */ for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1); count != 0; count--, addr1 -= 4, addr2 -= 4) outl(addr2, inl(addr1)); } } else { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1 += 4, addr2 += 4) *(volatile u_int32_t *)(addr2) = *(volatile u_int32_t *)(addr1); } else { /* dest after src: copy backwards */ for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1); count != 0; count--, addr1 -= 4, addr2 -= 4) *(volatile u_int32_t *)(addr2) = *(volatile u_int32_t *)(addr1); } } } #if 0 /* Cause a link error for bus_space_copy_8 */ #define bus_space_copy_region_8 !!! bus_space_copy_region_8 unimplemented !!! #endif /* * Bus read/write barrier methods. * * void bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t bsh, * bus_size_t offset, bus_size_t len, int flags); * * * Note that BUS_SPACE_BARRIER_WRITE doesn't do anything other than * prevent reordering by the compiler; all Intel x86 processors currently * retire operations outside the CPU in program order. */ static __inline void bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused, bus_size_t offset __unused, bus_size_t len __unused, int flags) { #ifdef __GNUCLIKE_ASM if (flags & BUS_SPACE_BARRIER_READ) #ifdef __amd64__ __asm __volatile("lock; addl $0,0(%%rsp)" : : : "memory"); #else __asm __volatile("lock; addl $0,0(%%esp)" : : : "memory"); #endif else __compiler_membar(); #endif } #ifdef BUS_SPACE_NO_LEGACY #undef inb #undef outb #define inb(a) compiler_error #define inw(a) compiler_error #define inl(a) compiler_error #define outb(a, b) compiler_error #define outw(a, b) compiler_error #define outl(a, b) compiler_error #endif /* * Stream accesses are the same as normal accesses on x86; there are no * supported bus systems with an endianess different from the host one. */ #define bus_space_read_stream_1(t, h, o) bus_space_read_1((t), (h), (o)) #define bus_space_read_stream_2(t, h, o) bus_space_read_2((t), (h), (o)) #define bus_space_read_stream_4(t, h, o) bus_space_read_4((t), (h), (o)) #define bus_space_read_multi_stream_1(t, h, o, a, c) \ bus_space_read_multi_1((t), (h), (o), (a), (c)) #define bus_space_read_multi_stream_2(t, h, o, a, c) \ bus_space_read_multi_2((t), (h), (o), (a), (c)) #define bus_space_read_multi_stream_4(t, h, o, a, c) \ bus_space_read_multi_4((t), (h), (o), (a), (c)) #define bus_space_write_stream_1(t, h, o, v) \ bus_space_write_1((t), (h), (o), (v)) #define bus_space_write_stream_2(t, h, o, v) \ bus_space_write_2((t), (h), (o), (v)) #define bus_space_write_stream_4(t, h, o, v) \ bus_space_write_4((t), (h), (o), (v)) #define bus_space_write_multi_stream_1(t, h, o, a, c) \ bus_space_write_multi_1((t), (h), (o), (a), (c)) #define bus_space_write_multi_stream_2(t, h, o, a, c) \ bus_space_write_multi_2((t), (h), (o), (a), (c)) #define bus_space_write_multi_stream_4(t, h, o, a, c) \ bus_space_write_multi_4((t), (h), (o), (a), (c)) #define bus_space_set_multi_stream_1(t, h, o, v, c) \ bus_space_set_multi_1((t), (h), (o), (v), (c)) #define bus_space_set_multi_stream_2(t, h, o, v, c) \ bus_space_set_multi_2((t), (h), (o), (v), (c)) #define bus_space_set_multi_stream_4(t, h, o, v, c) \ bus_space_set_multi_4((t), (h), (o), (v), (c)) #define bus_space_read_region_stream_1(t, h, o, a, c) \ bus_space_read_region_1((t), (h), (o), (a), (c)) #define bus_space_read_region_stream_2(t, h, o, a, c) \ bus_space_read_region_2((t), (h), (o), (a), (c)) #define bus_space_read_region_stream_4(t, h, o, a, c) \ bus_space_read_region_4((t), (h), (o), (a), (c)) #define bus_space_write_region_stream_1(t, h, o, a, c) \ bus_space_write_region_1((t), (h), (o), (a), (c)) #define bus_space_write_region_stream_2(t, h, o, a, c) \ bus_space_write_region_2((t), (h), (o), (a), (c)) #define bus_space_write_region_stream_4(t, h, o, a, c) \ bus_space_write_region_4((t), (h), (o), (a), (c)) #define bus_space_set_region_stream_1(t, h, o, v, c) \ bus_space_set_region_1((t), (h), (o), (v), (c)) #define bus_space_set_region_stream_2(t, h, o, v, c) \ bus_space_set_region_2((t), (h), (o), (v), (c)) #define bus_space_set_region_stream_4(t, h, o, v, c) \ bus_space_set_region_4((t), (h), (o), (v), (c)) #define bus_space_copy_region_stream_1(t, h1, o1, h2, o2, c) \ bus_space_copy_region_1((t), (h1), (o1), (h2), (o2), (c)) #define bus_space_copy_region_stream_2(t, h1, o1, h2, o2, c) \ bus_space_copy_region_2((t), (h1), (o1), (h2), (o2), (c)) #define bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c) \ bus_space_copy_region_4((t), (h1), (o1), (h2), (o2), (c)) #define BUS_PEEK_FUNC(width, type) \ static inline int \ bus_space_peek_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type *value) \ { \ type tmp; \ tmp = bus_space_read_##width(tag, hnd, offset); \ *value = (type)tmp; \ return (0); \ } BUS_PEEK_FUNC(1, uint8_t) BUS_PEEK_FUNC(2, uint16_t) BUS_PEEK_FUNC(4, uint32_t) #ifdef __amd64__ BUS_PEEK_FUNC(8, uint64_t) #endif #define BUS_POKE_FUNC(width, type) \ static inline int \ bus_space_poke_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ bus_space_write_##width(tag, hnd, offset, value); \ return (0); \ } BUS_POKE_FUNC(1, uint8_t) BUS_POKE_FUNC(2, uint16_t) BUS_POKE_FUNC(4, uint32_t) #ifdef __amd64__ BUS_POKE_FUNC(8, uint64_t) #endif -#endif /* !BUS_SAN_PREFIX */ +#endif /* !SAN_NEEDS_INTERCEPTORS && SAN_RUNTIME */ #endif /* !_MACHINE_BUS_H_ */