diff --git a/sys/kern/kern_vnodedumper.c b/sys/kern/kern_vnodedumper.c index 0104369b9d67..78979f906abd 100644 --- a/sys/kern/kern_vnodedumper.c +++ b/sys/kern/kern_vnodedumper.c @@ -1,212 +1,214 @@ /*- * Copyright (c) 2021-2022 Juniper Networks * * This software was developed by Mitchell Horne * under sponsorship from Juniper Networks and Klara Systems. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static dumper_start_t vnode_dumper_start; static dumper_t vnode_dump; static dumper_hdr_t vnode_write_headers; static struct sx livedump_sx; SX_SYSINIT(livedump, &livedump_sx, "Livedump sx"); /* * Invoke a live minidump on the system. */ int livedump_start(int fd, int flags, uint8_t compression) { #if MINIDUMP_PAGE_TRACKING == 1 struct dumperinfo di, *livedi; struct diocskerneldump_arg kda; struct vnode *vp; struct file *fp; void *rl_cookie; int error; error = priv_check(curthread, PRIV_KMEM_READ); if (error != 0) return (error); if (flags != 0) return (EINVAL); error = getvnode(curthread, fd, &cap_write_rights, &fp); if (error != 0) return (error); vp = fp->f_vnode; if ((fp->f_flag & FWRITE) == 0) { error = EBADF; goto drop; } /* Set up a new dumper. */ bzero(&di, sizeof(di)); di.dumper_start = vnode_dumper_start; di.dumper = vnode_dump; di.dumper_hdr = vnode_write_headers; di.blocksize = PAGE_SIZE; /* Arbitrary. */ di.maxiosize = MAXDUMPPGS * PAGE_SIZE; bzero(&kda, sizeof(kda)); kda.kda_compression = compression; error = dumper_create(&di, "livedump", &kda, &livedi); if (error != 0) goto drop; /* Only allow one livedump to proceed at a time. */ if (sx_try_xlock(&livedump_sx) == 0) { dumper_destroy(livedi); error = EBUSY; goto drop; } /* To be used by the callback functions. */ livedi->priv = vp; /* Lock the entire file range and vnode. */ rl_cookie = vn_rangelock_wlock(vp, 0, OFF_MAX); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); EVENTHANDLER_INVOKE(livedumper_start, &error); if (error != 0) goto out; + curthread->td_pflags2 |= TDP2_SAN_QUIET; dump_savectx(); error = minidumpsys(livedi, true); + curthread->td_pflags2 &= ~TDP2_SAN_QUIET; EVENTHANDLER_INVOKE(livedumper_finish); out: VOP_UNLOCK(vp); vn_rangelock_unlock(vp, rl_cookie); sx_xunlock(&livedump_sx); dumper_destroy(livedi); drop: fdrop(fp, curthread); return (error); #else return (EOPNOTSUPP); #endif /* MINIDUMP_PAGE_TRACKING == 1 */ } int vnode_dumper_start(struct dumperinfo *di, void *key, uint32_t keysize) { /* Always begin with an offset of zero. */ di->dumpoff = 0; KASSERT(keysize == 0, ("encryption not supported for livedumps")); return (0); } /* * Callback from dumpsys() to dump a chunk of memory. * * Parameters: * arg Opaque private pointer to vnode * virtual Virtual address (where to read the data from) * offset Offset from start of core file * length Data length * * Return value: * 0 on success * errno on error */ int vnode_dump(void *arg, void *virtual, off_t offset, size_t length) { struct vnode *vp; int error = 0; vp = arg; MPASS(vp != NULL); ASSERT_VOP_LOCKED(vp, __func__); EVENTHANDLER_INVOKE(livedumper_dump, virtual, offset, length, &error); if (error != 0) return (error); /* Done? */ if (virtual == NULL) return (0); error = vn_rdwr(UIO_WRITE, vp, virtual, length, offset, UIO_SYSSPACE, IO_NODELOCKED, curthread->td_ucred, NOCRED, NULL, curthread); if (error != 0) uprintf("%s: error writing livedump block at offset %jx: %d\n", __func__, (uintmax_t)offset, error); return (error); } /* * Callback from dumpsys() to write out the dump header, placed at the end. */ int vnode_write_headers(struct dumperinfo *di, struct kerneldumpheader *kdh) { struct vnode *vp; int error; off_t offset; vp = di->priv; MPASS(vp != NULL); ASSERT_VOP_LOCKED(vp, __func__); /* Compensate for compression/encryption adjustment of dumpoff. */ offset = roundup2(di->dumpoff, di->blocksize); /* Write the kernel dump header to the end of the file. */ error = vn_rdwr(UIO_WRITE, vp, kdh, sizeof(*kdh), offset, UIO_SYSSPACE, IO_NODELOCKED, curthread->td_ucred, NOCRED, NULL, curthread); if (error != 0) uprintf("%s: error writing livedump header: %d\n", __func__, error); return (error); } diff --git a/sys/kern/subr_asan.c b/sys/kern/subr_asan.c index bd143a816932..c934ff83874e 100644 --- a/sys/kern/subr_asan.c +++ b/sys/kern/subr_asan.c @@ -1,1229 +1,1232 @@ /* $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $ */ /* * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net * All rights reserved. * * This code is part of the KASAN subsystem of the NetBSD kernel. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #define SAN_RUNTIME #include #if 0 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $"); #endif #include #include #include #include #include #include #include #include /* ASAN constants. Part of the compiler ABI. */ #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE - 1) #define KASAN_ALLOCA_SCALE_SIZE 32 /* ASAN ABI version. */ #if defined(__clang__) && (__clang_major__ - 0 >= 6) #define ASAN_ABI_VERSION 8 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__) #define ASAN_ABI_VERSION 8 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__) #define ASAN_ABI_VERSION 6 #else #error "Unsupported compiler version" #endif #define __RET_ADDR (unsigned long)__builtin_return_address(0) /* Global variable descriptor. Part of the compiler ABI. */ struct __asan_global_source_location { const char *filename; int line_no; int column_no; }; struct __asan_global { const void *beg; /* address of the global variable */ size_t size; /* size of the global variable */ size_t size_with_redzone; /* size with the redzone */ const void *name; /* name of the variable */ const void *module_name; /* name of the module where the var is declared */ unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */ struct __asan_global_source_location *location; #if ASAN_ABI_VERSION >= 7 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */ #endif }; FEATURE(kasan, "Kernel address sanitizer"); static SYSCTL_NODE(_debug, OID_AUTO, kasan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "KASAN options"); static int panic_on_violation = 1; SYSCTL_INT(_debug_kasan, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN, &panic_on_violation, 0, "Panic if an invalid access is detected"); #define kasan_enabled (!kasan_disabled) static bool kasan_disabled __read_mostly = true; SYSCTL_BOOL(_debug_kasan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &kasan_disabled, 0, "KASAN is disabled"); /* -------------------------------------------------------------------------- */ void kasan_shadow_map(vm_offset_t addr, size_t size) { size_t sz, npages, i; vm_offset_t sva, eva; KASSERT(addr % KASAN_SHADOW_SCALE == 0, ("%s: invalid address %#lx", __func__, addr)); sz = roundup(size, KASAN_SHADOW_SCALE) / KASAN_SHADOW_SCALE; sva = kasan_md_addr_to_shad(addr); eva = kasan_md_addr_to_shad(addr) + sz; sva = rounddown(sva, PAGE_SIZE); eva = roundup(eva, PAGE_SIZE); npages = (eva - sva) / PAGE_SIZE; KASSERT(sva >= KASAN_MIN_ADDRESS && eva < KASAN_MAX_ADDRESS, ("%s: invalid address range %#lx-%#lx", __func__, sva, eva)); for (i = 0; i < npages; i++) pmap_san_enter(sva + ptoa(i)); } void kasan_init(void) { int disabled; disabled = 0; TUNABLE_INT_FETCH("debug.kasan.disabled", &disabled); if (disabled) return; /* MD initialization. */ kasan_md_init(); /* Now officially enabled. */ kasan_disabled = false; } void kasan_init_early(vm_offset_t stack, size_t size) { kasan_md_init_early(stack, size); } static inline const char * kasan_code_name(uint8_t code) { switch (code) { case KASAN_GENERIC_REDZONE: return "GenericRedZone"; case KASAN_MALLOC_REDZONE: return "MallocRedZone"; case KASAN_KMEM_REDZONE: return "KmemRedZone"; case KASAN_UMA_FREED: return "UMAUseAfterFree"; case KASAN_KSTACK_FREED: return "KernelStack"; case KASAN_EXEC_ARGS_FREED: return "ExecKVA"; case 1 ... 7: return "RedZonePartial"; case KASAN_STACK_LEFT: return "StackLeft"; case KASAN_STACK_MID: return "StackMiddle"; case KASAN_STACK_RIGHT: return "StackRight"; case KASAN_USE_AFTER_RET: return "UseAfterRet"; case KASAN_USE_AFTER_SCOPE: return "UseAfterScope"; default: return "Unknown"; } } #define REPORT(f, ...) do { \ if (panic_on_violation) { \ kasan_disabled = true; \ panic(f, __VA_ARGS__); \ } else { \ struct stack st; \ \ stack_save(&st); \ printf(f "\n", __VA_ARGS__); \ stack_print_ddb(&st); \ } \ } while (0) static void kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc, uint8_t code) { REPORT("ASan: Invalid access, %zu-byte %s at %#lx, %s(%x)", size, (write ? "write" : "read"), addr, kasan_code_name(code), code); } static __always_inline void kasan_shadow_1byte_markvalid(unsigned long addr) { int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr); int8_t last = (addr & KASAN_SHADOW_MASK) + 1; *byte = last; } static __always_inline void kasan_shadow_Nbyte_markvalid(const void *addr, size_t size) { size_t i; for (i = 0; i < size; i++) { kasan_shadow_1byte_markvalid((unsigned long)addr + i); } } static __always_inline void kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code) { void *shad; if (__predict_false(size == 0)) return; if (__predict_false(kasan_md_unsupported((vm_offset_t)addr))) return; KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0, ("%s: invalid address %p", __func__, addr)); KASSERT(size % KASAN_SHADOW_SCALE == 0, ("%s: invalid size %zu", __func__, size)); shad = (void *)kasan_md_addr_to_shad((uintptr_t)addr); size = size >> KASAN_SHADOW_SCALE_SHIFT; __builtin_memset(shad, code, size); } /* * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid, * and the rest as invalid. There are generally two use cases: * * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks * the redzone at the end of the buffer as invalid. If the entire is to be * marked invalid, origsize will be 0. * * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid. */ void kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code) { size_t i, n, redz; int8_t *shad; if (__predict_false(!kasan_enabled)) return; if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS && (vm_offset_t)addr < DMAP_MAX_ADDRESS) return; KASSERT((vm_offset_t)addr >= VM_MIN_KERNEL_ADDRESS && (vm_offset_t)addr < VM_MAX_KERNEL_ADDRESS, ("%s: invalid address %p", __func__, addr)); KASSERT((vm_offset_t)addr % KASAN_SHADOW_SCALE == 0, ("%s: invalid address %p", __func__, addr)); redz = redzsize - roundup(size, KASAN_SHADOW_SCALE); KASSERT(redz % KASAN_SHADOW_SCALE == 0, ("%s: invalid size %zu", __func__, redz)); shad = (int8_t *)kasan_md_addr_to_shad((uintptr_t)addr); /* Chunks of 8 bytes, valid. */ n = size / KASAN_SHADOW_SCALE; for (i = 0; i < n; i++) { *shad++ = 0; } /* Possibly one chunk, mid. */ if ((size & KASAN_SHADOW_MASK) != 0) { *shad++ = (size & KASAN_SHADOW_MASK); } /* Chunks of 8 bytes, invalid. */ n = redz / KASAN_SHADOW_SCALE; for (i = 0; i < n; i++) { *shad++ = code; } } /* -------------------------------------------------------------------------- */ #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \ (addr >> KASAN_SHADOW_SCALE_SHIFT) != \ ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT) static __always_inline bool kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code) { int8_t *byte = (int8_t *)kasan_md_addr_to_shad(addr); int8_t last = (addr & KASAN_SHADOW_MASK) + 1; if (__predict_true(*byte == 0 || last <= *byte)) { return (true); } *code = *byte; return (false); } static __always_inline bool kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code) { int8_t *byte, last; if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) { return (kasan_shadow_1byte_isvalid(addr, code) && kasan_shadow_1byte_isvalid(addr+1, code)); } byte = (int8_t *)kasan_md_addr_to_shad(addr); last = ((addr + 1) & KASAN_SHADOW_MASK) + 1; if (__predict_true(*byte == 0 || last <= *byte)) { return (true); } *code = *byte; return (false); } static __always_inline bool kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code) { int8_t *byte, last; if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) { return (kasan_shadow_2byte_isvalid(addr, code) && kasan_shadow_2byte_isvalid(addr+2, code)); } byte = (int8_t *)kasan_md_addr_to_shad(addr); last = ((addr + 3) & KASAN_SHADOW_MASK) + 1; if (__predict_true(*byte == 0 || last <= *byte)) { return (true); } *code = *byte; return (false); } static __always_inline bool kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code) { int8_t *byte, last; if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) { return (kasan_shadow_4byte_isvalid(addr, code) && kasan_shadow_4byte_isvalid(addr+4, code)); } byte = (int8_t *)kasan_md_addr_to_shad(addr); last = ((addr + 7) & KASAN_SHADOW_MASK) + 1; if (__predict_true(*byte == 0 || last <= *byte)) { return (true); } *code = *byte; return (false); } static __always_inline bool kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code) { size_t i; for (i = 0; i < size; i++) { if (!kasan_shadow_1byte_isvalid(addr+i, code)) return (false); } return (true); } static __always_inline void kasan_shadow_check(unsigned long addr, size_t size, bool write, unsigned long retaddr) { uint8_t code; bool valid; if (__predict_false(!kasan_enabled)) return; + if (__predict_false(curthread != NULL && + (curthread->td_pflags2 & TDP2_SAN_QUIET) != 0)) + return; if (__predict_false(size == 0)) return; if (__predict_false(kasan_md_unsupported(addr))) return; if (KERNEL_PANICKED()) return; if (__builtin_constant_p(size)) { switch (size) { case 1: valid = kasan_shadow_1byte_isvalid(addr, &code); break; case 2: valid = kasan_shadow_2byte_isvalid(addr, &code); break; case 4: valid = kasan_shadow_4byte_isvalid(addr, &code); break; case 8: valid = kasan_shadow_8byte_isvalid(addr, &code); break; default: valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); break; } } else { valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); } if (__predict_false(!valid)) { kasan_report(addr, size, write, retaddr, code); } } /* -------------------------------------------------------------------------- */ void * kasan_memcpy(void *dst, const void *src, size_t len) { kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); return (__builtin_memcpy(dst, src, len)); } int kasan_memcmp(const void *b1, const void *b2, size_t len) { kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR); kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR); return (__builtin_memcmp(b1, b2, len)); } void * kasan_memset(void *b, int c, size_t len) { kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR); return (__builtin_memset(b, c, len)); } void * kasan_memmove(void *dst, const void *src, size_t len) { kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); return (__builtin_memmove(dst, src, len)); } size_t kasan_strlen(const char *str) { const char *s; s = str; while (1) { kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR); if (*s == '\0') break; s++; } return (s - str); } char * kasan_strcpy(char *dst, const char *src) { char *save = dst; while (1) { kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR); kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR); *dst = *src; if (*src == '\0') break; src++, dst++; } return save; } int kasan_strcmp(const char *s1, const char *s2) { while (1) { kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR); kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR); if (*s1 != *s2) break; if (*s1 == '\0') return 0; s1++, s2++; } return (*(const unsigned char *)s1 - *(const unsigned char *)s2); } int kasan_copyin(const void *uaddr, void *kaddr, size_t len) { kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); return (copyin(uaddr, kaddr, len)); } int kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) { kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); return (copyinstr(uaddr, kaddr, len, done)); } int kasan_copyout(const void *kaddr, void *uaddr, size_t len) { kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR); return (copyout(kaddr, uaddr, len)); } /* -------------------------------------------------------------------------- */ int kasan_fubyte(volatile const void *base) { return (fubyte(base)); } int kasan_fuword16(volatile const void *base) { return (fuword16(base)); } int kasan_fueword(volatile const void *base, long *val) { kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR); return (fueword(base, val)); } int kasan_fueword32(volatile const void *base, int32_t *val) { kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR); return (fueword32(base, val)); } int kasan_fueword64(volatile const void *base, int64_t *val) { kasan_shadow_check((unsigned long)val, sizeof(*val), true, __RET_ADDR); return (fueword64(base, val)); } int kasan_subyte(volatile void *base, int byte) { return (subyte(base, byte)); } int kasan_suword(volatile void *base, long word) { return (suword(base, word)); } int kasan_suword16(volatile void *base, int word) { return (suword16(base, word)); } int kasan_suword32(volatile void *base, int32_t word) { return (suword32(base, word)); } int kasan_suword64(volatile void *base, int64_t word) { return (suword64(base, word)); } int kasan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, uint32_t newval) { kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true, __RET_ADDR); return (casueword32(base, oldval, oldvalp, newval)); } int kasan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval) { kasan_shadow_check((unsigned long)oldvalp, sizeof(*oldvalp), true, __RET_ADDR); return (casueword(base, oldval, oldvalp, newval)); } /* -------------------------------------------------------------------------- */ #include #include #define _ASAN_ATOMIC_FUNC_ADD(name, type) \ void kasan_atomic_add_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_add_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_ADD(name, type) \ _ASAN_ATOMIC_FUNC_ADD(name, type) \ _ASAN_ATOMIC_FUNC_ADD(acq_##name, type) \ _ASAN_ATOMIC_FUNC_ADD(rel_##name, type) #define _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ void kasan_atomic_subtract_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_subtract_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _ASAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _ASAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ _ASAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) #define _ASAN_ATOMIC_FUNC_SET(name, type) \ void kasan_atomic_set_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_set_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_SET(name, type) \ _ASAN_ATOMIC_FUNC_SET(name, type) \ _ASAN_ATOMIC_FUNC_SET(acq_##name, type) \ _ASAN_ATOMIC_FUNC_SET(rel_##name, type) #define _ASAN_ATOMIC_FUNC_CLEAR(name, type) \ void kasan_atomic_clear_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_clear_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_CLEAR(name, type) \ _ASAN_ATOMIC_FUNC_CLEAR(name, type) \ _ASAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ _ASAN_ATOMIC_FUNC_CLEAR(rel_##name, type) #define ASAN_ATOMIC_FUNC_FETCHADD(name, type) \ type kasan_atomic_fetchadd_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_fetchadd_##name(ptr, val)); \ } #define ASAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ type kasan_atomic_readandclear_##name(volatile type *ptr) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_readandclear_##name(ptr)); \ } #define ASAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ int kasan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_testandclear_##name(ptr, v)); \ } #define ASAN_ATOMIC_FUNC_TESTANDSET(name, type) \ int kasan_atomic_testandset_##name(volatile type *ptr, u_int v) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_testandset_##name(ptr, v)); \ } #define ASAN_ATOMIC_FUNC_SWAP(name, type) \ type kasan_atomic_swap_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_swap_##name(ptr, val)); \ } #define _ASAN_ATOMIC_FUNC_CMPSET(name, type) \ int kasan_atomic_cmpset_##name(volatile type *ptr, type oval, \ type nval) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_cmpset_##name(ptr, oval, nval)); \ } #define ASAN_ATOMIC_FUNC_CMPSET(name, type) \ _ASAN_ATOMIC_FUNC_CMPSET(name, type) \ _ASAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ _ASAN_ATOMIC_FUNC_CMPSET(rel_##name, type) #define _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ int kasan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ type nval) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_fcmpset_##name(ptr, oval, nval)); \ } #define ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ _ASAN_ATOMIC_FUNC_FCMPSET(name, type) \ _ASAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ _ASAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) #define ASAN_ATOMIC_FUNC_THREAD_FENCE(name) \ void kasan_atomic_thread_fence_##name(void) \ { \ atomic_thread_fence_##name(); \ } #define _ASAN_ATOMIC_FUNC_LOAD(name, type) \ type kasan_atomic_load_##name(volatile type *ptr) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ return (atomic_load_##name(ptr)); \ } #define ASAN_ATOMIC_FUNC_LOAD(name, type) \ _ASAN_ATOMIC_FUNC_LOAD(name, type) \ _ASAN_ATOMIC_FUNC_LOAD(acq_##name, type) #define _ASAN_ATOMIC_FUNC_STORE(name, type) \ void kasan_atomic_store_##name(volatile type *ptr, type val) \ { \ kasan_shadow_check((uintptr_t)ptr, sizeof(type), true, \ __RET_ADDR); \ atomic_store_##name(ptr, val); \ } #define ASAN_ATOMIC_FUNC_STORE(name, type) \ _ASAN_ATOMIC_FUNC_STORE(name, type) \ _ASAN_ATOMIC_FUNC_STORE(rel_##name, type) ASAN_ATOMIC_FUNC_ADD(8, uint8_t); ASAN_ATOMIC_FUNC_ADD(16, uint16_t); ASAN_ATOMIC_FUNC_ADD(32, uint32_t); ASAN_ATOMIC_FUNC_ADD(64, uint64_t); ASAN_ATOMIC_FUNC_ADD(int, u_int); ASAN_ATOMIC_FUNC_ADD(long, u_long); ASAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); ASAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); ASAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); ASAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); ASAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); ASAN_ATOMIC_FUNC_SUBTRACT(int, u_int); ASAN_ATOMIC_FUNC_SUBTRACT(long, u_long); ASAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); ASAN_ATOMIC_FUNC_SET(8, uint8_t); ASAN_ATOMIC_FUNC_SET(16, uint16_t); ASAN_ATOMIC_FUNC_SET(32, uint32_t); ASAN_ATOMIC_FUNC_SET(64, uint64_t); ASAN_ATOMIC_FUNC_SET(int, u_int); ASAN_ATOMIC_FUNC_SET(long, u_long); ASAN_ATOMIC_FUNC_SET(ptr, uintptr_t); ASAN_ATOMIC_FUNC_CLEAR(8, uint8_t); ASAN_ATOMIC_FUNC_CLEAR(16, uint16_t); ASAN_ATOMIC_FUNC_CLEAR(32, uint32_t); ASAN_ATOMIC_FUNC_CLEAR(64, uint64_t); ASAN_ATOMIC_FUNC_CLEAR(int, u_int); ASAN_ATOMIC_FUNC_CLEAR(long, u_long); ASAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); ASAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); ASAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); ASAN_ATOMIC_FUNC_FETCHADD(int, u_int); ASAN_ATOMIC_FUNC_FETCHADD(long, u_long); ASAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); ASAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); ASAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); ASAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); ASAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); ASAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); ASAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); ASAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); ASAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); ASAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t); ASAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); ASAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); ASAN_ATOMIC_FUNC_TESTANDSET(int, u_int); ASAN_ATOMIC_FUNC_TESTANDSET(long, u_long); ASAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t); ASAN_ATOMIC_FUNC_SWAP(32, uint32_t); ASAN_ATOMIC_FUNC_SWAP(64, uint64_t); ASAN_ATOMIC_FUNC_SWAP(int, u_int); ASAN_ATOMIC_FUNC_SWAP(long, u_long); ASAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); ASAN_ATOMIC_FUNC_CMPSET(8, uint8_t); ASAN_ATOMIC_FUNC_CMPSET(16, uint16_t); ASAN_ATOMIC_FUNC_CMPSET(32, uint32_t); ASAN_ATOMIC_FUNC_CMPSET(64, uint64_t); ASAN_ATOMIC_FUNC_CMPSET(int, u_int); ASAN_ATOMIC_FUNC_CMPSET(long, u_long); ASAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); ASAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); ASAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); ASAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); ASAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); ASAN_ATOMIC_FUNC_FCMPSET(int, u_int); ASAN_ATOMIC_FUNC_FCMPSET(long, u_long); ASAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); _ASAN_ATOMIC_FUNC_LOAD(bool, bool); ASAN_ATOMIC_FUNC_LOAD(8, uint8_t); ASAN_ATOMIC_FUNC_LOAD(16, uint16_t); ASAN_ATOMIC_FUNC_LOAD(32, uint32_t); ASAN_ATOMIC_FUNC_LOAD(64, uint64_t); ASAN_ATOMIC_FUNC_LOAD(char, u_char); ASAN_ATOMIC_FUNC_LOAD(short, u_short); ASAN_ATOMIC_FUNC_LOAD(int, u_int); ASAN_ATOMIC_FUNC_LOAD(long, u_long); ASAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); _ASAN_ATOMIC_FUNC_STORE(bool, bool); ASAN_ATOMIC_FUNC_STORE(8, uint8_t); ASAN_ATOMIC_FUNC_STORE(16, uint16_t); ASAN_ATOMIC_FUNC_STORE(32, uint32_t); ASAN_ATOMIC_FUNC_STORE(64, uint64_t); ASAN_ATOMIC_FUNC_STORE(char, u_char); ASAN_ATOMIC_FUNC_STORE(short, u_short); ASAN_ATOMIC_FUNC_STORE(int, u_int); ASAN_ATOMIC_FUNC_STORE(long, u_long); ASAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); ASAN_ATOMIC_FUNC_THREAD_FENCE(acq); ASAN_ATOMIC_FUNC_THREAD_FENCE(rel); ASAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); ASAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); void kasan_atomic_interrupt_fence(void) { } /* -------------------------------------------------------------------------- */ #include #include #include int kasan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, int flags, bus_space_handle_t *handlep) { return (bus_space_map(tag, hnd, size, flags, handlep)); } void kasan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_unmap(tag, hnd, size); } int kasan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) { return (bus_space_subregion(tag, hnd, offset, size, handlep)); } void kasan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_free(tag, hnd, size); } void kasan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, int flags) { bus_space_barrier(tag, hnd, offset, size, flags); } #define ASAN_BUS_READ_FUNC(func, width, type) \ type kasan_bus_space_read##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset) \ { \ return (bus_space_read##func##_##width(tag, hnd, \ offset)); \ } \ #define ASAN_BUS_READ_PTR_FUNC(func, width, type) \ void kasan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, type *buf, \ bus_size_t count) \ { \ kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ false, __RET_ADDR); \ bus_space_read_##func##_##width(tag, hnd, size, buf, \ count); \ } ASAN_BUS_READ_FUNC(, 1, uint8_t) ASAN_BUS_READ_FUNC(_stream, 1, uint8_t) ASAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) ASAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) ASAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) ASAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) ASAN_BUS_READ_FUNC(, 2, uint16_t) ASAN_BUS_READ_FUNC(_stream, 2, uint16_t) ASAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) ASAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) ASAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) ASAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) ASAN_BUS_READ_FUNC(, 4, uint32_t) ASAN_BUS_READ_FUNC(_stream, 4, uint32_t) ASAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) ASAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) ASAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) ASAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) ASAN_BUS_READ_FUNC(, 8, uint64_t) #if defined(__aarch64__) ASAN_BUS_READ_FUNC(_stream, 8, uint64_t) ASAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) ASAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) ASAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) ASAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) #endif #define ASAN_BUS_WRITE_FUNC(func, width, type) \ void kasan_bus_space_write##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ bus_space_write##func##_##width(tag, hnd, offset, value);\ } \ #define ASAN_BUS_WRITE_PTR_FUNC(func, width, type) \ void kasan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ bus_space_handle_t hnd, bus_size_t size, const type *buf, \ bus_size_t count) \ { \ kasan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ true, __RET_ADDR); \ bus_space_write_##func##_##width(tag, hnd, size, buf, \ count); \ } ASAN_BUS_WRITE_FUNC(, 1, uint8_t) ASAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) ASAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) ASAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) ASAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) ASAN_BUS_WRITE_FUNC(, 2, uint16_t) ASAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) ASAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) ASAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) ASAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) ASAN_BUS_WRITE_FUNC(, 4, uint32_t) ASAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) ASAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) ASAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) ASAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) ASAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) ASAN_BUS_WRITE_FUNC(, 8, uint64_t) #define ASAN_BUS_SET_FUNC(func, width, type) \ void kasan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value, \ bus_size_t count) \ { \ bus_space_set_##func##_##width(tag, hnd, offset, value, \ count); \ } ASAN_BUS_SET_FUNC(multi, 1, uint8_t) ASAN_BUS_SET_FUNC(region, 1, uint8_t) ASAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) ASAN_BUS_SET_FUNC(region_stream, 1, uint8_t) ASAN_BUS_SET_FUNC(multi, 2, uint16_t) ASAN_BUS_SET_FUNC(region, 2, uint16_t) ASAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) ASAN_BUS_SET_FUNC(region_stream, 2, uint16_t) ASAN_BUS_SET_FUNC(multi, 4, uint32_t) ASAN_BUS_SET_FUNC(region, 4, uint32_t) ASAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) ASAN_BUS_SET_FUNC(region_stream, 4, uint32_t) #define ASAN_BUS_PEEK_FUNC(width, type) \ int kasan_bus_space_peek_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type *valuep) \ { \ return (bus_space_peek_##width(tag, hnd, offset, \ valuep)); \ } ASAN_BUS_PEEK_FUNC(1, uint8_t) ASAN_BUS_PEEK_FUNC(2, uint16_t) ASAN_BUS_PEEK_FUNC(4, uint32_t) ASAN_BUS_PEEK_FUNC(8, uint64_t) #define ASAN_BUS_POKE_FUNC(width, type) \ int kasan_bus_space_poke_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ return (bus_space_poke_##width(tag, hnd, offset, \ value)); \ } ASAN_BUS_POKE_FUNC(1, uint8_t) ASAN_BUS_POKE_FUNC(2, uint16_t) ASAN_BUS_POKE_FUNC(4, uint32_t) ASAN_BUS_POKE_FUNC(8, uint64_t) /* -------------------------------------------------------------------------- */ void __asan_register_globals(struct __asan_global *, size_t); void __asan_unregister_globals(struct __asan_global *, size_t); void __asan_register_globals(struct __asan_global *globals, size_t n) { size_t i; for (i = 0; i < n; i++) { kasan_mark(globals[i].beg, globals[i].size, globals[i].size_with_redzone, KASAN_GENERIC_REDZONE); } } void __asan_unregister_globals(struct __asan_global *globals, size_t n) { size_t i; for (i = 0; i < n; i++) { kasan_mark(globals[i].beg, globals[i].size_with_redzone, globals[i].size_with_redzone, 0); } } #define ASAN_LOAD_STORE(size) \ void __asan_load##size(unsigned long); \ void __asan_load##size(unsigned long addr) \ { \ kasan_shadow_check(addr, size, false, __RET_ADDR);\ } \ void __asan_load##size##_noabort(unsigned long); \ void __asan_load##size##_noabort(unsigned long addr) \ { \ kasan_shadow_check(addr, size, false, __RET_ADDR);\ } \ void __asan_store##size(unsigned long); \ void __asan_store##size(unsigned long addr) \ { \ kasan_shadow_check(addr, size, true, __RET_ADDR);\ } \ void __asan_store##size##_noabort(unsigned long); \ void __asan_store##size##_noabort(unsigned long addr) \ { \ kasan_shadow_check(addr, size, true, __RET_ADDR);\ } ASAN_LOAD_STORE(1); ASAN_LOAD_STORE(2); ASAN_LOAD_STORE(4); ASAN_LOAD_STORE(8); ASAN_LOAD_STORE(16); void __asan_loadN(unsigned long, size_t); void __asan_loadN_noabort(unsigned long, size_t); void __asan_storeN(unsigned long, size_t); void __asan_storeN_noabort(unsigned long, size_t); void __asan_handle_no_return(void); void __asan_loadN(unsigned long addr, size_t size) { kasan_shadow_check(addr, size, false, __RET_ADDR); } void __asan_loadN_noabort(unsigned long addr, size_t size) { kasan_shadow_check(addr, size, false, __RET_ADDR); } void __asan_storeN(unsigned long addr, size_t size) { kasan_shadow_check(addr, size, true, __RET_ADDR); } void __asan_storeN_noabort(unsigned long addr, size_t size) { kasan_shadow_check(addr, size, true, __RET_ADDR); } void __asan_handle_no_return(void) { /* nothing */ } #define ASAN_SET_SHADOW(byte) \ void __asan_set_shadow_##byte(void *, size_t); \ void __asan_set_shadow_##byte(void *addr, size_t size) \ { \ __builtin_memset((void *)addr, 0x##byte, size); \ } ASAN_SET_SHADOW(00); ASAN_SET_SHADOW(f1); ASAN_SET_SHADOW(f2); ASAN_SET_SHADOW(f3); ASAN_SET_SHADOW(f5); ASAN_SET_SHADOW(f8); void __asan_poison_stack_memory(const void *, size_t); void __asan_unpoison_stack_memory(const void *, size_t); void __asan_poison_stack_memory(const void *addr, size_t size) { size = roundup(size, KASAN_SHADOW_SCALE); kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE); } void __asan_unpoison_stack_memory(const void *addr, size_t size) { kasan_shadow_Nbyte_markvalid(addr, size); } void __asan_alloca_poison(const void *, size_t); void __asan_allocas_unpoison(const void *, const void *); void __asan_alloca_poison(const void *addr, size_t size) { const void *l, *r; KASSERT((vm_offset_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0, ("%s: invalid address %p", __func__, addr)); l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE; r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE); kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT); kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE), KASAN_STACK_MID); kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT); } void __asan_allocas_unpoison(const void *stkbegin, const void *stkend) { size_t size; if (__predict_false(!stkbegin)) return; if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend)) return; size = (uintptr_t)stkend - (uintptr_t)stkbegin; kasan_shadow_Nbyte_fill(stkbegin, size, 0); } void __asan_poison_memory_region(const void *addr, size_t size); void __asan_unpoison_memory_region(const void *addr, size_t size); void __asan_poison_memory_region(const void *addr, size_t size) { } void __asan_unpoison_memory_region(const void *addr, size_t size) { } diff --git a/sys/kern/subr_msan.c b/sys/kern/subr_msan.c index 029c5ed8df96..b596443de2aa 100644 --- a/sys/kern/subr_msan.c +++ b/sys/kern/subr_msan.c @@ -1,1553 +1,1559 @@ /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */ /* * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net * All rights reserved. * Copyright (c) 2021 The FreeBSD Foundation * * Portions of this software were developed by Mark Johnston under sponsorship * from the FreeBSD Foundation. * * This code is part of the KMSAN subsystem of the NetBSD kernel. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #define SAN_RUNTIME #include #if 0 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $"); #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include void kmsan_init_arg(size_t); void kmsan_init_ret(size_t); /* -------------------------------------------------------------------------- */ /* * Part of the compiler ABI. */ typedef struct { uint8_t *shad; msan_orig_t *orig; } msan_meta_t; #define MSAN_PARAM_SIZE 800 #define MSAN_RETVAL_SIZE 800 typedef struct { uint8_t param_shadow[MSAN_PARAM_SIZE]; uint8_t retval_shadow[MSAN_RETVAL_SIZE]; uint8_t va_arg_shadow[MSAN_PARAM_SIZE]; uint8_t va_arg_origin[MSAN_PARAM_SIZE]; uint64_t va_arg_overflow_size; msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)]; msan_orig_t retval_origin; } msan_tls_t; /* -------------------------------------------------------------------------- */ #define MSAN_NCONTEXT 4 #define MSAN_ORIG_MASK (~0x3) typedef struct kmsan_td { size_t ctx; msan_tls_t tls[MSAN_NCONTEXT]; } msan_td_t; static msan_tls_t dummy_tls; /* * Use separate dummy regions for loads and stores: stores may mark the region * as uninitialized, and that can trigger false positives. */ static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE); static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); static msan_td_t msan_thread0; static bool kmsan_reporting = false; /* * Avoid clobbering any thread-local state before we panic. */ #define kmsan_panic(f, ...) do { \ kmsan_disabled = true; \ panic(f, __VA_ARGS__); \ } while (0) #define REPORT(f, ...) do { \ if (panic_on_violation) { \ kmsan_panic(f, __VA_ARGS__); \ } else { \ struct stack st; \ \ stack_save(&st); \ printf(f "\n", __VA_ARGS__); \ stack_print_ddb(&st); \ } \ } while (0) FEATURE(kmsan, "Kernel memory sanitizer"); static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "KMSAN options"); static bool panic_on_violation = 1; SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN, &panic_on_violation, 0, "Panic if an invalid access is detected"); static bool kmsan_disabled __read_mostly = true; #define kmsan_enabled (!kmsan_disabled) SYSCTL_BOOL(_debug_kmsan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &kmsan_disabled, 0, "KMSAN is disabled"); static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer"); /* -------------------------------------------------------------------------- */ static inline const char * kmsan_orig_name(int type) { switch (type) { case KMSAN_TYPE_STACK: return ("stack"); case KMSAN_TYPE_KMEM: return ("kmem"); case KMSAN_TYPE_MALLOC: return ("malloc"); case KMSAN_TYPE_UMA: return ("UMA"); default: return ("unknown"); } } static void kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off, const char *hook) { const char *typename; char *var, *fn; uintptr_t ptr; long foff; char buf[128]; int type; if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) return; + if (__predict_false(curthread != NULL && + (curthread->td_pflags2 & TDP2_SAN_QUIET) != 0)) + return; kmsan_reporting = true; __compiler_membar(); if (*orig == 0) { REPORT("MSan: Uninitialized memory in %s, offset %zu", hook, off); goto out; } kmsan_md_orig_decode(*orig, &type, &ptr); typename = kmsan_orig_name(type); if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, sizeof(buf), &foff) == 0) { REPORT("MSan: Uninitialized %s memory in %s, " "offset %zu/%zu, addr %p, from %s+%#lx", typename, hook, off, size, addr, buf, foff); } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { /* * The format of the string is: "----var@function". Parse it to * display a nice warning. */ var = (char *)ptr + 4; strlcpy(buf, var, sizeof(buf)); var = buf; fn = strchr(buf, '@'); *fn++ = '\0'; REPORT("MSan: Uninitialized %s memory in %s, offset %zu, " "variable '%s' from %s", typename, hook, off, var, fn); } else { REPORT("MSan: Uninitialized %s memory in %s, " "offset %zu/%zu, addr %p, PC %p", typename, hook, off, size, addr, (void *)ptr); } out: __compiler_membar(); kmsan_reporting = false; } static void kmsan_report_inline(msan_orig_t orig, unsigned long pc) { const char *typename; char *var, *fn; uintptr_t ptr; char buf[128]; long foff; int type; if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) return; + if (__predict_false(curthread != NULL && + (curthread->td_pflags2 & TDP2_SAN_QUIET) != 0)) + return; kmsan_reporting = true; __compiler_membar(); if (orig == 0) { REPORT("MSan: uninitialized variable in %p", (void *)pc); goto out; } kmsan_md_orig_decode(orig, &type, &ptr); typename = kmsan_orig_name(type); if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, sizeof(buf), &foff) == 0) { REPORT("MSan: Uninitialized %s memory from %s+%#lx", typename, buf, foff); } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { /* * The format of the string is: "----var@function". Parse it to * display a nice warning. */ var = (char *)ptr + 4; strlcpy(buf, var, sizeof(buf)); var = buf; fn = strchr(buf, '@'); *fn++ = '\0'; REPORT("MSan: Uninitialized variable '%s' from %s", var, fn); } else { REPORT("MSan: Uninitialized %s memory, origin %x", typename, orig); } out: __compiler_membar(); kmsan_reporting = false; } /* -------------------------------------------------------------------------- */ static inline msan_meta_t kmsan_meta_get(const void *addr, size_t size, const bool write) { msan_meta_t ret; if (__predict_false(!kmsan_enabled)) { ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; ret.orig = (msan_orig_t *)msan_dummy_orig; } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) { ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; ret.orig = (msan_orig_t *)msan_dummy_orig; } else { ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr); ret.orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & MSAN_ORIG_MASK); } return (ret); } static inline void kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size) { msan_orig_t *orig; size_t i; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) return; orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); size += ((uintptr_t)orig & (sizeof(*orig) - 1)); orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); for (i = 0; i < size; i += 4) { orig[i / 4] = o; } } static inline void kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size) { uint8_t *shad; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(kmsan_md_unsupported(addr))) return; shad = (uint8_t *)kmsan_md_addr_to_shad(addr); __builtin_memset(shad, c, size); } static inline void kmsan_meta_copy(void *dst, const void *src, size_t size) { uint8_t *orig_src, *orig_dst; uint8_t *shad_src, *shad_dst; msan_orig_t *_src, *_dst; size_t i; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst))) return; if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) { kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size); return; } shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src); shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst); __builtin_memmove(shad_dst, shad_src, size); orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src); orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst); for (i = 0; i < size; i++) { _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK); _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK); *_dst = *_src; orig_src++; orig_dst++; } } static inline void kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook) { msan_orig_t *orig; uint8_t *shad; size_t i; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(kmsan_md_unsupported(addr))) return; shad = (uint8_t *)kmsan_md_addr_to_shad(addr); for (i = 0; i < size; i++) { if (__predict_true(shad[i] == 0)) continue; orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr + i); orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); kmsan_report_hook((const char *)addr + i, orig, size, i, hook); break; } } void kmsan_init_arg(size_t n) { msan_td_t *mtd; uint8_t *arg; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(curthread == NULL)) return; mtd = curthread->td_kmsan; arg = mtd->tls[mtd->ctx].param_shadow; __builtin_memset(arg, 0, n); } void kmsan_init_ret(size_t n) { msan_td_t *mtd; uint8_t *arg; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(curthread == NULL)) return; mtd = curthread->td_kmsan; arg = mtd->tls[mtd->ctx].retval_shadow; __builtin_memset(arg, 0, n); } static void kmsan_check_arg(size_t size, const char *hook) { msan_orig_t *orig; msan_td_t *mtd; uint8_t *arg; size_t ctx, i; if (__predict_false(!kmsan_enabled)) return; if (__predict_false(curthread == NULL)) return; mtd = curthread->td_kmsan; ctx = mtd->ctx; arg = mtd->tls[ctx].param_shadow; for (i = 0; i < size; i++) { if (__predict_true(arg[i] == 0)) continue; orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)]; kmsan_report_hook((const char *)arg + i, orig, size, i, hook); break; } } void kmsan_thread_alloc(struct thread *td) { msan_td_t *mtd; if (!kmsan_enabled) return; mtd = td->td_kmsan; if (mtd == NULL) { /* We might be recycling a thread. */ kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) + sizeof(int)); mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK); } __builtin_memset(mtd, 0, sizeof(*mtd)); mtd->ctx = 0; if (td->td_kstack != 0) kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages), KMSAN_STATE_UNINIT); td->td_kmsan = mtd; } void kmsan_thread_free(struct thread *td) { msan_td_t *mtd; if (!kmsan_enabled) return; if (__predict_false(td == curthread)) kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__); mtd = td->td_kmsan; kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *)); free(mtd, M_KMSAN); td->td_kmsan = NULL; } void kmsan_intr_enter(void); void kmsan_intr_leave(void); void kmsan_intr_enter(void) { msan_td_t *mtd; if (__predict_false(!kmsan_enabled)) return; mtd = curthread->td_kmsan; mtd->ctx++; if (__predict_false(mtd->ctx >= MSAN_NCONTEXT)) kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); } void kmsan_intr_leave(void) { msan_td_t *mtd; if (__predict_false(!kmsan_enabled)) return; mtd = curthread->td_kmsan; if (__predict_false(mtd->ctx == 0)) kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); mtd->ctx--; } /* -------------------------------------------------------------------------- */ void kmsan_shadow_map(vm_offset_t addr, size_t size) { size_t npages, i; vm_offset_t va; MPASS(addr % PAGE_SIZE == 0); MPASS(size % PAGE_SIZE == 0); if (!kmsan_enabled) return; npages = atop(size); va = kmsan_md_addr_to_shad(addr); for (i = 0; i < npages; i++) { pmap_san_enter(va + ptoa(i)); } va = kmsan_md_addr_to_orig(addr); for (i = 0; i < npages; i++) { pmap_san_enter(va + ptoa(i)); } } void kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc) { msan_orig_t orig; orig = kmsan_md_orig_encode(type, pc); kmsan_origin_fill(addr, orig, size); } void kmsan_mark(const void *addr, size_t size, uint8_t c) { kmsan_shadow_fill((uintptr_t)addr, c, size); } void kmsan_mark_bio(const struct bio *bp, uint8_t c) { kmsan_mark(bp->bio_data, bp->bio_length, c); } void kmsan_mark_mbuf(const struct mbuf *m, uint8_t c) { do { if ((m->m_flags & M_EXTPG) == 0) kmsan_mark(m->m_data, m->m_len, c); m = m->m_next; } while (m != NULL); } void kmsan_check(const void *p, size_t sz, const char *descr) { kmsan_shadow_check((uintptr_t)p, sz, descr); } void kmsan_check_bio(const struct bio *bp, const char *descr) { kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr); } void kmsan_check_mbuf(const struct mbuf *m, const char *descr) { do { kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr); } while ((m = m->m_next) != NULL); } void kmsan_check_uio(const struct uio *uio, const char *descr) { for (int i = 0; i < uio->uio_iovcnt; i++) { kmsan_check(uio->uio_iov[i].iov_base, uio->uio_iov[i].iov_len, descr); } } void kmsan_init(void) { int disabled; disabled = 0; TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled); if (disabled) return; /* Initialize the TLS for curthread. */ msan_thread0.ctx = 0; thread0.td_kmsan = &msan_thread0; /* Now officially enabled. */ kmsan_disabled = false; } /* -------------------------------------------------------------------------- */ msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); msan_meta_t __msan_metadata_ptr_for_load_n(void *addr, size_t size) { return (kmsan_meta_get(addr, size, false)); } msan_meta_t __msan_metadata_ptr_for_store_n(void *addr, size_t size) { return (kmsan_meta_get(addr, size, true)); } #define MSAN_META_FUNC(size) \ msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ { \ return (kmsan_meta_get(addr, size, false)); \ } \ msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ { \ return (kmsan_meta_get(addr, size, true)); \ } MSAN_META_FUNC(1) MSAN_META_FUNC(2) MSAN_META_FUNC(4) MSAN_META_FUNC(8) void __msan_instrument_asm_store(const void *, size_t); msan_orig_t __msan_chain_origin(msan_orig_t); void __msan_poison(const void *, size_t); void __msan_unpoison(const void *, size_t); void __msan_poison_alloca(const void *, uint64_t, const char *); void __msan_unpoison_alloca(const void *, uint64_t); void __msan_warning(msan_orig_t); msan_tls_t *__msan_get_context_state(void); void __msan_instrument_asm_store(const void *addr, size_t size) { kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); } msan_orig_t __msan_chain_origin(msan_orig_t origin) { return (origin); } void __msan_poison(const void *addr, size_t size) { kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); } void __msan_unpoison(const void *addr, size_t size) { kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); } void __msan_poison_alloca(const void *addr, uint64_t size, const char *descr) { msan_orig_t orig; orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); kmsan_origin_fill(addr, orig, size); kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); } void __msan_unpoison_alloca(const void *addr, uint64_t size) { kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); } void __msan_warning(msan_orig_t origin) { if (__predict_false(!kmsan_enabled)) return; kmsan_report_inline(origin, KMSAN_RET_ADDR); } msan_tls_t * __msan_get_context_state(void) { msan_td_t *mtd; /* * When APs are started, they execute some C code before curthread is * set. We have to handle that here. */ if (__predict_false(!kmsan_enabled || curthread == NULL)) return (&dummy_tls); mtd = curthread->td_kmsan; return (&mtd->tls[mtd->ctx]); } /* -------------------------------------------------------------------------- */ /* * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle * initialized areas properly. */ void * kmsan_memcpy(void *dst, const void *src, size_t len) { /* No kmsan_check_arg, because inlined. */ kmsan_init_ret(sizeof(void *)); if (__predict_true(len != 0)) { kmsan_meta_copy(dst, src, len); } return (__builtin_memcpy(dst, src, len)); } int kmsan_memcmp(const void *b1, const void *b2, size_t len) { const uint8_t *_b1 = b1, *_b2 = b2; size_t i; kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), "memcmp():args"); kmsan_init_ret(sizeof(int)); for (i = 0; i < len; i++) { if (*_b1 != *_b2) { kmsan_shadow_check((uintptr_t)b1, i + 1, "memcmp():arg1"); kmsan_shadow_check((uintptr_t)b2, i + 1, "memcmp():arg2"); return (*_b1 - *_b2); } _b1++, _b2++; } return (0); } void * kmsan_memset(void *dst, int c, size_t len) { /* No kmsan_check_arg, because inlined. */ kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len); kmsan_init_ret(sizeof(void *)); return (__builtin_memset(dst, c, len)); } void * kmsan_memmove(void *dst, const void *src, size_t len) { /* No kmsan_check_arg, because inlined. */ if (__predict_true(len != 0)) { kmsan_meta_copy(dst, src, len); } kmsan_init_ret(sizeof(void *)); return (__builtin_memmove(dst, src, len)); } __strong_reference(kmsan_memcpy, __msan_memcpy); __strong_reference(kmsan_memset, __msan_memset); __strong_reference(kmsan_memmove, __msan_memmove); char * kmsan_strcpy(char *dst, const char *src) { const char *_src = src; char *_dst = dst; size_t len = 0; kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); while (1) { len++; *dst = *src; if (*src == '\0') break; src++, dst++; } kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2"); kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len); kmsan_init_ret(sizeof(char *)); return (_dst); } int kmsan_strcmp(const char *s1, const char *s2) { const char *_s1 = s1, *_s2 = s2; size_t len = 0; kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); kmsan_init_ret(sizeof(int)); while (1) { len++; if (*s1 != *s2) break; if (*s1 == '\0') { kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); return (0); } s1++, s2++; } kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); return (*(const unsigned char *)s1 - *(const unsigned char *)s2); } size_t kmsan_strlen(const char *str) { const char *s; kmsan_check_arg(sizeof(str), "strlen():args"); s = str; while (1) { if (*s == '\0') break; s++; } kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1"); kmsan_init_ret(sizeof(size_t)); return (s - str); } int kmsan_copyin(const void *, void *, size_t); int kmsan_copyout(const void *, void *, size_t); int kmsan_copyinstr(const void *, void *, size_t, size_t *); int kmsan_copyin(const void *uaddr, void *kaddr, size_t len) { int ret; kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), "copyin():args"); ret = copyin(uaddr, kaddr, len); if (ret == 0) kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_copyout(const void *kaddr, void *uaddr, size_t len) { kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), "copyout():args"); kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1"); kmsan_init_ret(sizeof(int)); return (copyout(kaddr, uaddr, len)); } int kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) { size_t _done; int ret; kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len) + sizeof(done), "copyinstr():args"); ret = copyinstr(uaddr, kaddr, len, &_done); if (ret == 0) kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done); if (done != NULL) { *done = _done; kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t)); } kmsan_init_ret(sizeof(int)); return (ret); } /* -------------------------------------------------------------------------- */ int kmsan_fubyte(volatile const void *base) { int ret; kmsan_check_arg(sizeof(base), "fubyte(): args"); ret = fubyte(base); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_fuword16(volatile const void *base) { int ret; kmsan_check_arg(sizeof(base), "fuword16(): args"); ret = fuword16(base); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_fueword(volatile const void *base, long *val) { int ret; kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args"); ret = fueword(base, val); if (ret == 0) kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, sizeof(*val)); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_fueword32(volatile const void *base, int32_t *val) { int ret; kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args"); ret = fueword32(base, val); if (ret == 0) kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, sizeof(*val)); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_fueword64(volatile const void *base, int64_t *val) { int ret; kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args"); ret = fueword64(base, val); if (ret == 0) kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, sizeof(*val)); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_subyte(volatile void *base, int byte) { int ret; kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args"); ret = subyte(base, byte); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_suword(volatile void *base, long word) { int ret; kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args"); ret = suword(base, word); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_suword16(volatile void *base, int word) { int ret; kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args"); ret = suword16(base, word); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_suword32(volatile void *base, int32_t word) { int ret; kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args"); ret = suword32(base, word); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_suword64(volatile void *base, int64_t word) { int ret; kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args"); ret = suword64(base, word); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, uint32_t newval) { int ret; kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + sizeof(newval), "casueword32(): args"); ret = casueword32(base, oldval, oldvalp, newval); kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, sizeof(*oldvalp)); kmsan_init_ret(sizeof(int)); return (ret); } int kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval) { int ret; kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + sizeof(newval), "casueword32(): args"); ret = casueword(base, oldval, oldvalp, newval); kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, sizeof(*oldvalp)); kmsan_init_ret(sizeof(int)); return (ret); } /* -------------------------------------------------------------------------- */ #include #include #define _MSAN_ATOMIC_FUNC_ADD(name, type) \ void kmsan_atomic_add_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_add_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_add_" #name "():ptr"); \ atomic_add_##name(ptr, val); \ } #define MSAN_ATOMIC_FUNC_ADD(name, type) \ _MSAN_ATOMIC_FUNC_ADD(name, type) \ _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ _MSAN_ATOMIC_FUNC_ADD(rel_##name, type) #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_subtract_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_subtract_" #name "():ptr"); \ atomic_subtract_##name(ptr, val); \ } #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) #define _MSAN_ATOMIC_FUNC_SET(name, type) \ void kmsan_atomic_set_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_set_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_set_" #name "():ptr"); \ atomic_set_##name(ptr, val); \ } #define MSAN_ATOMIC_FUNC_SET(name, type) \ _MSAN_ATOMIC_FUNC_SET(name, type) \ _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \ _MSAN_ATOMIC_FUNC_SET(rel_##name, type) #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ void kmsan_atomic_clear_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_clear_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_clear_" #name "():ptr"); \ atomic_clear_##name(ptr, val); \ } #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \ _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \ type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_fetchadd_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_fetchadd_" #name "():ptr"); \ kmsan_init_ret(sizeof(type)); \ return (atomic_fetchadd_##name(ptr, val)); \ } #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ type kmsan_atomic_readandclear_##name(volatile type *ptr) \ { \ kmsan_check_arg(sizeof(ptr), \ "atomic_readandclear_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_readandclear_" #name "():ptr"); \ kmsan_init_ret(sizeof(type)); \ return (atomic_readandclear_##name(ptr)); \ } #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(v), \ "atomic_testandclear_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_testandclear_" #name "():ptr"); \ kmsan_init_ret(sizeof(int)); \ return (atomic_testandclear_##name(ptr, v)); \ } #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(v), \ "atomic_testandset_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_testandset_" #name "():ptr"); \ kmsan_init_ret(sizeof(int)); \ return (atomic_testandset_##name(ptr, v)); \ } #define MSAN_ATOMIC_FUNC_SWAP(name, type) \ type kmsan_atomic_swap_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_swap_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_swap_" #name "():ptr"); \ kmsan_init_ret(sizeof(type)); \ return (atomic_swap_##name(ptr, val)); \ } #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \ type nval) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ sizeof(nval), "atomic_cmpset_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_cmpset_" #name "():ptr"); \ kmsan_init_ret(sizeof(int)); \ return (atomic_cmpset_##name(ptr, oval, nval)); \ } #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \ _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ type nval) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ sizeof(nval), "atomic_fcmpset_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_fcmpset_" #name "():ptr"); \ kmsan_init_ret(sizeof(int)); \ return (atomic_fcmpset_##name(ptr, oval, nval)); \ } #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ void kmsan_atomic_thread_fence_##name(void) \ { \ atomic_thread_fence_##name(); \ } #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \ type kmsan_atomic_load_##name(volatile type *ptr) \ { \ kmsan_check_arg(sizeof(ptr), \ "atomic_load_" #name "():args"); \ kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ "atomic_load_" #name "():ptr"); \ kmsan_init_ret(sizeof(type)); \ return (atomic_load_##name(ptr)); \ } #define MSAN_ATOMIC_FUNC_LOAD(name, type) \ _MSAN_ATOMIC_FUNC_LOAD(name, type) \ _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type) #define _MSAN_ATOMIC_FUNC_STORE(name, type) \ void kmsan_atomic_store_##name(volatile type *ptr, type val) \ { \ kmsan_check_arg(sizeof(ptr) + sizeof(val), \ "atomic_store_" #name "():args"); \ kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \ sizeof(type)); \ atomic_store_##name(ptr, val); \ } #define MSAN_ATOMIC_FUNC_STORE(name, type) \ _MSAN_ATOMIC_FUNC_STORE(name, type) \ _MSAN_ATOMIC_FUNC_STORE(rel_##name, type) MSAN_ATOMIC_FUNC_ADD(8, uint8_t); MSAN_ATOMIC_FUNC_ADD(16, uint16_t); MSAN_ATOMIC_FUNC_ADD(32, uint32_t); MSAN_ATOMIC_FUNC_ADD(64, uint64_t); MSAN_ATOMIC_FUNC_ADD(int, u_int); MSAN_ATOMIC_FUNC_ADD(long, u_long); MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int); MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long); MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); MSAN_ATOMIC_FUNC_SET(8, uint8_t); MSAN_ATOMIC_FUNC_SET(16, uint16_t); MSAN_ATOMIC_FUNC_SET(32, uint32_t); MSAN_ATOMIC_FUNC_SET(64, uint64_t); MSAN_ATOMIC_FUNC_SET(int, u_int); MSAN_ATOMIC_FUNC_SET(long, u_long); MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t); MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t); MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t); MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t); MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t); MSAN_ATOMIC_FUNC_CLEAR(int, u_int); MSAN_ATOMIC_FUNC_CLEAR(long, u_long); MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); MSAN_ATOMIC_FUNC_FETCHADD(int, u_int); MSAN_ATOMIC_FUNC_FETCHADD(long, u_long); MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); MSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t); MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int); MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long); MSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t); MSAN_ATOMIC_FUNC_SWAP(32, uint32_t); MSAN_ATOMIC_FUNC_SWAP(64, uint64_t); MSAN_ATOMIC_FUNC_SWAP(int, u_int); MSAN_ATOMIC_FUNC_SWAP(long, u_long); MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t); MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t); MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t); MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t); MSAN_ATOMIC_FUNC_CMPSET(int, u_int); MSAN_ATOMIC_FUNC_CMPSET(long, u_long); MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); MSAN_ATOMIC_FUNC_FCMPSET(int, u_int); MSAN_ATOMIC_FUNC_FCMPSET(long, u_long); MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); _MSAN_ATOMIC_FUNC_LOAD(bool, bool); MSAN_ATOMIC_FUNC_LOAD(8, uint8_t); MSAN_ATOMIC_FUNC_LOAD(16, uint16_t); MSAN_ATOMIC_FUNC_LOAD(32, uint32_t); MSAN_ATOMIC_FUNC_LOAD(64, uint64_t); MSAN_ATOMIC_FUNC_LOAD(char, u_char); MSAN_ATOMIC_FUNC_LOAD(short, u_short); MSAN_ATOMIC_FUNC_LOAD(int, u_int); MSAN_ATOMIC_FUNC_LOAD(long, u_long); MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); _MSAN_ATOMIC_FUNC_STORE(bool, bool); MSAN_ATOMIC_FUNC_STORE(8, uint8_t); MSAN_ATOMIC_FUNC_STORE(16, uint16_t); MSAN_ATOMIC_FUNC_STORE(32, uint32_t); MSAN_ATOMIC_FUNC_STORE(64, uint64_t); MSAN_ATOMIC_FUNC_STORE(char, u_char); MSAN_ATOMIC_FUNC_STORE(short, u_short); MSAN_ATOMIC_FUNC_STORE(int, u_int); MSAN_ATOMIC_FUNC_STORE(long, u_long); MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); MSAN_ATOMIC_FUNC_THREAD_FENCE(acq); MSAN_ATOMIC_FUNC_THREAD_FENCE(rel); MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); void kmsan_atomic_interrupt_fence(void) { atomic_interrupt_fence(); } /* -------------------------------------------------------------------------- */ #include #include #include int kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, int flags, bus_space_handle_t *handlep) { return (bus_space_map(tag, hnd, size, flags, handlep)); } void kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_unmap(tag, hnd, size); } int kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) { return (bus_space_subregion(tag, hnd, offset, size, handlep)); } void kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_free(tag, hnd, size); } void kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, int flags) { bus_space_barrier(tag, hnd, offset, size, flags); } /* XXXMJ x86-specific */ #define MSAN_BUS_READ_FUNC(func, width, type) \ type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset) \ { \ type ret; \ if ((tag) != X86_BUS_SPACE_IO) \ kmsan_shadow_fill((uintptr_t)(hnd + offset), \ KMSAN_STATE_INITED, (width)); \ ret = bus_space_read##func##_##width(tag, hnd, offset); \ kmsan_init_ret(sizeof(type)); \ return (ret); \ } \ #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \ void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, type *buf, \ bus_size_t count) \ { \ kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \ (width) * count); \ bus_space_read_##func##_##width(tag, hnd, size, buf, \ count); \ } MSAN_BUS_READ_FUNC(, 1, uint8_t) MSAN_BUS_READ_FUNC(_stream, 1, uint8_t) MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) MSAN_BUS_READ_FUNC(, 2, uint16_t) MSAN_BUS_READ_FUNC(_stream, 2, uint16_t) MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) MSAN_BUS_READ_FUNC(, 4, uint32_t) MSAN_BUS_READ_FUNC(_stream, 4, uint32_t) MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) MSAN_BUS_READ_FUNC(, 8, uint64_t) #define MSAN_BUS_WRITE_FUNC(func, width, type) \ void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ bus_space_write##func##_##width(tag, hnd, offset, value);\ } \ #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ bus_space_handle_t hnd, bus_size_t size, const type *buf, \ bus_size_t count) \ { \ kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ "bus_space_write()"); \ bus_space_write_##func##_##width(tag, hnd, size, buf, \ count); \ } MSAN_BUS_WRITE_FUNC(, 1, uint8_t) MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) MSAN_BUS_WRITE_FUNC(, 2, uint16_t) MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) MSAN_BUS_WRITE_FUNC(, 4, uint32_t) MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) MSAN_BUS_WRITE_FUNC(, 8, uint64_t) #define MSAN_BUS_SET_FUNC(func, width, type) \ void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value, \ bus_size_t count) \ { \ bus_space_set_##func##_##width(tag, hnd, offset, value, \ count); \ } MSAN_BUS_SET_FUNC(multi, 1, uint8_t) MSAN_BUS_SET_FUNC(region, 1, uint8_t) MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) MSAN_BUS_SET_FUNC(multi, 2, uint16_t) MSAN_BUS_SET_FUNC(region, 2, uint16_t) MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) MSAN_BUS_SET_FUNC(multi, 4, uint32_t) MSAN_BUS_SET_FUNC(region, 4, uint32_t) MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) /* -------------------------------------------------------------------------- */ void kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op) { /* * Some drivers, e.g., nvme, use the same code path for loading device * read and write requests, and will thus specify both flags. In this * case we should not do any checking since it will generally lead to * false positives. */ if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) == BUS_DMASYNC_PREWRITE) { switch (desc->md_type) { case MEMDESC_VADDR: kmsan_check(desc->u.md_vaddr, desc->md_len, "dmasync"); break; case MEMDESC_MBUF: kmsan_check_mbuf(desc->u.md_mbuf, "dmasync"); break; case 0: break; default: kmsan_panic("%s: unhandled memdesc type %d", __func__, desc->md_type); } } if ((op & BUS_DMASYNC_POSTREAD) != 0) { switch (desc->md_type) { case MEMDESC_VADDR: kmsan_mark(desc->u.md_vaddr, desc->md_len, KMSAN_STATE_INITED); break; case MEMDESC_MBUF: kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED); break; case 0: break; default: kmsan_panic("%s: unhandled memdesc type %d", __func__, desc->md_type); } } } diff --git a/sys/sys/proc.h b/sys/sys/proc.h index 3815571cd46f..c43d00a223f7 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -1,1364 +1,1365 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1986, 1989, 1991, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)proc.h 8.15 (Berkeley) 5/19/95 */ #ifndef _SYS_PROC_H_ #define _SYS_PROC_H_ #include /* For struct callout. */ #include /* For struct klist. */ #ifdef _KERNEL #include #endif #include #ifndef _KERNEL #include #endif #include #include #include #include #include #include #include /* XXX. */ #include #include #include #include #include #ifndef _KERNEL #include /* For structs itimerval, timeval. */ #else #include #include #endif #include #include #include #include #include /* Machine-dependent proc substruct. */ #ifdef _KERNEL #include #endif /* * One structure allocated per session. * * List of locks * (m) locked by s_mtx mtx * (e) locked by proctree_lock sx * (c) const until freeing */ struct session { u_int s_count; /* Ref cnt; pgrps in session - atomic. */ struct proc *s_leader; /* (m + e) Session leader. */ struct vnode *s_ttyvp; /* (m) Vnode of controlling tty. */ struct cdev_priv *s_ttydp; /* (m) Device of controlling tty. */ struct tty *s_ttyp; /* (e) Controlling tty. */ pid_t s_sid; /* (c) Session ID. */ /* (m) Setlogin() name: */ char s_login[roundup(MAXLOGNAME, sizeof(long))]; struct mtx s_mtx; /* Mutex to protect members. */ }; /* * One structure allocated per process group. * * List of locks * (m) locked by pg_mtx mtx * (e) locked by proctree_lock sx * (c) const until freeing */ struct pgrp { LIST_ENTRY(pgrp) pg_hash; /* (e) Hash chain. */ LIST_HEAD(, proc) pg_members; /* (m + e) Pointer to pgrp members. */ struct session *pg_session; /* (c) Pointer to session. */ struct sigiolst pg_sigiolst; /* (m) List of sigio sources. */ pid_t pg_id; /* (c) Process group id. */ struct mtx pg_mtx; /* Mutex to protect members */ int pg_flags; /* (m) PGRP_ flags */ struct sx pg_killsx; /* Mutual exclusion between group member * fork() and killpg() */ }; #define PGRP_ORPHANED 0x00000001 /* Group is orphaned */ /* * pargs, used to hold a copy of the command line, if it had a sane length. */ struct pargs { u_int ar_ref; /* Reference count. */ u_int ar_length; /* Length. */ u_char ar_args[1]; /* Arguments. */ }; /*- * Description of a process. * * This structure contains the information needed to manage a thread of * control, known in UN*X as a process; it has references to substructures * containing descriptions of things that the process uses, but may share * with related processes. The process structure and the substructures * are always addressable except for those marked "(CPU)" below, * which might be addressable only on a processor on which the process * is running. * * Below is a key of locks used to protect each member of struct proc. The * lock is indicated by a reference to a specific character in parens in the * associated comment. * * - not yet protected * a - only touched by curproc or parent during fork/wait * b - created at fork, never changes * (exception aiods switch vmspaces, but they are also * marked 'P_SYSTEM' so hopefully it will be left alone) * c - locked by proc mtx * d - locked by allproc_lock lock * e - locked by proctree_lock lock * f - session mtx * g - process group mtx * h - callout_lock mtx * i - by curproc or the master session mtx * j - locked by proc slock * k - only accessed by curthread * k*- only accessed by curthread and from an interrupt * kx- only accessed by curthread and by debugger * l - the attaching proc or attaching proc parent * n - not locked, lazy * o - ktrace lock * q - td_contested lock * r - p_peers lock * s - see sleepq_switch(), sleeping_on_old_rtc(), and sleep(9) * t - thread lock * u - process stat lock * w - process timer lock * x - created at fork, only changes during single threading in exec * y - created at first aio, doesn't change until exit or exec at which * point we are single-threaded and only curthread changes it * * If the locking key specifies two identifiers (for example, p_pptr) then * either lock is sufficient for read access, but both locks must be held * for write access. */ struct cpuset; struct filecaps; struct filemon; struct kaioinfo; struct kaudit_record; struct kcov_info; struct kdtrace_proc; struct kdtrace_thread; struct kmsan_td; struct kq_timer_cb_data; struct mqueue_notifier; struct p_sched; struct proc; struct procdesc; struct racct; struct sbuf; struct sleepqueue; struct socket; struct td_sched; struct thread; struct trapframe; struct turnstile; struct vm_map; struct vm_map_entry; struct epoch_tracker; struct syscall_args { u_int code; u_int original_code; struct sysent *callp; register_t args[8]; }; /* * XXX: Does this belong in resource.h or resourcevar.h instead? * Resource usage extension. The times in rusage structs in the kernel are * never up to date. The actual times are kept as runtimes and tick counts * (with control info in the "previous" times), and are converted when * userland asks for rusage info. Backwards compatibility prevents putting * this directly in the user-visible rusage struct. * * Locking for p_rux: (cu) means (u) for p_rux and (c) for p_crux. * Locking for td_rux: (t) for all fields. */ struct rusage_ext { uint64_t rux_runtime; /* (cu) Real time. */ uint64_t rux_uticks; /* (cu) Statclock hits in user mode. */ uint64_t rux_sticks; /* (cu) Statclock hits in sys mode. */ uint64_t rux_iticks; /* (cu) Statclock hits in intr mode. */ uint64_t rux_uu; /* (c) Previous user time in usec. */ uint64_t rux_su; /* (c) Previous sys time in usec. */ uint64_t rux_tu; /* (c) Previous total time in usec. */ }; /* * Kernel runnable context (thread). * This is what is put to sleep and reactivated. * Thread context. Processes may have multiple threads. */ struct thread { struct mtx *volatile td_lock; /* replaces sched lock */ struct proc *td_proc; /* (*) Associated process. */ TAILQ_ENTRY(thread) td_plist; /* (*) All threads in this proc. */ TAILQ_ENTRY(thread) td_runq; /* (t) Run queue. */ union { TAILQ_ENTRY(thread) td_slpq; /* (t) Sleep queue. */ struct thread *td_zombie; /* Zombie list linkage */ }; TAILQ_ENTRY(thread) td_lockq; /* (t) Lock queue. */ LIST_ENTRY(thread) td_hash; /* (d) Hash chain. */ struct cpuset *td_cpuset; /* (t) CPU affinity mask. */ struct domainset_ref td_domain; /* (a) NUMA policy */ struct seltd *td_sel; /* Select queue/channel. */ struct sleepqueue *td_sleepqueue; /* (k) Associated sleep queue. */ struct turnstile *td_turnstile; /* (k) Associated turnstile. */ struct rl_q_entry *td_rlqe; /* (k) Associated range lock entry. */ struct umtx_q *td_umtxq; /* (c?) Link for when we're blocked. */ lwpid_t td_tid; /* (b) Thread ID. */ sigqueue_t td_sigqueue; /* (c) Sigs arrived, not delivered. */ #define td_siglist td_sigqueue.sq_signals u_char td_lend_user_pri; /* (t) Lend user pri. */ u_char td_allocdomain; /* (b) NUMA domain backing this struct thread. */ u_char td_base_ithread_pri; /* (t) Base ithread pri */ struct kmsan_td *td_kmsan; /* (k) KMSAN state */ /* Cleared during fork1(), thread_create(), or kthread_add(). */ #define td_startzero td_flags int td_flags; /* (t) TDF_* flags. */ int td_ast; /* (t) TDA_* indicators */ int td_inhibitors; /* (t) Why can not run. */ int td_pflags; /* (k) Private thread (TDP_*) flags. */ int td_pflags2; /* (k) Private thread (TDP2_*) flags. */ int td_dupfd; /* (k) Ret value from fdopen. XXX */ int td_sqqueue; /* (t) Sleepqueue queue blocked on. */ const void *td_wchan; /* (t) Sleep address. */ const char *td_wmesg; /* (t) Reason for sleep. */ volatile u_char td_owepreempt; /* (k*) Preempt on last critical_exit */ u_char td_tsqueue; /* (t) Turnstile queue blocked on. */ u_char td_stopsched; /* (k) Scheduler stopped. */ int td_locks; /* (k) Debug: count of non-spin locks */ int td_rw_rlocks; /* (k) Count of rwlock read locks. */ int td_sx_slocks; /* (k) Count of sx shared locks. */ int td_lk_slocks; /* (k) Count of lockmgr shared locks. */ struct turnstile *td_blocked; /* (t) Lock thread is blocked on. */ const char *td_lockname; /* (t) Name of lock blocked on. */ LIST_HEAD(, turnstile) td_contested; /* (q) Contested locks. */ struct lock_list_entry *td_sleeplocks; /* (k) Held sleep locks. */ int td_intr_nesting_level; /* (k) Interrupt recursion. */ int td_pinned; /* (k) Temporary cpu pin count. */ struct ucred *td_realucred; /* (k) Reference to credentials. */ struct ucred *td_ucred; /* (k) Used credentials, temporarily switchable. */ struct plimit *td_limit; /* (k) Resource limits. */ int td_slptick; /* (t) Time at sleep. */ int td_blktick; /* (t) Time spent blocked. */ int td_swvoltick; /* (t) Time at last SW_VOL switch. */ int td_swinvoltick; /* (t) Time at last SW_INVOL switch. */ u_int td_cow; /* (*) Number of copy-on-write faults */ struct rusage td_ru; /* (t) rusage information. */ struct rusage_ext td_rux; /* (t) Internal rusage information. */ uint64_t td_incruntime; /* (t) Cpu ticks to transfer to proc. */ uint64_t td_runtime; /* (t) How many cpu ticks we've run. */ u_int td_pticks; /* (t) Statclock hits for profiling */ u_int td_sticks; /* (t) Statclock hits in system mode. */ u_int td_iticks; /* (t) Statclock hits in intr mode. */ u_int td_uticks; /* (t) Statclock hits in user mode. */ int td_intrval; /* (t) Return value for sleepq. */ sigset_t td_oldsigmask; /* (k) Saved mask from pre sigpause. */ volatile u_int td_generation; /* (k) For detection of preemption */ stack_t td_sigstk; /* (k) Stack ptr and on-stack flag. */ int td_xsig; /* (c) Signal for ptrace */ u_long td_profil_addr; /* (k) Temporary addr until AST. */ u_int td_profil_ticks; /* (k) Temporary ticks until AST. */ char td_name[MAXCOMLEN + 1]; /* (*) Thread name. */ struct file *td_fpop; /* (k) file referencing cdev under op */ int td_dbgflags; /* (c) Userland debugger flags */ siginfo_t td_si; /* (c) For debugger or core file */ int td_ng_outbound; /* (k) Thread entered ng from above. */ struct osd td_osd; /* (k) Object specific data. */ struct vm_map_entry *td_map_def_user; /* (k) Deferred entries. */ pid_t td_dbg_forked; /* (c) Child pid for debugger. */ struct vnode *td_vp_reserved;/* (k) Preallocated vnode. */ u_int td_no_sleeping; /* (k) Sleeping disabled count. */ void *td_su; /* (k) FFS SU private */ sbintime_t td_sleeptimo; /* (t) Sleep timeout. */ int td_rtcgen; /* (s) rtc_generation of abs. sleep */ int td_errno; /* (k) Error from last syscall. */ size_t td_vslock_sz; /* (k) amount of vslock-ed space */ struct kcov_info *td_kcov_info; /* (*) Kernel code coverage data */ long td_ucredref; /* (k) references on td_realucred */ #define td_endzero td_sigmask /* Copied during fork1(), thread_create(), or kthread_add(). */ #define td_startcopy td_endzero sigset_t td_sigmask; /* (c) Current signal mask. */ u_char td_rqindex; /* (t) Run queue index. */ u_char td_base_pri; /* (t) Thread base kernel priority. */ u_char td_priority; /* (t) Thread active priority. */ u_char td_pri_class; /* (t) Scheduling class. */ u_char td_user_pri; /* (t) User pri from estcpu and nice. */ u_char td_base_user_pri; /* (t) Base user pri */ uintptr_t td_rb_list; /* (k) Robust list head. */ uintptr_t td_rbp_list; /* (k) Robust priv list head. */ uintptr_t td_rb_inact; /* (k) Current in-action mutex loc. */ struct syscall_args td_sa; /* (kx) Syscall parameters. Copied on fork for child tracing. */ void *td_sigblock_ptr; /* (k) uptr for fast sigblock. */ uint32_t td_sigblock_val; /* (k) fast sigblock value read at td_sigblock_ptr on kern entry */ #define td_endcopy td_pcb /* * Fields that must be manually set in fork1(), thread_create(), kthread_add(), * or already have been set in the allocator, constructor, etc. */ struct pcb *td_pcb; /* (k) Kernel VA of pcb and kstack. */ enum td_states { TDS_INACTIVE = 0x0, TDS_INHIBITED, TDS_CAN_RUN, TDS_RUNQ, TDS_RUNNING } td_state; /* (t) thread state */ /* Note: td_state must be accessed using TD_{GET,SET}_STATE(). */ union { syscallarg_t tdu_retval[2]; off_t tdu_off; } td_uretoff; /* (k) Syscall aux returns. */ #define td_retval td_uretoff.tdu_retval u_int td_cowgen; /* (k) Generation of COW pointers. */ /* LP64 hole */ struct callout td_slpcallout; /* (h) Callout for sleep. */ struct trapframe *td_frame; /* (k) */ vm_offset_t td_kstack; /* (a) Kernel VA of kstack. */ int td_kstack_pages; /* (a) Size of the kstack. */ volatile u_int td_critnest; /* (k*) Critical section nest level. */ struct mdthread td_md; /* (k) Any machine-dependent fields. */ struct kaudit_record *td_ar; /* (k) Active audit record, if any. */ struct lpohead td_lprof[2]; /* (a) lock profiling objects. */ struct kdtrace_thread *td_dtrace; /* (*) DTrace-specific data. */ struct vnet *td_vnet; /* (k) Effective vnet. */ const char *td_vnet_lpush; /* (k) Debugging vnet push / pop. */ struct trapframe *td_intr_frame;/* (k) Frame of the current irq */ struct proc *td_rfppwait_p; /* (k) The vforked child */ struct vm_page **td_ma; /* (k) uio pages held */ int td_ma_cnt; /* (k) size of *td_ma */ /* LP64 hole */ void *td_emuldata; /* Emulator state data */ int td_lastcpu; /* (t) Last cpu we were on. */ int td_oncpu; /* (t) Which cpu we are on. */ void *td_lkpi_task; /* LinuxKPI task struct pointer */ int td_pmcpend; void *td_remotereq; /* (c) dbg remote request. */ off_t td_ktr_io_lim; /* (k) limit for ktrace file size */ #ifdef EPOCH_TRACE SLIST_HEAD(, epoch_tracker) td_epochs; #endif }; struct thread0_storage { struct thread t0st_thread; uint64_t t0st_sched[10]; }; struct mtx *thread_lock_block(struct thread *); void thread_lock_block_wait(struct thread *); void thread_lock_set(struct thread *, struct mtx *); void thread_lock_unblock(struct thread *, struct mtx *); #define THREAD_LOCK_ASSERT(td, type) \ mtx_assert((td)->td_lock, (type)) #define THREAD_LOCK_BLOCKED_ASSERT(td, type) \ do { \ struct mtx *__m = (td)->td_lock; \ if (__m != &blocked_lock) \ mtx_assert(__m, (type)); \ } while (0) #ifdef INVARIANTS #define THREAD_LOCKPTR_ASSERT(td, lock) \ do { \ struct mtx *__m; \ __m = (td)->td_lock; \ KASSERT(__m == (lock), \ ("Thread %p lock %p does not match %p", td, __m, (lock))); \ } while (0) #define THREAD_LOCKPTR_BLOCKED_ASSERT(td, lock) \ do { \ struct mtx *__m; \ __m = (td)->td_lock; \ KASSERT(__m == (lock) || __m == &blocked_lock, \ ("Thread %p lock %p does not match %p", td, __m, (lock))); \ } while (0) #define TD_LOCKS_INC(td) ((td)->td_locks++) #define TD_LOCKS_DEC(td) do { \ KASSERT(SCHEDULER_STOPPED_TD(td) || (td)->td_locks > 0, \ ("Thread %p owns no locks", (td))); \ (td)->td_locks--; \ } while (0) #else #define THREAD_LOCKPTR_ASSERT(td, lock) #define THREAD_LOCKPTR_BLOCKED_ASSERT(td, lock) #define TD_LOCKS_INC(td) #define TD_LOCKS_DEC(td) #endif /* * Flags kept in td_flags: * To change these you MUST have the scheduler lock. */ #define TDF_BORROWING 0x00000001 /* Thread is borrowing pri from another. */ #define TDF_INPANIC 0x00000002 /* Caused a panic, let it drive crashdump. */ #define TDF_INMEM 0x00000004 /* Thread's stack is in memory. */ #define TDF_SINTR 0x00000008 /* Sleep is interruptible. */ #define TDF_TIMEOUT 0x00000010 /* Timing out during sleep. */ #define TDF_IDLETD 0x00000020 /* This is a per-CPU idle thread. */ #define TDF_CANSWAP 0x00000040 /* Thread can be swapped. */ #define TDF_SIGWAIT 0x00000080 /* Ignore ignored signals */ #define TDF_KTH_SUSP 0x00000100 /* kthread is suspended */ #define TDF_ALLPROCSUSP 0x00000200 /* suspended by SINGLE_ALLPROC */ #define TDF_BOUNDARY 0x00000400 /* Thread suspended at user boundary */ #define TDF_UNUSED1 0x00000800 /* Available */ #define TDF_UNUSED2 0x00001000 /* Available */ #define TDF_SBDRY 0x00002000 /* Stop only on usermode boundary. */ #define TDF_UPIBLOCKED 0x00004000 /* Thread blocked on user PI mutex. */ #define TDF_UNUSED3 0x00008000 /* Available */ #define TDF_UNUSED4 0x00010000 /* Available */ #define TDF_UNUSED5 0x00020000 /* Available */ #define TDF_NOLOAD 0x00040000 /* Ignore during load avg calculations. */ #define TDF_SERESTART 0x00080000 /* ERESTART on stop attempts. */ #define TDF_THRWAKEUP 0x00100000 /* Libthr thread must not suspend itself. */ #define TDF_SEINTR 0x00200000 /* EINTR on stop attempts. */ #define TDF_SWAPINREQ 0x00400000 /* Swapin request due to wakeup. */ #define TDF_UNUSED6 0x00800000 /* Available */ #define TDF_SCHED0 0x01000000 /* Reserved for scheduler private use */ #define TDF_SCHED1 0x02000000 /* Reserved for scheduler private use */ #define TDF_SCHED2 0x04000000 /* Reserved for scheduler private use */ #define TDF_SCHED3 0x08000000 /* Reserved for scheduler private use */ #define TDF_UNUSED7 0x10000000 /* Available */ #define TDF_UNUSED8 0x20000000 /* Available */ #define TDF_UNUSED9 0x40000000 /* Available */ #define TDF_UNUSED10 0x80000000 /* Available */ enum { TDA_AST = 0, /* Special: call all non-flagged AST handlers */ TDA_OWEUPC, TDA_HWPMC, TDA_VFORK, TDA_ALRM, TDA_PROF, TDA_MAC, TDA_SCHED, TDA_UFS, TDA_GEOM, TDA_KQUEUE, TDA_RACCT, TDA_MOD1, /* For third party use, before signals are */ TAD_MOD2, /* processed .. */ TDA_SIG, TDA_KTRACE, TDA_SUSPEND, TDA_SIGSUSPEND, TDA_MOD3, /* .. and after */ TAD_MOD4, TDA_MAX, }; #define TDAI(tda) (1U << (tda)) #define td_ast_pending(td, tda) ((td->td_ast & TDAI(tda)) != 0) /* Userland debug flags */ #define TDB_SUSPEND 0x00000001 /* Thread is suspended by debugger */ #define TDB_XSIG 0x00000002 /* Thread is exchanging signal under trace */ #define TDB_USERWR 0x00000004 /* Debugger modified memory or registers */ #define TDB_SCE 0x00000008 /* Thread performs syscall enter */ #define TDB_SCX 0x00000010 /* Thread performs syscall exit */ #define TDB_EXEC 0x00000020 /* TDB_SCX from exec(2) family */ #define TDB_FORK 0x00000040 /* TDB_SCX from fork(2) that created new process */ #define TDB_STOPATFORK 0x00000080 /* Stop at the return from fork (child only) */ #define TDB_CHILD 0x00000100 /* New child indicator for ptrace() */ #define TDB_BORN 0x00000200 /* New LWP indicator for ptrace() */ #define TDB_EXIT 0x00000400 /* Exiting LWP indicator for ptrace() */ #define TDB_VFORK 0x00000800 /* vfork indicator for ptrace() */ #define TDB_FSTP 0x00001000 /* The thread is PT_ATTACH leader */ #define TDB_STEP 0x00002000 /* (x86) PSL_T set for PT_STEP */ #define TDB_SSWITCH 0x00004000 /* Suspended in ptracestop */ #define TDB_BOUNDARY 0x00008000 /* ptracestop() at boundary */ #define TDB_COREDUMPREQ 0x00010000 /* Coredump request */ #define TDB_SCREMOTEREQ 0x00020000 /* Remote syscall request */ /* * "Private" flags kept in td_pflags: * These are only written by curthread and thus need no locking. */ #define TDP_OLDMASK 0x00000001 /* Need to restore mask after suspend. */ #define TDP_INKTR 0x00000002 /* Thread is currently in KTR code. */ #define TDP_INKTRACE 0x00000004 /* Thread is currently in KTRACE code. */ #define TDP_BUFNEED 0x00000008 /* Do not recurse into the buf flush */ #define TDP_COWINPROGRESS 0x00000010 /* Snapshot copy-on-write in progress. */ #define TDP_ALTSTACK 0x00000020 /* Have alternate signal stack. */ #define TDP_DEADLKTREAT 0x00000040 /* Lock acquisition - deadlock treatment. */ #define TDP_NOFAULTING 0x00000080 /* Do not handle page faults. */ #define TDP_SIGFASTBLOCK 0x00000100 /* Fast sigblock active */ #define TDP_OWEUPC 0x00000200 /* Call addupc() at next AST. */ #define TDP_ITHREAD 0x00000400 /* Thread is an interrupt thread. */ #define TDP_SYNCIO 0x00000800 /* Local override, disable async i/o. */ #define TDP_SCHED1 0x00001000 /* Reserved for scheduler private use */ #define TDP_SCHED2 0x00002000 /* Reserved for scheduler private use */ #define TDP_SCHED3 0x00004000 /* Reserved for scheduler private use */ #define TDP_SCHED4 0x00008000 /* Reserved for scheduler private use */ #define TDP_GEOM 0x00010000 /* Settle GEOM before finishing syscall */ #define TDP_SOFTDEP 0x00020000 /* Stuck processing softdep worklist */ #define TDP_NORUNNINGBUF 0x00040000 /* Ignore runningbufspace check */ #define TDP_WAKEUP 0x00080000 /* Don't sleep in umtx cond_wait */ #define TDP_INBDFLUSH 0x00100000 /* Already in BO_BDFLUSH, do not recurse */ #define TDP_KTHREAD 0x00200000 /* This is an official kernel thread */ #define TDP_CALLCHAIN 0x00400000 /* Capture thread's callchain */ #define TDP_IGNSUSP 0x00800000 /* Permission to ignore the MNTK_SUSPEND* */ #define TDP_AUDITREC 0x01000000 /* Audit record pending on thread */ #define TDP_RFPPWAIT 0x02000000 /* Handle RFPPWAIT on syscall exit */ #define TDP_RESETSPUR 0x04000000 /* Reset spurious page fault history. */ #define TDP_NERRNO 0x08000000 /* Last errno is already in td_errno */ #define TDP_UIOHELD 0x10000000 /* Current uio has pages held in td_ma */ #define TDP_INTCPCALLOUT 0x20000000 /* used by netinet/tcp_timer.c */ #define TDP_EXECVMSPC 0x40000000 /* Execve destroyed old vmspace */ #define TDP_SIGFASTPENDING 0x80000000 /* Pending signal due to sigfastblock */ #define TDP2_SBPAGES 0x00000001 /* Owns sbusy on some pages */ #define TDP2_COMPAT32RB 0x00000002 /* compat32 ABI for robust lists */ #define TDP2_ACCT 0x00000004 /* Doing accounting */ #define TDP2_EFIRT 0x20000000 /* In firmware (EFI RT) call */ +#define TDP2_SAN_QUIET 0x00000008 /* Disable warnings from K(A|M)SAN */ /* * Reasons that the current thread can not be run yet. * More than one may apply. */ #define TDI_SUSPENDED 0x0001 /* On suspension queue. */ #define TDI_SLEEPING 0x0002 /* Actually asleep! (tricky). */ #define TDI_SWAPPED 0x0004 /* Stack not in mem. Bad juju if run. */ #define TDI_LOCK 0x0008 /* Stopped on a lock. */ #define TDI_IWAIT 0x0010 /* Awaiting interrupt. */ #define TD_IS_SLEEPING(td) ((td)->td_inhibitors & TDI_SLEEPING) #define TD_ON_SLEEPQ(td) ((td)->td_wchan != NULL) #define TD_IS_SUSPENDED(td) ((td)->td_inhibitors & TDI_SUSPENDED) #define TD_IS_SWAPPED(td) ((td)->td_inhibitors & TDI_SWAPPED) #define TD_ON_LOCK(td) ((td)->td_inhibitors & TDI_LOCK) #define TD_AWAITING_INTR(td) ((td)->td_inhibitors & TDI_IWAIT) #ifdef _KERNEL #define TD_GET_STATE(td) atomic_load_int(&(td)->td_state) #else #define TD_GET_STATE(td) ((td)->td_state) #endif #define TD_IS_RUNNING(td) (TD_GET_STATE(td) == TDS_RUNNING) #define TD_ON_RUNQ(td) (TD_GET_STATE(td) == TDS_RUNQ) #define TD_CAN_RUN(td) (TD_GET_STATE(td) == TDS_CAN_RUN) #define TD_IS_INHIBITED(td) (TD_GET_STATE(td) == TDS_INHIBITED) #define TD_ON_UPILOCK(td) ((td)->td_flags & TDF_UPIBLOCKED) #define TD_IS_IDLETHREAD(td) ((td)->td_flags & TDF_IDLETD) #define TD_CAN_ABORT(td) (TD_ON_SLEEPQ((td)) && \ ((td)->td_flags & TDF_SINTR) != 0) #define KTDSTATE(td) \ (((td)->td_inhibitors & TDI_SLEEPING) != 0 ? "sleep" : \ ((td)->td_inhibitors & TDI_SUSPENDED) != 0 ? "suspended" : \ ((td)->td_inhibitors & TDI_SWAPPED) != 0 ? "swapped" : \ ((td)->td_inhibitors & TDI_LOCK) != 0 ? "blocked" : \ ((td)->td_inhibitors & TDI_IWAIT) != 0 ? "iwait" : "yielding") #define TD_SET_INHIB(td, inhib) do { \ TD_SET_STATE(td, TDS_INHIBITED); \ (td)->td_inhibitors |= (inhib); \ } while (0) #define TD_CLR_INHIB(td, inhib) do { \ if (((td)->td_inhibitors & (inhib)) && \ (((td)->td_inhibitors &= ~(inhib)) == 0)) \ TD_SET_STATE(td, TDS_CAN_RUN); \ } while (0) #define TD_SET_SLEEPING(td) TD_SET_INHIB((td), TDI_SLEEPING) #define TD_SET_SWAPPED(td) TD_SET_INHIB((td), TDI_SWAPPED) #define TD_SET_LOCK(td) TD_SET_INHIB((td), TDI_LOCK) #define TD_SET_SUSPENDED(td) TD_SET_INHIB((td), TDI_SUSPENDED) #define TD_SET_IWAIT(td) TD_SET_INHIB((td), TDI_IWAIT) #define TD_SET_EXITING(td) TD_SET_INHIB((td), TDI_EXITING) #define TD_CLR_SLEEPING(td) TD_CLR_INHIB((td), TDI_SLEEPING) #define TD_CLR_SWAPPED(td) TD_CLR_INHIB((td), TDI_SWAPPED) #define TD_CLR_LOCK(td) TD_CLR_INHIB((td), TDI_LOCK) #define TD_CLR_SUSPENDED(td) TD_CLR_INHIB((td), TDI_SUSPENDED) #define TD_CLR_IWAIT(td) TD_CLR_INHIB((td), TDI_IWAIT) #ifdef _KERNEL #define TD_SET_STATE(td, state) atomic_store_int(&(td)->td_state, state) #else #define TD_SET_STATE(td, state) (td)->td_state = state #endif #define TD_SET_RUNNING(td) TD_SET_STATE(td, TDS_RUNNING) #define TD_SET_RUNQ(td) TD_SET_STATE(td, TDS_RUNQ) #define TD_SET_CAN_RUN(td) TD_SET_STATE(td, TDS_CAN_RUN) #define TD_SBDRY_INTR(td) \ (((td)->td_flags & (TDF_SEINTR | TDF_SERESTART)) != 0) #define TD_SBDRY_ERRNO(td) \ (((td)->td_flags & TDF_SEINTR) != 0 ? EINTR : ERESTART) /* * Process structure. */ struct proc { LIST_ENTRY(proc) p_list; /* (d) List of all processes. */ TAILQ_HEAD(, thread) p_threads; /* (c) all threads. */ struct mtx p_slock; /* process spin lock */ struct ucred *p_ucred; /* (c) Process owner's identity. */ struct filedesc *p_fd; /* (b) Open files. */ struct filedesc_to_leader *p_fdtol; /* (b) Tracking node */ struct pwddesc *p_pd; /* (b) Cwd, chroot, jail, umask */ struct pstats *p_stats; /* (b) Accounting/statistics (CPU). */ struct plimit *p_limit; /* (c) Resource limits. */ struct callout p_limco; /* (c) Limit callout handle */ struct sigacts *p_sigacts; /* (x) Signal actions, state (CPU). */ int p_flag; /* (c) P_* flags. */ int p_flag2; /* (c) P2_* flags. */ enum p_states { PRS_NEW = 0, /* In creation */ PRS_NORMAL, /* threads can be run. */ PRS_ZOMBIE } p_state; /* (j/c) Process status. */ pid_t p_pid; /* (b) Process identifier. */ LIST_ENTRY(proc) p_hash; /* (d) Hash chain. */ LIST_ENTRY(proc) p_pglist; /* (g + e) List of processes in pgrp. */ struct proc *p_pptr; /* (c + e) Pointer to parent process. */ LIST_ENTRY(proc) p_sibling; /* (e) List of sibling processes. */ LIST_HEAD(, proc) p_children; /* (e) Pointer to list of children. */ struct proc *p_reaper; /* (e) My reaper. */ LIST_HEAD(, proc) p_reaplist; /* (e) List of my descendants (if I am reaper). */ LIST_ENTRY(proc) p_reapsibling; /* (e) List of siblings - descendants of the same reaper. */ struct mtx p_mtx; /* (n) Lock for this struct. */ struct mtx p_statmtx; /* Lock for the stats */ struct mtx p_itimmtx; /* Lock for the virt/prof timers */ struct mtx p_profmtx; /* Lock for the profiling */ struct ksiginfo *p_ksi; /* Locked by parent proc lock */ sigqueue_t p_sigqueue; /* (c) Sigs not delivered to a td. */ #define p_siglist p_sigqueue.sq_signals pid_t p_oppid; /* (c + e) Real parent pid. */ /* The following fields are all zeroed upon creation in fork. */ #define p_startzero p_vmspace struct vmspace *p_vmspace; /* (b) Address space. */ u_int p_swtick; /* (c) Tick when swapped in or out. */ u_int p_cowgen; /* (c) Generation of COW pointers. */ struct itimerval p_realtimer; /* (c) Alarm timer. */ struct rusage p_ru; /* (a) Exit information. */ struct rusage_ext p_rux; /* (cu) Internal resource usage. */ struct rusage_ext p_crux; /* (c) Internal child resource usage. */ int p_profthreads; /* (c) Num threads in addupc_task. */ volatile int p_exitthreads; /* (j) Number of threads exiting */ int p_traceflag; /* (o) Kernel trace points. */ struct ktr_io_params *p_ktrioparms; /* (c + o) Params for ktrace. */ struct vnode *p_textvp; /* (b) Vnode of executable. */ struct vnode *p_textdvp; /* (b) Dir containing textvp. */ char *p_binname; /* (b) Binary hardlink name. */ u_int p_lock; /* (c) Proclock (prevent swap) count. */ struct sigiolst p_sigiolst; /* (c) List of sigio sources. */ int p_sigparent; /* (c) Signal to parent on exit. */ int p_sig; /* (n) For core dump/debugger XXX. */ u_int p_ptevents; /* (c + e) ptrace() event mask. */ struct kaioinfo *p_aioinfo; /* (y) ASYNC I/O info. */ struct thread *p_singlethread;/* (c + j) If single threading this is it */ int p_suspcount; /* (j) Num threads in suspended mode. */ struct thread *p_xthread; /* (c) Trap thread */ int p_boundary_count;/* (j) Num threads at user boundary */ int p_pendingcnt; /* (c) how many signals are pending */ struct itimers *p_itimers; /* (c) POSIX interval timers. */ struct procdesc *p_procdesc; /* (e) Process descriptor, if any. */ u_int p_treeflag; /* (e) P_TREE flags */ int p_pendingexits; /* (c) Count of pending thread exits. */ struct filemon *p_filemon; /* (c) filemon-specific data. */ int p_pdeathsig; /* (c) Signal from parent on exit. */ /* End area that is zeroed on creation. */ #define p_endzero p_magic /* The following fields are all copied upon creation in fork. */ #define p_startcopy p_endzero u_int p_magic; /* (b) Magic number. */ int p_osrel; /* (x) osreldate for the binary (from ELF note, if any) */ uint32_t p_fctl0; /* (x) ABI feature control, ELF note */ char p_comm[MAXCOMLEN + 1]; /* (x) Process name. */ struct sysentvec *p_sysent; /* (b) Syscall dispatch info. */ struct pargs *p_args; /* (c) Process arguments. */ rlim_t p_cpulimit; /* (c) Current CPU limit in seconds. */ signed char p_nice; /* (c) Process "nice" value. */ int p_fibnum; /* in this routing domain XXX MRT */ pid_t p_reapsubtree; /* (e) Pid of the direct child of the reaper which spawned our subtree. */ uint64_t p_elf_flags; /* (x) ELF flags */ void *p_elf_brandinfo; /* (x) Elf_Brandinfo, NULL for non ELF binaries. */ sbintime_t p_umtx_min_timeout; /* End area that is copied on creation. */ #define p_endcopy p_xexit u_int p_xexit; /* (c) Exit code. */ u_int p_xsig; /* (c) Stop/kill sig. */ struct pgrp *p_pgrp; /* (c + e) Pointer to process group. */ struct knlist *p_klist; /* (c) Knotes attached to this proc. */ int p_numthreads; /* (c) Number of threads. */ struct mdproc p_md; /* Any machine-dependent fields. */ struct callout p_itcallout; /* (h + c) Interval timer callout. */ u_short p_acflag; /* (c) Accounting flags. */ struct proc *p_peers; /* (r) */ struct proc *p_leader; /* (b) */ void *p_emuldata; /* (c) Emulator state data. */ struct label *p_label; /* (*) Proc (not subject) MAC label. */ STAILQ_HEAD(, ktr_request) p_ktr; /* (o) KTR event queue. */ LIST_HEAD(, mqueue_notifier) p_mqnotifier; /* (c) mqueue notifiers.*/ struct kdtrace_proc *p_dtrace; /* (*) DTrace-specific data. */ struct cv p_pwait; /* (*) wait cv for exit/exec. */ uint64_t p_prev_runtime; /* (c) Resource usage accounting. */ struct racct *p_racct; /* (b) Resource accounting. */ int p_throttled; /* (c) Flag for racct pcpu throttling */ /* * An orphan is the child that has been re-parented to the * debugger as a result of attaching to it. Need to keep * track of them for parent to be able to collect the exit * status of what used to be children. */ LIST_ENTRY(proc) p_orphan; /* (e) List of orphan processes. */ LIST_HEAD(, proc) p_orphans; /* (e) Pointer to list of orphans. */ TAILQ_HEAD(, kq_timer_cb_data) p_kqtim_stop; /* (c) */ LIST_ENTRY(proc) p_jaillist; /* (d) Jail process linkage. */ }; #define p_session p_pgrp->pg_session #define p_pgid p_pgrp->pg_id #define NOCPU (-1) /* For when we aren't on a CPU. */ #define NOCPU_OLD (255) #define MAXCPU_OLD (254) #define PROC_SLOCK(p) mtx_lock_spin(&(p)->p_slock) #define PROC_SUNLOCK(p) mtx_unlock_spin(&(p)->p_slock) #define PROC_SLOCK_ASSERT(p, type) mtx_assert(&(p)->p_slock, (type)) #define PROC_STATLOCK(p) mtx_lock_spin(&(p)->p_statmtx) #define PROC_STATUNLOCK(p) mtx_unlock_spin(&(p)->p_statmtx) #define PROC_STATLOCK_ASSERT(p, type) mtx_assert(&(p)->p_statmtx, (type)) #define PROC_ITIMLOCK(p) mtx_lock_spin(&(p)->p_itimmtx) #define PROC_ITIMUNLOCK(p) mtx_unlock_spin(&(p)->p_itimmtx) #define PROC_ITIMLOCK_ASSERT(p, type) mtx_assert(&(p)->p_itimmtx, (type)) #define PROC_PROFLOCK(p) mtx_lock_spin(&(p)->p_profmtx) #define PROC_PROFUNLOCK(p) mtx_unlock_spin(&(p)->p_profmtx) #define PROC_PROFLOCK_ASSERT(p, type) mtx_assert(&(p)->p_profmtx, (type)) /* These flags are kept in p_flag. */ #define P_ADVLOCK 0x00000001 /* Process may hold a POSIX advisory lock. */ #define P_CONTROLT 0x00000002 /* Has a controlling terminal. */ #define P_KPROC 0x00000004 /* Kernel process. */ #define P_UNUSED3 0x00000008 /* --available-- */ #define P_PPWAIT 0x00000010 /* Parent is waiting for child to exec/exit. */ #define P_PROFIL 0x00000020 /* Has started profiling. */ #define P_STOPPROF 0x00000040 /* Has thread requesting to stop profiling. */ #define P_HADTHREADS 0x00000080 /* Has had threads (no cleanup shortcuts) */ #define P_SUGID 0x00000100 /* Had set id privileges since last exec. */ #define P_SYSTEM 0x00000200 /* System proc: no sigs, stats or swapping. */ #define P_SINGLE_EXIT 0x00000400 /* Threads suspending should exit, not wait. */ #define P_TRACED 0x00000800 /* Debugged process being traced. */ #define P_WAITED 0x00001000 /* Someone is waiting for us. */ #define P_WEXIT 0x00002000 /* Working on exiting. */ #define P_EXEC 0x00004000 /* Process called exec. */ #define P_WKILLED 0x00008000 /* Killed, go to kernel/user boundary ASAP. */ #define P_CONTINUED 0x00010000 /* Proc has continued from a stopped state. */ #define P_STOPPED_SIG 0x00020000 /* Stopped due to SIGSTOP/SIGTSTP. */ #define P_STOPPED_TRACE 0x00040000 /* Stopped because of tracing. */ #define P_STOPPED_SINGLE 0x00080000 /* Only 1 thread can continue (not to user). */ #define P_PROTECTED 0x00100000 /* Do not kill on memory overcommit. */ #define P_SIGEVENT 0x00200000 /* Process pending signals changed. */ #define P_SINGLE_BOUNDARY 0x00400000 /* Threads should suspend at user boundary. */ #define P_HWPMC 0x00800000 /* Process is using HWPMCs */ #define P_JAILED 0x01000000 /* Process is in jail. */ #define P_TOTAL_STOP 0x02000000 /* Stopped in stop_all_proc. */ #define P_INEXEC 0x04000000 /* Process is in execve(). */ #define P_STATCHILD 0x08000000 /* Child process stopped or exited. */ #define P_INMEM 0x10000000 /* Loaded into memory. */ #define P_SWAPPINGOUT 0x20000000 /* Process is being swapped out. */ #define P_SWAPPINGIN 0x40000000 /* Process is being swapped in. */ #define P_PPTRACE 0x80000000 /* PT_TRACEME by vforked child. */ #define P_STOPPED (P_STOPPED_SIG|P_STOPPED_SINGLE|P_STOPPED_TRACE) #define P_SHOULDSTOP(p) ((p)->p_flag & P_STOPPED) #define P_KILLED(p) ((p)->p_flag & P_WKILLED) /* These flags are kept in p_flag2. */ #define P2_INHERIT_PROTECTED 0x00000001 /* New children get P_PROTECTED. */ #define P2_NOTRACE 0x00000002 /* No ptrace(2) attach or coredumps. */ #define P2_NOTRACE_EXEC 0x00000004 /* Keep P2_NOPTRACE on exec(2). */ #define P2_AST_SU 0x00000008 /* Handles SU ast for kthreads. */ #define P2_PTRACE_FSTP 0x00000010 /* SIGSTOP from PT_ATTACH not yet handled. */ #define P2_TRAPCAP 0x00000020 /* SIGTRAP on ENOTCAPABLE */ #define P2_ASLR_ENABLE 0x00000040 /* Force enable ASLR. */ #define P2_ASLR_DISABLE 0x00000080 /* Force disable ASLR. */ #define P2_ASLR_IGNSTART 0x00000100 /* Enable ASLR to consume sbrk area. */ #define P2_PROTMAX_ENABLE 0x00000200 /* Force enable implied PROT_MAX. */ #define P2_PROTMAX_DISABLE 0x00000400 /* Force disable implied PROT_MAX. */ #define P2_STKGAP_DISABLE 0x00000800 /* Disable stack gap for MAP_STACK */ #define P2_STKGAP_DISABLE_EXEC 0x00001000 /* Stack gap disabled after exec */ #define P2_ITSTOPPED 0x00002000 /* itimers stopped */ #define P2_PTRACEREQ 0x00004000 /* Active ptrace req */ #define P2_NO_NEW_PRIVS 0x00008000 /* Ignore setuid */ #define P2_WXORX_DISABLE 0x00010000 /* WX mappings enabled */ #define P2_WXORX_ENABLE_EXEC 0x00020000 /* WXORX enabled after exec */ #define P2_WEXIT 0x00040000 /* exit just started, no external thread_single() is permitted */ #define P2_REAPKILLED 0x00080000 /* REAP_KILL pass touched me */ #define P2_MEMBAR_PRIVE 0x00100000 /* membar private expedited registered */ #define P2_MEMBAR_PRIVE_SYNCORE 0x00200000 /* membar private expedited sync core registered */ #define P2_MEMBAR_GLOBE 0x00400000 /* membar global expedited registered */ /* Flags protected by proctree_lock, kept in p_treeflags. */ #define P_TREE_ORPHANED 0x00000001 /* Reparented, on orphan list */ #define P_TREE_FIRST_ORPHAN 0x00000002 /* First element of orphan list */ #define P_TREE_REAPER 0x00000004 /* Reaper of subtree */ #define P_TREE_GRPEXITED 0x00000008 /* exit1() done with job ctl */ /* * These were process status values (p_stat), now they are only used in * legacy conversion code. */ #define SIDL 1 /* Process being created by fork. */ #define SRUN 2 /* Currently runnable. */ #define SSLEEP 3 /* Sleeping on an address. */ #define SSTOP 4 /* Process debugging or suspension. */ #define SZOMB 5 /* Awaiting collection by parent. */ #define SWAIT 6 /* Waiting for interrupt. */ #define SLOCK 7 /* Blocked on a lock. */ #define P_MAGIC 0xbeefface #ifdef _KERNEL /* Types and flags for mi_switch(9). */ #define SW_TYPE_MASK 0xff /* First 8 bits are switch type */ #define SWT_OWEPREEMPT 1 /* Switching due to owepreempt. */ #define SWT_TURNSTILE 2 /* Turnstile contention. */ #define SWT_SLEEPQ 3 /* Sleepq wait. */ #define SWT_RELINQUISH 4 /* yield call. */ #define SWT_NEEDRESCHED 5 /* NEEDRESCHED was set. */ #define SWT_IDLE 6 /* Switching from the idle thread. */ #define SWT_IWAIT 7 /* Waiting for interrupts. */ #define SWT_SUSPEND 8 /* Thread suspended. */ #define SWT_REMOTEPREEMPT 9 /* Remote processor preempted. */ #define SWT_REMOTEWAKEIDLE 10 /* Remote processor preempted idle. */ #define SWT_BIND 11 /* Thread bound to a new CPU. */ #define SWT_COUNT 12 /* Number of switch types. */ /* Flags */ #define SW_VOL 0x0100 /* Voluntary switch. */ #define SW_INVOL 0x0200 /* Involuntary switch. */ #define SW_PREEMPT 0x0400 /* The invol switch is a preemption */ /* How values for thread_single(). */ #define SINGLE_NO_EXIT 0 #define SINGLE_EXIT 1 #define SINGLE_BOUNDARY 2 #define SINGLE_ALLPROC 3 #ifdef MALLOC_DECLARE MALLOC_DECLARE(M_PARGS); MALLOC_DECLARE(M_SESSION); MALLOC_DECLARE(M_SUBPROC); #endif #define FOREACH_PROC_IN_SYSTEM(p) \ LIST_FOREACH((p), &allproc, p_list) #define FOREACH_THREAD_IN_PROC(p, td) \ TAILQ_FOREACH((td), &(p)->p_threads, td_plist) #define FIRST_THREAD_IN_PROC(p) TAILQ_FIRST(&(p)->p_threads) /* * We use process IDs <= pid_max <= PID_MAX; PID_MAX + 1 must also fit * in a pid_t, as it is used to represent "no process group". */ #define PID_MAX 99999 #define NO_PID 100000 #define THREAD0_TID NO_PID extern pid_t pid_max; #define SESS_LEADER(p) ((p)->p_session->s_leader == (p)) /* Lock and unlock a process. */ #define PROC_LOCK(p) mtx_lock(&(p)->p_mtx) #define PROC_TRYLOCK(p) mtx_trylock(&(p)->p_mtx) #define PROC_UNLOCK(p) mtx_unlock(&(p)->p_mtx) #define PROC_LOCKED(p) mtx_owned(&(p)->p_mtx) #define PROC_WAIT_UNLOCKED(p) mtx_wait_unlocked(&(p)->p_mtx) #define PROC_LOCK_ASSERT(p, type) mtx_assert(&(p)->p_mtx, (type)) /* Lock and unlock a process group. */ #define PGRP_LOCK(pg) mtx_lock(&(pg)->pg_mtx) #define PGRP_UNLOCK(pg) mtx_unlock(&(pg)->pg_mtx) #define PGRP_LOCKED(pg) mtx_owned(&(pg)->pg_mtx) #define PGRP_LOCK_ASSERT(pg, type) mtx_assert(&(pg)->pg_mtx, (type)) #define PGRP_LOCK_PGSIGNAL(pg) do { \ if ((pg) != NULL) \ PGRP_LOCK(pg); \ } while (0) #define PGRP_UNLOCK_PGSIGNAL(pg) do { \ if ((pg) != NULL) \ PGRP_UNLOCK(pg); \ } while (0) /* Lock and unlock a session. */ #define SESS_LOCK(s) mtx_lock(&(s)->s_mtx) #define SESS_UNLOCK(s) mtx_unlock(&(s)->s_mtx) #define SESS_LOCKED(s) mtx_owned(&(s)->s_mtx) #define SESS_LOCK_ASSERT(s, type) mtx_assert(&(s)->s_mtx, (type)) /* * Non-zero p_lock ensures that: * - exit1() is not performed until p_lock reaches zero; * - the process' threads stack are not swapped out if they are currently * not (P_INMEM). * * PHOLD() asserts that the process (except the current process) is * not exiting, increments p_lock and swaps threads stacks into memory, * if needed. * _PHOLD() is same as PHOLD(), it takes the process locked. * _PHOLD_LITE() also takes the process locked, but comparing with * _PHOLD(), it only guarantees that exit1() is not executed, * faultin() is not called. */ #define PHOLD(p) do { \ PROC_LOCK(p); \ _PHOLD(p); \ PROC_UNLOCK(p); \ } while (0) #define _PHOLD(p) do { \ PROC_LOCK_ASSERT((p), MA_OWNED); \ KASSERT(!((p)->p_flag & P_WEXIT) || (p) == curproc, \ ("PHOLD of exiting process %p", p)); \ (p)->p_lock++; \ if (((p)->p_flag & P_INMEM) == 0) \ faultin((p)); \ } while (0) #define _PHOLD_LITE(p) do { \ PROC_LOCK_ASSERT((p), MA_OWNED); \ KASSERT(!((p)->p_flag & P_WEXIT) || (p) == curproc, \ ("PHOLD of exiting process %p", p)); \ (p)->p_lock++; \ } while (0) #define PROC_ASSERT_HELD(p) do { \ KASSERT((p)->p_lock > 0, ("process %p not held", p)); \ } while (0) #define PRELE(p) do { \ PROC_LOCK((p)); \ _PRELE((p)); \ PROC_UNLOCK((p)); \ } while (0) #define _PRELE(p) do { \ PROC_LOCK_ASSERT((p), MA_OWNED); \ PROC_ASSERT_HELD(p); \ (--(p)->p_lock); \ if (((p)->p_flag & P_WEXIT) && (p)->p_lock == 0) \ wakeup(&(p)->p_lock); \ } while (0) #define PROC_ASSERT_NOT_HELD(p) do { \ KASSERT((p)->p_lock == 0, ("process %p held", p)); \ } while (0) #define PROC_UPDATE_COW(p) do { \ struct proc *_p = (p); \ PROC_LOCK_ASSERT((_p), MA_OWNED); \ atomic_store_int(&_p->p_cowgen, _p->p_cowgen + 1); \ } while (0) #define PROC_COW_CHANGECOUNT(td, p) ({ \ struct thread *_td = (td); \ struct proc *_p = (p); \ MPASS(_td == curthread); \ PROC_LOCK_ASSERT(_p, MA_OWNED); \ _p->p_cowgen - _td->td_cowgen; \ }) /* Check whether a thread is safe to be swapped out. */ #define thread_safetoswapout(td) ((td)->td_flags & TDF_CANSWAP) /* Control whether or not it is safe for curthread to sleep. */ #define THREAD_NO_SLEEPING() do { \ curthread->td_no_sleeping++; \ MPASS(curthread->td_no_sleeping > 0); \ } while (0) #define THREAD_SLEEPING_OK() do { \ MPASS(curthread->td_no_sleeping > 0); \ curthread->td_no_sleeping--; \ } while (0) #define THREAD_CAN_SLEEP() ((curthread)->td_no_sleeping == 0) #define PIDHASH(pid) (&pidhashtbl[(pid) & pidhash]) #define PIDHASHLOCK(pid) (&pidhashtbl_lock[((pid) & pidhashlock)]) extern LIST_HEAD(pidhashhead, proc) *pidhashtbl; extern struct sx *pidhashtbl_lock; extern u_long pidhash; extern u_long pidhashlock; #define PGRPHASH(pgid) (&pgrphashtbl[(pgid) & pgrphash]) extern LIST_HEAD(pgrphashhead, pgrp) *pgrphashtbl; extern u_long pgrphash; extern struct sx allproc_lock; extern int allproc_gen; extern struct sx proctree_lock; extern struct mtx ppeers_lock; extern struct mtx procid_lock; extern struct proc proc0; /* Process slot for swapper. */ extern struct thread0_storage thread0_st; /* Primary thread in proc0. */ #define thread0 (thread0_st.t0st_thread) extern struct vmspace vmspace0; /* VM space for proc0. */ extern int hogticks; /* Limit on kernel cpu hogs. */ extern int lastpid; extern int nprocs, maxproc; /* Current and max number of procs. */ extern int maxprocperuid; /* Max procs per uid. */ extern u_long ps_arg_cache_limit; LIST_HEAD(proclist, proc); TAILQ_HEAD(procqueue, proc); TAILQ_HEAD(threadqueue, thread); extern struct proclist allproc; /* List of all processes. */ extern struct proc *initproc, *pageproc; /* Process slots for init, pager. */ extern struct uma_zone *proc_zone; extern struct uma_zone *pgrp_zone; struct proc *pfind(pid_t); /* Find process by id. */ struct proc *pfind_any(pid_t); /* Find (zombie) process by id. */ struct proc *pfind_any_locked(pid_t pid); /* Find process by id, locked. */ struct pgrp *pgfind(pid_t); /* Find process group by id. */ void pidhash_slockall(void); /* Shared lock all pid hash lists. */ void pidhash_sunlockall(void); /* Shared unlock all pid hash lists. */ struct fork_req { int fr_flags; int fr_pages; int *fr_pidp; struct proc **fr_procp; int *fr_pd_fd; int fr_pd_flags; struct filecaps *fr_pd_fcaps; int fr_flags2; #define FR2_DROPSIG_CAUGHT 0x00000001 /* Drop caught non-DFL signals */ #define FR2_SHARE_PATHS 0x00000002 /* Invert sense of RFFDG for paths */ #define FR2_KPROC 0x00000004 /* Create a kernel process */ }; /* * pget() flags. */ #define PGET_HOLD 0x00001 /* Hold the process. */ #define PGET_CANSEE 0x00002 /* Check against p_cansee(). */ #define PGET_CANDEBUG 0x00004 /* Check against p_candebug(). */ #define PGET_ISCURRENT 0x00008 /* Check that the found process is current. */ #define PGET_NOTWEXIT 0x00010 /* Check that the process is not in P_WEXIT. */ #define PGET_NOTINEXEC 0x00020 /* Check that the process is not in P_INEXEC. */ #define PGET_NOTID 0x00040 /* Do not assume tid if pid > PID_MAX. */ #define PGET_WANTREAD (PGET_HOLD | PGET_CANDEBUG | PGET_NOTWEXIT) int pget(pid_t pid, int flags, struct proc **pp); /* ast_register() flags */ #define ASTR_ASTF_REQUIRED 0x0001 /* td_ast TDAI(TDA_X) flag set is required for call */ #define ASTR_TDP 0x0002 /* td_pflags flag set is required */ #define ASTR_KCLEAR 0x0004 /* call me on ast_kclear() */ #define ASTR_UNCOND 0x0008 /* call me always */ void ast(struct trapframe *framep); void ast_kclear(struct thread *td); void ast_register(int ast, int ast_flags, int tdp, void (*f)(struct thread *td, int asts)); void ast_deregister(int tda); void ast_sched_locked(struct thread *td, int tda); void ast_sched_mask(struct thread *td, int ast); void ast_sched(struct thread *td, int tda); void ast_unsched_locked(struct thread *td, int tda); struct thread *choosethread(void); int cr_bsd_visible(struct ucred *u1, struct ucred *u2); int cr_cansee(struct ucred *u1, struct ucred *u2); int cr_canseesocket(struct ucred *cred, struct socket *so); int cr_cansignal(struct ucred *cred, struct proc *proc, int signum); int enterpgrp(struct proc *p, pid_t pgid, struct pgrp *pgrp, struct session *sess); int enterthispgrp(struct proc *p, struct pgrp *pgrp); void faultin(struct proc *p); int fork1(struct thread *, struct fork_req *); void fork_exit(void (*)(void *, struct trapframe *), void *, struct trapframe *); void fork_return(struct thread *, struct trapframe *); int inferior(struct proc *p); void itimer_proc_continue(struct proc *p); void kqtimer_proc_continue(struct proc *p); void kern_proc_vmmap_resident(struct vm_map *map, struct vm_map_entry *entry, int *resident_count, bool *super); void kern_yield(int); void kick_proc0(void); void killjobc(void); int leavepgrp(struct proc *p); int maybe_preempt(struct thread *td); void maybe_yield(void); void mi_switch(int flags); int p_candebug(struct thread *td, struct proc *p); int p_cansee(struct thread *td, struct proc *p); int p_cansched(struct thread *td, struct proc *p); int p_cansignal(struct thread *td, struct proc *p, int signum); int p_canwait(struct thread *td, struct proc *p); struct pargs *pargs_alloc(int len); void pargs_drop(struct pargs *pa); void pargs_hold(struct pargs *pa); void proc_add_orphan(struct proc *child, struct proc *parent); int proc_get_binpath(struct proc *p, char *binname, char **fullpath, char **freepath); int proc_getargv(struct thread *td, struct proc *p, struct sbuf *sb); int proc_getauxv(struct thread *td, struct proc *p, struct sbuf *sb); int proc_getenvv(struct thread *td, struct proc *p, struct sbuf *sb); void procinit(void); int proc_iterate(int (*cb)(struct proc *, void *), void *cbarg); void proc_linkup0(struct proc *p, struct thread *td); void proc_linkup(struct proc *p, struct thread *td); struct proc *proc_realparent(struct proc *child); void proc_reap(struct thread *td, struct proc *p, int *status, int options); void proc_reparent(struct proc *child, struct proc *newparent, bool set_oppid); void proc_set_p2_wexit(struct proc *p); void proc_set_traced(struct proc *p, bool stop); void proc_wkilled(struct proc *p); struct pstats *pstats_alloc(void); void pstats_fork(struct pstats *src, struct pstats *dst); void pstats_free(struct pstats *ps); void proc_clear_orphan(struct proc *p); void reaper_abandon_children(struct proc *p, bool exiting); int securelevel_ge(struct ucred *cr, int level); int securelevel_gt(struct ucred *cr, int level); void sess_hold(struct session *); void sess_release(struct session *); int setrunnable(struct thread *, int); void setsugid(struct proc *p); bool should_yield(void); int sigonstack(size_t sp); void stopevent(struct proc *, u_int, u_int); struct thread *tdfind(lwpid_t, pid_t); void threadinit(void); void tidhash_add(struct thread *); void tidhash_remove(struct thread *); void cpu_idle(int); int cpu_idle_wakeup(int); extern void (*cpu_idle_hook)(sbintime_t); /* Hook to machdep CPU idler. */ void cpu_switch(struct thread *, struct thread *, struct mtx *); void cpu_sync_core(void); void cpu_throw(struct thread *, struct thread *) __dead2; bool curproc_sigkilled(void); void userret(struct thread *, struct trapframe *); void cpu_exit(struct thread *); void exit1(struct thread *, int, int) __dead2; void cpu_copy_thread(struct thread *td, struct thread *td0); bool cpu_exec_vmspace_reuse(struct proc *p, struct vm_map *map); int cpu_fetch_syscall_args(struct thread *td); void cpu_fork(struct thread *, struct proc *, struct thread *, int); void cpu_fork_kthread_handler(struct thread *, void (*)(void *), void *); int cpu_procctl(struct thread *td, int idtype, id_t id, int com, void *data); void cpu_set_syscall_retval(struct thread *, int); int cpu_set_upcall(struct thread *, void (*)(void *), void *, stack_t *); int cpu_set_user_tls(struct thread *, void *tls_base); void cpu_thread_alloc(struct thread *); void cpu_thread_clean(struct thread *); void cpu_thread_exit(struct thread *); void cpu_thread_free(struct thread *); void cpu_thread_swapin(struct thread *); void cpu_thread_swapout(struct thread *); struct thread *thread_alloc(int pages); int thread_alloc_stack(struct thread *, int pages); int thread_check_susp(struct thread *td, bool sleep); void thread_cow_get_proc(struct thread *newtd, struct proc *p); void thread_cow_get(struct thread *newtd, struct thread *td); void thread_cow_free(struct thread *td); void thread_cow_update(struct thread *td); void thread_cow_synced(struct thread *td); int thread_create(struct thread *td, struct rtprio *rtp, int (*initialize_thread)(struct thread *, void *), void *thunk); void thread_exit(void) __dead2; void thread_free(struct thread *td); void thread_link(struct thread *td, struct proc *p); void thread_reap_barrier(void); int thread_single(struct proc *p, int how); void thread_single_end(struct proc *p, int how); void thread_stash(struct thread *td); void thread_stopped(struct proc *p); void childproc_stopped(struct proc *child, int reason); void childproc_continued(struct proc *child); void childproc_exited(struct proc *child); void thread_run_flash(struct thread *td); int thread_suspend_check(int how); bool thread_suspend_check_needed(void); void thread_suspend_switch(struct thread *, struct proc *p); void thread_suspend_one(struct thread *td); void thread_unlink(struct thread *td); void thread_unsuspend(struct proc *p); void thread_wait(struct proc *p); bool stop_all_proc_block(void); void stop_all_proc_unblock(void); void stop_all_proc(void); void resume_all_proc(void); static __inline int curthread_pflags_set(int flags) { struct thread *td; int save; td = curthread; save = ~flags | (td->td_pflags & flags); td->td_pflags |= flags; return (save); } static __inline void curthread_pflags_restore(int save) { curthread->td_pflags &= save; } static __inline int curthread_pflags2_set(int flags) { struct thread *td; int save; td = curthread; save = ~flags | (td->td_pflags2 & flags); td->td_pflags2 |= flags; return (save); } static __inline void curthread_pflags2_restore(int save) { curthread->td_pflags2 &= save; } static __inline __pure2 struct td_sched * td_get_sched(struct thread *td) { return ((struct td_sched *)&td[1]); } #define PROC_ID_PID 0 #define PROC_ID_GROUP 1 #define PROC_ID_SESSION 2 #define PROC_ID_REAP 3 void proc_id_set(int type, pid_t id); void proc_id_set_cond(int type, pid_t id); void proc_id_clear(int type, pid_t id); EVENTHANDLER_LIST_DECLARE(process_ctor); EVENTHANDLER_LIST_DECLARE(process_dtor); EVENTHANDLER_LIST_DECLARE(process_init); EVENTHANDLER_LIST_DECLARE(process_fini); EVENTHANDLER_LIST_DECLARE(process_exit); EVENTHANDLER_LIST_DECLARE(process_fork); EVENTHANDLER_LIST_DECLARE(process_exec); EVENTHANDLER_LIST_DECLARE(thread_ctor); EVENTHANDLER_LIST_DECLARE(thread_dtor); EVENTHANDLER_LIST_DECLARE(thread_init); #endif /* _KERNEL */ #endif /* !_SYS_PROC_H_ */