Index: sys/amd64/amd64/machdep.c =================================================================== --- sys/amd64/amd64/machdep.c +++ sys/amd64/amd64/machdep.c @@ -77,6 +77,7 @@ #include #include #include +#include #include #include #include @@ -1920,6 +1921,7 @@ thread0.td_critnest = 0; kasan_init(); + kmsan_init((void *)thread0.td_kstack); TSEXIT(); Index: sys/amd64/include/msan.h =================================================================== --- /dev/null +++ sys/amd64/include/msan.h @@ -0,0 +1,85 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 The FreeBSD Foundation + * + * This software was developed by Mark Johnston under sponsorship from the + * FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _MACHINE_MSAN_H_ +#define _MACHINE_MSAN_H_ + +#ifdef KMSAN + +#include +#include +#include +#include + +typedef uint32_t msan_orig_t; + +static inline msan_orig_t +kmsan_md_orig_encode(int type, uintptr_t ptr) +{ + /* XXXMJ comments and symbolic constants */ + return ((type << 29u) | ((ptr & 0x1fffffff))); +} + +static inline void +kmsan_md_orig_decode(msan_orig_t orig, int *type, uintptr_t *ptr) +{ + *type = orig >> 29u; + *ptr = (orig & 0x1fffffff) | KERNBASE; +} + +static inline vm_offset_t +kmsan_md_addr_to_shad(vm_offset_t addr) +{ + return (addr - VM_MIN_KERNEL_ADDRESS + KMSAN_SHAD_MIN_ADDRESS); +} + +static inline vm_offset_t +kmsan_md_addr_to_orig(vm_offset_t addr) +{ + return (addr - VM_MIN_KERNEL_ADDRESS + KMSAN_ORIG_MIN_ADDRESS); +} + +static inline bool +kmsan_md_unsupported(vm_offset_t addr) +{ + return (addr < VM_MIN_KERNEL_ADDRESS || addr >= KERNBASE); +} + +extern char btext[], etext[]; + +static inline bool +kmsan_md_is_pc(uintptr_t pc) +{ + return (pc >= (uintptr_t)btext);// XXXMJ && pc < (uintptr_t)etext); +} + +#endif /* KMSAN */ + +#endif /* !_MACHINE_MSAN_H_ */ Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -3924,6 +3924,8 @@ kern/subr_log.c standard kern/subr_mchain.c optional libmchain kern/subr_module.c standard +kern/subr_msan.c optional kmsan \ + compile-with "${NORMAL_C:N-fsanitize*}" kern/subr_msgbuf.c standard kern/subr_param.c standard kern/subr_pcpu.c standard Index: sys/kern/init_main.c =================================================================== --- sys/kern/init_main.c +++ sys/kern/init_main.c @@ -64,6 +64,7 @@ #include #include #include +#include #include #include #include @@ -615,6 +616,7 @@ #endif EVENTHANDLER_DIRECT_INVOKE(process_ctor, p); EVENTHANDLER_DIRECT_INVOKE(thread_ctor, td); + kmsan_thread_alloc(td); /* * Charge root for one process. Index: sys/kern/kern_thread.c =================================================================== --- sys/kern/kern_thread.c +++ sys/kern/kern_thread.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -84,11 +85,11 @@ * structures. */ #ifdef __amd64__ -_Static_assert(offsetof(struct thread, td_flags) == 0xfc, +_Static_assert(offsetof(struct thread, td_flags) == 0x108, "struct thread KBI td_flags"); -_Static_assert(offsetof(struct thread, td_pflags) == 0x104, +_Static_assert(offsetof(struct thread, td_pflags) == 0x110, "struct thread KBI td_pflags"); -_Static_assert(offsetof(struct thread, td_frame) == 0x4a0, +_Static_assert(offsetof(struct thread, td_frame) == 0x4a8, "struct thread KBI td_frame"); _Static_assert(offsetof(struct thread, td_emuldata) == 0x6b0, "struct thread KBI td_emuldata"); @@ -761,6 +762,7 @@ return (NULL); } td->td_tid = tid; + kmsan_thread_alloc(td); cpu_thread_alloc(td); EVENTHANDLER_DIRECT_INVOKE(thread_ctor, td); return (td); @@ -774,6 +776,7 @@ ("thread_alloc_stack called on a thread with kstack")); if (!vm_thread_new(td, pages)) return (0); + kmsan_thread_alloc(td); cpu_thread_alloc(td); return (1); } @@ -797,6 +800,7 @@ * Freeing handled by the caller. */ td->td_tid = -1; + kmsan_thread_free(td); uma_zfree(thread_zone, td); } Index: sys/kern/subr_msan.c =================================================================== --- /dev/null +++ sys/kern/subr_msan.c @@ -0,0 +1,1600 @@ +/* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */ + +/* + * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net + * All rights reserved. + * + * This code is part of the KMSAN subsystem of the NetBSD kernel. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#define SAN_RUNTIME + +#include +__FBSDID("$FreeBSD$"); +#if 0 +__KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $"); +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include + +void kmsan_init_arg(size_t); +void kmsan_init_ret(size_t); + +/* -------------------------------------------------------------------------- */ + +/* + * Part of the compiler ABI. + */ + +typedef struct { + uint8_t *shad; + msan_orig_t *orig; +} msan_meta_t; + +#define MSAN_PARAM_SIZE 800 +#define MSAN_RETVAL_SIZE 800 +typedef struct { + uint8_t param_shadow[MSAN_PARAM_SIZE]; + uint8_t retval_shadow[MSAN_RETVAL_SIZE]; + uint8_t va_arg_shadow[MSAN_PARAM_SIZE]; + uint8_t va_arg_origin[MSAN_PARAM_SIZE]; + uint64_t va_arg_overflow_size; + msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)]; + msan_orig_t retval_origin; +} msan_tls_t; + +/* -------------------------------------------------------------------------- */ + +/* The MD code. */ +#include + +/* -------------------------------------------------------------------------- */ + +#define MSAN_NCONTEXT 4 +#define MSAN_ORIG_MASK (~0x3) + +typedef struct kmsan_td { + size_t ctx; + msan_tls_t tls[MSAN_NCONTEXT]; +} msan_td_t; + +static msan_tls_t dummy_tls; + +static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); +static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); +static msan_td_t msan_thread0; +static bool kmsan_enabled __read_mostly; + +static bool kmsan_reporting = false; + +/* + * Avoid clobbering any thread-local state before we panic. + */ +#define kmsan_panic(f, ...) do { \ + kmsan_enabled = false; \ + panic(f, __VA_ARGS__); \ +} while (0) + +#define REPORT(f, ...) do { \ + if (panic_on_violation) { \ + kmsan_panic(f, __VA_ARGS__); \ + } else { \ + struct stack st; \ + \ + stack_save(&st); \ + printf(f "\n", __VA_ARGS__); \ + stack_print_ddb(&st); \ + } \ +} while (0) + +FEATURE(kmsan, "Kernel memory sanitizer"); + +static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "KMSAN options"); + +static bool panic_on_violation = 1; +SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN, + &panic_on_violation, 0, + "Panic if an invalid access is detected"); + +static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer"); + +/* -------------------------------------------------------------------------- */ + +static inline const char * +kmsan_orig_name(int type) +{ + switch (type) { + case KMSAN_TYPE_STACK: + return "stack"; + case KMSAN_TYPE_KMEM: + return "kmem"; + case KMSAN_TYPE_MALLOC: + return "malloc"; + case KMSAN_TYPE_UMA: + return "UMA"; + default: + return "unknown"; + } +} + +/* + * The format of the string is: "----var@function". Parse it to display a nice + * warning. + */ +static void +kmsan_report_hook(const void *addr, size_t size, size_t off, const char *hook) +{ + msan_orig_t *orig; + const char *typename; + char *var, *fn; + uintptr_t ptr; + long foff; + char buf[128]; + int type; + + if (__predict_false(panicstr != NULL || kdb_active || kmsan_reporting)) + return; + + kmsan_reporting = true; + __compiler_membar(); + + orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); + orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); + + if (*orig == 0) { + REPORT("KMSAN: Uninitialized memory in %s, offset %zu", + hook, off); + goto out; + } + + kmsan_md_orig_decode(*orig, &type, &ptr); + typename = kmsan_orig_name(type); + + if (kmsan_md_is_pc(ptr)) { + if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, + sizeof(buf), &foff) != 0) { + REPORT("KMSAN: Uninitialized %s memory in %s, " + "offset %zu/%zu, addr %p, PC %p", + typename, hook, off, size, addr, (void *)ptr); + } else { + REPORT("KMSAN: Uninitialized %s memory in %s, " + "offset %zu/%zu, addr %p, from %s+%#lx", + typename, hook, off, size, addr, buf, foff); + } + } else { + var = (char *)ptr + 4; + strlcpy(buf, var, sizeof(buf)); + var = buf; + fn = __builtin_strchr(buf, '@'); + *fn++ = '\0'; + REPORT("KMSAN: Uninitialized %s memory in %s, offset %zu, " + "variable '%s' from %s", typename, hook, off, var, fn); + } + +out: + __compiler_membar(); + kmsan_reporting = false; +} + +static void +kmsan_report_inline(msan_orig_t orig, unsigned long pc) +{ + const char *typename; + char *var, *fn; + uintptr_t ptr; + char buf[128]; + long foff; + int type; + + if (__predict_false(panicstr != NULL || kdb_active || kmsan_reporting)) + return; + + kmsan_reporting = true; + __compiler_membar(); + + if (orig == 0) { + REPORT("KMSAN: uninitialized variable in %p", (void *)pc); + goto out; + } + + kmsan_md_orig_decode(orig, &type, &ptr); + typename = kmsan_orig_name(type); + + if (kmsan_md_is_pc(ptr)) { + if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, + sizeof(buf), &foff) != 0) { + REPORT("KMSAN: Uninitialized %s memory, origin %x", + typename, orig); + } else { + REPORT("KMSAN: Uninitialized %s memory from %s+%#lx", + typename, buf, foff); + } + } else { + var = (char *)ptr + 4; + strlcpy(buf, var, sizeof(buf)); + var = buf; + fn = __builtin_strchr(buf, '@'); + *fn++ = '\0'; + REPORT("KMSAN: Uninitialized variable '%s' from %s", var, fn); + } + +out: + __compiler_membar(); + kmsan_reporting = false; +} + +/* -------------------------------------------------------------------------- */ + +static inline msan_meta_t +kmsan_meta_get(const void *addr, size_t size) +{ + msan_meta_t ret; + + if (__predict_false(!kmsan_enabled)) { + ret.shad = msan_dummy_shad; + ret.orig = (msan_orig_t *)msan_dummy_orig; + } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) { + ret.shad = msan_dummy_shad; + ret.orig = (msan_orig_t *)msan_dummy_orig; + } else { + ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr); + ret.orig = + (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); + ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & + MSAN_ORIG_MASK); + } + + return ret; +} + +static inline void +kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size) +{ + msan_orig_t *orig; + size_t i; + + if (__predict_false(!kmsan_enabled)) + return; + if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) + return; + + orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); + size += ((uintptr_t)orig & (sizeof(*orig) - 1)); + orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); + + for (i = 0; i < size; i += 4) { + orig[i / 4] = o; + } +} + +static inline void +kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size) +{ + uint8_t *shad; + + if (__predict_false(!kmsan_enabled)) + return; + if (__predict_false(kmsan_md_unsupported(addr))) + return; + + shad = (uint8_t *)kmsan_md_addr_to_shad(addr); + __builtin_memset(shad, c, size); +} + +static inline void +kmsan_meta_copy(void *dst, const void *src, size_t size) +{ + uint8_t *orig_src, *orig_dst; + uint8_t *shad_src, *shad_dst; + msan_orig_t *_src, *_dst; + size_t i; + + if (__predict_false(!kmsan_enabled)) + return; + if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst))) + return; + if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) { + kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size); + return; + } + + shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src); + shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst); + __builtin_memmove(shad_dst, shad_src, size); + + orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src); + orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst); + for (i = 0; i < size; i++) { + _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK); + _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK); + *_dst = *_src; + orig_src++; + orig_dst++; + } +} + +static inline void +kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook) +{ + uint8_t *shad; + size_t i; + + if (__predict_false(!kmsan_enabled)) + return; + if (__predict_false(kmsan_md_unsupported(addr))) + return; + + shad = (uint8_t *)kmsan_md_addr_to_shad(addr); + for (i = 0; i < size; i++) { + if (__predict_true(shad[i] == 0)) + continue; + kmsan_report_hook((const char *)addr + i, size, i, hook); + break; + } +} + +void +kmsan_init_arg(size_t n) +{ + msan_td_t *mtd; + uint8_t *arg; + + if (__predict_false(!kmsan_enabled)) + return; + if (__predict_false(curthread == NULL)) + return; + mtd = curthread->td_kmsan; + arg = mtd->tls[mtd->ctx].param_shadow; + __builtin_memset(arg, 0, n); +} + +void +kmsan_init_ret(size_t n) +{ + msan_td_t *mtd; + uint8_t *arg; + + if (__predict_false(!kmsan_enabled)) + return; + if (__predict_false(curthread == NULL)) + return; + mtd = curthread->td_kmsan; + arg = mtd->tls[mtd->ctx].retval_shadow; + __builtin_memset(arg, 0, n); +} + +static void +kmsan_check_arg(size_t size, const char *hook) +{ + msan_td_t *mtd; + uint8_t *arg; + size_t i; + + if (__predict_false(!kmsan_enabled)) + return; + if (__predict_false(curthread == NULL)) + return; + mtd = curthread->td_kmsan; + arg = mtd->tls[mtd->ctx].param_shadow; + + for (i = 0; i < size; i++) { + if (__predict_true(arg[i] == 0)) + continue; + kmsan_report_hook((const char *)arg + i, size, i, hook); + break; + } +} + +void +kmsan_thread_alloc(struct thread *td) +{ + msan_td_t *mtd; + + if (!kmsan_enabled) + return; + + mtd = td->td_kmsan; + if (mtd == NULL) { + /* We might be recycling a thread. */ + kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) + + sizeof(int)); + mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK); + kmsan_mark(mtd, sizeof(*mtd), KMSAN_STATE_INITED); + } + __builtin_memset(mtd, 0, sizeof(*mtd)); + mtd->ctx = 1; + + td->td_kmsan = mtd; +} + +void +kmsan_thread_free(struct thread *td) +{ + msan_td_t *mtd; + + if (!kmsan_enabled) + return; + if (__predict_false(td == curthread)) + kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__); + + mtd = td->td_kmsan; + kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *)); + free(mtd, M_KMSAN); + td->td_kmsan = NULL; +} + +void kmsan_intr_enter(size_t sz); +void kmsan_intr_leave(void); + +void +kmsan_intr_enter(size_t sz) +{ + msan_td_t *mtd; + + if (__predict_false(!kmsan_enabled)) + return; + + mtd = curthread->td_kmsan; + + mtd->ctx++; + if (__predict_false(mtd->ctx >= MSAN_NCONTEXT)) + kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); + + kmsan_init_arg(sz); +} + +void +kmsan_intr_leave(void) +{ + msan_td_t *mtd; + + if (__predict_false(!kmsan_enabled)) + return; + mtd = curthread->td_kmsan; + + if (__predict_false(mtd->ctx == 0)) + kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); + mtd->ctx--; +} + +/* -------------------------------------------------------------------------- */ + +void +kmsan_shadow_map(vm_offset_t addr, size_t size) +{ + size_t npages, i; + vm_offset_t va; + + MPASS(addr % PAGE_SIZE == 0); + MPASS(size % PAGE_SIZE == 0); + + if (!kmsan_enabled) + return; + + npages = atop(size); + + va = kmsan_md_addr_to_shad(addr); + for (i = 0; i < npages; i++) { + pmap_kmsan_enter(va + ptoa(i)); + } + + va = kmsan_md_addr_to_orig(addr); + for (i = 0; i < npages; i++) { + pmap_kmsan_enter(va + ptoa(i)); + } +} + +void +kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc) +{ + msan_orig_t orig; + + orig = kmsan_md_orig_encode(type, pc); + kmsan_origin_fill(addr, orig, size); +} + +void +kmsan_mark(const void *addr, size_t size, uint8_t c) +{ + kmsan_shadow_fill((uintptr_t)addr, c, size); +} + +static void +kmsan_mark_ccb(const union ccb *ccb, uint8_t c) +{ + /* XXXMJ CAM_DATA_MASK */ + if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN) + return; + switch (ccb->ccb_h.func_code) { + case XPT_SCSI_IO: { + const struct ccb_scsiio *scsiio; + + scsiio = &ccb->ctio; + kmsan_mark(scsiio->data_ptr, scsiio->dxfer_len, c); + break; + } + case XPT_ATA_IO: { + const struct ccb_ataio *ataio; + + ataio = &ccb->ataio; + kmsan_mark(ataio->data_ptr, ataio->dxfer_len, c); + break; + } + case XPT_NVME_IO: { + const struct ccb_nvmeio *nvmeio; + + nvmeio = &ccb->nvmeio; + kmsan_mark(nvmeio->data_ptr, nvmeio->dxfer_len, c); + break; + } + default: + kmsan_panic("%s: unhandled CCB type %d", __func__, + ccb->ccb_h.func_code); + } +} + +static void +kmsan_mark_mbuf(const struct mbuf *m, uint8_t c) +{ + do { + if ((m->m_flags & M_EXTPG) == 0) + kmsan_mark(m->m_data, m->m_len, c); + m = m->m_next; + } while (m != NULL); +} + +void +kmsan_check(const void *p, size_t sz, const char *descr) +{ + kmsan_shadow_check((uintptr_t)p, sz, descr); +} + +void +kmsan_check_bio(const struct bio *bp, const char *descr) +{ + kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr); +} + +void +kmsan_check_ccb(const union ccb *ccb, const char *descr) +{ + if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) + return; + switch (ccb->ccb_h.func_code) { + case XPT_SCSI_IO: { + const struct ccb_scsiio *scsiio; + + scsiio = &ccb->ctio; + kmsan_check(scsiio->data_ptr, scsiio->dxfer_len, descr); + break; + } + case XPT_ATA_IO: { + const struct ccb_ataio *ataio; + + ataio = &ccb->ataio; + kmsan_check(ataio->data_ptr, ataio->dxfer_len, descr); + break; + } + case XPT_NVME_IO: { + const struct ccb_nvmeio *nvmeio; + + nvmeio = &ccb->nvmeio; + kmsan_check(nvmeio->data_ptr, nvmeio->dxfer_len, descr); + break; + } + default: + kmsan_panic("%s: unhandled CCB type %d", __func__, + ccb->ccb_h.func_code); + } +} + +void +kmsan_check_mbuf(const struct mbuf *m, const char *descr) +{ + do { + kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr); + } while ((m = m->m_next) != NULL); +} + +void +kmsan_init(void *stack) +{ + int disabled; + + disabled = 0; + TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled); + if (disabled) + return; + + /* Map the stack. XXXMJ */ + //kmsan_shadow_map(stack, KSTACK_PAGES * PAGE_SIZE); + + /* Initialize the TLS for curthread. */ + msan_thread0.ctx = 1; + thread0.td_kmsan = &msan_thread0; + + /* Now officially enabled. */ + kmsan_enabled = true; +} + +/* -------------------------------------------------------------------------- */ + +msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); +msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); + +msan_meta_t +__msan_metadata_ptr_for_load_n(void *addr, size_t size) +{ + return kmsan_meta_get(addr, size); +} + +msan_meta_t +__msan_metadata_ptr_for_store_n(void *addr, size_t size) +{ + return kmsan_meta_get(addr, size); +} + +#define MSAN_META_FUNC(size) \ + msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ + msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ + { \ + return kmsan_meta_get(addr, size); \ + } \ + msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ + msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ + { \ + return kmsan_meta_get(addr, size); \ + } + +MSAN_META_FUNC(1) +MSAN_META_FUNC(2) +MSAN_META_FUNC(4) +MSAN_META_FUNC(8) + +void __msan_instrument_asm_store(const void *, size_t); +msan_orig_t __msan_chain_origin(msan_orig_t); +void __msan_poison(const void *, size_t); +void __msan_unpoison(const void *, size_t); +void __msan_poison_alloca(const void *, uint64_t, const char *); +void __msan_unpoison_alloca(const void *, uint64_t); +void __msan_warning(msan_orig_t); +msan_tls_t *__msan_get_context_state(void); + +void +__msan_instrument_asm_store(const void *addr, size_t size) +{ + kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); +} + +msan_orig_t +__msan_chain_origin(msan_orig_t origin) +{ + return origin; +} + +void +__msan_poison(const void *addr, size_t size) +{ + kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); +} + +void +__msan_unpoison(const void *addr, size_t size) +{ + kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); +} + +void +__msan_poison_alloca(const void *addr, uint64_t size, const char *descr) +{ + msan_orig_t orig; + + orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); + kmsan_origin_fill(addr, orig, size); + kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); +} + +void +__msan_unpoison_alloca(const void *addr, uint64_t size) +{ + kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); +} + +void +__msan_warning(msan_orig_t origin) +{ + if (__predict_false(!kmsan_enabled)) + return; + kmsan_report_inline(origin, KMSAN_RET_ADDR); +} + +msan_tls_t * +__msan_get_context_state(void) +{ + msan_td_t *mtd; + + /* + * When APs are started, they execute some C code before curthread is + * set. We have to handle that here. + */ + if (__predict_false(!kmsan_enabled || curthread == NULL)) + return &dummy_tls; + mtd = curthread->td_kmsan; + return (&mtd->tls[mtd->ctx]); +} + +/* -------------------------------------------------------------------------- */ + +/* + * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle + * initialized areas properly. + */ + +void * +kmsan_memcpy(void *dst, const void *src, size_t len) +{ + /* No kmsan_check_arg, because inlined. */ + kmsan_init_ret(sizeof(void *)); + if (__predict_true(len != 0)) { + kmsan_meta_copy(dst, src, len); + } + return __builtin_memcpy(dst, src, len); +} + +int +kmsan_memcmp(const void *b1, const void *b2, size_t len) +{ + const uint8_t *_b1 = b1, *_b2 = b2; + size_t i; + + kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), + "memcmp():args"); + kmsan_init_ret(sizeof(int)); + + for (i = 0; i < len; i++) { + if (*_b1 != *_b2) { + kmsan_shadow_check((uintptr_t)b1, i + 1, "memcmp():arg1"); + kmsan_shadow_check((uintptr_t)b2, i + 1, "memcmp():arg2"); + return *_b1 - *_b2; + } + _b1++, _b2++; + } + + return 0; +} + +void * +kmsan_memset(void *dst, int c, size_t len) +{ + /* No kmsan_check_arg, because inlined. */ + kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len); + kmsan_init_ret(sizeof(void *)); + return __builtin_memset(dst, c, len); +} + +void * +kmsan_memmove(void *dst, const void *src, size_t len) +{ + /* No kmsan_check_arg, because inlined. */ + if (__predict_true(len != 0)) { + kmsan_meta_copy(dst, src, len); + } + kmsan_init_ret(sizeof(void *)); + return __builtin_memmove(dst, src, len); +} + +__strong_reference(kmsan_memcpy, __msan_memcpy); +__strong_reference(kmsan_memset, __msan_memset); +__strong_reference(kmsan_memmove, __msan_memmove); + +char * +kmsan_strcpy(char *dst, const char *src) +{ + const char *_src = src; + char *_dst = dst; + size_t len = 0; + + kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); + + while (1) { + len++; + *dst = *src; + if (*src == '\0') + break; + src++, dst++; + } + + kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2"); + kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len); + kmsan_init_ret(sizeof(char *)); + return _dst; +} + +int +kmsan_strcmp(const char *s1, const char *s2) +{ + const char *_s1 = s1, *_s2 = s2; + size_t len = 0; + + kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); + kmsan_init_ret(sizeof(int)); + + while (1) { + len++; + if (*s1 != *s2) + break; + if (*s1 == '\0') { + kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); + kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); + return 0; + } + s1++, s2++; + } + + kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); + kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); + + return (*(const unsigned char *)s1 - *(const unsigned char *)s2); +} + +size_t +kmsan_strlen(const char *str) +{ + const char *s; + + kmsan_check_arg(sizeof(str), "strlen():args"); + + s = str; + while (1) { + if (*s == '\0') + break; + s++; + } + + kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1"); + kmsan_init_ret(sizeof(size_t)); + return (s - str); +} + +int kmsan_copyin(const void *, void *, size_t); +int kmsan_copyout(const void *, void *, size_t); +int kmsan_copyinstr(const void *, void *, size_t, size_t *); + +int +kmsan_copyin(const void *uaddr, void *kaddr, size_t len) +{ + int ret; + + kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), + "copyin():args"); + ret = copyin(uaddr, kaddr, len); + if (ret == 0) + kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len); + kmsan_init_ret(sizeof(int)); + + return ret; +} + +int +kmsan_copyout(const void *kaddr, void *uaddr, size_t len) +{ + kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), + "copyout():args"); + kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1"); + kmsan_init_ret(sizeof(int)); + return copyout(kaddr, uaddr, len); +} + +int +kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) +{ + size_t _done; + int ret; + + kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + + sizeof(len) + sizeof(done), "copyinstr():args"); + ret = copyinstr(uaddr, kaddr, len, &_done); + if (ret == 0) + kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done); + if (done != NULL) { + *done = _done; + kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t)); + } + kmsan_init_ret(sizeof(int)); + + return ret; +} + +/* -------------------------------------------------------------------------- */ + +int +kmsan_fubyte(volatile const void *base) +{ + int ret; + + kmsan_check_arg(sizeof(base), "fubyte(): args"); + ret = fubyte(base); + kmsan_init_ret(sizeof(int)); + return ret; +} + +int +kmsan_fuword16(volatile const void *base) +{ + int ret; + + kmsan_check_arg(sizeof(base), "fuword16(): args"); + ret = fuword16(base); + kmsan_init_ret(sizeof(int)); + return ret; +} + +int +kmsan_fueword(volatile const void *base, long *val) +{ + int ret; + + kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args"); + ret = fueword(base, val); + if (ret == 0) + kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, + sizeof(*val)); + kmsan_init_ret(sizeof(int)); + return ret; +} + +int +kmsan_fueword32(volatile const void *base, int32_t *val) +{ + int ret; + + kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args"); + ret = fueword32(base, val); + if (ret == 0) + kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, + sizeof(*val)); + kmsan_init_ret(sizeof(int)); + return ret; +} + +int +kmsan_fueword64(volatile const void *base, int64_t *val) +{ + int ret; + + kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args"); + ret = fueword64(base, val); + if (ret == 0) + kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, + sizeof(*val)); + kmsan_init_ret(sizeof(int)); + return ret; +} + +int +kmsan_subyte(volatile void *base, int byte) +{ + int ret; + + kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args"); + ret = subyte(base, byte); + kmsan_init_ret(sizeof(int)); + return (ret); +} + +int +kmsan_suword(volatile void *base, long word) +{ + int ret; + + kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args"); + ret = suword(base, word); + kmsan_init_ret(sizeof(int)); + return (ret); +} + +int +kmsan_suword16(volatile void *base, int word) +{ + int ret; + + kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args"); + ret = suword16(base, word); + kmsan_init_ret(sizeof(int)); + return (ret); +} + +int +kmsan_suword32(volatile void *base, int32_t word) +{ + int ret; + + kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args"); + ret = suword32(base, word); + kmsan_init_ret(sizeof(int)); + return (ret); +} + +int +kmsan_suword64(volatile void *base, int64_t word) +{ + int ret; + + kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args"); + ret = suword64(base, word); + kmsan_init_ret(sizeof(int)); + return (ret); +} + +int +kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, + uint32_t newval) +{ + int ret; + + kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + + sizeof(newval), "casueword32(): args"); + ret = casueword32(base, oldval, oldvalp, newval); + kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, + sizeof(*oldvalp)); + kmsan_init_ret(sizeof(int)); + return (ret); +} + +int +kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, + u_long newval) +{ + int ret; + + kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + + sizeof(newval), "casueword32(): args"); + ret = casueword(base, oldval, oldvalp, newval); + kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, + sizeof(*oldvalp)); + kmsan_init_ret(sizeof(int)); + return (ret); +} + +/* -------------------------------------------------------------------------- */ + +#include +#include + +#define _MSAN_ATOMIC_FUNC_ADD(name, type) \ + void kmsan_atomic_add_##name(volatile type *ptr, type val) \ + { \ + kmsan_check_arg(sizeof(ptr) + sizeof(val), \ + "atomic_add_" #name "():args"); \ + kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ + "atomic_add_" #name "():ptr"); \ + atomic_add_##name(ptr, val); \ + } + +#define MSAN_ATOMIC_FUNC_ADD(name, type) \ + _MSAN_ATOMIC_FUNC_ADD(name, type) \ + _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ + _MSAN_ATOMIC_FUNC_ADD(rel_##name, type) + +#define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ + void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \ + { \ + kmsan_check_arg(sizeof(ptr) + sizeof(val), \ + "atomic_subtract_" #name "():args"); \ + kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ + "atomic_subtract_" #name "():ptr"); \ + atomic_subtract_##name(ptr, val); \ + } + +#define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ + _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ + _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ + _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) + +#define _MSAN_ATOMIC_FUNC_SET(name, type) \ + void kmsan_atomic_set_##name(volatile type *ptr, type val) \ + { \ + kmsan_check_arg(sizeof(ptr) + sizeof(val), \ + "atomic_set_" #name "():args"); \ + kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ + "atomic_set_" #name "():ptr"); \ + atomic_set_##name(ptr, val); \ + } + +#define MSAN_ATOMIC_FUNC_SET(name, type) \ + _MSAN_ATOMIC_FUNC_SET(name, type) \ + _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \ + _MSAN_ATOMIC_FUNC_SET(rel_##name, type) + +#define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ + void kmsan_atomic_clear_##name(volatile type *ptr, type val) \ + { \ + kmsan_check_arg(sizeof(ptr) + sizeof(val), \ + "atomic_clear_" #name "():args"); \ + kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ + "atomic_clear_" #name "():ptr"); \ + atomic_clear_##name(ptr, val); \ + } + +#define MSAN_ATOMIC_FUNC_CLEAR(name, type) \ + _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ + _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ + _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) + +#define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \ + type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ + { \ + kmsan_check_arg(sizeof(ptr) + sizeof(val), \ + "atomic_fetchadd_" #name "():args"); \ + kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ + "atomic_fetchadd_" #name "():ptr"); \ + kmsan_init_ret(sizeof(type)); \ + return (atomic_fetchadd_##name(ptr, val)); \ + } + +#define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ + type kmsan_atomic_readandclear_##name(volatile type *ptr) \ + { \ + kmsan_check_arg(sizeof(ptr), \ + "atomic_readandclear_" #name "():args"); \ + kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ + "atomic_readandclear_" #name "():ptr"); \ + kmsan_init_ret(sizeof(type)); \ + return (atomic_readandclear_##name(ptr)); \ + } + +#define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ + int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ + { \ + kmsan_check_arg(sizeof(ptr) + sizeof(v), \ + "atomic_testandclear_" #name "():args"); \ + kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ + "atomic_testandclear_" #name "():ptr"); \ + kmsan_init_ret(sizeof(int)); \ + return (atomic_testandclear_##name(ptr, v)); \ + } + +#define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ + int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \ + { \ + kmsan_check_arg(sizeof(ptr) + sizeof(v), \ + "atomic_testandset_" #name "():args"); \ + kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ + "atomic_testandset_" #name "():ptr"); \ + kmsan_init_ret(sizeof(int)); \ + return (atomic_testandset_##name(ptr, v)); \ + } + +#define MSAN_ATOMIC_FUNC_SWAP(name, type) \ + type kmsan_atomic_swap_##name(volatile type *ptr, type val) \ + { \ + kmsan_check_arg(sizeof(ptr) + sizeof(val), \ + "atomic_swap_" #name "():args"); \ + kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ + "atomic_swap_" #name "():ptr"); \ + kmsan_init_ret(sizeof(type)); \ + return (atomic_swap_##name(ptr, val)); \ + } + +#define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ + int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \ + type nval) \ + { \ + kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ + sizeof(nval), "atomic_cmpset_" #name "():args"); \ + kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ + "atomic_cmpset_" #name "():ptr"); \ + kmsan_init_ret(sizeof(int)); \ + return (atomic_cmpset_##name(ptr, oval, nval)); \ + } + +#define MSAN_ATOMIC_FUNC_CMPSET(name, type) \ + _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ + _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ + _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) + +#define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ + int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ + type nval) \ + { \ + kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ + sizeof(nval), "atomic_fcmpset_" #name "():args"); \ + kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ + "atomic_fcmpset_" #name "():ptr"); \ + kmsan_init_ret(sizeof(int)); \ + return (atomic_fcmpset_##name(ptr, oval, nval)); \ + } + +#define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ + _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ + _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ + _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) + +#define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ + void kmsan_atomic_thread_fence_##name(void) \ + { \ + atomic_thread_fence_##name(); \ + } + +#define _MSAN_ATOMIC_FUNC_LOAD(name, type) \ + type kmsan_atomic_load_##name(volatile type *ptr) \ + { \ + kmsan_check_arg(sizeof(ptr), \ + "atomic_load_" #name "():args"); \ + kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ + "atomic_load_" #name "():ptr"); \ + kmsan_init_ret(sizeof(type)); \ + return (atomic_load_##name(ptr)); \ + } + +#define MSAN_ATOMIC_FUNC_LOAD(name, type) \ + _MSAN_ATOMIC_FUNC_LOAD(name, type) \ + _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type) + +#define _MSAN_ATOMIC_FUNC_STORE(name, type) \ + void kmsan_atomic_store_##name(volatile type *ptr, type val) \ + { \ + kmsan_check_arg(sizeof(ptr) + sizeof(val), \ + "atomic_store_" #name "():args"); \ + kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \ + sizeof(type)); \ + atomic_store_##name(ptr, val); \ + } + +#define MSAN_ATOMIC_FUNC_STORE(name, type) \ + _MSAN_ATOMIC_FUNC_STORE(name, type) \ + _MSAN_ATOMIC_FUNC_STORE(rel_##name, type) + +MSAN_ATOMIC_FUNC_ADD(8, uint8_t); +MSAN_ATOMIC_FUNC_ADD(16, uint16_t); +MSAN_ATOMIC_FUNC_ADD(32, uint32_t); +MSAN_ATOMIC_FUNC_ADD(64, uint64_t); +MSAN_ATOMIC_FUNC_ADD(int, u_int); +MSAN_ATOMIC_FUNC_ADD(long, u_long); +MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); + +MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); +MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); +MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); +MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); +MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int); +MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long); +MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); + +MSAN_ATOMIC_FUNC_SET(8, uint8_t); +MSAN_ATOMIC_FUNC_SET(16, uint16_t); +MSAN_ATOMIC_FUNC_SET(32, uint32_t); +MSAN_ATOMIC_FUNC_SET(64, uint64_t); +MSAN_ATOMIC_FUNC_SET(int, u_int); +MSAN_ATOMIC_FUNC_SET(long, u_long); +MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t); + +MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t); +MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t); +MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t); +MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t); +MSAN_ATOMIC_FUNC_CLEAR(int, u_int); +MSAN_ATOMIC_FUNC_CLEAR(long, u_long); +MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); + +MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); +MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); +MSAN_ATOMIC_FUNC_FETCHADD(int, u_int); +MSAN_ATOMIC_FUNC_FETCHADD(long, u_long); + +MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); +MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); +MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); +MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); +MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); + +MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); +MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); +MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); +MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); + +MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); +MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); +MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int); +MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long); + +MSAN_ATOMIC_FUNC_SWAP(32, uint32_t); +MSAN_ATOMIC_FUNC_SWAP(64, uint64_t); +MSAN_ATOMIC_FUNC_SWAP(int, u_int); +MSAN_ATOMIC_FUNC_SWAP(long, u_long); +MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); + +MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t); +MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t); +MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t); +MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t); +MSAN_ATOMIC_FUNC_CMPSET(int, u_int); +MSAN_ATOMIC_FUNC_CMPSET(long, u_long); +MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); + +MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); +MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); +MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); +MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); +MSAN_ATOMIC_FUNC_FCMPSET(int, u_int); +MSAN_ATOMIC_FUNC_FCMPSET(long, u_long); +MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); + +MSAN_ATOMIC_FUNC_LOAD(8, uint8_t); +MSAN_ATOMIC_FUNC_LOAD(16, uint16_t); +MSAN_ATOMIC_FUNC_LOAD(32, uint32_t); +MSAN_ATOMIC_FUNC_LOAD(64, uint64_t); +MSAN_ATOMIC_FUNC_LOAD(char, u_char); +MSAN_ATOMIC_FUNC_LOAD(short, u_short); +MSAN_ATOMIC_FUNC_LOAD(int, u_int); +MSAN_ATOMIC_FUNC_LOAD(long, u_long); +MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); + +MSAN_ATOMIC_FUNC_STORE(8, uint8_t); +MSAN_ATOMIC_FUNC_STORE(16, uint16_t); +MSAN_ATOMIC_FUNC_STORE(32, uint32_t); +MSAN_ATOMIC_FUNC_STORE(64, uint64_t); +MSAN_ATOMIC_FUNC_STORE(char, u_char); +MSAN_ATOMIC_FUNC_STORE(short, u_short); +MSAN_ATOMIC_FUNC_STORE(int, u_int); +MSAN_ATOMIC_FUNC_STORE(long, u_long); +MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); + +MSAN_ATOMIC_FUNC_THREAD_FENCE(acq); +MSAN_ATOMIC_FUNC_THREAD_FENCE(rel); +MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); +MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); + +void +kmsan_atomic_interrupt_fence(void) +{ + atomic_interrupt_fence(); +} + +/* -------------------------------------------------------------------------- */ + +#include +#include +#include + +int +kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, + int flags, bus_space_handle_t *handlep) +{ + return (bus_space_map(tag, hnd, size, flags, handlep)); +} + +void +kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, + bus_size_t size) +{ + bus_space_unmap(tag, hnd, size); +} + +int +kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, + bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) +{ + return (bus_space_subregion(tag, hnd, offset, size, handlep)); +} + +void +kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, + bus_size_t size) +{ + bus_space_free(tag, hnd, size); +} + +void +kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, + bus_size_t offset, bus_size_t size, int flags) +{ + bus_space_barrier(tag, hnd, offset, size, flags); +} + +/* XXXMJ x86-specific */ +#define MSAN_BUS_READ_FUNC(func, width, type) \ + type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t offset) \ + { \ + type ret; \ + if ((tag) != X86_BUS_SPACE_IO) \ + kmsan_shadow_fill((uintptr_t)(hnd + offset), \ + KMSAN_STATE_INITED, (width)); \ + ret = bus_space_read##func##_##width(tag, hnd, offset); \ + kmsan_init_ret(sizeof(type)); \ + return (ret); \ + } \ + +#define MSAN_BUS_READ_PTR_FUNC(func, width, type) \ + void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t size, type *buf, \ + bus_size_t count) \ + { \ + kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \ + (width) * count); \ + bus_space_read_##func##_##width(tag, hnd, size, buf, \ + count); \ + } + +MSAN_BUS_READ_FUNC(, 1, uint8_t) +MSAN_BUS_READ_FUNC(_stream, 1, uint8_t) +MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) +MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) +MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) +MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) + +MSAN_BUS_READ_FUNC(, 2, uint16_t) +MSAN_BUS_READ_FUNC(_stream, 2, uint16_t) +MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) +MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) +MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) +MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) + +MSAN_BUS_READ_FUNC(, 4, uint32_t) +MSAN_BUS_READ_FUNC(_stream, 4, uint32_t) +MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) +MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) +MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) +MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) + +MSAN_BUS_READ_FUNC(, 8, uint64_t) + +#define MSAN_BUS_WRITE_FUNC(func, width, type) \ + void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t offset, type value) \ + { \ + bus_space_write##func##_##width(tag, hnd, offset, value);\ + } \ + +#define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ + void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ + bus_space_handle_t hnd, bus_size_t size, const type *buf, \ + bus_size_t count) \ + { \ + kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ + "bus_space_write()"); \ + bus_space_write_##func##_##width(tag, hnd, size, buf, \ + count); \ + } + +MSAN_BUS_WRITE_FUNC(, 1, uint8_t) +MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) +MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) +MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) +MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) +MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) + +MSAN_BUS_WRITE_FUNC(, 2, uint16_t) +MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) +MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) +MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) +MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) +MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) + +MSAN_BUS_WRITE_FUNC(, 4, uint32_t) +MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) +MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) +MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) +MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) +MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) + +MSAN_BUS_WRITE_FUNC(, 8, uint64_t) + +#define MSAN_BUS_SET_FUNC(func, width, type) \ + void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ + bus_space_handle_t hnd, bus_size_t offset, type value, \ + bus_size_t count) \ + { \ + bus_space_set_##func##_##width(tag, hnd, offset, value, \ + count); \ + } + +MSAN_BUS_SET_FUNC(multi, 1, uint8_t) +MSAN_BUS_SET_FUNC(region, 1, uint8_t) +MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) +MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) + +MSAN_BUS_SET_FUNC(multi, 2, uint16_t) +MSAN_BUS_SET_FUNC(region, 2, uint16_t) +MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) +MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) + +MSAN_BUS_SET_FUNC(multi, 4, uint32_t) +MSAN_BUS_SET_FUNC(region, 4, uint32_t) +MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) +MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) + +/* -------------------------------------------------------------------------- */ + +void +kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op) +{ + if ((op & BUS_DMASYNC_PREWRITE) != 0) { + switch (desc->md_type) { + case MEMDESC_VADDR: + kmsan_check(desc->u.md_vaddr, desc->md_opaque, + "dmasync"); + break; + case MEMDESC_BIO: + kmsan_check_bio(desc->u.md_bio, "dmasync"); + break; + case MEMDESC_MBUF: + kmsan_check_mbuf(desc->u.md_mbuf, "dmasync"); + break; + case MEMDESC_CCB: + kmsan_check_ccb(desc->u.md_ccb, "dmasync"); + break; + case 0: + break; + default: + kmsan_panic("%s: unhandled memdesc type %d", __func__, + desc->md_type); + } + } + if ((op & BUS_DMASYNC_POSTREAD) != 0) { + switch (desc->md_type) { + case MEMDESC_VADDR: + kmsan_mark(desc->u.md_vaddr, desc->md_opaque, + KMSAN_STATE_INITED); + break; + case MEMDESC_MBUF: + kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED); + break; + case MEMDESC_CCB: + kmsan_mark_ccb(desc->u.md_ccb, KMSAN_STATE_INITED); + break; + case 0: + break; + default: + kmsan_panic("%s: unhandled memdesc type %d", __func__, + desc->md_type); + } + } +} Index: sys/sys/msan.h =================================================================== --- /dev/null +++ sys/sys/msan.h @@ -0,0 +1,86 @@ +/* $NetBSD: msan.h,v 1.2 2020/09/09 16:29:59 maxv Exp $ */ + +/* + * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net + * All rights reserved. + * + * This code is part of the KMSAN subsystem of the NetBSD kernel. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _SYS_MSAN_H_ +#define _SYS_MSAN_H_ + +#ifdef KMSAN +#include +#include + +#define KMSAN_STATE_UNINIT 0xFF +#define KMSAN_STATE_INITED 0x00 + +#define KMSAN_TYPE_STACK 0 +#define KMSAN_TYPE_KMEM 1 +#define KMSAN_TYPE_MALLOC 2 +#define KMSAN_TYPE_UMA 3 + +#define KMSAN_RET_ADDR (uintptr_t)__builtin_return_address(0) + +union ccb; +struct bio; +struct mbuf; +struct memdesc; + +void kmsan_init(void *); + +void kmsan_shadow_map(vm_offset_t, size_t); + +void kmsan_thread_alloc(struct thread *); +void kmsan_thread_free(struct thread *); + +void kmsan_bus_dmamap_sync(struct memdesc *, bus_dmasync_op_t); + +void kmsan_orig(const void *, size_t, int, uintptr_t); +void kmsan_mark(const void *, size_t, uint8_t); + +void kmsan_check(const void *, size_t, const char *); +void kmsan_check_bio(const struct bio *, const char *); +void kmsan_check_ccb(const union ccb *, const char *); +void kmsan_check_mbuf(const struct mbuf *, const char *); + +#else +#define kmsan_init(u) +#define kmsan_shadow_map(a, s) +#define kmsan_thread_alloc(td) +#define kmsan_thread_free(l) +#define kmsan_dma_sync(m, a, s, o) +#define kmsan_dma_load(m, b, s, o) +#define kmsan_orig(p, l, c, a) +#define kmsan_mark(p, l, c) +#define kmsan_check(b, s, d) +#define kmsan_check_bio(b, d) +#define kmsan_check_ccb(c, d) +#define kmsan_check_mbuf(m, d) +#define kmsan_bus_dmamap_sync(d, op) +#endif + +#endif /* !_SYS_MSAN_H_ */ Index: sys/sys/proc.h =================================================================== --- sys/sys/proc.h +++ sys/sys/proc.h @@ -183,6 +183,7 @@ struct kcov_info; struct kdtrace_proc; struct kdtrace_thread; +struct kmsan_td; struct kq_timer_cb_data; struct mqueue_notifier; struct p_sched; @@ -250,6 +251,7 @@ #define td_siglist td_sigqueue.sq_signals u_char td_lend_user_pri; /* (t) Lend user pri. */ u_char td_allocdomain; /* (b) NUMA domain backing this struct thread. */ + struct kmsan_td *td_kmsan; /* (k) KMSAN state */ /* Cleared during fork1() */ #define td_startzero td_flags