Index: sys/amd64/conf/GENERIC =================================================================== --- sys/amd64/conf/GENERIC +++ sys/amd64/conf/GENERIC @@ -100,9 +100,12 @@ options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones options VERBOSE_SYSINIT=0 # Support debug.verbose_sysinit, off by default + +# Kernel Sanitizers +options COVERAGE # Generic kernel coverage. Used by KCOV +options KCOV # Kernel Coverage Sanitizer # Warning: KUBSAN can result in a kernel too large for loader to load #options KUBSAN # Kernel Undefined Behavior Sanitizer -#options KCOV # Kernel Coverage Sanitizer # Kernel dump features. options EKCD # Support for encrypted kernel dumps Index: sys/amd64/conf/GENERIC-NODEBUG =================================================================== --- sys/amd64/conf/GENERIC-NODEBUG +++ sys/amd64/conf/GENERIC-NODEBUG @@ -37,4 +37,5 @@ nooptions BUF_TRACKING nooptions DEADLKRES nooptions FULL_BUF_TRACKING - +nooptions COVERAGE +nooptions KCOV Index: sys/arm64/conf/GENERIC =================================================================== --- sys/arm64/conf/GENERIC +++ sys/arm64/conf/GENERIC @@ -92,9 +92,12 @@ options ALT_BREAK_TO_DEBUGGER # Enter debugger on keyboard escape sequence options USB_DEBUG # enable debug msgs options VERBOSE_SYSINIT=0 # Support debug.verbose_sysinit, off by default + +# Kernel Sanitizers +options COVERAGE # Generic kernel coverage. Used by KCOV +options KCOV # Kernel Coverage Sanitizer # Warning: KUBSAN can result in a kernel too large for loader to load #options KUBSAN # Kernel Undefined Behavior Sanitizer -#options KCOV # Kernel Coverage Sanitizer # Kernel dump features. options EKCD # Support for encrypted kernel dumps Index: sys/arm64/conf/GENERIC-NODEBUG =================================================================== --- sys/arm64/conf/GENERIC-NODEBUG +++ sys/arm64/conf/GENERIC-NODEBUG @@ -36,3 +36,5 @@ nooptions WITNESS_SKIPSPIN nooptions DEADLKRES nooptions USB_DEBUG +nooptions COVERAGE +nooptions KCOV Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -3883,6 +3883,8 @@ kern/subr_clock.c standard kern/subr_compressor.c standard \ compile-with "${NORMAL_C} -I$S/contrib/zstd/lib/freebsd" +kern/subr_coverage.c optional coverage \ + compile-with "${NORMAL_C} -fno-sanitize=all" kern/subr_counter.c standard kern/subr_devstat.c standard kern/subr_disk.c standard Index: sys/conf/kern.pre.mk =================================================================== --- sys/conf/kern.pre.mk +++ sys/conf/kern.pre.mk @@ -118,8 +118,8 @@ SAN_CFLAGS+= -fsanitize=undefined .endif -KCOV_ENABLED!= grep KCOV opt_kcov.h || true ; echo -.if !empty(KCOV_ENABLED) +COVERAGE_ENABLED!= grep COVERAGE opt_global.h || true ; echo +.if !empty(COVERAGE_ENABLED) SAN_CFLAGS+= -fsanitize-coverage=trace-pc,trace-cmp .endif Index: sys/conf/options =================================================================== --- sys/conf/options +++ sys/conf/options @@ -57,7 +57,6 @@ DDB_NUMSYM opt_ddb.h FULL_BUF_TRACKING opt_global.h GDB -KCOV opt_kcov.h KDB opt_global.h KDB_TRACE opt_kdb.h KDB_UNATTENDED opt_kdb.h @@ -234,6 +233,8 @@ ZSTDIO opt_zstdio.h # Sanitizers +COVERAGE opt_global.h +KCOV KUBSAN opt_global.h # POSIX kernel options Index: sys/kern/kern_kcov.c =================================================================== --- sys/kern/kern_kcov.c +++ sys/kern/kern_kcov.c @@ -39,29 +39,26 @@ __FBSDID("$FreeBSD$"); #include +#include #include -#include #include #include +#include #include #include #include #include #include #include -#include #include -#include -#include #include +#include #include #include #include #include -#include - MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type"); #define KCOV_ELEMENT_SIZE sizeof(uint64_t) @@ -138,17 +135,6 @@ static d_mmap_single_t kcov_mmap_single; static d_ioctl_t kcov_ioctl; -void __sanitizer_cov_trace_pc(void); -void __sanitizer_cov_trace_cmp1(uint8_t, uint8_t); -void __sanitizer_cov_trace_cmp2(uint16_t, uint16_t); -void __sanitizer_cov_trace_cmp4(uint32_t, uint32_t); -void __sanitizer_cov_trace_cmp8(uint64_t, uint64_t); -void __sanitizer_cov_trace_const_cmp1(uint8_t, uint8_t); -void __sanitizer_cov_trace_const_cmp2(uint16_t, uint16_t); -void __sanitizer_cov_trace_const_cmp4(uint32_t, uint32_t); -void __sanitizer_cov_trace_const_cmp8(uint64_t, uint64_t); -void __sanitizer_cov_trace_switch(uint64_t, uint64_t *); - static int kcov_alloc(struct kcov_info *info, size_t entries); static void kcov_init(const void *unused); @@ -169,6 +155,7 @@ "Maximum number of entries in the kcov buffer"); static struct mtx kcov_lock; +static int open_count; static struct kcov_info * get_kinfo(struct thread *td) @@ -197,25 +184,13 @@ return (info); } -/* - * Main entry point. A call to this function will be inserted - * at every edge, and if coverage is enabled for the thread - * this function will add the PC to the buffer. - */ -void -__sanitizer_cov_trace_pc(void) +static void +trace_pc(uintptr_t ret) { struct thread *td; struct kcov_info *info; uint64_t *buf, index; - /* - * To guarantee curthread is properly set, we exit early - * until the driver has been initialized - */ - if (cold) - return; - td = curthread; info = get_kinfo(td); if (info == NULL) @@ -237,7 +212,7 @@ if (index + 2 > info->entries) return; - buf[index + 1] = (uint64_t)__builtin_return_address(0); + buf[index + 1] = ret; buf[0] = index + 1; } @@ -248,13 +223,6 @@ struct kcov_info *info; uint64_t *buf, index; - /* - * To guarantee curthread is properly set, we exit early - * until the driver has been initialized - */ - if (cold) - return (false); - td = curthread; info = get_kinfo(td); if (info == NULL) @@ -287,108 +255,6 @@ return (true); } -void -__sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(0), arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(1), arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(2), arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(3), arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -/* - * val is the switch operand - * cases[0] is the number of case constants - * cases[1] is the size of val in bits - * cases[2..n] are the case constants - */ -void -__sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases) -{ - uint64_t i, count, ret, type; - - count = cases[0]; - ret = (uint64_t)__builtin_return_address(0); - - switch (cases[1]) { - case 8: - type = KCOV_CMP_SIZE(0); - break; - case 16: - type = KCOV_CMP_SIZE(1); - break; - case 32: - type = KCOV_CMP_SIZE(2); - break; - case 64: - type = KCOV_CMP_SIZE(3); - break; - default: - return; - } - - val |= KCOV_CMP_CONST; - - for (i = 0; i < count; i++) - if (!trace_cmp(type, val, cases[i + 2], ret)) - return; -} - /* * The fd is being closed, cleanup everything we can. */ @@ -444,6 +310,16 @@ info->mode = -1; info->mmap = false; + mtx_lock_spin(&kcov_lock); + /* Lets hope nobody opens this 2 billion times */ + KASSERT(open_count < INT_MAX, ("kcov_open: Open too many times")); + open_count++; + if (open_count == 1) { + cov_register_pc(&trace_pc); + cov_register_cmp(&trace_cmp); + } + mtx_unlock_spin(&kcov_lock); + if ((error = devfs_set_cdevpriv(info, kcov_mmap_cleanup)) != 0) kcov_mmap_cleanup(info); @@ -456,6 +332,15 @@ struct kcov_info *info; int error; + mtx_lock_spin(&kcov_lock); + KASSERT(open_count > 0, ("kcov_closed: Open count is zero")); + open_count--; + if (open_count == 0) { + cov_register_pc(&trace_pc); + cov_register_cmp(&trace_cmp); + } + mtx_unlock_spin(&kcov_lock); + if ((error = devfs_get_cdevpriv((void **)&info)) != 0) return (error); @@ -658,6 +543,7 @@ struct cdev *dev; mtx_init(&kcov_lock, "kcov lock", NULL, MTX_SPIN); + open_count = 0; make_dev_args_init(&args); args.mda_devsw = &kcov_cdevsw; @@ -673,4 +559,4 @@ EVENTHANDLER_PRI_ANY); } -SYSINIT(kcovdev, SI_SUB_DEVFS, SI_ORDER_ANY, kcov_init, NULL); +SYSINIT(kcovdev, SI_SUB_LAST, SI_ORDER_ANY, kcov_init, NULL); Index: sys/kern/subr_coverage.c =================================================================== --- /dev/null +++ sys/kern/subr_coverage.c @@ -0,0 +1,237 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (C) 2018 The FreeBSD Foundation. All rights reserved. + * Copyright (C) 2018, 2019 Andrew Turner + * + * This software was developed by Mitchell Horne under sponsorship of + * the FreeBSD Foundation. + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 + * ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include + +void __sanitizer_cov_trace_pc(void); +void __sanitizer_cov_trace_cmp1(uint8_t, uint8_t); +void __sanitizer_cov_trace_cmp2(uint16_t, uint16_t); +void __sanitizer_cov_trace_cmp4(uint32_t, uint32_t); +void __sanitizer_cov_trace_cmp8(uint64_t, uint64_t); +void __sanitizer_cov_trace_const_cmp1(uint8_t, uint8_t); +void __sanitizer_cov_trace_const_cmp2(uint16_t, uint16_t); +void __sanitizer_cov_trace_const_cmp4(uint32_t, uint32_t); +void __sanitizer_cov_trace_const_cmp8(uint64_t, uint64_t); +void __sanitizer_cov_trace_switch(uint64_t, uint64_t *); + +static cov_trace_pc_t cov_trace_pc; +static cov_trace_cmp_t cov_trace_cmp; + +void +cov_register_pc(cov_trace_pc_t trace_pc) +{ + + atomic_store_ptr(&cov_trace_pc, trace_pc); +} + +void +cov_unregister_pc(void) +{ + + atomic_store_ptr(&cov_trace_pc, NULL); +} + +void +cov_register_cmp(cov_trace_cmp_t trace_cmp) +{ + + atomic_store_ptr(&cov_trace_cmp, trace_cmp); +} + +void +cov_unregister_cmp(void) +{ + + atomic_store_ptr(&cov_trace_cmp, NULL); +} + +/* + * Main entry point. A call to this function will be inserted + * at every edge, and if coverage is enabled for the thread + * this function will add the PC to the buffer. + */ +void +__sanitizer_cov_trace_pc(void) +{ + cov_trace_pc_t trace_pc; + + trace_pc = (cov_trace_pc_t)atomic_load_ptr(&cov_trace_pc); + if (trace_pc != NULL) + trace_pc((uint64_t)__builtin_return_address(0)); +} + +/* + * Comparison entry points. When the kernel performs a comparison + * operation the compiler inserts a call to one of the following + * functions to record the operation. + */ +void +__sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(0), arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(1), arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(2), arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(3), arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(0) | COV_CMP_CONST, arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(1) | COV_CMP_CONST, arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(2) | COV_CMP_CONST, arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(3) | COV_CMP_CONST, arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +/* + * val is the switch operand + * cases[0] is the number of case constants + * cases[1] is the size of val in bits + * cases[2..n] are the case constants + */ +void +__sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases) +{ + uint64_t i, count, ret, type; + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp == NULL) + return; + + count = cases[0]; + ret = (uint64_t)__builtin_return_address(0); + + switch (cases[1]) { + case 8: + type = COV_CMP_SIZE(0); + break; + case 16: + type = COV_CMP_SIZE(1); + break; + case 32: + type = COV_CMP_SIZE(2); + break; + case 64: + type = COV_CMP_SIZE(3); + break; + default: + return; + } + + val |= COV_CMP_CONST; + + for (i = 0; i < count; i++) + if (!trace_cmp(type, val, cases[i + 2], ret)) + return; +} Index: sys/sys/coverage.h =================================================================== --- sys/sys/coverage.h +++ sys/sys/coverage.h @@ -35,25 +35,26 @@ * $FreeBSD$ */ -#ifndef _SYS_KCOV_H_ -#define _SYS_KCOV_H_ +#ifndef _SYS_COVERAGE_H_ +#define _SYS_COVERAGE_H_ -#include +#if !defined(_KERNEL) && !defined(_SYS_KCOV_H_) +#error Do not include this file directly in userspace, use sys/kcov.h +#endif -#define KCOV_MAXENTRIES (1 << 24) /* 16M */ -#define KCOV_ENTRY_SIZE 8 +#define COV_CMP_CONST (1 << 0) +#define COV_CMP_SIZE(x) ((x) << 1) +#define COV_CMP_MASK (3 << 1) +#define COV_CMP_GET_SIZE(x) (((x) >> 1) & 3) -#define KCOV_MODE_TRACE_PC 0 -#define KCOV_MODE_TRACE_CMP 1 +#ifdef _KERNEL +typedef void (*cov_trace_pc_t)(uintptr_t); +typedef bool (*cov_trace_cmp_t)(uint64_t, uint64_t, uint64_t, uint64_t); -/* KCOV ioctls */ -#define KIOENABLE _IOWINT('c', 2) /* Enable coverage recording */ -#define KIODISABLE _IO('c', 3) /* Disable coverage recording */ -#define KIOSETBUFSIZE _IOWINT('c', 4) /* Set the buffer size */ +void cov_register_cmp(cov_trace_cmp_t); +void cov_unregister_cmp(void); +void cov_register_pc(cov_trace_pc_t); +void cov_unregister_pc(void); +#endif -#define KCOV_CMP_CONST (1 << 0) -#define KCOV_CMP_SIZE(x) ((x) << 1) -#define KCOV_CMP_MASK (3 << 1) -#define KCOV_CMP_GET_SIZE(x) (((x) >> 1) & 3) - -#endif /* _SYS_KCOV_H_ */ +#endif /* _SYS_COVERAGE_H_ */ Index: sys/sys/kcov.h =================================================================== --- sys/sys/kcov.h +++ sys/sys/kcov.h @@ -38,6 +38,7 @@ #ifndef _SYS_KCOV_H_ #define _SYS_KCOV_H_ +#include #include #define KCOV_MAXENTRIES (1 << 24) /* 16M */ @@ -51,9 +52,9 @@ #define KIODISABLE _IO('c', 3) /* Disable coverage recording */ #define KIOSETBUFSIZE _IOWINT('c', 4) /* Set the buffer size */ -#define KCOV_CMP_CONST (1 << 0) -#define KCOV_CMP_SIZE(x) ((x) << 1) -#define KCOV_CMP_MASK (3 << 1) -#define KCOV_CMP_GET_SIZE(x) (((x) >> 1) & 3) +#define KCOV_CMP_CONST COV_CMP_CONST +#define KCOV_CMP_SIZE(x) COV_CMP_SIZE(x) +#define KCOV_CMP_MASK COV_CMP_MASK +#define KCOV_CMP_GET_SIZE(x) COV_CMP_GET_SIZE(x) #endif /* _SYS_KCOV_H_ */