Index: head/sys/amd64/conf/GENERIC =================================================================== --- head/sys/amd64/conf/GENERIC +++ head/sys/amd64/conf/GENERIC @@ -100,9 +100,12 @@ options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones options VERBOSE_SYSINIT=0 # Support debug.verbose_sysinit, off by default + +# Kernel Sanitizers +#options COVERAGE # Generic kernel coverage. Used by KCOV +#options KCOV # Kernel Coverage Sanitizer # Warning: KUBSAN can result in a kernel too large for loader to load #options KUBSAN # Kernel Undefined Behavior Sanitizer -#options KCOV # Kernel Coverage Sanitizer # Kernel dump features. options EKCD # Support for encrypted kernel dumps Index: head/sys/amd64/conf/GENERIC-NODEBUG =================================================================== --- head/sys/amd64/conf/GENERIC-NODEBUG +++ head/sys/amd64/conf/GENERIC-NODEBUG @@ -37,4 +37,5 @@ nooptions BUF_TRACKING nooptions DEADLKRES nooptions FULL_BUF_TRACKING - +nooptions COVERAGE +nooptions KCOV Index: head/sys/arm64/conf/GENERIC =================================================================== --- head/sys/arm64/conf/GENERIC +++ head/sys/arm64/conf/GENERIC @@ -92,9 +92,12 @@ options ALT_BREAK_TO_DEBUGGER # Enter debugger on keyboard escape sequence options USB_DEBUG # enable debug msgs options VERBOSE_SYSINIT=0 # Support debug.verbose_sysinit, off by default + +# Kernel Sanitizers +#options COVERAGE # Generic kernel coverage. Used by KCOV +#options KCOV # Kernel Coverage Sanitizer # Warning: KUBSAN can result in a kernel too large for loader to load #options KUBSAN # Kernel Undefined Behavior Sanitizer -#options KCOV # Kernel Coverage Sanitizer # Kernel dump features. options EKCD # Support for encrypted kernel dumps Index: head/sys/arm64/conf/GENERIC-NODEBUG =================================================================== --- head/sys/arm64/conf/GENERIC-NODEBUG +++ head/sys/arm64/conf/GENERIC-NODEBUG @@ -36,3 +36,5 @@ nooptions WITNESS_SKIPSPIN nooptions DEADLKRES nooptions USB_DEBUG +nooptions COVERAGE +nooptions KCOV Index: head/sys/conf/files =================================================================== --- head/sys/conf/files +++ head/sys/conf/files @@ -3883,6 +3883,8 @@ kern/subr_clock.c standard kern/subr_compressor.c standard \ compile-with "${NORMAL_C} -I$S/contrib/zstd/lib/freebsd" +kern/subr_coverage.c optional coverage \ + compile-with "${NORMAL_C} -fno-sanitize=all" kern/subr_counter.c standard kern/subr_devstat.c standard kern/subr_disk.c standard Index: head/sys/conf/kern.pre.mk =================================================================== --- head/sys/conf/kern.pre.mk +++ head/sys/conf/kern.pre.mk @@ -118,8 +118,8 @@ SAN_CFLAGS+= -fsanitize=undefined .endif -KCOV_ENABLED!= grep KCOV opt_kcov.h || true ; echo -.if !empty(KCOV_ENABLED) +COVERAGE_ENABLED!= grep COVERAGE opt_global.h || true ; echo +.if !empty(COVERAGE_ENABLED) SAN_CFLAGS+= -fsanitize-coverage=trace-pc,trace-cmp .endif Index: head/sys/conf/options =================================================================== --- head/sys/conf/options +++ head/sys/conf/options @@ -57,7 +57,6 @@ DDB_NUMSYM opt_ddb.h FULL_BUF_TRACKING opt_global.h GDB -KCOV opt_kcov.h KDB opt_global.h KDB_TRACE opt_kdb.h KDB_UNATTENDED opt_kdb.h @@ -234,6 +233,8 @@ ZSTDIO opt_zstdio.h # Sanitizers +COVERAGE opt_global.h +KCOV KUBSAN opt_global.h # POSIX kernel options Index: head/sys/kern/kern_kcov.c =================================================================== --- head/sys/kern/kern_kcov.c +++ head/sys/kern/kern_kcov.c @@ -39,29 +39,26 @@ __FBSDID("$FreeBSD$"); #include +#include #include -#include #include #include +#include #include #include #include #include #include #include -#include #include -#include -#include #include +#include #include #include #include #include -#include - MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type"); #define KCOV_ELEMENT_SIZE sizeof(uint64_t) @@ -138,17 +135,6 @@ static d_mmap_single_t kcov_mmap_single; static d_ioctl_t kcov_ioctl; -void __sanitizer_cov_trace_pc(void); -void __sanitizer_cov_trace_cmp1(uint8_t, uint8_t); -void __sanitizer_cov_trace_cmp2(uint16_t, uint16_t); -void __sanitizer_cov_trace_cmp4(uint32_t, uint32_t); -void __sanitizer_cov_trace_cmp8(uint64_t, uint64_t); -void __sanitizer_cov_trace_const_cmp1(uint8_t, uint8_t); -void __sanitizer_cov_trace_const_cmp2(uint16_t, uint16_t); -void __sanitizer_cov_trace_const_cmp4(uint32_t, uint32_t); -void __sanitizer_cov_trace_const_cmp8(uint64_t, uint64_t); -void __sanitizer_cov_trace_switch(uint64_t, uint64_t *); - static int kcov_alloc(struct kcov_info *info, size_t entries); static void kcov_init(const void *unused); @@ -169,6 +155,7 @@ "Maximum number of entries in the kcov buffer"); static struct mtx kcov_lock; +static int active_count; static struct kcov_info * get_kinfo(struct thread *td) @@ -197,25 +184,13 @@ return (info); } -/* - * Main entry point. A call to this function will be inserted - * at every edge, and if coverage is enabled for the thread - * this function will add the PC to the buffer. - */ -void -__sanitizer_cov_trace_pc(void) +static void +trace_pc(uintptr_t ret) { struct thread *td; struct kcov_info *info; uint64_t *buf, index; - /* - * To guarantee curthread is properly set, we exit early - * until the driver has been initialized - */ - if (cold) - return; - td = curthread; info = get_kinfo(td); if (info == NULL) @@ -237,7 +212,7 @@ if (index + 2 > info->entries) return; - buf[index + 1] = (uint64_t)__builtin_return_address(0); + buf[index + 1] = ret; buf[0] = index + 1; } @@ -248,13 +223,6 @@ struct kcov_info *info; uint64_t *buf, index; - /* - * To guarantee curthread is properly set, we exit early - * until the driver has been initialized - */ - if (cold) - return (false); - td = curthread; info = get_kinfo(td); if (info == NULL) @@ -287,109 +255,7 @@ return (true); } -void -__sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(0), arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(1), arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(2), arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(3), arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - -void -__sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2) -{ - - trace_cmp(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2, - (uint64_t)__builtin_return_address(0)); -} - /* - * val is the switch operand - * cases[0] is the number of case constants - * cases[1] is the size of val in bits - * cases[2..n] are the case constants - */ -void -__sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases) -{ - uint64_t i, count, ret, type; - - count = cases[0]; - ret = (uint64_t)__builtin_return_address(0); - - switch (cases[1]) { - case 8: - type = KCOV_CMP_SIZE(0); - break; - case 16: - type = KCOV_CMP_SIZE(1); - break; - case 32: - type = KCOV_CMP_SIZE(2); - break; - case 64: - type = KCOV_CMP_SIZE(3); - break; - default: - return; - } - - val |= KCOV_CMP_CONST; - - for (i = 0; i < count; i++) - if (!trace_cmp(type, val, cases[i + 2], ret)) - return; -} - -/* * The fd is being closed, cleanup everything we can. */ static void @@ -456,6 +322,7 @@ struct kcov_info *info; int error; + if ((error = devfs_get_cdevpriv((void **)&info)) != 0) return (error); @@ -571,6 +438,16 @@ error = EINVAL; break; } + + /* Lets hope nobody opens this 2 billion times */ + KASSERT(active_count < INT_MAX, + ("%s: Open too many times", __func__)); + active_count++; + if (active_count == 1) { + cov_register_pc(&trace_pc); + cov_register_cmp(&trace_cmp); + } + KASSERT(info->thread == NULL, ("Enabling kcov when already enabled")); info->thread = td; @@ -589,6 +466,13 @@ error = EINVAL; break; } + KASSERT(active_count > 0, ("%s: Open count is zero", __func__)); + active_count--; + if (active_count == 0) { + cov_register_pc(&trace_pc); + cov_register_cmp(&trace_cmp); + } + td->td_kcov_info = NULL; atomic_store_int(&info->state, KCOV_STATE_READY); /* @@ -618,6 +502,12 @@ return; mtx_lock_spin(&kcov_lock); + KASSERT(active_count > 0, ("%s: Open count is zero", __func__)); + active_count--; + if (active_count == 0) { + cov_register_pc(&trace_pc); + cov_register_cmp(&trace_cmp); + } td->td_kcov_info = NULL; if (info->state != KCOV_STATE_DYING) { /* @@ -673,4 +563,4 @@ EVENTHANDLER_PRI_ANY); } -SYSINIT(kcovdev, SI_SUB_DEVFS, SI_ORDER_ANY, kcov_init, NULL); +SYSINIT(kcovdev, SI_SUB_LAST, SI_ORDER_ANY, kcov_init, NULL); Index: head/sys/kern/subr_coverage.c =================================================================== --- head/sys/kern/subr_coverage.c +++ head/sys/kern/subr_coverage.c @@ -0,0 +1,237 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (C) 2018 The FreeBSD Foundation. All rights reserved. + * Copyright (C) 2018, 2019 Andrew Turner + * + * This software was developed by Mitchell Horne under sponsorship of + * the FreeBSD Foundation. + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 + * ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include + +void __sanitizer_cov_trace_pc(void); +void __sanitizer_cov_trace_cmp1(uint8_t, uint8_t); +void __sanitizer_cov_trace_cmp2(uint16_t, uint16_t); +void __sanitizer_cov_trace_cmp4(uint32_t, uint32_t); +void __sanitizer_cov_trace_cmp8(uint64_t, uint64_t); +void __sanitizer_cov_trace_const_cmp1(uint8_t, uint8_t); +void __sanitizer_cov_trace_const_cmp2(uint16_t, uint16_t); +void __sanitizer_cov_trace_const_cmp4(uint32_t, uint32_t); +void __sanitizer_cov_trace_const_cmp8(uint64_t, uint64_t); +void __sanitizer_cov_trace_switch(uint64_t, uint64_t *); + +static cov_trace_pc_t cov_trace_pc; +static cov_trace_cmp_t cov_trace_cmp; + +void +cov_register_pc(cov_trace_pc_t trace_pc) +{ + + atomic_store_ptr(&cov_trace_pc, trace_pc); +} + +void +cov_unregister_pc(void) +{ + + atomic_store_ptr(&cov_trace_pc, NULL); +} + +void +cov_register_cmp(cov_trace_cmp_t trace_cmp) +{ + + atomic_store_ptr(&cov_trace_cmp, trace_cmp); +} + +void +cov_unregister_cmp(void) +{ + + atomic_store_ptr(&cov_trace_cmp, NULL); +} + +/* + * Main entry point. A call to this function will be inserted + * at every edge, and if coverage is enabled for the thread + * this function will add the PC to the buffer. + */ +void +__sanitizer_cov_trace_pc(void) +{ + cov_trace_pc_t trace_pc; + + trace_pc = (cov_trace_pc_t)atomic_load_ptr(&cov_trace_pc); + if (trace_pc != NULL) + trace_pc((uint64_t)__builtin_return_address(0)); +} + +/* + * Comparison entry points. When the kernel performs a comparison + * operation the compiler inserts a call to one of the following + * functions to record the operation. + */ +void +__sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(0), arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(1), arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(2), arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(3), arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(0) | COV_CMP_CONST, arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(1) | COV_CMP_CONST, arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(2) | COV_CMP_CONST, arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +void +__sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2) +{ + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp != NULL) + trace_cmp(COV_CMP_SIZE(3) | COV_CMP_CONST, arg1, arg2, + (uint64_t)__builtin_return_address(0)); +} + +/* + * val is the switch operand + * cases[0] is the number of case constants + * cases[1] is the size of val in bits + * cases[2..n] are the case constants + */ +void +__sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases) +{ + uint64_t i, count, ret, type; + cov_trace_cmp_t trace_cmp; + + trace_cmp = (cov_trace_cmp_t)atomic_load_ptr(&cov_trace_cmp); + if (trace_cmp == NULL) + return; + + count = cases[0]; + ret = (uint64_t)__builtin_return_address(0); + + switch (cases[1]) { + case 8: + type = COV_CMP_SIZE(0); + break; + case 16: + type = COV_CMP_SIZE(1); + break; + case 32: + type = COV_CMP_SIZE(2); + break; + case 64: + type = COV_CMP_SIZE(3); + break; + default: + return; + } + + val |= COV_CMP_CONST; + + for (i = 0; i < count; i++) + if (!trace_cmp(type, val, cases[i + 2], ret)) + return; +} Index: head/sys/sys/coverage.h =================================================================== --- head/sys/sys/coverage.h +++ head/sys/sys/coverage.h @@ -0,0 +1,60 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (C) 2018 The FreeBSD Foundation. All rights reserved. + * Copyright (C) 2018, 2019 Andrew Turner. + * + * This software was developed by Mitchell Horne under sponsorship of + * the FreeBSD Foundation. + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 + * ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_COVERAGE_H_ +#define _SYS_COVERAGE_H_ + +#if !defined(_KERNEL) && !defined(_SYS_KCOV_H_) +#error Do not include this file directly in userspace, use sys/kcov.h +#endif + +#define COV_CMP_CONST (1 << 0) +#define COV_CMP_SIZE(x) ((x) << 1) +#define COV_CMP_MASK (3 << 1) +#define COV_CMP_GET_SIZE(x) (((x) >> 1) & 3) + +#ifdef _KERNEL +typedef void (*cov_trace_pc_t)(uintptr_t); +typedef bool (*cov_trace_cmp_t)(uint64_t, uint64_t, uint64_t, uint64_t); + +void cov_register_cmp(cov_trace_cmp_t); +void cov_unregister_cmp(void); +void cov_register_pc(cov_trace_pc_t); +void cov_unregister_pc(void); +#endif + +#endif /* _SYS_COVERAGE_H_ */ Index: head/sys/sys/kcov.h =================================================================== --- head/sys/sys/kcov.h +++ head/sys/sys/kcov.h @@ -38,6 +38,7 @@ #ifndef _SYS_KCOV_H_ #define _SYS_KCOV_H_ +#include #include #define KCOV_MAXENTRIES (1 << 24) /* 16M */ @@ -51,9 +52,9 @@ #define KIODISABLE _IO('c', 3) /* Disable coverage recording */ #define KIOSETBUFSIZE _IOWINT('c', 4) /* Set the buffer size */ -#define KCOV_CMP_CONST (1 << 0) -#define KCOV_CMP_SIZE(x) ((x) << 1) -#define KCOV_CMP_MASK (3 << 1) -#define KCOV_CMP_GET_SIZE(x) (((x) >> 1) & 3) +#define KCOV_CMP_CONST COV_CMP_CONST +#define KCOV_CMP_SIZE(x) COV_CMP_SIZE(x) +#define KCOV_CMP_MASK COV_CMP_MASK +#define KCOV_CMP_GET_SIZE(x) COV_CMP_GET_SIZE(x) #endif /* _SYS_KCOV_H_ */