Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -3783,6 +3783,8 @@ kern/kern_idle.c standard kern/kern_intr.c standard kern/kern_jail.c standard +kern/kern_kcov.c optional kcov \ + compile-with "${NORMAL_C} -fno-sanitize-coverage=trace-pc" kern/kern_khelp.c standard kern/kern_kthread.c standard kern/kern_ktr.c optional ktr Index: sys/conf/kern.opts.mk =================================================================== --- sys/conf/kern.opts.mk +++ sys/conf/kern.opts.mk @@ -47,6 +47,7 @@ __DEFAULT_NO_OPTIONS = \ EXTRA_TCP_STACKS \ + KCOV \ KERNEL_RETPOLINE \ NAND \ OFED \ Index: sys/conf/kern.pre.mk =================================================================== --- sys/conf/kern.pre.mk +++ sys/conf/kern.pre.mk @@ -112,6 +112,10 @@ .endif DEFINED_PROF= ${PROF} +.if ${MK_KCOV} != "no" +CFLAGS+= -fsanitize-coverage=trace-pc +.endif + # Put configuration-specific C flags last (except for ${PROF}) so that they # can override the others. CFLAGS+= ${CONF_CFLAGS} Index: sys/conf/options =================================================================== --- sys/conf/options +++ sys/conf/options @@ -58,6 +58,7 @@ DDB_NUMSYM opt_ddb.h FULL_BUF_TRACKING opt_global.h GDB +KCOV opt_kcov.h KDB opt_global.h KDB_TRACE opt_kdb.h KDB_UNATTENDED opt_kdb.h Index: sys/kern/kern_kcov.c =================================================================== --- /dev/null +++ sys/kern/kern_kcov.c @@ -0,0 +1,332 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (C) 2018 The FreeBSD Foundation. All rights reserved. + * + * This software was developed by Mitchell Horne under sponsorship of + * the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#define BUF_SIZE_BYTES(info) \ + (info != NULL ? (size_t)info->size * sizeof(uintptr_t) : 0) +#define BUF_AVAIL_BYTES(info) \ + (info != NULL ? (size_t)info->index * sizeof(uintptr_t) : 0) + +MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type"); +MALLOC_DEFINE(M_KCOV_BUF, "kcovbuffer", "KCOV buffer type"); + +struct kcov_info { + struct sx lock; + struct thread *td; + uintptr_t *buf; + u_int size; + u_int index; + int mode; +}; + +/* Prototypes */ +static d_open_t kcov_open; +static d_close_t kcov_close; +static d_read_t kcov_read; +static d_mmap_t kcov_mmap; +static d_ioctl_t kcov_ioctl; + +static void kcov_info_reset(struct kcov_info *info); +static int kcov_alloc(struct kcov_info *info, u_int entries); +static void kcov_init(const void *unused); + +static bool kcov_initialized = false; + +static struct cdevsw kcov_cdevsw = { + .d_version = D_VERSION, + .d_open = kcov_open, + .d_close = kcov_close, + .d_read = kcov_read, + .d_mmap = kcov_mmap, + .d_ioctl = kcov_ioctl, + .d_name = "kcov", +}; + +static u_int kcov_max_entries = KCOV_MAXENTRIES; +SYSCTL_UINT(_kern, OID_AUTO, kcov_max_entries, CTLFLAG_RW, + &kcov_max_entries, 0, + "Maximum number of entries that can be stored in a kcov buffer"); + +/* + * Main entry point. A call to this function will be inserted + * at every edge, and if coverage is enabled for the thread + * this function will add the PC to the buffer. + */ +void +__sanitizer_cov_trace_pc(void) +{ + struct thread *td; + struct kcov_info *info; + + /* + * To guarantee curthread is properly set, we exit early + * until the driver has been initialized + */ + if (!kcov_initialized) + return; + + td = curthread; + info = td->td_kcov_info; + + /* + * Check first that KCOV is enabled for the current thread. + * Additionally, we want to exclude (for now) all code that + * is not explicitly part of syscall call chain, such as + * interrupt handlers, since we are mainly interested in + * finding non-trivial paths through the syscall. + */ + if (info == NULL || info->buf == NULL || + info->mode != KCOV_MODE_TRACE_PC || + td->td_intr_nesting_level > 0 || !interrupts_enabled()) + return; + + if (info->index < info->size) { + info->buf[info->index] = + (uintptr_t)__builtin_return_address(0); + info->index++; + } +} + +static int +kcov_open(struct cdev *dev, int oflags, int devtype, struct thread *td) +{ + struct kcov_info *info; + + info = malloc(sizeof(struct kcov_info), M_KCOV_INFO, + M_ZERO | M_WAITOK); + kcov_info_reset(info); + sx_init(&info->lock, "kcov_lock"); + dev->si_drv1 = info; + + return (0); +} + +static int +kcov_close(struct cdev *dev, int fflag, int devtype, struct thread *td) +{ + struct kcov_info *info; + + info = dev->si_drv1; + if (info == NULL) + return (EINVAL); + + td->td_kcov_info = NULL; + dev->si_drv1 = NULL; + sx_destroy(&info->lock); + free(info->buf, M_KCOV_BUF); + free(info, M_KCOV_INFO); + + return (0); +} + +static int +kcov_read(struct cdev *dev, struct uio *uio, int ioflag) +{ + struct kcov_info *info; + size_t len; + int error; + + info = dev->si_drv1; + if (info == NULL || info->buf == NULL) { + return (EINVAL); + } + + sx_slock(&info->lock); + len = uio->uio_resid <= BUF_AVAIL_BYTES(info) - uio->uio_offset ? + uio->uio_resid : BUF_AVAIL_BYTES(info) - uio->uio_offset; + + error = uiomove(info->buf, len, uio); + sx_sunlock(&info->lock); + return (error); +} + +static int +kcov_mmap(struct cdev *dev, vm_ooffset_t offset, vm_paddr_t *paddr, + int prot, vm_memattr_t *memattr __unused) +{ + struct kcov_info *info; + + if (prot & PROT_EXEC) + return (EINVAL); + + info = dev->si_drv1; + if (offset < 0 || offset >= BUF_SIZE_BYTES(info)) + return (EINVAL); + + *paddr = vtophys(info->buf) + offset; + return (0); +} + +static void +kcov_info_reset(struct kcov_info *info) +{ + + if (info == NULL) + return; + + free(info->buf, M_KCOV_BUF); + info->buf = NULL; + info->mode = KCOV_MODE_NONE; + info->size = 0; + info->index = 0; +} + +static int +kcov_alloc(struct kcov_info *info, u_int entries) +{ + size_t buf_size; + + if (entries > kcov_max_entries) + return (EINVAL); + + /* Align to page size so mmap can't access other kernel memory */ + buf_size = roundup2(entries, PAGE_SIZE) * sizeof(uintptr_t); + + kcov_info_reset(info); + info->buf = malloc(buf_size, M_KCOV_BUF, M_WAITOK); + info->size = entries; + + return (0); +} + +static int +kcov_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag __unused, + struct thread *td) +{ + struct kcov_info *info; + int error; + + error = 0; + info = dev->si_drv1; + + sx_xlock(&info->lock); + switch (cmd) { + case KIOSETBUFSIZE: + /* + * Set the size of the coverage buffer. Should be called + * before enabling coverage collection for that thread. + */ + if (info->td != NULL) { + error = EBUSY; + break; + } + error = kcov_alloc(info, *(u_int *)data); + break; + case KIOENABLE: + /* Only enable if not currently owned */ + if (info->td != NULL) { + error = EBUSY; + break; + } + info->mode = *(int *)data; + td->td_kcov_info = info; + info->td = td; + break; + case KIODISABLE: + /* Only the currently enabled thread may disable itself */ + if (info->td != td) { + error = EINVAL; + } + info->mode = KCOV_MODE_NONE; + td->td_kcov_info = NULL; + info->td = NULL; + break; + case KIORESET: + info->index = 0; + break; + case KIONREAD: + /* Return the number of entries available to be read */ + *(u_int *)data = info->index; + break; + default: + error = EINVAL; + break; + } + sx_xunlock(&info->lock); + + return (error); +} + +static void +kcov_init(const void *unused) +{ + struct make_dev_args args; + struct cdev *dev; + + make_dev_args_init(&args); + args.mda_devsw = &kcov_cdevsw; + args.mda_uid = UID_ROOT; + args.mda_gid = GID_WHEEL; + args.mda_mode = 0660; + if (make_dev_s(&args, &dev, "kcov") != 0) { + printf("%s", "Failed to create kcov device"); + return; + } + + kcov_initialized = true; +} + +/* + * thread_exit() hook + */ +void +kcov_thread_exit(struct thread *td) +{ + + if (td->td_kcov_info != NULL) { + td->td_kcov_info->td = NULL; + td->td_kcov_info = NULL; + } +} + +SYSINIT(kcovdev, SI_SUB_DEVFS, SI_ORDER_ANY, kcov_init, NULL); Index: sys/kern/kern_thread.c =================================================================== --- sys/kern/kern_thread.c +++ sys/kern/kern_thread.c @@ -30,12 +30,14 @@ #include "opt_witness.h" #include "opt_hwpmc_hooks.h" +#include "opt_kcov.h" #include __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include @@ -81,9 +83,9 @@ "struct thread KBI td_flags"); _Static_assert(offsetof(struct thread, td_pflags) == 0x104, "struct thread KBI td_pflags"); -_Static_assert(offsetof(struct thread, td_frame) == 0x468, +_Static_assert(offsetof(struct thread, td_frame) == 0x470, "struct thread KBI td_frame"); -_Static_assert(offsetof(struct thread, td_emuldata) == 0x510, +_Static_assert(offsetof(struct thread, td_emuldata) == 0x518, "struct thread KBI td_emuldata"); _Static_assert(offsetof(struct proc, p_flag) == 0xb0, "struct proc KBI p_flag"); @@ -101,9 +103,9 @@ "struct thread KBI td_flags"); _Static_assert(offsetof(struct thread, td_pflags) == 0xa0, "struct thread KBI td_pflags"); -_Static_assert(offsetof(struct thread, td_frame) == 0x2e4, +_Static_assert(offsetof(struct thread, td_frame) == 0x2e8, "struct thread KBI td_frame"); -_Static_assert(offsetof(struct thread, td_emuldata) == 0x330, +_Static_assert(offsetof(struct thread, td_emuldata) == 0x334, "struct thread KBI td_emuldata"); _Static_assert(offsetof(struct proc, p_flag) == 0x68, "struct proc KBI p_flag"); @@ -532,6 +534,9 @@ SDT_PROBE0(proc, , , lwp__exit); KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); +#ifdef KCOV + kcov_thread_exit(td); +#endif #ifdef AUDIT AUDIT_SYSCALL_EXIT(0, td); #endif Index: sys/sys/kcov.h =================================================================== --- /dev/null +++ sys/sys/kcov.h @@ -0,0 +1,57 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (C) 2018 The FreeBSD Foundation. All rights reserved. + * + * This software was developed by Mitchell Horne under sponsorship of + * the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_KCOV_H_ +#define _SYS_KCOV_H_ + +#include + +#define KCOV_MAXENTRIES (1 << 24) /* 16M */ + +#define KCOV_MODE_NONE -1 +#define KCOV_MODE_TRACE_PC 0 +#define KCOV_MODE_TRACE_CMP 1 + +/* KCOV ioctls */ +#define KIOENABLE _IOW('c', 2, int) /* Enable coverage recording */ +#define KIODISABLE _IO('c', 3) /* Disable coverage recording */ +#define KIOSETBUFSIZE _IOW('c', 4, u_int) /* Set the buffer size */ +#define KIONREAD _IOR('c', 5, u_int) /* Entries available for read */ +#define KIORESET _IO('c', 6) /* Reset the buffer index */ + +#ifdef _KERNEL + +void kcov_thread_exit(struct thread *); +void __sanitizer_cov_trace_pc(void); + +#endif /* _KERNEL */ +#endif /* _SYS_KCOV_H_ */ Index: sys/sys/proc.h =================================================================== --- sys/sys/proc.h +++ sys/sys/proc.h @@ -176,6 +176,7 @@ struct filemon; struct kaioinfo; struct kaudit_record; +struct kcov_info; struct kdtrace_proc; struct kdtrace_thread; struct mqueue_notifier; @@ -297,6 +298,7 @@ void *td_su; /* (k) FFS SU private */ sbintime_t td_sleeptimo; /* (t) Sleep timeout. */ int td_rtcgen; /* (s) rtc_generation of abs. sleep */ + struct kcov_info *td_kcov_info; /* (*) Kernel code coverage data */ #define td_endzero td_sigmask /* Copied during fork1() or create_thread(). */