diff --git a/stand/common/load_elf.c b/stand/common/load_elf.c --- a/stand/common/load_elf.c +++ b/stand/common/load_elf.c @@ -545,6 +545,9 @@ Elf_Dyn *dp; Elf_Addr adp; Elf_Addr ctors; +#ifdef KCFI + Elf_Addr kcfi; +#endif int ndp; int symstrindex; int symtabindex; @@ -731,15 +734,30 @@ shdr[ehdr->e_shstrndx].sh_offset, chunk); if (shstr) { for (i = 0; i < ehdr->e_shnum; i++) { + if (strcmp(shstr + shdr[i].sh_name, ".ctors") == + 0) { + ctors = shdr[i].sh_addr; + file_addmetadata(fp, + MODINFOMD_CTORS_ADDR, sizeof(ctors), + &ctors); + size = shdr[i].sh_size; + file_addmetadata(fp, + MODINFOMD_CTORS_SIZE, sizeof(size), + &size); + } +#ifdef KCFI if (strcmp(shstr + shdr[i].sh_name, - ".ctors") != 0) - continue; - ctors = shdr[i].sh_addr; - file_addmetadata(fp, MODINFOMD_CTORS_ADDR, - sizeof(ctors), &ctors); - size = shdr[i].sh_size; - file_addmetadata(fp, MODINFOMD_CTORS_SIZE, - sizeof(size), &size); + ".kcfi_traps") == 0) { + kcfi = shdr[i].sh_addr; + file_addmetadata(fp, + MODINFOMD_KCFI_ADDR, sizeof(kcfi), + &kcfi); + size = shdr[i].sh_size; + file_addmetadata(fp, + MODINFOMD_KCFI_SIZE, sizeof(size), + &size); + } +#endif break; } free(shstr); diff --git a/sys/amd64/amd64/cfi.c b/sys/amd64/amd64/cfi.c new file mode 100644 --- /dev/null +++ b/sys/amd64/amd64/cfi.c @@ -0,0 +1,73 @@ +#include + +#include +#include + +#include +#include + +static int +regoff(int reg) +{ +#define _MATCH_REG(i, reg) \ + case i: \ + return (offsetof(struct trapframe, tf_ ## reg) / \ + sizeof(register_t)) + switch (reg) { + _MATCH_REG( 0, rax); + _MATCH_REG( 1, rcx); + _MATCH_REG( 2, rdx); + _MATCH_REG( 3, rbx); + _MATCH_REG( 4, rsp); /* SIB when mod != 3 */ + _MATCH_REG( 5, rbp); + _MATCH_REG( 6, rsi); + _MATCH_REG( 7, rdi); + _MATCH_REG( 8, r8); /* REX.R is set */ + _MATCH_REG( 9, r9); + _MATCH_REG(10, r10); + _MATCH_REG(11, r11); + _MATCH_REG(12, r12); + _MATCH_REG(13, r13); + _MATCH_REG(14, r14); + _MATCH_REG(15, r15); + } +#undef _MATCH_REG + return (0); +} + + +bool +decode_cfi(struct trapframe *tf, register_t *addr, uint32_t *type) { + unsigned char buffer[13]; + + strlcpy(buffer, (char *)(tf->tf_rip - 12), sizeof(buffer)); + /* + * clang generates following instructions: + * + * mov $typeid, %reg + * add $callee-16, %reg + * je .Lcorrect + * ud2 + * .Lcorrect + * + * What we do is to compare if the previous context is trigger by mov(0xba) and following with one add(0x03), + * if it is, we can identified it maybe CFI fault + */ + + if(buffer[1] != 0xba) + return (false); + + if(buffer[7] != 0x03) + return (false); + + if(buffer[7] != 0x03) + return (false); + + *type = *((uint32_t *)(buffer + 2)); + + // Decode register(prefix & REX.R) | (MODRM.reg(3bit)) + *addr = ((register_t *)tf)[regoff((((buffer[0] >> 2) & 0x1) << 3) | ((buffer[2] >> 3) & 0x7))]; + return (true); +} + + diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -440,6 +441,14 @@ (void)trap_pfault(frame, false, NULL, NULL); return; + case T_PRIVINFLT: + /* Triggered by ud2 with kCFI enabled */ + #ifdef KCFI + if(cfi_handler(frame)) + return; + #endif + break; + case T_DNA: if (PCB_USER_FPU(td->td_pcb)) panic("Unregistered use of FPU in kernel"); diff --git a/sys/amd64/include/cfi.h b/sys/amd64/include/cfi.h new file mode 100644 --- /dev/null +++ b/sys/amd64/include/cfi.h @@ -0,0 +1,14 @@ +#ifndef _MACHINE_CFI_H +#define _MACHINE_CFI_H + +#include + + +#include +#include + + +bool +decode_cfi(struct trapframe *, register_t *, uint32_t *); + +#endif diff --git a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c --- a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c +++ b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c @@ -345,21 +345,26 @@ { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON }, }; -static void -dtrace_nullop(void) -{} +#define DEFINE_NULLOP_HANDLER(name, ...) \ + static void name(__VA_ARGS__) \ + { \ + } + +DEFINE_NULLOP_HANDLER(dtrace_provide_handler, void *_1, dtrace_probedesc_t *_2); +DEFINE_NULLOP_HANDLER(dtrace_provide_module_handler, void *_1, modctl_t *_2); +DEFINE_NULLOP_HANDLER(dtrace_vpdtid_vp_handler, void *_1, dtrace_id_t _2, void *_3); static dtrace_pops_t dtrace_provider_ops = { - .dtps_provide = (void (*)(void *, dtrace_probedesc_t *))dtrace_nullop, - .dtps_provide_module = (void (*)(void *, modctl_t *))dtrace_nullop, - .dtps_enable = (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, - .dtps_disable = (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, - .dtps_suspend = (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, - .dtps_resume = (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, + .dtps_provide = dtrace_provide_handler, + .dtps_provide_module = dtrace_provide_module_handler, + .dtps_enable = dtrace_vpdtid_vp_handler, + .dtps_disable = dtrace_vpdtid_vp_handler, + .dtps_suspend = dtrace_vpdtid_vp_handler, + .dtps_resume = dtrace_vpdtid_vp_handler, .dtps_getargdesc = NULL, .dtps_getargval = NULL, .dtps_usermode = NULL, - .dtps_destroy = (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, + .dtps_destroy = dtrace_vpdtid_vp_handler, }; static dtrace_id_t dtrace_probeid_begin; /* special BEGIN probe */ @@ -8785,21 +8790,21 @@ if (pops->dtps_provide == NULL) { ASSERT(pops->dtps_provide_module != NULL); provider->dtpv_pops.dtps_provide = - (void (*)(void *, dtrace_probedesc_t *))dtrace_nullop; + dtrace_provide_handler; } if (pops->dtps_provide_module == NULL) { ASSERT(pops->dtps_provide != NULL); provider->dtpv_pops.dtps_provide_module = - (void (*)(void *, modctl_t *))dtrace_nullop; + dtrace_provide_module_handler; } if (pops->dtps_suspend == NULL) { ASSERT(pops->dtps_resume == NULL); provider->dtpv_pops.dtps_suspend = - (void (*)(void *, dtrace_id_t, void *))dtrace_nullop; + dtrace_vpdtid_vp_handler; provider->dtpv_pops.dtps_resume = - (void (*)(void *, dtrace_id_t, void *))dtrace_nullop; + dtrace_vpdtid_vp_handler; } provider->dtpv_arg = arg; @@ -8866,8 +8871,7 @@ int i, self = 0, noreap = 0; dtrace_probe_t *probe, *first = NULL; - if (old->dtpv_pops.dtps_enable == - (void (*)(void *, dtrace_id_t, void *))dtrace_nullop) { + if (old->dtpv_pops.dtps_enable == dtrace_vpdtid_vp_handler) { /* * If DTrace itself is the provider, we're called with locks * already held. @@ -9045,8 +9049,7 @@ { dtrace_provider_t *pvp = (dtrace_provider_t *)id; - ASSERT(pvp->dtpv_pops.dtps_enable != - (void (*)(void *, dtrace_id_t, void *))dtrace_nullop); + ASSERT(pvp->dtpv_pops.dtps_enable != dtrace_vpdtid_vp_handler); mutex_enter(&dtrace_provider_lock); mutex_enter(&dtrace_lock); @@ -9086,8 +9089,7 @@ /* * Make sure this isn't the dtrace provider itself. */ - ASSERT(prov->dtpv_pops.dtps_enable != - (void (*)(void *, dtrace_id_t, void *))dtrace_nullop); + ASSERT(prov->dtpv_pops.dtps_enable != dtrace_vpdtid_vp_handler); mutex_enter(&dtrace_provider_lock); mutex_enter(&dtrace_lock); diff --git a/sys/compat/linuxkpi/common/include/linux/module.h b/sys/compat/linuxkpi/common/include/linux/module.h --- a/sys/compat/linuxkpi/common/include/linux/module.h +++ b/sys/compat/linuxkpi/common/include/linux/module.h @@ -82,9 +82,11 @@ #define SI_SUB_OFED_MODINIT (SI_SUB_ROOT_CONF - 1) #include +#include static inline void _module_run(void *arg) +__NOCFI { void (*fn)(void); #ifdef OFED_DEBUG_INIT diff --git a/sys/conf/files b/sys/conf/files --- a/sys/conf/files +++ b/sys/conf/files @@ -3895,6 +3895,7 @@ kern/subr_bus_dma.c standard kern/subr_bufring.c standard kern/subr_capability.c standard +kern/subr_cfi.c optional kcfi kern/subr_clock.c standard kern/subr_compressor.c standard \ compile-with "${NORMAL_C} -I$S/contrib/zstd/lib/freebsd" diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -64,6 +64,7 @@ amd64/amd64/bios.c standard amd64/amd64/bpf_jit_machdep.c optional bpf_jitter amd64/amd64/copyout.c standard +amd64/amd64/cfi.c optional kcfi amd64/amd64/cpu_switch.S standard amd64/amd64/db_disasm.c optional ddb amd64/amd64/db_interface.c optional ddb diff --git a/sys/conf/kern.mk b/sys/conf/kern.mk --- a/sys/conf/kern.mk +++ b/sys/conf/kern.mk @@ -291,6 +291,10 @@ -fsanitize=thread .endif +.if !empty(KCFI_ENABLED) +SAN_CFLAGS+= -fsanitize=kcfi +.endif + # # Kernel Memory SANitizer support # diff --git a/sys/conf/kern.post.mk b/sys/conf/kern.post.mk --- a/sys/conf/kern.post.mk +++ b/sys/conf/kern.post.mk @@ -37,6 +37,10 @@ MKMODULESENV+= KCSAN_ENABLED="yes" .endif +.if !empty(KCFI_ENABLED) +MKMODULESENV+= KCFI_ENABLED="yes" +.endif + .if defined(GCOV_CFLAGS) MKMODULESENV+= GCOV_CFLAGS="${GCOV_CFLAGS}" .endif @@ -210,11 +214,15 @@ OBJS_DEPEND_GUESS+= offset.inc assym.inc vnode_if.h ${BEFORE_DEPEND:M*.h} \ ${MFILES:T:S/.m$/.h/} +.if !empty(KCFI_ENABLED) +MAKEOBJOPT= "-s" +.endif + .for mfile in ${MFILES} # XXX the low quality .m.o rules gnerated by config are normally used # instead of the .m.c rules here. ${mfile:T:S/.m$/.c/}: ${mfile} - ${AWK} -f $S/tools/makeobjops.awk ${mfile} -c + ${AWK} -f $S/tools/makeobjops.awk ${mfile} -c ${MAKEOBJOPT} ${mfile:T:S/.m$/.h/}: ${mfile} ${AWK} -f $S/tools/makeobjops.awk ${mfile} -h .endfor diff --git a/sys/conf/kern.pre.mk b/sys/conf/kern.pre.mk --- a/sys/conf/kern.pre.mk +++ b/sys/conf/kern.pre.mk @@ -92,6 +92,7 @@ COMPAT_FREEBSD32_ENABLED!= grep COMPAT_FREEBSD32 opt_global.h || true ; echo +KCFI_ENABLED!= grep KCFI opt_global.h || true ; echo KASAN_ENABLED!= grep KASAN opt_global.h || true ; echo KCSAN_ENABLED!= grep KCSAN opt_global.h || true ; echo KMSAN_ENABLED!= grep KMSAN opt_global.h || true ; echo @@ -105,6 +106,10 @@ .endif .endif +.if !empty(KCFI_ENABLED) +MAKEOBJOPT= "-s" +.endif + CFLAGS+= ${GCOV_CFLAGS} # Put configuration-specific C flags last so that they can override @@ -149,7 +154,7 @@ NORMAL_S= ${CC:N${CCACHE_BIN}} -c ${ASM_CFLAGS} ${WERROR} ${.IMPSRC} NORMAL_C_NOWERROR= ${CC} -c ${CFLAGS} ${.IMPSRC} -NORMAL_M= ${AWK} -f $S/tools/makeobjops.awk ${.IMPSRC} -c ; \ +NORMAL_M= ${AWK} -f $S/tools/makeobjops.awk ${.IMPSRC} -c ${MAKEOBJOPT} ; \ ${CC} -c ${CFLAGS} ${WERROR} ${.PREFIX}.c NORMAL_FW= uudecode -o ${.TARGET} ${.ALLSRC} diff --git a/sys/conf/kmod.mk b/sys/conf/kmod.mk --- a/sys/conf/kmod.mk +++ b/sys/conf/kmod.mk @@ -406,6 +406,7 @@ .endfor .endif +KCFI_ENABLED= ${KERN_OPTS:MKCFI} KASAN_ENABLED= ${KERN_OPTS:MKASAN} KCSAN_ENABLED= ${KERN_OPTS:MKCSAN} KMSAN_ENABLED= ${KERN_OPTS:MKMSAN} @@ -435,6 +436,10 @@ ${AWK} -f ${SYSDIR}/tools/vnode_if.awk ${SYSDIR}/kern/vnode_if.src -q .endif +.if !empty(KCFI_ENABLED) +MAKEOBJOPT= "-s" +.endif + # Build _if.[ch] from _if.m, and clean them when we're done. # __MPATH defined in config.mk _MFILES=${__MPATH:T:O} @@ -448,7 +453,7 @@ .endif .endfor # _i .m.c: ${SYSDIR}/tools/makeobjops.awk - ${AWK} -f ${SYSDIR}/tools/makeobjops.awk ${.IMPSRC} -c + ${AWK} -f ${SYSDIR}/tools/makeobjops.awk ${.IMPSRC} -c ${MAKEOBJOPT} .m.h: ${SYSDIR}/tools/makeobjops.awk ${AWK} -f ${SYSDIR}/tools/makeobjops.awk ${.IMPSRC} -h diff --git a/sys/conf/options b/sys/conf/options --- a/sys/conf/options +++ b/sys/conf/options @@ -249,6 +249,7 @@ COVERAGE opt_global.h KASAN opt_global.h KCOV +KCFI opt_global.h KCSAN opt_global.h KMSAN opt_global.h KUBSAN opt_global.h diff --git a/sys/dev/null/null.c b/sys/dev/null/null.c --- a/sys/dev/null/null.c +++ b/sys/dev/null/null.c @@ -48,6 +48,7 @@ static struct cdev *null_dev; static struct cdev *zero_dev; +static d_read_t null_read; static d_write_t full_write; static d_write_t null_write; static d_ioctl_t null_ioctl; @@ -64,7 +65,7 @@ static struct cdevsw null_cdevsw = { .d_version = D_VERSION, - .d_read = (d_read_t *)nullop, + .d_read = null_read, .d_write = null_write, .d_ioctl = null_ioctl, .d_name = "null", @@ -79,6 +80,13 @@ .d_flags = D_MMAP_ANON, }; + +static int +null_read(struct cdev *dev __unused, struct uio *uio __unused, int flags __unused) +{ + return (0); +} + /* ARGSUSED */ static int full_write(struct cdev *dev __unused, struct uio *uio __unused, int flags __unused) diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -97,6 +97,7 @@ #include #include +#include void mi_startup(void); /* Should be elsewhere */ @@ -255,6 +256,7 @@ */ void mi_startup(void) +__NOCFI { struct sysinit *sip; int last; diff --git a/sys/kern/kern_conf.c b/sys/kern/kern_conf.c --- a/sys/kern/kern_conf.c +++ b/sys/kern/kern_conf.c @@ -257,41 +257,44 @@ atomic_subtract_rel_long(&dev->si_threadcount, 1); } -int -nullop(void) -{ - - return (0); -} - -int -eopnotsupp(void) -{ - - return (EOPNOTSUPP); -} +#define DEFINE_ENXIO_HANDLER(name, ...) \ + static int name(__VA_ARGS__) \ + { \ + return (ENXIO); \ + } -static int -enxio(void) -{ - return (ENXIO); -} +#define DEFINE_ENODEV_HANDLER(name, ...) \ + static int name(__VA_ARGS__) \ + { \ + return (ENODEV); \ + } -static int -enodev(void) -{ - return (ENODEV); -} +#define DEFINE_NULLOP_HANDLER(name, ...) \ + static int \ + name (__VA_ARGS__) \ + { \ + return (0); \ + } +DEFINE_ENXIO_HANDLER(dead_open_handler, struct cdev *dev, int oflags, int devtype, struct thread *td); +DEFINE_ENXIO_HANDLER(dead_close_handler, struct cdev *dev, int fflag, int devtype, struct thread *td); +DEFINE_ENXIO_HANDLER(dead_read_handler, struct cdev *dev, struct uio *uio, int ioflag); +DEFINE_ENXIO_HANDLER(dead_write_handler, struct cdev *dev, struct uio *uio, int ioflag); +DEFINE_ENXIO_HANDLER(dead_ioctl_handler, struct cdev *dev, u_long cmd, caddr_t data, + int fflag, struct thread *td); +DEFINE_ENODEV_HANDLER(dead_poll_handler, struct cdev *dev, int events, + struct thread *td); +DEFINE_ENODEV_HANDLER(dead_mmap_handler, struct cdev *dev, vm_ooffset_t offset, vm_paddr_t *paddr, + int nprot, vm_memattr_t *memattr); /* Define a dead_cdevsw for use when devices leave unexpectedly. */ -#define dead_open (d_open_t *)enxio -#define dead_close (d_close_t *)enxio -#define dead_read (d_read_t *)enxio -#define dead_write (d_write_t *)enxio -#define dead_ioctl (d_ioctl_t *)enxio -#define dead_poll (d_poll_t *)enodev -#define dead_mmap (d_mmap_t *)enodev +#define dead_open dead_open_handler +#define dead_close dead_close_handler +#define dead_read dead_read_handler +#define dead_write dead_write_handler +#define dead_ioctl dead_ioctl_handler +#define dead_poll dead_poll_handler +#define dead_mmap dead_mmap_handler static void dead_strategy(struct bio *bp) @@ -300,8 +303,12 @@ biofinish(bp, NULL, ENXIO); } -#define dead_kqfilter (d_kqfilter_t *)enxio -#define dead_mmap_single (d_mmap_single_t *)enodev +DEFINE_ENXIO_HANDLER(dead_kqfilter_handler, struct cdev *dev, struct knote *kn); +DEFINE_ENODEV_HANDLER(dead_mmap_single_handler, struct cdev *cdev, vm_ooffset_t *offset, + vm_size_t size, struct vm_object **object, int nprot); + +#define dead_kqfilter dead_kqfilter_handler +#define dead_mmap_single dead_mmap_single_handler static struct cdevsw dead_cdevsw = { .d_version = D_VERSION, @@ -320,14 +327,22 @@ /* Default methods if driver does not specify method */ -#define null_open (d_open_t *)nullop -#define null_close (d_close_t *)nullop -#define no_read (d_read_t *)enodev -#define no_write (d_write_t *)enodev -#define no_ioctl (d_ioctl_t *)enodev -#define no_mmap (d_mmap_t *)enodev -#define no_kqfilter (d_kqfilter_t *)enodev -#define no_mmap_single (d_mmap_single_t *)enodev +DEFINE_NULLOP_HANDLER(no_open_handler, struct cdev *dev, int oflags, int devtype, struct thread *td); +DEFINE_NULLOP_HANDLER(no_close_handler, struct cdev *dev, int fflag, int devtype, struct thread *td); +DEFINE_ENODEV_HANDLER(no_read_handler, struct cdev *dev, struct uio *uio, int ioflag); +DEFINE_ENODEV_HANDLER(no_write_handler, struct cdev *dev, struct uio *uio, int ioflag); +DEFINE_ENODEV_HANDLER(no_ioctl_handler, struct cdev *dev, u_long cmd, caddr_t data, + int fflag, struct thread *td); +DEFINE_ENODEV_HANDLER(no_kqfilter_handler, struct cdev *dev, struct knote *kn); + +#define null_open (d_open_t *)no_open_handler +#define null_close (d_close_t *)no_close_handler +#define no_read (d_read_t *)no_read_handler +#define no_write (d_write_t *)no_write_handler +#define no_ioctl (d_ioctl_t *)no_ioctl_handler +#define no_mmap (d_mmap_t *)dead_mmap_handler +#define no_kqfilter (d_kqfilter_t *)no_kqfilter_handler +#define no_mmap_single (d_mmap_single_t *)dead_mmap_single_handler static void no_strategy(struct bio *bp) diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -71,6 +71,7 @@ #include #include #include +#include #include #include @@ -1113,8 +1114,8 @@ * is called from the MD fork_trampoline() entry point. */ void -fork_exit(void (*callout)(void *, struct trapframe *), void *arg, - struct trapframe *frame) +fork_exit(void (*callout)(struct thread *, struct trapframe *), void *arg, + struct trapframe *frame) __NOCFI { struct proc *p; struct thread *td; diff --git a/sys/kern/kern_intr.c b/sys/kern/kern_intr.c --- a/sys/kern/kern_intr.c +++ b/sys/kern/kern_intr.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -61,6 +62,7 @@ #include #include #include + #ifdef DDB #include #include @@ -1188,6 +1190,7 @@ static void ithread_execute_handlers(struct proc *p, struct intr_event *ie) +__NOCFI { /* Interrupt handlers should not sleep. */ @@ -1334,7 +1337,7 @@ * o EINVAL: stray interrupt. */ int -intr_event_handle(struct intr_event *ie, struct trapframe *frame) +intr_event_handle(struct intr_event *ie, struct trapframe *frame) __NOCFI { struct intr_handler *ih; struct trapframe *oldframe; diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c --- a/sys/kern/kern_linker.c +++ b/sys/kern/kern_linker.c @@ -26,11 +26,11 @@ * SUCH DAMAGE. */ -#include #include "opt_ddb.h" -#include "opt_kld.h" #include "opt_hwpmc_hooks.h" +#include "opt_kld.h" +#include #include #include #include @@ -672,6 +672,10 @@ #ifdef __arm__ lf->exidx_addr = 0; lf->exidx_size = 0; +#endif +#ifdef KCFI + lf->kcfi_traps_addr = 0; + lf->kcfi_traps_size = 0; #endif STAILQ_INIT(&lf->common); TAILQ_INIT(&lf->modules); diff --git a/sys/kern/link_elf.c b/sys/kern/link_elf.c --- a/sys/kern/link_elf.c +++ b/sys/kern/link_elf.c @@ -47,6 +47,7 @@ #include #include #include +#include #include @@ -359,6 +360,7 @@ static void link_elf_invoke_cbs(caddr_t addr, size_t size) +__NOCFI { void (**ctor)(void); size_t i, cnt; @@ -876,6 +878,38 @@ #endif /* __arm__ */ +#ifdef KCFI + +static void +link_elf_locate_kcfi_traps(struct linker_file *lf, Elf_Shdr *shdr, int nhdr, char* shstrs) { + int i; + + for (i = 0; i < nhdr; ++i) { + if (strcmp(shstrs + shdr[i].sh_name, ".kcfi_traps") == 0) { + lf->kcfi_traps_addr = shdr[i].sh_addr + lf->address; + lf->kcfi_traps_size = shdr[i].sh_size; + break; + } + } +} + +static void +link_elf_locate_kcfi_traps_preload(struct linker_file *lf, caddr_t modptr) { + Elf_Addr *kcfi_addrp; + Elf_Size *kcfi_sizep; + + kcfi_addrp = (Elf_Addr *)preload_search_info(modptr, + MODINFO_METADATA | MODINFOMD_KCFI_ADDR); + kcfi_sizep = (Elf_Size *)preload_search_info(modptr, + MODINFO_METADATA | MODINFOMD_KCFI_SIZE); + if (kcfi_addrp != NULL && kcfi_sizep != NULL) { + lf->kcfi_traps_addr = (caddr_t)*kcfi_addrp; + lf->kcfi_traps_size = (size_t)*kcfi_sizep; + } +} + +#endif /* KCFI */ + static int link_elf_link_preload(linker_class_t cls, const char *filename, linker_file_t *result) @@ -935,6 +969,10 @@ link_elf_locate_exidx_preload(lf, modptr); #endif +#ifdef KCFI + link_elf_locate_kcfi_traps_preload(lf, modptr); +#endif + error = parse_dynamic(ef); if (error == 0) error = parse_dpcpu(ef); @@ -1330,6 +1368,10 @@ link_elf_locate_exidx(lf, shdr, hdr->e_shnum); #endif +#ifdef KCFI + link_elf_locate_kcfi_traps(lf, shdr, hdr->e_shnum, shstrs); +#endif + error = link_elf_link_common_finish(lf); if (error != 0) goto out; @@ -1636,7 +1678,7 @@ static int link_elf_symbol_values1(linker_file_t lf, c_linker_sym_t sym, - linker_symval_t *symval, bool see_local) + linker_symval_t *symval, bool see_local) __NOCFI { elf_file_t ef; const Elf_Sym *es; @@ -1669,7 +1711,7 @@ static int link_elf_debug_symbol_values(linker_file_t lf, c_linker_sym_t sym, - linker_symval_t *symval) + linker_symval_t *symval) __NOCFI { elf_file_t ef = (elf_file_t)lf; const Elf_Sym *es = (const Elf_Sym *)sym; @@ -1999,7 +2041,7 @@ */ static int elf_lookup_ifunc(linker_file_t lf, Elf_Size symidx, int deps __unused, - Elf_Addr *res) + Elf_Addr *res) __NOCFI { elf_file_t ef; const Elf_Sym *symp; diff --git a/sys/kern/link_elf_obj.c b/sys/kern/link_elf_obj.c --- a/sys/kern/link_elf_obj.c +++ b/sys/kern/link_elf_obj.c @@ -43,6 +43,7 @@ #include #include #include +#include #include @@ -576,7 +577,13 @@ lf->dtors_size = shdr[i].sh_size; } } - +#ifdef KCFI + else if (ef->progtab[pb].name != NULL && + strcmp(ef->progtab[pb].name, ".kcfi_traps") == 0) { + lf->kcfi_traps_addr = ef->progtab[pb].addr; + lf->kcfi_traps_size = shdr[i].sh_size; + } +#endif /* Update all symbol values with the offset. */ for (j = 0; j < ef->ddbsymcnt; j++) { es = &ef->ddbsymtab[j]; @@ -644,6 +651,7 @@ static void link_elf_invoke_cbs(caddr_t addr, size_t size) +__NOCFI { void (**ctor)(void); size_t i, cnt; @@ -1141,6 +1149,16 @@ error = EINVAL; goto out; } +#ifdef KCFI + if (ef->progtab[pb].name != NULL && + !strcmp(ef->progtab[pb].name, + ".kcfi_traps")) { + lf->kcfi_traps_addr = + ef->progtab[pb].addr; + lf->kcfi_traps_size = + ef->progtab[pb].size; + } +#endif /* Initialize the per-cpu area. */ if (ef->progtab[pb].addr != (void *)mapbase && !strcmp(ef->progtab[pb].name, DPCPU_SETNAME)) @@ -1515,7 +1533,7 @@ static int link_elf_symbol_values1(linker_file_t lf, c_linker_sym_t sym, - linker_symval_t *symval, bool see_local) + linker_symval_t *symval, bool see_local) __NOCFI { elf_file_t ef; const Elf_Sym *es; @@ -1690,6 +1708,7 @@ */ static int elf_obj_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res) +__NOCFI { elf_file_t ef = (elf_file_t)lf; Elf_Sym *sym; diff --git a/sys/kern/subr_cfi.c b/sys/kern/subr_cfi.c new file mode 100644 --- /dev/null +++ b/sys/kern/subr_cfi.c @@ -0,0 +1,108 @@ +#include + +#include "opt_printf.h" + +#ifdef _KERNEL +#include +#include +#include +#include +#include +#include +#endif + +#ifdef _KERNEL +#include +#include +#endif + +FEATURE(kcfi, "Kernel control flow integration"); + +static SYSCTL_NODE(_debug, OID_AUTO, kcfi, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "KCFI options"); + +static bool cfi_panic __read_mostly = false; +SYSCTL_BOOL(_debug_kcfi, OID_AUTO, panic_on_violation, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, + &cfi_panic, 0, "KCFI panic on violation is disabled"); + +static bool cfi_disabled __read_mostly = false; +SYSCTL_BOOL(_debug_kcfi, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, + &cfi_disabled, 0, "Disable KCFI message"); + +#ifdef _KERNEL +static void +report_cfi(struct trapframe *frame, register_t func_addr, uint32_t type) { + static char *cfi_format_str = "CFI chcek failed on ip %lx for callee %lx and type %x\n"; + register_t addr = TRAPF_PC(frame); + + if (cfi_panic) + panic(cfi_format_str, addr, func_addr, type); + else + printf(cfi_format_str, addr, func_addr, type); +} +#endif + +#ifdef _KERNEL +static int +kcfi_linker_file_cb(linker_file_t lf, void *arg) +{ + linker_file_t *module = (linker_file_t *)arg; + + if (lf->address <= (caddr_t)*module && + (lf->address + lf->size) >= (caddr_t)*module) { + *module = lf; + return 1; + } + + return 0; +} +#endif + +#ifdef _KERNEL +static bool +is_cfi_exception(caddr_t address) { + linker_file_t module; + caddr_t trap_addr; + caddr_t *traps_start, *traps_end; + + module = (linker_file_t)address; + linker_file_foreach(&kcfi_linker_file_cb, &module); + + if (module == NULL) + return false; + + traps_start = (caddr_t *)(module->kcfi_traps_addr); + traps_end = (caddr_t *)(module->kcfi_traps_addr + + module->kcfi_traps_size); + + if (traps_start == 0) + return false; + + while(traps_start != traps_end) { + trap_addr = *traps_start; + if (trap_addr == address) + return true; + traps_start++; + } + + return false; +} +#endif + +#ifdef _KERNEL +bool +cfi_handler(struct trapframe *frame) { + register_t func_addr; + uint32_t type; + + if (decode_cfi(frame, &func_addr, &type) && + is_cfi_exception((caddr_t)func_addr)) { + report_cfi(frame, func_addr, type); + /* needs to be platform dependent */ + TRAPF_PC(frame) += 2; + return (true); + } + + return (false); +} +#endif diff --git a/sys/kern/subr_epoch.c b/sys/kern/subr_epoch.c --- a/sys/kern/subr_epoch.c +++ b/sys/kern/subr_epoch.c @@ -43,6 +43,7 @@ #include #include #include +#include #ifdef EPOCH_TRACE #include #include @@ -802,6 +803,7 @@ static void epoch_call_task(void *arg __unused) +__NOCFI { ck_stack_entry_t *cursor, *head, *next; ck_epoch_record_t *record; diff --git a/sys/kern/subr_module.c b/sys/kern/subr_module.c --- a/sys/kern/subr_module.c +++ b/sys/kern/subr_module.c @@ -385,6 +385,14 @@ case MODINFOMD_KEYBUF: sbuf_cat(sbp, "MODINFOMD_KEYBUF"); break; +#ifdef KCFI + case MODINFOMD_KCFI_ADDR: + sbuf_cat(sbp, "MODINFOMD_KCFI_ADDR"); + break; + case MODINFOMD_KCFI_SIZE: + sbuf_cat(sbp, "MODINFOMD_KCFI_SIZE"); + break; +#endif #ifdef MODINFOMD_SMAP case MODINFOMD_SMAP: sbuf_cat(sbp, "MODINFOMD_SMAP"); @@ -464,6 +472,10 @@ case MODINFO_METADATA | MODINFOMD_KERNEND: case MODINFO_METADATA | MODINFOMD_ENVP: case MODINFO_METADATA | MODINFOMD_CTORS_ADDR: +#ifdef KCFI + case MODINFO_METADATA | MODINFOMD_KCFI_ADDR: + case MODINFO_METADATA | MODINFOMD_KCFI_SIZE: +#endif #ifdef MODINFOMD_SMAP case MODINFO_METADATA | MODINFOMD_SMAP: #endif diff --git a/sys/kern/subr_pctrie.c b/sys/kern/subr_pctrie.c --- a/sys/kern/subr_pctrie.c +++ b/sys/kern/subr_pctrie.c @@ -49,6 +49,7 @@ #include #include "opt_ddb.h" +#include #include #include #include @@ -1159,7 +1160,7 @@ */ static __always_inline struct pctrie_node * pctrie_reclaim_prune(struct pctrie_node **pnode, struct pctrie_node *parent, - pctrie_cb_t callback, int keyoff, void *arg) + pctrie_cb_t callback, int keyoff, void *arg) __NOCFI { struct pctrie_node *child, *node; int slot; @@ -1188,11 +1189,11 @@ } /* - * Recover the node parent from its first child and continue pruning. + * Recover the node parent from its first child and cont=inue pruning. */ static __always_inline struct pctrie_node * pctrie_reclaim_resume_compound(struct pctrie_node **pnode, - pctrie_cb_t callback, int keyoff, void *arg) + pctrie_cb_t callback, int keyoff, void *arg) __NOCFI { struct pctrie_node *parent, *node; @@ -1212,7 +1213,7 @@ static __always_inline struct pctrie_node * pctrie_reclaim_begin_compound(struct pctrie_node **pnode, struct pctrie *ptree, - pctrie_cb_t callback, int keyoff, void *arg) + pctrie_cb_t callback, int keyoff, void *arg) __NOCFI { struct pctrie_node *node; diff --git a/sys/kern/subr_scanf.c b/sys/kern/subr_scanf.c --- a/sys/kern/subr_scanf.c +++ b/sys/kern/subr_scanf.c @@ -39,6 +39,7 @@ #include #include #include +#include /* * Note that stdarg.h and the ANSI style va_start macro is used for both @@ -101,6 +102,7 @@ int vsscanf(const char *inp, char const *fmt0, va_list ap) +__NOCFI { int inr; const u_char *fmt = (const u_char *)fmt0; diff --git a/sys/kern/subr_syscall.c b/sys/kern/subr_syscall.c --- a/sys/kern/subr_syscall.c +++ b/sys/kern/subr_syscall.c @@ -43,6 +43,7 @@ #include #include #include +#include #ifdef KTRACE #include #include @@ -50,7 +51,7 @@ #include static inline void -syscallenter(struct thread *td) +syscallenter(struct thread *td) __NOCFI { struct proc *p; struct syscall_args *sa; diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c --- a/sys/kern/vfs_subr.c +++ b/sys/kern/vfs_subr.c @@ -5298,7 +5298,7 @@ /* * Routine to create and manage a filesystem syncer vnode. */ -#define sync_close ((int (*)(struct vop_close_args *))nullop) +static int sync_close(struct vop_close_args *); static int sync_fsync(struct vop_fsync_args *); static int sync_inactive(struct vop_inactive_args *); static int sync_reclaim(struct vop_reclaim_args *); @@ -5392,6 +5392,10 @@ vrele(vp); } +static int sync_close(struct vop_close_args *args __unused) { + return (0); +} + /* * Do a lazy sync of the filesystem. */ diff --git a/sys/net/radix.c b/sys/net/radix.c --- a/sys/net/radix.c +++ b/sys/net/radix.c @@ -33,7 +33,8 @@ * Routines to build and maintain radix trees for routing lookups. */ #include -#ifdef _KERNEL +#ifdef _KERNEL +#include #include #include #include @@ -1056,7 +1057,7 @@ int rn_walktree(struct radix_head *h, walktree_f_t *f, void *w) -{ +__NOCFI { int error; struct radix_node *base, *next; struct radix_node *rn = h->rnh_treetop; diff --git a/sys/net/vnet.c b/sys/net/vnet.c --- a/sys/net/vnet.c +++ b/sys/net/vnet.c @@ -53,6 +53,7 @@ #include #include #include +#include #include @@ -512,6 +513,7 @@ */ void vnet_register_sysinit(void *arg) +__NOCFI { struct vnet_sysinit *vs, *vs2; struct vnet *vnet; diff --git a/sys/sys/cfi.h b/sys/sys/cfi.h new file mode 100644 --- /dev/null +++ b/sys/sys/cfi.h @@ -0,0 +1,22 @@ +/* + */ +#ifndef _SYS_CFI_H_ +#define _SYS_CFI_H_ + +#ifdef _KERNEL + +#include +#include + +#ifdef KCFI +#define __NOCFI __attribute__((__no_sanitize__("kcfi"))) +#else +#define __NOCFI +#endif + +bool +cfi_handler(struct trapframe *); + +#endif /* _KERNEL */ + +#endif /* _SYS_CFI_H_ */ diff --git a/sys/sys/linker.h b/sys/sys/linker.h --- a/sys/sys/linker.h +++ b/sys/sys/linker.h @@ -109,6 +109,11 @@ caddr_t exidx_addr; /* Unwind data index table start */ size_t exidx_size; /* Unwind data index table size */ #endif + +#ifdef KCFI + caddr_t kcfi_traps_addr; /* KCFI trap table start */ + size_t kcfi_traps_size; /* KCFI trap table size */ +#endif }; /* @@ -251,7 +256,11 @@ #define MODINFOMD_KEYBUF 0x000d /* Crypto key intake buffer */ #define MODINFOMD_FONT 0x000e /* Console font */ #define MODINFOMD_SPLASH 0x000f /* Console splash screen */ -#define MODINFOMD_NOCOPY 0x8000 /* don't copy this metadata to the kernel */ +#define MODINFOMD_NOCOPY 0x8000 /* don't copy this metadata to the kernel */ +#ifdef KCFI +#define MODINFOMD_KCFI_ADDR 0x0010 /* address of .kcfi_traps */ +#define MODINFOMD_KCFI_SIZE 0x0020 /* size of .kcfi_traps */ +#endif #define MODINFOMD_DEPLIST (0x4001 | MODINFOMD_NOCOPY) /* depends on */ diff --git a/sys/sys/proc.h b/sys/sys/proc.h --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -1154,7 +1154,7 @@ struct session *sess); int enterthispgrp(struct proc *p, struct pgrp *pgrp); int fork1(struct thread *, struct fork_req *); -void fork_exit(void (*)(void *, struct trapframe *), void *, +void fork_exit(void (*)(struct thread*, struct trapframe *), void *, struct trapframe *); void fork_return(struct thread *, struct trapframe *); int inferior(struct proc *p); diff --git a/sys/sys/systm.h b/sys/sys/systm.h --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -144,7 +144,6 @@ int setjmp(struct _jmp_buf *) __returns_twice; void longjmp(struct _jmp_buf *, int) __dead2; int dumpstatus(vm_offset_t addr, off_t count); -int nullop(void); int eopnotsupp(void); int ureadc(int, struct uio *); void hashdestroy(void *, struct malloc_type *, u_long); diff --git a/sys/tools/vnode_if.awk b/sys/tools/vnode_if.awk --- a/sys/tools/vnode_if.awk +++ b/sys/tools/vnode_if.awk @@ -199,6 +199,7 @@ "#include \n" \ "#include \n" \ "#include \n" \ + "#include \n" \ "\n" \ "SDT_PROVIDER_DECLARE(vfs);\n" \ "\n" \ @@ -389,7 +390,7 @@ printc(""); printc("\treturn(" uname "_APV(a->a_" args[0] "->v_op, a));"); printc("}"); - printc("\nint\n" uname "_APV(const struct vop_vector *vop, struct " name "_args *a)"); + printc("\nint\n" uname "_APV(const struct vop_vector *vop, struct " name "_args *a) __NOCFI"); printc("{"); printc("\tint rc;"); printc("");