Index: lib/libc/gen/arc4random.h =================================================================== --- lib/libc/gen/arc4random.h +++ lib/libc/gen/arc4random.h @@ -24,9 +24,33 @@ /* * Stub functions for portability. */ +#include +#include #include +#include /* for sys/vdso.h only. */ +#include +#include +#include +#include #include +#include +#include + +/* + * The kernel root seed version is a 64-bit counter, but we truncate it to a + * 32-bit value in userspace for the convenience of 32-bit platforms. 32-bit + * rollover is not possible with the current reseed interval (1 hour at limit) + * without dynamic addition of new random devices (which also force a reseed in + * the FXRNG design). We don't have any dynamic device mechanism at this + * time, and anyway something else is very wrong if billions of new devices are + * being added. + * + * As is, it takes roughly 456,000 years of runtime to overflow the 32-bit + * version. + */ +#define fxrng_load_acq_generation(x) atomic_load_acq_32(x) +static struct vdso_fxrng_generation_1 *vdso_fxrngp; static pthread_mutex_t arc4random_mtx = PTHREAD_MUTEX_INITIALIZER; #define _ARC4_LOCK() \ @@ -47,6 +71,28 @@ raise(SIGKILL); } +static inline void +_rs_initialize_fxrng(void) +{ + struct vdso_fxrng_generation_1 *fxrngp; + int error; + + error = _elf_aux_info(AT_FXRNG, &fxrngp, sizeof(fxrngp)); + if (error != 0) { + /* + * New userspace on an old or !RANDOM_FENESTRASX kernel; or an + * arch that does not have a VDSO page. + */ + return; + } + + /* Old userspace on newer kernel, or disabled FXRNG VDSO. */ + if (fxrngp->fx_vdso_version != VDSO_FXRNG_VER_1) + return; + + vdso_fxrngp = fxrngp; +} + static inline int _rs_allocate(struct _rs **rsp, struct _rsx **rsxp) { @@ -65,12 +111,33 @@ return (-1); } #endif + + _rs_initialize_fxrng(); + *rsp = &p->rs; *rsxp = &p->rsx; return (0); } +/* + * This isn't only detecting fork. We're also using the existing callback from + * _rs_stir_if_needed() to force arc4random(3) to reseed if the fenestrasX root + * seed version has changed. (That is, the root random(4) has reseeded from + * pooled entropy.) + */ static inline void _rs_forkdetect(void) { + /* Detect fork (minherit(2) INHERIT_ZERO). */ + if (__predict_false(rs == NULL || rsx == NULL)) + return; + /* If present, detect kernel FenestrasX seed version change. */ + if (vdso_fxrngp == NULL) + return; + if (__predict_true(rsx->rs_seed_generation == + fxrng_load_acq_generation(&vdso_fxrngp->fx_generation32))) + return; + + /* Invalidate rs_buf to force "stir" (reseed). */ + memset(rs, 0, sizeof(*rs)); } Index: lib/libc/gen/arc4random.c =================================================================== --- lib/libc/gen/arc4random.c +++ lib/libc/gen/arc4random.c @@ -27,6 +27,9 @@ __FBSDID("$FreeBSD$"); #include "namespace.h" +#if defined(__FreeBSD__) +#include +#endif #include #include #include @@ -68,6 +71,9 @@ static struct _rsx { chacha_ctx rs_chacha; /* chacha context for random keystream */ u_char rs_buf[RSBUFSZ]; /* keystream blocks */ +#ifdef __FreeBSD__ + uint32_t rs_seed_generation; /* 32-bit userspace RNG version */ +#endif } *rsx; static inline int _rs_allocate(struct _rs **, struct _rsx **); @@ -96,11 +102,43 @@ { u_char rnd[KEYSZ + IVSZ]; +#if defined(__FreeBSD__) + bool need_init; + + /* + * De-couple allocation (which locates the vdso_fxrngp pointer in + * auxinfo) from initialization. This allows us to read the root seed + * version before we fetch system entropy, maintaining the invariant + * that the PRF was seeded with entropy from rs_seed_generation or a + * later generation. But never seeded from an earlier generation. + * This invariant prevents us from missing a root reseed event. + */ + need_init = false; + if (rs == NULL) { + if (_rs_allocate(&rs, &rsx) == -1) + abort(); + need_init = true; + } + /* + * Transition period: new userspace on old kernel. This should become + * a hard error at some point, if the scheme is adopted. + */ + if (vdso_fxrngp != NULL) + rsx->rs_seed_generation = + fxrng_load_acq_generation(&vdso_fxrngp->fx_generation32); +#endif + if (getentropy(rnd, sizeof rnd) == -1) _getentropy_fail(); +#if !defined(__FreeBSD__) if (!rs) _rs_init(rnd, sizeof(rnd)); +#else /* __FreeBSD__ */ + assert(rs != NULL); + if (need_init) + _rs_init(rnd, sizeof(rnd)); +#endif else _rs_rekey(rnd, sizeof(rnd)); explicit_bzero(rnd, sizeof(rnd)); /* discard source seed */ Index: lib/libc/gen/auxv.c =================================================================== --- lib/libc/gen/auxv.c +++ lib/libc/gen/auxv.c @@ -72,6 +72,7 @@ static char *canary, *pagesizes, *execpath; static void *ps_strings, *timekeep; static u_long hwcap, hwcap2; +static void *fxrng_seed_version; #ifdef __powerpc__ static int powerpc_new_auxv_format = 0; @@ -139,6 +140,10 @@ case AT_PS_STRINGS: ps_strings = aux->a_un.a_ptr; break; + + case AT_FXRNG: + fxrng_seed_version = aux->a_un.a_ptr; + break; #ifdef __powerpc__ /* * Since AT_STACKPROT is always set, and the common @@ -355,6 +360,16 @@ } else res = EINVAL; break; + case AT_FXRNG: + if (buflen == sizeof(void *)) { + if (fxrng_seed_version != NULL) { + *(void **)buf = fxrng_seed_version; + res = 0; + } else + res = ENOENT; + } else + res = EINVAL; + break; default: res = ENOENT; break; Index: sys/amd64/amd64/elf_machdep.c =================================================================== --- sys/amd64/amd64/elf_machdep.c +++ sys/amd64/amd64/elf_machdep.c @@ -72,7 +72,7 @@ .sv_fixlimit = NULL, .sv_maxssiz = NULL, .sv_flags = SV_ABI_FREEBSD | SV_ASLR | SV_LP64 | SV_SHP | - SV_TIMEKEEP, + SV_TIMEKEEP | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, @@ -107,7 +107,7 @@ .sv_fixlimit = NULL, .sv_maxssiz = NULL, .sv_flags = SV_ABI_FREEBSD | SV_ASLR | SV_LP64 | SV_SHP | - SV_TIMEKEEP, + SV_TIMEKEEP | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, Index: sys/arm/arm/elf_machdep.c =================================================================== --- sys/arm/arm/elf_machdep.c +++ sys/arm/arm/elf_machdep.c @@ -86,7 +86,7 @@ .sv_maxssiz = NULL, .sv_flags = #if __ARM_ARCH >= 6 - SV_ASLR | SV_SHP | SV_TIMEKEEP | + SV_ASLR | SV_SHP | SV_TIMEKEEP | SV_RNG_SEED_VER | #endif SV_ABI_FREEBSD | SV_ILP32 | SV_ASLR, .sv_set_syscall_retval = cpu_set_syscall_retval, Index: sys/arm64/arm64/elf32_machdep.c =================================================================== --- sys/arm64/arm64/elf32_machdep.c +++ sys/arm64/arm64/elf32_machdep.c @@ -96,7 +96,8 @@ .sv_setregs = freebsd32_setregs, .sv_fixlimit = NULL, // XXX .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_ILP32 | SV_SHP | SV_TIMEKEEP, + .sv_flags = SV_ABI_FREEBSD | SV_ILP32 | SV_SHP | SV_TIMEKEEP | + SV_RNG_SEED_VER, .sv_set_syscall_retval = freebsd32_set_syscall_retval, .sv_fetch_syscall_args = freebsd32_fetch_syscall_args, .sv_syscallnames = freebsd32_syscallnames, Index: sys/arm64/arm64/elf_machdep.c =================================================================== --- sys/arm64/arm64/elf_machdep.c +++ sys/arm64/arm64/elf_machdep.c @@ -81,7 +81,7 @@ .sv_fixlimit = NULL, .sv_maxssiz = NULL, .sv_flags = SV_SHP | SV_TIMEKEEP | SV_ABI_FREEBSD | SV_LP64 | - SV_ASLR, + SV_ASLR | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, Index: sys/compat/ia32/ia32_sysvec.c =================================================================== --- sys/compat/ia32/ia32_sysvec.c +++ sys/compat/ia32/ia32_sysvec.c @@ -118,7 +118,7 @@ .sv_fixlimit = ia32_fixlimit, .sv_maxssiz = &ia32_maxssiz, .sv_flags = SV_ABI_FREEBSD | SV_ASLR | SV_IA32 | SV_ILP32 | - SV_SHP | SV_TIMEKEEP, + SV_SHP | SV_TIMEKEEP | SV_RNG_SEED_VER, .sv_set_syscall_retval = ia32_set_syscall_retval, .sv_fetch_syscall_args = ia32_fetch_syscall_args, .sv_syscallnames = freebsd32_syscallnames, Index: sys/dev/random/fenestrasX/fx_brng.c =================================================================== --- sys/dev/random/fenestrasX/fx_brng.c +++ sys/dev/random/fenestrasX/fx_brng.c @@ -39,6 +39,7 @@ #include #include #include +#include #include @@ -108,6 +109,8 @@ */ rng->brng_generation++; atomic_store_rel_64(&fxrng_root_generation, rng->brng_generation); + /* Update VDSO version. */ + fxrng_push_seed_generation(rng->brng_generation); FXRNG_BRNG_UNLOCK(rng); } @@ -129,9 +132,26 @@ rng->brng_generation++; atomic_store_rel_64(&fxrng_root_generation, rng->brng_generation); + /* Update VDSO version. */ + fxrng_push_seed_generation(rng->brng_generation); FXRNG_BRNG_UNLOCK(rng); } +/* + * Sysentvec and VDSO are initialized much later than SI_SUB_RANDOM. When + * they're online, go ahead and push an initial root seed version. + * INIT_SYSENTVEC runs at SI_SUB_EXEC:SI_ORDER_ANY, and SI_ORDER_ANY is the + * maximum value, so we must run at SI_SUB_EXEC+1. + */ +static void +fxrng_vdso_sysinit(void *dummy __unused) +{ + FXRNG_BRNG_LOCK(&fxrng_root); + fxrng_push_seed_generation(fxrng_root.brng_generation); + FXRNG_BRNG_UNLOCK(&fxrng_root); +} +SYSINIT(fxrng_vdso, SI_SUB_EXEC + 1, SI_ORDER_ANY, fxrng_vdso_sysinit, NULL); + /* * Grab some bytes off an initialized, current generation RNG. * Index: sys/dev/random/fenestrasX/fx_main.c =================================================================== --- sys/dev/random/fenestrasX/fx_main.c +++ sys/dev/random/fenestrasX/fx_main.c @@ -88,7 +88,8 @@ * a while). * * Not yet implemented, not in scope, or todo: - * - Userspace portions -- shared page, like timehands vdso? + * - Various initial seeding sources we don't have yet + * - In particular, VM migration/copy detection */ #include Index: sys/i386/i386/elf_machdep.c =================================================================== --- sys/i386/i386/elf_machdep.c +++ sys/i386/i386/elf_machdep.c @@ -74,7 +74,7 @@ .sv_fixlimit = NULL, .sv_maxssiz = NULL, .sv_flags = SV_ABI_FREEBSD | SV_ASLR | SV_IA32 | SV_ILP32 | - SV_SHP | SV_TIMEKEEP, + SV_SHP | SV_TIMEKEEP | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, Index: sys/kern/imgact_elf.c =================================================================== --- sys/kern/imgact_elf.c +++ sys/kern/imgact_elf.c @@ -1389,6 +1389,8 @@ AUXARGS_ENTRY(pos, AT_ENVC, imgp->args->envc); AUXARGS_ENTRY_PTR(pos, AT_ENVV, imgp->envv); AUXARGS_ENTRY_PTR(pos, AT_PS_STRINGS, imgp->ps_strings); + if (imgp->sysent->sv_fxrng_gen_base != 0) + AUXARGS_ENTRY(pos, AT_FXRNG, imgp->sysent->sv_fxrng_gen_base); AUXARGS_ENTRY(pos, AT_NULL, 0); free(imgp->auxargs, M_TEMP); Index: sys/kern/kern_sharedpage.c =================================================================== --- sys/kern/kern_sharedpage.c +++ sys/kern/kern_sharedpage.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -60,6 +61,10 @@ static int shared_page_free; char *shared_page_mapping; +#ifdef RANDOM_FENESTRASX +static struct vdso_fxrng_generation *fxrng_shpage_mapping; +#endif + void shared_page_write(int base, int size, const void *data) { @@ -256,10 +261,72 @@ } #endif +#ifdef RANDOM_FENESTRASX +void +fxrng_push_seed_generation(uint64_t gen) +{ + if (fxrng_shpage_mapping == NULL || + fxrng_shpage_mapping->fx_vdso_version == VDSO_FXRNG_DISABLED) + return; + KASSERT(gen < INT32_MAX, + ("fxrng seed version shouldn't roll over a 32-bit counter " + "for approximately 456,000 years")); + atomic_store_rel_32(&fxrng_shpage_mapping->fx_generation32, + (uint32_t)gen); +} + +static void +alloc_sv_fxrng_generation(void) +{ + int base; + + /* + * Allocate a full cache line for the fxrng root generation (64-bit + * counter, or truncated 32-bit counter on ILP32 userspace). It is + * important that the line is not shared with frequently dirtied data, + * and the shared page allocator lacks a __read_mostly mechanism. + * However, PAGE_SIZE is typically large relative to the amount of + * stuff we've got in it so far, so maybe the possible waste isn't an + * issue. + */ + base = shared_page_alloc(CACHE_LINE_SIZE, CACHE_LINE_SIZE); + KASSERT(base != -1, ("%s: base allocation failed", __func__)); + fxrng_shpage_mapping = (void *)(shared_page_mapping + base); + *fxrng_shpage_mapping = (struct vdso_fxrng_generation) { + .fx_vdso_version = VDSO_FXRNG_VER_CURR, + }; +} + +static int +sysctl_fxrng_vdso(SYSCTL_HANDLER_ARGS) +{ + int vdso_enable, error; + + if (fxrng_shpage_mapping == NULL) + return (ENXIO); + + vdso_enable = + (fxrng_shpage_mapping->fx_vdso_version != VDSO_FXRNG_DISABLED); + error = sysctl_handle_int(oidp, &vdso_enable, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + + fxrng_shpage_mapping->fx_vdso_version = (vdso_enable != 0) ? + VDSO_FXRNG_VER_CURR : VDSO_FXRNG_DISABLED; + return (0); +} +SYSCTL_PROC(_debug, OID_AUTO, fxrng_vdso_enable, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, + NULL, 0, sysctl_fxrng_vdso, "I", "Enable FXRNG VDSO"); +#endif /* RANDOM_FENESTRASX */ + void exec_sysvec_init(void *param) { struct sysentvec *sv; +#ifdef RANDOM_FENESTRASX + ptrdiff_t base; +#endif sv = (struct sysentvec *)param; if ((sv->sv_flags & SV_SHP) == 0) @@ -287,6 +354,18 @@ } #endif } +#ifdef RANDOM_FENESTRASX + if ((sv->sv_flags & SV_RNG_SEED_VER) != 0) { + /* + * Only allocate a single VDSO entry for multiple sysentvecs, + * i.e., native and COMPAT32. + */ + if (fxrng_shpage_mapping == NULL) + alloc_sv_fxrng_generation(); + base = (char *)fxrng_shpage_mapping - shared_page_mapping; + sv->sv_fxrng_gen_base = sv->sv_shared_page_base + base; + } +#endif } void @@ -305,4 +384,10 @@ sv2->sv_timekeep_base = sv2->sv_shared_page_base + (sv->sv_timekeep_base - sv->sv_shared_page_base); } +#ifdef RANDOM_FENESTRASX + if ((sv2->sv_flags & SV_RNG_SEED_VER) != 0) { + sv2->sv_fxrng_gen_base = sv2->sv_shared_page_base + + (sv->sv_fxrng_gen_base - sv->sv_shared_page_base); + } +#endif } Index: sys/mips/mips/elf_machdep.c =================================================================== --- sys/mips/mips/elf_machdep.c +++ sys/mips/mips/elf_machdep.c @@ -77,10 +77,11 @@ .sv_setregs = exec_setregs, .sv_fixlimit = NULL, .sv_maxssiz = NULL, + .sv_flags = SV_ABI_FREEBSD | SV_ASLR | SV_RNG_SEED_VER | #ifdef __mips_n64 - .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_ASLR, + SV_LP64, #else - .sv_flags = SV_ABI_FREEBSD | SV_ILP32 | SV_ASLR, + SV_ILP32, #endif .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, Index: sys/mips/mips/freebsd32_machdep.c =================================================================== --- sys/mips/mips/freebsd32_machdep.c +++ sys/mips/mips/freebsd32_machdep.c @@ -97,7 +97,7 @@ .sv_setregs = exec_setregs, .sv_fixlimit = NULL, .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_ILP32, + .sv_flags = SV_ABI_FREEBSD | SV_ILP32 | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = freebsd32_syscallnames, Index: sys/powerpc/powerpc/elf32_machdep.c =================================================================== --- sys/powerpc/powerpc/elf32_machdep.c +++ sys/powerpc/powerpc/elf32_machdep.c @@ -121,7 +121,7 @@ #endif .sv_maxssiz = NULL, .sv_flags = SV_ABI_FREEBSD | SV_ILP32 | SV_SHP | SV_ASLR | - SV_TIMEKEEP, + SV_TIMEKEEP | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_shared_page_base = FREEBSD32_SHAREDPAGE, Index: sys/powerpc/powerpc/elf64_machdep.c =================================================================== --- sys/powerpc/powerpc/elf64_machdep.c +++ sys/powerpc/powerpc/elf64_machdep.c @@ -82,7 +82,7 @@ .sv_fixlimit = NULL, .sv_maxssiz = NULL, .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP | SV_ASLR | - SV_TIMEKEEP, + SV_TIMEKEEP | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, @@ -118,7 +118,7 @@ .sv_fixlimit = NULL, .sv_maxssiz = NULL, .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP | - SV_TIMEKEEP, + SV_TIMEKEEP | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, Index: sys/riscv/riscv/elf_machdep.c =================================================================== --- sys/riscv/riscv/elf_machdep.c +++ sys/riscv/riscv/elf_machdep.c @@ -84,7 +84,8 @@ .sv_setregs = exec_setregs, .sv_fixlimit = NULL, .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP | SV_ASLR, + .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP | SV_ASLR | + SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, Index: sys/sys/elf_common.h =================================================================== --- sys/sys/elf_common.h +++ sys/sys/elf_common.h @@ -967,8 +967,9 @@ #define AT_ENVC 30 /* Environment count */ #define AT_ENVV 31 /* Environment vector */ #define AT_PS_STRINGS 32 /* struct ps_strings */ +#define AT_FXRNG 99 /* Pointer to root RNG seed version. */ -#define AT_COUNT 33 /* Count of defined aux entry types. */ +#define AT_COUNT 100 /* Count of defined aux entry types. */ /* * Relocation types. Index: sys/sys/sysent.h =================================================================== --- sys/sys/sysent.h +++ sys/sys/sysent.h @@ -144,6 +144,7 @@ u_long *sv_hwcap; /* Value passed in AT_HWCAP. */ u_long *sv_hwcap2; /* Value passed in AT_HWCAP2. */ const char *(*sv_machine_arch)(struct proc *); + vm_offset_t sv_fxrng_gen_base; }; #define SV_ILP32 0x000100 /* 32-bit executable. */ @@ -154,6 +155,7 @@ #define SV_CAPSICUM 0x020000 /* Force cap_enter() on startup. */ #define SV_TIMEKEEP 0x040000 /* Shared page timehands. */ #define SV_ASLR 0x080000 /* ASLR allowed. */ +#define SV_RNG_SEED_VER 0x100000 /* random(4) reseed generation. */ #define SV_ABI_MASK 0xff #define SV_PROC_FLAG(p, x) ((p)->p_sysent->sv_flags & (x)) Index: sys/sys/vdso.h =================================================================== --- sys/sys/vdso.h +++ sys/sys/vdso.h @@ -59,6 +59,19 @@ #define VDSO_TH_ALGO_3 0x3 #define VDSO_TH_ALGO_4 0x4 +struct vdso_fxrng_generation_1 { + uint32_t fx_vdso_version; /* 1 */ + uint32_t fx_generation32; + uint64_t _fx_reserved; +}; +_Static_assert(sizeof(struct vdso_fxrng_generation_1) == 16, ""); +#define vdso_fxrng_generation vdso_fxrng_generation_1 + +/* fx_vdso_version values: */ +#define VDSO_FXRNG_DISABLED 0x0 +#define VDSO_FXRNG_VER_1 0x1 +#define VDSO_FXRNG_VER_CURR VDSO_FXRNG_VER_1 + #ifndef _KERNEL struct timespec; @@ -82,6 +95,9 @@ uint32_t sv_timekeep_gen; }; +#ifdef RANDOM_FENESTRASX +void fxrng_push_seed_generation(uint64_t gen); +#endif void timekeep_push_vdso(void); uint32_t tc_fill_vdso_timehands(struct vdso_timehands *vdso_th);