Index: lib/libc/gen/arc4random.h =================================================================== --- lib/libc/gen/arc4random.h +++ lib/libc/gen/arc4random.h @@ -24,9 +24,30 @@ /* * Stub functions for portability. */ +#include +#include #include +#include /* for sys/vdso.h only. */ +#include +#include +#include +#include #include +#include +#include + +/* + * The root seed version is a 64-bit counter, but we truncate it to a 32-bit + * value on ILP32 userspace (including compat32). + */ +#ifdef __LP64__ +#define fxrng_load_acq_generation(x) atomic_load_acq_64(x) +static uint64_t *fxrng_root_generationp; +#else +#define fxrng_load_acq_generation(x) atomic_load_acq_32(x) +static uint32_t *fxrng_root_generationp; +#endif static pthread_mutex_t arc4random_mtx = PTHREAD_MUTEX_INITIALIZER; #define _ARC4_LOCK() \ @@ -50,10 +71,12 @@ static inline int _rs_allocate(struct _rs **rsp, struct _rsx **rsxp) { + struct vdso_fxrng_generation *vdso_fxrngp; struct { struct _rs rs; struct _rsx rsx; } *p; + int error; if ((p = mmap(NULL, sizeof(*p), PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0)) == MAP_FAILED) @@ -62,13 +85,46 @@ munmap(p, sizeof(*p)); return (-1); } + error = _elf_aux_info(AT_FXRNG, &vdso_fxrngp, sizeof(vdso_fxrngp)); + if (error == 0) { +#ifdef __LP64__ + fxrng_root_generationp = &vdso_fxrngp->fx_generation; +#else + fxrng_root_generationp = &vdso_fxrngp->fx_generation32; +#endif + } else { +#ifdef NOTYET + /* + * Transition period: new userspace on old kernel. Should + * become a hard error at some point, if the scheme is adopted. + */ + errno = error; + return (-1); +#endif + } *rsp = &p->rs; *rsxp = &p->rsx; return (0); } +/* + * This isn't detecting fork; we're just using the existing callback from + * _rs_stir_if_needed() to force arc4random(3) to reseed if the fenestrasX root + * seed version has changed. (That is, the root random(4) has reseeded from + * pooled entropy.) + */ static inline void _rs_forkdetect(void) { + if (__predict_false(rs == NULL || rsx == NULL)) + return; + if (fxrng_root_generationp == NULL) + return; + if (__predict_true(rsx->rs_seed_generation == + fxrng_load_acq_generation(fxrng_root_generationp))) + return; + + /* Invalidate rs_buf to force "stir" (reseed). */ + memset(rs, 0, sizeof(*rs)); } Index: lib/libc/gen/arc4random.c =================================================================== --- lib/libc/gen/arc4random.c +++ lib/libc/gen/arc4random.c @@ -27,6 +27,9 @@ __FBSDID("$FreeBSD$"); #include "namespace.h" +#if defined(__FreeBSD__) +#include +#endif #include #include #include @@ -68,6 +71,9 @@ static struct _rsx { chacha_ctx rs_chacha; /* chacha context for random keystream */ u_char rs_buf[RSBUFSZ]; /* keystream blocks */ +#ifdef __FreeBSD__ + uint64_t rs_seed_generation; /* FXRNG seed version */ +#endif } *rsx; static inline int _rs_allocate(struct _rs **, struct _rsx **); @@ -96,11 +102,43 @@ { u_char rnd[KEYSZ + IVSZ]; +#if defined(__FreeBSD__) + bool need_init; + + /* + * De-couple allocation (which locates the fxrng_root_generationp in + * auxinfo) from initialization. This allows us to read the root seed + * version before we fetch system entropy, maintaining the invariant + * that the PRF was seeded with entropy from rs_seed_generation or a + * later generation. But never seeded from an earlier generation. + * This invariant prevents us from missing a root reseed event. + */ + need_init = false; + if (rs == NULL) { + if (_rs_allocate(&rs, &rsx) == -1) + abort(); + need_init = true; + } + /* + * Transition period: new userspace on old kernel. This should become + * a hard error at some point, if the scheme is adopted. + */ + if (fxrng_root_generationp != NULL) + rsx->rs_seed_generation = + fxrng_load_acq_generation(fxrng_root_generationp); +#endif + if (getentropy(rnd, sizeof rnd) == -1) _getentropy_fail(); +#if !defined(__FreeBSD__) if (!rs) _rs_init(rnd, sizeof(rnd)); +#else /* __FreeBSD__ */ + assert(rs != NULL); + if (need_init) + _rs_init(rnd, sizeof(rnd)); +#endif else _rs_rekey(rnd, sizeof(rnd)); explicit_bzero(rnd, sizeof(rnd)); /* discard source seed */ Index: lib/libc/gen/auxv.c =================================================================== --- lib/libc/gen/auxv.c +++ lib/libc/gen/auxv.c @@ -72,6 +72,7 @@ static char *canary, *pagesizes, *execpath; static void *timekeep; static u_long hwcap, hwcap2; +static void *fxrng_seed_version; static void init_aux(void) @@ -125,6 +126,10 @@ case AT_TIMEKEEP: timekeep = aux->a_un.a_ptr; break; + + case AT_FXRNG: + fxrng_seed_version = aux->a_un.a_ptr; + break; } } } @@ -224,6 +229,16 @@ } else res = EINVAL; break; + case AT_FXRNG: + if (buflen == sizeof(void *)) { + if (fxrng_seed_version != NULL) { + *(void **)buf = fxrng_seed_version; + res = 0; + } else + res = ENOENT; + } else + res = EINVAL; + break; default: res = ENOENT; break; Index: sys/amd64/amd64/elf_machdep.c =================================================================== --- sys/amd64/amd64/elf_machdep.c +++ sys/amd64/amd64/elf_machdep.c @@ -74,7 +74,7 @@ .sv_fixlimit = NULL, .sv_maxssiz = NULL, .sv_flags = SV_ABI_FREEBSD | SV_ASLR | SV_LP64 | SV_SHP | - SV_TIMEKEEP, + SV_TIMEKEEP | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, Index: sys/arm/arm/elf_machdep.c =================================================================== --- sys/arm/arm/elf_machdep.c +++ sys/arm/arm/elf_machdep.c @@ -87,7 +87,7 @@ .sv_maxssiz = NULL, .sv_flags = #if __ARM_ARCH >= 6 - SV_ASLR | SV_SHP | SV_TIMEKEEP | + SV_ASLR | SV_SHP | SV_TIMEKEEP | SV_RNG_SEED_VER | #endif SV_ABI_FREEBSD | SV_ILP32 | SV_ASLR, .sv_set_syscall_retval = cpu_set_syscall_retval, Index: sys/arm64/arm64/elf32_machdep.c =================================================================== --- sys/arm64/arm64/elf32_machdep.c +++ sys/arm64/arm64/elf32_machdep.c @@ -97,7 +97,8 @@ .sv_setregs = freebsd32_setregs, .sv_fixlimit = NULL, // XXX .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_ILP32 | SV_SHP | SV_TIMEKEEP, + .sv_flags = SV_ABI_FREEBSD | SV_ILP32 | SV_SHP | SV_TIMEKEEP | + SV_RNG_SEED_VER, .sv_set_syscall_retval = freebsd32_set_syscall_retval, .sv_fetch_syscall_args = freebsd32_fetch_syscall_args, .sv_syscallnames = freebsd32_syscallnames, Index: sys/arm64/arm64/elf_machdep.c =================================================================== --- sys/arm64/arm64/elf_machdep.c +++ sys/arm64/arm64/elf_machdep.c @@ -82,7 +82,7 @@ .sv_fixlimit = NULL, .sv_maxssiz = NULL, .sv_flags = SV_SHP | SV_TIMEKEEP | SV_ABI_FREEBSD | SV_LP64 | - SV_ASLR, + SV_ASLR | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, Index: sys/compat/ia32/ia32_sysvec.c =================================================================== --- sys/compat/ia32/ia32_sysvec.c +++ sys/compat/ia32/ia32_sysvec.c @@ -120,7 +120,7 @@ .sv_fixlimit = ia32_fixlimit, .sv_maxssiz = &ia32_maxssiz, .sv_flags = SV_ABI_FREEBSD | SV_ASLR | SV_IA32 | SV_ILP32 | - SV_SHP | SV_TIMEKEEP, + SV_SHP | SV_TIMEKEEP | SV_RNG_SEED_VER, .sv_set_syscall_retval = ia32_set_syscall_retval, .sv_fetch_syscall_args = ia32_fetch_syscall_args, .sv_syscallnames = freebsd32_syscallnames, Index: sys/dev/random/fenestrasX/fx_brng.c =================================================================== --- sys/dev/random/fenestrasX/fx_brng.c +++ sys/dev/random/fenestrasX/fx_brng.c @@ -39,6 +39,7 @@ #include #include #include +#include #include @@ -108,6 +109,8 @@ */ rng->brng_generation++; atomic_store_rel_64(&fxrng_root_generation, rng->brng_generation); + /* Update VDSO version. */ + fxrng_push_seed_generation(rng->brng_generation); FXRNG_BRNG_UNLOCK(rng); } @@ -129,9 +132,26 @@ rng->brng_generation++; atomic_store_rel_64(&fxrng_root_generation, rng->brng_generation); + /* Update VDSO version. */ + fxrng_push_seed_generation(rng->brng_generation); FXRNG_BRNG_UNLOCK(rng); } +/* + * Sysentvec and VDSO are initialized much later than SI_SUB_RANDOM. When + * they're online, go ahead and push an initial root seed version. + * INIT_SYSENTVEC runs at SI_SUB_EXEC:SI_ORDER_ANY, and SI_ORDER_ANY is the + * maximum value, so we must run at SI_SUB_EXEC+1. + */ +static void +fxrng_vdso_sysinit(void *dummy __unused) +{ + FXRNG_BRNG_LOCK(&fxrng_root); + fxrng_push_seed_generation(fxrng_root.brng_generation); + FXRNG_BRNG_UNLOCK(&fxrng_root); +} +SYSINIT(fxrng_vdso, SI_SUB_EXEC + 1, SI_ORDER_ANY, fxrng_vdso_sysinit, NULL); + /* * Grab some bytes off an initialized, current generation RNG. * Index: sys/dev/random/fenestrasX/fx_main.c =================================================================== --- sys/dev/random/fenestrasX/fx_main.c +++ sys/dev/random/fenestrasX/fx_main.c @@ -88,7 +88,8 @@ * a while). * * Not yet implemented, not in scope, or todo: - * - Userspace portions -- shared page, like timehands vdso? + * - Various initial seeding sources we don't have yet + * - In particular, VM migration/copy detection */ #include Index: sys/i386/i386/elf_machdep.c =================================================================== --- sys/i386/i386/elf_machdep.c +++ sys/i386/i386/elf_machdep.c @@ -76,7 +76,7 @@ .sv_fixlimit = NULL, .sv_maxssiz = NULL, .sv_flags = SV_ABI_FREEBSD | SV_ASLR | SV_IA32 | SV_ILP32 | - SV_SHP | SV_TIMEKEEP, + SV_SHP | SV_TIMEKEEP | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, Index: sys/kern/imgact_elf.c =================================================================== --- sys/kern/imgact_elf.c +++ sys/kern/imgact_elf.c @@ -1367,6 +1367,8 @@ AUXARGS_ENTRY(pos, AT_HWCAP, *imgp->sysent->sv_hwcap); if (imgp->sysent->sv_hwcap2 != NULL) AUXARGS_ENTRY(pos, AT_HWCAP2, *imgp->sysent->sv_hwcap2); + if (imgp->sysent->sv_fxrng_gen_base != 0) + AUXARGS_ENTRY(pos, AT_FXRNG, imgp->sysent->sv_fxrng_gen_base); AUXARGS_ENTRY(pos, AT_NULL, 0); free(imgp->auxargs, M_TEMP); Index: sys/kern/kern_sharedpage.c =================================================================== --- sys/kern/kern_sharedpage.c +++ sys/kern/kern_sharedpage.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -60,6 +61,10 @@ static int shared_page_free; char *shared_page_mapping; +#ifdef RANDOM_FENESTRASX +static struct vdso_fxrng_generation *fxrng_shpage_mapping; +#endif + void shared_page_write(int base, int size, const void *data) { @@ -256,10 +261,53 @@ } #endif +#ifdef RANDOM_FENESTRASX +void +fxrng_push_seed_generation(uint64_t gen) +{ + if (fxrng_shpage_mapping == NULL) + return; +#if defined(__LP64__) + atomic_store_rel_64(&fxrng_shpage_mapping->fx_generation, gen); +#endif +#if defined(COMPAT_FREEBSD32) || defined(__ILP32__) + KASSERT(gen < INT32_MAX, + ("fxrng seed version shouldn't roll over a 32-bit counter " + "for approximately 456,000 years")); + atomic_store_rel_32(&fxrng_shpage_mapping->fx_generation32, + (uint32_t)gen); +#endif +} + +static void +alloc_sv_fxrng_generation(void) +{ + int base; + + /* + * Allocate a full cache line for the fxrng root generation (64-bit + * counter, or truncated 32-bit counter on ILP32 userspace). It is + * important that the line is not shared with frequently dirtied data, + * and the shared page allocator lacks a __read_mostly mechanism. + * However, PAGE_SIZE is typically large relative to the amount of + * stuff we've got in it so far, so maybe the possible waste isn't an + * issue. + */ + base = shared_page_alloc(CACHE_LINE_SIZE, CACHE_LINE_SIZE); + KASSERT(base != -1, ("%s: base allocation failed", __func__)); + shared_page_write(base, sizeof(struct vdso_fxrng_generation), + zero_region); + fxrng_shpage_mapping = (void *)(shared_page_mapping + base); +} +#endif + void exec_sysvec_init(void *param) { struct sysentvec *sv; +#ifdef RANDOM_FENESTRASX + ptrdiff_t base; +#endif sv = (struct sysentvec *)param; if ((sv->sv_flags & SV_SHP) == 0) @@ -287,4 +335,16 @@ } #endif } +#ifdef RANDOM_FENESTRASX + if ((sv->sv_flags & SV_RNG_SEED_VER) != 0) { + /* + * Only allocate a single VDSO entry for multiple sysentvecs, + * i.e., native and COMPAT32. + */ + if (fxrng_shpage_mapping == NULL) + alloc_sv_fxrng_generation(); + base = (char *)fxrng_shpage_mapping - shared_page_mapping; + sv->sv_fxrng_gen_base = sv->sv_shared_page_base + base; + } +#endif } Index: sys/mips/mips/elf_machdep.c =================================================================== --- sys/mips/mips/elf_machdep.c +++ sys/mips/mips/elf_machdep.c @@ -79,10 +79,11 @@ .sv_setregs = exec_setregs, .sv_fixlimit = NULL, .sv_maxssiz = NULL, + .sv_flags = SV_ABI_FREEBSD | SV_ASLR | SV_RNG_SEED_VER | #ifdef __mips_n64 - .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_ASLR, + SV_LP64, #else - .sv_flags = SV_ABI_FREEBSD | SV_ILP32 | SV_ASLR, + SV_ILP32, #endif .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, Index: sys/mips/mips/freebsd32_machdep.c =================================================================== --- sys/mips/mips/freebsd32_machdep.c +++ sys/mips/mips/freebsd32_machdep.c @@ -100,7 +100,7 @@ .sv_setregs = freebsd32_exec_setregs, .sv_fixlimit = NULL, .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_ILP32, + .sv_flags = SV_ABI_FREEBSD | SV_ILP32 | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = freebsd32_syscallnames, Index: sys/powerpc/powerpc/elf32_machdep.c =================================================================== --- sys/powerpc/powerpc/elf32_machdep.c +++ sys/powerpc/powerpc/elf32_machdep.c @@ -121,7 +121,8 @@ .sv_fixlimit = NULL, #endif .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_ILP32 | SV_SHP | SV_ASLR, + .sv_flags = SV_ABI_FREEBSD | SV_ILP32 | SV_SHP | SV_ASLR | + SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_shared_page_base = FREEBSD32_SHAREDPAGE, Index: sys/powerpc/powerpc/elf64_machdep.c =================================================================== --- sys/powerpc/powerpc/elf64_machdep.c +++ sys/powerpc/powerpc/elf64_machdep.c @@ -83,7 +83,8 @@ .sv_setregs = exec_setregs_funcdesc, .sv_fixlimit = NULL, .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP | SV_ASLR, + .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP | SV_ASLR | + SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, @@ -121,7 +122,7 @@ .sv_setregs = exec_setregs, .sv_fixlimit = NULL, .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP, + .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, Index: sys/riscv/riscv/elf_machdep.c =================================================================== --- sys/riscv/riscv/elf_machdep.c +++ sys/riscv/riscv/elf_machdep.c @@ -84,7 +84,8 @@ .sv_setregs = exec_setregs, .sv_fixlimit = NULL, .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP | SV_ASLR, + .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_SHP | SV_ASLR | + SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, Index: sys/sparc64/sparc64/elf_machdep.c =================================================================== --- sys/sparc64/sparc64/elf_machdep.c +++ sys/sparc64/sparc64/elf_machdep.c @@ -80,7 +80,7 @@ .sv_setregs = exec_setregs, .sv_fixlimit = NULL, .sv_maxssiz = NULL, - .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_ASLR, + .sv_flags = SV_ABI_FREEBSD | SV_LP64 | SV_ASLR | SV_RNG_SEED_VER, .sv_set_syscall_retval = cpu_set_syscall_retval, .sv_fetch_syscall_args = cpu_fetch_syscall_args, .sv_syscallnames = syscallnames, Index: sys/sys/elf_common.h =================================================================== --- sys/sys/elf_common.h +++ sys/sys/elf_common.h @@ -949,8 +949,9 @@ #define AT_EHDRFLAGS 24 /* e_flags field from elf hdr */ #define AT_HWCAP 25 /* CPU feature flags. */ #define AT_HWCAP2 26 /* CPU feature flags 2. */ +#define AT_FXRNG 27 /* Pointer to root RNG seed version. */ -#define AT_COUNT 27 /* Count of defined aux entry types. */ +#define AT_COUNT 28 /* Count of defined aux entry types. */ /* * Relocation types. Index: sys/sys/sysent.h =================================================================== --- sys/sys/sysent.h +++ sys/sys/sysent.h @@ -138,6 +138,7 @@ int (*sv_trap)(struct thread *); u_long *sv_hwcap; /* Value passed in AT_HWCAP. */ u_long *sv_hwcap2; /* Value passed in AT_HWCAP2. */ + vm_offset_t sv_fxrng_gen_base; }; #define SV_ILP32 0x000100 /* 32-bit executable. */ @@ -148,6 +149,7 @@ #define SV_CAPSICUM 0x020000 /* Force cap_enter() on startup. */ #define SV_TIMEKEEP 0x040000 /* Shared page timehands. */ #define SV_ASLR 0x080000 /* ASLR allowed. */ +#define SV_RNG_SEED_VER 0x100000 /* random(4) reseed generation. */ #define SV_ABI_MASK 0xff #define SV_ABI_ERRNO(p, e) ((p)->p_sysent->sv_errsize <= 0 ? e : \ Index: sys/sys/vdso.h =================================================================== --- sys/sys/vdso.h +++ sys/sys/vdso.h @@ -59,6 +59,18 @@ #define VDSO_TH_ALGO_3 0x3 #define VDSO_TH_ALGO_4 0x4 +struct vdso_fxrng_generation { + /* The first field is used only by LP64 kernels. */ + uint64_t fx_generation; + /* The latter is used by ILP32 kernels and COMPAT32 LP64 kernel. */ + uint32_t fx_generation32; + /* + * The sometimes unused 64-bit field allows 32-bit programs not to care + * if they are running on ILP32 or COMPAT32 LP64 kernel for such + * architectures. + */ +}; + #ifndef _KERNEL struct timespec; @@ -82,6 +94,9 @@ uint32_t sv_timekeep_gen; }; +#ifdef RANDOM_FENESTRASX +void fxrng_push_seed_generation(uint64_t gen); +#endif void timekeep_push_vdso(void); uint32_t tc_fill_vdso_timehands(struct vdso_timehands *vdso_th);