Index: head/sys/dev/random/fenestrasX/fx_brng.c
===================================================================
--- head/sys/dev/random/fenestrasX/fx_brng.c (revision 366620)
+++ head/sys/dev/random/fenestrasX/fx_brng.c (revision 366621)
@@ -1,295 +1,296 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2019 Conrad Meyer <cem@FreeBSD.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/fail.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/random.h>
#include <sys/sdt.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <machine/cpu.h>
#include <dev/random/randomdev.h>
#include <dev/random/random_harvestq.h>
#include <dev/random/uint128.h>
#include <dev/random/fenestrasX/fx_brng.h>
#include <dev/random/fenestrasX/fx_priv.h>
+#include <dev/random/fenestrasX/fx_pub.h>
#include <dev/random/fenestrasX/fx_rng.h>
/*
* Implementation of a buffered RNG, described in § 1.2-1.4 of the whitepaper.
*/
/*
* Initialize a buffered rng instance (either the static root instance, or a
* per-cpu instance on the heap. Both should be zero initialized before this
* routine.
*/
void
fxrng_brng_init(struct fxrng_buffered_rng *rng)
{
fxrng_rng_init(&rng->brng_rng, rng == &fxrng_root);
/* I.e., the buffer is empty. */
rng->brng_avail_idx = sizeof(rng->brng_buffer);
/*
* It is fine and correct for brng_generation and brng_buffer to be
* zero values.
*
* brng_prf and brng_generation must be initialized later.
* Initialization is special for the root BRNG. PCPU child instances
* use fxrng_brng_produce_seed_data_internal() below.
*/
}
/*
* Directly reseed the root BRNG from a first-time entropy source,
* incorporating the existing BRNG state. The main motivation for doing so "is
* to ensure that as soon as an entropy source produces data, PRNG output
* depends on the data from that source." (§ 3.1)
*
* The root BRNG is locked on entry and initial keying (brng_generation > 0)
* has already been performed. The root BRNG is unlocked on return.
*/
void
fxrng_brng_src_reseed(const struct harvest_event *event)
{
struct fxrng_buffered_rng *rng;
rng = &fxrng_root;
FXRNG_BRNG_ASSERT(rng);
ASSERT_DEBUG(rng->brng_generation > 0, "root RNG not seeded");
fxrng_rng_src_reseed(&rng->brng_rng, event);
FXRNG_BRNG_ASSERT(rng);
/*
* Bump root generation (which is costly) to force downstream BRNGs to
* reseed and quickly incorporate the new entropy. The intuition is
* that this tradeoff is worth it because new sources show up extremely
* rarely (limiting cost) and if they can contribute any entropy to a
* weak state, we want to propagate it to all generators ASAP.
*/
rng->brng_generation++;
atomic_store_rel_64(&fxrng_root_generation, rng->brng_generation);
FXRNG_BRNG_UNLOCK(rng);
}
/*
* Reseed a brng from some amount of pooled entropy (determined in fx_pool.c by
* fxent_timer_reseed_npools). For initial seeding, we pool entropy in a
* single pool and use this API as well (fxrng_alg_seeded).
*/
void
fxrng_brng_reseed(const void *entr, size_t sz)
{
struct fxrng_buffered_rng *rng;
rng = &fxrng_root;
FXRNG_BRNG_LOCK(rng);
fxrng_rng_reseed(&rng->brng_rng, (rng->brng_generation > 0), entr, sz);
FXRNG_BRNG_ASSERT(rng);
rng->brng_generation++;
atomic_store_rel_64(&fxrng_root_generation, rng->brng_generation);
FXRNG_BRNG_UNLOCK(rng);
}
/*
* Grab some bytes off an initialized, current generation RNG.
*
* (Does not handle reseeding if our generation is stale.)
*
* Locking protocol is a bit odd. The RNG is locked on entrance, but the lock
* is dropped on exit. This avoids holding a lock during expensive and slow
* RNG generation.
*/
static void
fxrng_brng_getbytes_internal(struct fxrng_buffered_rng *rng, void *buf,
size_t nbytes)
{
FXRNG_BRNG_ASSERT(rng);
/* Make the zero request impossible for the rest of the logic. */
if (__predict_false(nbytes == 0)) {
FXRNG_BRNG_UNLOCK(rng);
goto out;
}
/* Fast/easy case: Use some bytes from the buffer. */
if (rng->brng_avail_idx + nbytes <= sizeof(rng->brng_buffer)) {
memcpy(buf, &rng->brng_buffer[rng->brng_avail_idx], nbytes);
explicit_bzero(&rng->brng_buffer[rng->brng_avail_idx], nbytes);
rng->brng_avail_idx += nbytes;
FXRNG_BRNG_UNLOCK(rng);
goto out;
}
/* Buffer case: */
if (nbytes < sizeof(rng->brng_buffer)) {
size_t rem;
/* Drain anything left in the buffer first. */
if (rng->brng_avail_idx < sizeof(rng->brng_buffer)) {
rem = sizeof(rng->brng_buffer) - rng->brng_avail_idx;
ASSERT_DEBUG(nbytes > rem, "invariant");
memcpy(buf, &rng->brng_buffer[rng->brng_avail_idx], rem);
buf = (uint8_t*)buf + rem;
nbytes -= rem;
ASSERT_DEBUG(nbytes != 0, "invariant");
}
/*
* Partial fill from first buffer, have to rekey and generate a
* new buffer to do the rest.
*/
fxrng_rng_genrandom_internal(&rng->brng_rng, rng->brng_buffer,
sizeof(rng->brng_buffer), false);
FXRNG_BRNG_ASSERT(rng);
rng->brng_avail_idx = 0;
memcpy(buf, &rng->brng_buffer[rng->brng_avail_idx], nbytes);
explicit_bzero(&rng->brng_buffer[rng->brng_avail_idx], nbytes);
rng->brng_avail_idx += nbytes;
FXRNG_BRNG_UNLOCK(rng);
goto out;
}
/* Large request; skip the buffer. */
fxrng_rng_genrandom_internal(&rng->brng_rng, buf, nbytes, true);
out:
FXRNG_BRNG_ASSERT_NOT(rng);
return;
}
/*
* API to get a new key for a downstream RNG. Returns the new key in 'buf', as
* well as the generator's reseed_generation.
*
* 'rng' is locked on entry and unlocked on return.
*
* Only valid after confirming the caller's seed version or reseed_generation
* matches roots (or we are root). (For now, this is only used to reseed the
* per-CPU generators from root.)
*/
void
fxrng_brng_produce_seed_data_internal(struct fxrng_buffered_rng *rng,
void *buf, size_t keysz, uint64_t *seed_generation)
{
FXRNG_BRNG_ASSERT(rng);
ASSERT_DEBUG(keysz == FX_CHACHA20_KEYSIZE, "keysz: %zu", keysz);
*seed_generation = rng->brng_generation;
fxrng_brng_getbytes_internal(rng, buf, keysz);
FXRNG_BRNG_ASSERT_NOT(rng);
}
/*
* Read from an allocated and initialized buffered BRNG. This a high-level
* API, but doesn't handle PCPU BRNG allocation.
*
* BRNG is locked on entry. It is unlocked on return.
*/
void
fxrng_brng_read(struct fxrng_buffered_rng *rng, void *buf, size_t nbytes)
{
uint8_t newkey[FX_CHACHA20_KEYSIZE];
FXRNG_BRNG_ASSERT(rng);
/* Fast path: there hasn't been a global reseed since last read. */
if (rng->brng_generation == atomic_load_acq_64(&fxrng_root_generation))
goto done_reseeding;
ASSERT(rng != &fxrng_root, "root rng inconsistent seed version");
/*
* Slow path: We need to rekey from the parent BRNG to incorporate new
* entropy material.
*
* Lock order is always root -> percpu.
*/
FXRNG_BRNG_UNLOCK(rng);
FXRNG_BRNG_LOCK(&fxrng_root);
FXRNG_BRNG_LOCK(rng);
/*
* If we lost the reseeding race when the lock was dropped, don't
* duplicate work.
*/
if (__predict_false(rng->brng_generation ==
atomic_load_acq_64(&fxrng_root_generation))) {
FXRNG_BRNG_UNLOCK(&fxrng_root);
goto done_reseeding;
}
fxrng_brng_produce_seed_data_internal(&fxrng_root, newkey,
sizeof(newkey), &rng->brng_generation);
FXRNG_BRNG_ASSERT_NOT(&fxrng_root);
FXRNG_BRNG_ASSERT(rng);
fxrng_rng_setkey(&rng->brng_rng, newkey, sizeof(newkey));
explicit_bzero(newkey, sizeof(newkey));
/*
* A reseed invalidates any previous buffered contents. Here, we
* forward the available index to the end of the buffer, i.e., empty.
* Requests that would use the buffer (< 128 bytes) will refill its
* contents on demand.
*
* It is explicitly ok that we do not zero out any remaining buffer
* bytes; they will never be handed out to callers, and they reveal
* nothing about the reseeded key (which came from the root BRNG).
* (§ 1.3)
*/
rng->brng_avail_idx = sizeof(rng->brng_buffer);
done_reseeding:
if (rng != &fxrng_root)
FXRNG_BRNG_ASSERT_NOT(&fxrng_root);
FXRNG_BRNG_ASSERT(rng);
fxrng_brng_getbytes_internal(rng, buf, nbytes);
FXRNG_BRNG_ASSERT_NOT(rng);
}
Index: head/sys/dev/random/fenestrasX/fx_main.c
===================================================================
--- head/sys/dev/random/fenestrasX/fx_main.c (revision 366620)
+++ head/sys/dev/random/fenestrasX/fx_main.c (revision 366621)
@@ -1,274 +1,296 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2019 Conrad Meyer <cem@FreeBSD.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* This random algorithm is derived in part from the "Windows 10 random number
* generation infrastructure" whitepaper published by Niels Ferguson and
* Microsoft: https://aka.ms/win10rng
*
* It is also inspired by DJB's writing on buffered key-erasure PRNGs:
* https://blog.cr.yp.to/20170723-random.html
*
* The Windows 10 RNG bears some similarity to Fortuna, which Ferguson was also
* involved with. Notable differences include:
* - Extended to multi-CPU design
* - Extended to pre-buffer some PRNG output
* - Pool-based reseeding is solely time-based (rather than on-access w/
* pacing)
* - Extended to specify efficient userspace design
* - Always-available design (requires the equivalent of loader(8) for all
* boots; probably relatively easy given the limited platforms Windows 10
* supports)
*
* Some aspects of the design document I found confusing and may have
* misinterpreted:
* - Relationship between root PRNG seed version and periodic reseed pool use.
* I interpreted these as separate sequences. The root PRNG seed version is
* bumped both by the periodic pool based reseed, and also special
* conditions such as the first time an entropy source provides entropy. I
* don't think first-time entropy sources should cause us to skip an entropy
* pool reseed.
* - Initial seeding. The paper is pretty terse on the subject. My
* interpretation of the document is that the Windows RNG infrastructure
* relies on the loader(8)-provided material for initial seeding and either
* ignores or doesn't start entropy sources until after that time. So when
* the paper says that first-time entropy source material "bypasses the
* pools," the root PRNG state already has been keyed for the first time and
* can generate 256 bits, mix it with the first-time entropy, and reseed
* immediately.
*
* Some notable design choices in this implementation divergent from that
* specified in the document above:
* - Blake2b instead of SHA-2 512 for entropy pooling
* - Chacha20 instead of AES-CTR DRBG for PRF
* - Initial seeding. We treat the 0->1 seed version (brng_generation) edge
* as the transition from blocked to unblocked. That edge is also the first
* time the key of the root BRNG's PRF is set. We perform initial seeding
* when the first request for entropy arrives.
* • As a result: Entropy callbacks prior to this edge do not have a keyed
* root PRNG, so bypassing the pools is kind of meaningless. Instead,
* they feed into pool0. (They also do not set the root PRNG key or bump
* the root PRNG seed version.)
* • Entropy callbacks after the edge behave like the specification.
* • All one-off sources are fed into pool0 and the result used to seed the
* root BRNG during the initial seed step.
* • All memory needed for initial seeding must be preallocated or static or
* fit on the stack; random reads can occur in nonsleepable contexts and
* we cannot allocate M_WAITOK. (We also cannot fail to incorporate any
* present one-off source, to the extent it is in the control of
* software.)
* - Timer interval reseeding. We also start the timer-based reseeding at
* initial seed, but unlike the design, our initial seed is some time after
* load (usually within the order of micro- or milliseconds due to
* stack_guard on x86, but conceivably later if nothing reads from random for
* a while).
*
* Not yet implemented, not in scope, or todo:
- * - arc4random(9) injection/replacement
* - Userspace portions -- shared page, like timehands vdso?
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/domainset.h>
#include <sys/fail.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/random.h>
#include <sys/sdt.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <machine/cpu.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_page.h>
#include <vm/vm_phys.h>
#include <vm/vm_pagequeue.h>
#include <dev/random/randomdev.h>
#include <dev/random/random_harvestq.h>
#include <dev/random/uint128.h>
#include <dev/random/fenestrasX/fx_brng.h>
#include <dev/random/fenestrasX/fx_hash.h>
#include <dev/random/fenestrasX/fx_pool.h>
#include <dev/random/fenestrasX/fx_priv.h>
+#include <dev/random/fenestrasX/fx_pub.h>
#include <dev/random/fenestrasX/fx_rng.h>
struct fxrng_buffered_rng fxrng_root;
uint64_t __read_mostly fxrng_root_generation;
DPCPU_DEFINE_STATIC(struct fxrng_buffered_rng *, fxrng_brng);
/*
* Top-level read API from randomdev. Responsible for NOWAIT-allocating
* per-cpu NUMA-local BRNGs, if needed and satisfiable; subroutines handle
* reseeding if the local BRNG is stale and rekeying when necessary. In
* low-memory conditions when a local BRNG cannot be allocated, the request is
* simply forwarded to the root BRNG.
*
* It is a precondition is that the root BRNG initial seeding has completed and
* the root generation number >0.
*/
static void
-fxrng_alg_read(uint8_t *output, size_t nbytes)
+_fxrng_alg_read(uint8_t *output, size_t nbytes, uint64_t *seed_version_out)
{
struct fxrng_buffered_rng **pcpu_brng_p, *rng, *tmp;
struct pcpu *pcpu;
pcpu = get_pcpu();
/*
* The following statement directly accesses an implementation detail
* of DPCPU, but the macros cater only to pinned threads; we want to
* operate on our initial CPU, without pinning, *even if* we migrate.
*/
pcpu_brng_p = _DPCPU_PTR(pcpu->pc_dynamic, fxrng_brng);
rng = (void *)atomic_load_acq_ptr((uintptr_t *)pcpu_brng_p);
/*
* Usually the pcpu BRNG has already been allocated, but we do it
* on-demand and need to check first. BRNGs are never deallocated and
* are valid as soon as the pointer is initialized.
*/
if (__predict_false(rng == NULL)) {
uint8_t newkey[FX_CHACHA20_KEYSIZE];
struct domainset *ds;
int domain;
domain = pcpu->pc_domain;
/*
* Allocate pcpu BRNGs off-domain on weird NUMA machines like
* AMD Threadripper 2990WX, which has 2 NUMA nodes without
* local memory controllers. The PREF policy is automatically
* converted to something appropriate when domains are empty.
* (FIXED is not.)
*
* Otherwise, allocate strictly CPU-local memory. The
* rationale is this: if there is a memory shortage such that
* PREF policy would fallback to RR, we have no business
* wasting memory on a faster BRNG. So, use a FIXED domainset
* policy. If we cannot allocate, that's fine! We fall back
* to invoking the root BRNG.
*/
if (VM_DOMAIN_EMPTY(domain))
ds = DOMAINSET_PREF(domain);
else
ds = DOMAINSET_FIXED(domain);
rng = malloc_domainset(sizeof(*rng), M_ENTROPY, ds,
M_NOWAIT | M_ZERO);
if (rng == NULL) {
/* Relatively easy case: fall back to root BRNG. */
rng = &fxrng_root;
goto have_valid_rng;
}
fxrng_brng_init(rng);
/*
* The root BRNG is always up and available. Requests are
* always satisfiable. This is a design invariant.
*/
ASSERT_DEBUG(atomic_load_acq_64(&fxrng_root_generation) != 0,
"%s: attempting to seed child BRNG when root hasn't "
"been initialized yet.", __func__);
FXRNG_BRNG_LOCK(&fxrng_root);
#ifdef WITNESS
/* Establish lock order root->pcpu for WITNESS. */
FXRNG_BRNG_LOCK(rng);
FXRNG_BRNG_UNLOCK(rng);
#endif
fxrng_brng_produce_seed_data_internal(&fxrng_root, newkey,
sizeof(newkey), &rng->brng_generation);
FXRNG_BRNG_ASSERT_NOT(&fxrng_root);
fxrng_rng_setkey(&rng->brng_rng, newkey, sizeof(newkey));
explicit_bzero(newkey, sizeof(newkey));
/*
* We have a valid RNG. Try to install it, or grab the other
* one if we lost the race.
*/
tmp = NULL;
while (tmp == NULL)
if (atomic_fcmpset_ptr((uintptr_t *)pcpu_brng_p,
(uintptr_t *)&tmp, (uintptr_t)rng))
goto have_valid_rng;
/*
* We lost the race. There's nothing sensitive about
* our BRNG's PRF state, because it will never be used
* for anything and the key doesn't expose any
* information about the parent (root) generator's
* state -- it has already rekeyed. The generation
* number is public, and a zero counter isn't sensitive.
*/
free(rng, M_ENTROPY);
/*
* Use the winner's PCPU BRNG.
*/
rng = tmp;
}
have_valid_rng:
/* At this point we have a valid, initialized and seeded rng pointer. */
FXRNG_BRNG_LOCK(rng);
+ if (seed_version_out != NULL)
+ *seed_version_out = rng->brng_generation;
fxrng_brng_read(rng, output, nbytes);
FXRNG_BRNG_ASSERT_NOT(rng);
+}
+
+static void
+fxrng_alg_read(uint8_t *output, size_t nbytes)
+{
+ _fxrng_alg_read(output, nbytes, NULL);
+}
+
+/*
+ * External API for arc4random(9) to fetch new key material and associated seed
+ * version in chacha20_randomstir().
+ */
+void
+read_random_key(void *output, size_t nbytes, uint64_t *seed_version_out)
+{
+ /* Ensure _fxrng_alg_read invariant. */
+ if (__predict_false(atomic_load_acq_64(&fxrng_root_generation) == 0))
+ (void)fxrng_alg_seeded();
+
+ _fxrng_alg_read(output, nbytes, seed_version_out);
}
static void
fxrng_init_alg(void *dummy __unused)
{
DPCPU_ZERO(fxrng_brng);
fxrng_brng_init(&fxrng_root);
fxrng_pools_init();
}
SYSINIT(random_alg, SI_SUB_RANDOM, SI_ORDER_SECOND, fxrng_init_alg, NULL);
/*
* Public visibility struct referenced directly by other parts of randomdev.
*/
const struct random_algorithm random_alg_context = {
.ra_ident = "fenestrasX",
.ra_pre_read = (void (*)(void))nullop,
.ra_read = fxrng_alg_read,
.ra_seeded = fxrng_alg_seeded,
.ra_event_processor = fxrng_event_processor,
.ra_poolcount = FXRNG_NPOOLS,
};
Index: head/sys/dev/random/fenestrasX/fx_pool.c
===================================================================
--- head/sys/dev/random/fenestrasX/fx_pool.c (revision 366620)
+++ head/sys/dev/random/fenestrasX/fx_pool.c (revision 366621)
@@ -1,615 +1,616 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2019 Conrad Meyer <cem@FreeBSD.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/domainset.h>
#include <sys/fail.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/queue.h>
#include <sys/random.h>
#include <sys/sdt.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/taskqueue.h>
#include <machine/atomic.h>
#include <machine/smp.h>
#include <dev/random/randomdev.h>
#include <dev/random/random_harvestq.h>
#include <dev/random/fenestrasX/fx_brng.h>
#include <dev/random/fenestrasX/fx_hash.h>
#include <dev/random/fenestrasX/fx_pool.h>
#include <dev/random/fenestrasX/fx_priv.h>
+#include <dev/random/fenestrasX/fx_pub.h>
/*
* Timer-based reseed interval growth factor and limit in seconds. (§ 3.2)
*/
#define FXENT_RESSED_INTVL_GFACT 3
#define FXENT_RESEED_INTVL_MAX 3600
/*
* Pool reseed schedule. Initially, only pool 0 is active. Until the timer
* interval reaches INTVL_MAX, only pool 0 is used.
*
* After reaching INTVL_MAX, pool k is either activated (if inactive) or used
* (if active) every 3^k timer reseeds. (§ 3.3)
*
* (Entropy harvesting only round robins across active pools.)
*/
#define FXENT_RESEED_BASE 3
/*
* Number of bytes from high quality sources to allocate to pool 0 before
* normal round-robin allocation after each timer reseed. (§ 3.4)
*/
#define FXENT_HI_SRC_POOL0_BYTES 32
/*
* § 3.1
*
* Low sources provide unconditioned entropy, such as mouse movements; high
* sources are assumed to provide high-quality random bytes. Pull sources are
* those which can be polled, i.e., anything randomdev calls a "random_source."
*
* In the whitepaper, low sources are pull. For us, at least in the existing
* design, low-quality sources push into some global ring buffer and then get
* forwarded into the RNG by a thread that continually polls. Presumably their
* design batches low entopy signals in some way (SHA512?) and only requests
* them dynamically on reseed. I'm not sure what the benefit is vs feeding
* into the pools directly.
*/
enum fxrng_ent_access_cls {
FXRNG_PUSH,
FXRNG_PULL,
};
enum fxrng_ent_source_cls {
FXRNG_HI,
FXRNG_LO,
FXRNG_GARBAGE,
};
struct fxrng_ent_cls {
enum fxrng_ent_access_cls entc_axx_cls;
enum fxrng_ent_source_cls entc_src_cls;
};
static const struct fxrng_ent_cls fxrng_hi_pull = {
.entc_axx_cls = FXRNG_PULL,
.entc_src_cls = FXRNG_HI,
};
static const struct fxrng_ent_cls fxrng_hi_push = {
.entc_axx_cls = FXRNG_PUSH,
.entc_src_cls = FXRNG_HI,
};
static const struct fxrng_ent_cls fxrng_lo_push = {
.entc_axx_cls = FXRNG_PUSH,
.entc_src_cls = FXRNG_LO,
};
static const struct fxrng_ent_cls fxrng_garbage = {
.entc_axx_cls = FXRNG_PUSH,
.entc_src_cls = FXRNG_GARBAGE,
};
/*
* This table is a mapping of randomdev's current source abstractions to the
* designations above; at some point, if the design seems reasonable, it would
* make more sense to pull this up into the abstraction layer instead.
*/
static const struct fxrng_ent_char {
const struct fxrng_ent_cls *entc_cls;
} fxrng_ent_char[ENTROPYSOURCE] = {
[RANDOM_CACHED] = {
.entc_cls = &fxrng_hi_push,
},
[RANDOM_ATTACH] = {
.entc_cls = &fxrng_lo_push,
},
[RANDOM_KEYBOARD] = {
.entc_cls = &fxrng_lo_push,
},
[RANDOM_MOUSE] = {
.entc_cls = &fxrng_lo_push,
},
[RANDOM_NET_TUN] = {
.entc_cls = &fxrng_lo_push,
},
[RANDOM_NET_ETHER] = {
.entc_cls = &fxrng_lo_push,
},
[RANDOM_NET_NG] = {
.entc_cls = &fxrng_lo_push,
},
[RANDOM_INTERRUPT] = {
.entc_cls = &fxrng_lo_push,
},
[RANDOM_SWI] = {
.entc_cls = &fxrng_lo_push,
},
[RANDOM_FS_ATIME] = {
.entc_cls = &fxrng_lo_push,
},
[RANDOM_UMA] = {
.entc_cls = &fxrng_lo_push,
},
[RANDOM_PURE_OCTEON] = {
.entc_cls = &fxrng_hi_push, /* Could be made pull. */
},
[RANDOM_PURE_SAFE] = {
.entc_cls = &fxrng_hi_push,
},
[RANDOM_PURE_GLXSB] = {
.entc_cls = &fxrng_hi_push,
},
[RANDOM_PURE_HIFN] = {
.entc_cls = &fxrng_hi_push,
},
[RANDOM_PURE_RDRAND] = {
.entc_cls = &fxrng_hi_pull,
},
[RANDOM_PURE_NEHEMIAH] = {
.entc_cls = &fxrng_hi_pull,
},
[RANDOM_PURE_RNDTEST] = {
.entc_cls = &fxrng_garbage,
},
[RANDOM_PURE_VIRTIO] = {
.entc_cls = &fxrng_hi_pull,
},
[RANDOM_PURE_BROADCOM] = {
.entc_cls = &fxrng_hi_push,
},
[RANDOM_PURE_CCP] = {
.entc_cls = &fxrng_hi_pull,
},
[RANDOM_PURE_DARN] = {
.entc_cls = &fxrng_hi_pull,
},
[RANDOM_PURE_TPM] = {
.entc_cls = &fxrng_hi_push,
},
[RANDOM_PURE_VMGENID] = {
.entc_cls = &fxrng_hi_push,
},
};
/* Useful for single-bit-per-source state. */
BITSET_DEFINE(fxrng_bits, ENTROPYSOURCE);
/* XXX Borrowed from not-yet-committed D22702. */
#ifndef BIT_TEST_SET_ATOMIC_ACQ
#define BIT_TEST_SET_ATOMIC_ACQ(_s, n, p) \
(atomic_testandset_acq_long( \
&(p)->__bits[__bitset_word((_s), (n))], (n)) != 0)
#endif
#define FXENT_TEST_SET_ATOMIC_ACQ(n, p) \
BIT_TEST_SET_ATOMIC_ACQ(ENTROPYSOURCE, n, p)
/* For special behavior on first-time entropy sources. (§ 3.1) */
static struct fxrng_bits __read_mostly fxrng_seen;
/* For special behavior for high-entropy sources after a reseed. (§ 3.4) */
_Static_assert(FXENT_HI_SRC_POOL0_BYTES <= UINT8_MAX, "");
static uint8_t __read_mostly fxrng_reseed_seen[ENTROPYSOURCE];
/* Entropy pools. Lock order is ENT -> RNG(root) -> RNG(leaf). */
static struct mtx fxent_pool_lk;
MTX_SYSINIT(fx_pool, &fxent_pool_lk, "fx entropy pool lock", MTX_DEF);
#define FXENT_LOCK() mtx_lock(&fxent_pool_lk)
#define FXENT_UNLOCK() mtx_unlock(&fxent_pool_lk)
#define FXENT_ASSERT(rng) mtx_assert(&fxent_pool_lk, MA_OWNED)
#define FXENT_ASSERT_NOT(rng) mtx_assert(&fxent_pool_lk, MA_NOTOWNED)
static struct fxrng_hash fxent_pool[FXRNG_NPOOLS];
static unsigned __read_mostly fxent_nactpools = 1;
static struct timeout_task fxent_reseed_timer;
static int __read_mostly fxent_timer_ready;
/*
* Track number of bytes of entropy harvested from high-quality sources prior
* to initial keying. The idea is to collect more jitter entropy when fewer
* high-quality bytes were available and less if we had other good sources. We
* want to provide always-on availability but don't necessarily have *any*
* great sources on some platforms.
*
* Like fxrng_ent_char: at some point, if the design seems reasonable, it would
* make more sense to pull this up into the abstraction layer instead.
*
* Jitter entropy is unimplemented for now.
*/
static unsigned long fxrng_preseed_ent;
void
fxrng_pools_init(void)
{
size_t i;
for (i = 0; i < nitems(fxent_pool); i++)
fxrng_hash_init(&fxent_pool[i]);
}
static inline bool
fxrng_hi_source(enum random_entropy_source src)
{
return (fxrng_ent_char[src].entc_cls->entc_src_cls == FXRNG_HI);
}
/*
* A racy check that this high-entropy source's event should contribute to
* pool0 on the basis of per-source byte count. The check is racy for two
* reasons:
* - Performance: The vast majority of the time, we've already taken 32 bytes
* from any present high quality source and the racy check lets us avoid
* dirtying the cache for the global array.
* - Correctness: It's fine that the check is racy. The failure modes are:
* • False positive: We will detect when we take the lock.
* • False negative: We still collect the entropy; it just won't be
* preferentially placed in pool0 in this case.
*/
static inline bool
fxrng_hi_pool0_eligible_racy(enum random_entropy_source src)
{
return (atomic_load_acq_8(&fxrng_reseed_seen[src]) <
FXENT_HI_SRC_POOL0_BYTES);
}
/*
* Top level entropy processing API from randomdev.
*
* Invoked by the core randomdev subsystem both for preload entropy, "push"
* sources (like interrupts, keyboard, etc) and pull sources (RDRAND, etc).
*/
void
fxrng_event_processor(struct harvest_event *event)
{
enum random_entropy_source src;
unsigned pool;
bool first_time, first_32;
src = event->he_source;
ASSERT_DEBUG(event->he_size <= sizeof(event->he_entropy),
"%s: he_size: %u > sizeof(he_entropy): %zu", __func__,
(unsigned)event->he_size, sizeof(event->he_entropy));
/*
* Zero bytes of source entropy doesn't count as observing this source
* for the first time. We still harvest the counter entropy.
*/
first_time = event->he_size > 0 &&
!FXENT_TEST_SET_ATOMIC_ACQ(src, &fxrng_seen);
if (__predict_false(first_time)) {
/*
* "The first time [any source] provides entropy, it is used to
* directly reseed the root PRNG. The entropy pools are
* bypassed." (§ 3.1)
*
* Unlike Windows, we cannot rely on loader(8) seed material
* being present, so we perform initial keying in the kernel.
* We use brng_generation 0 to represent an unkeyed state.
*
* Prior to initial keying, it doesn't make sense to try to mix
* the entropy directly with the root PRNG state, as the root
* PRNG is unkeyed. Instead, we collect pre-keying dynamic
* entropy in pool0 and do not bump the root PRNG seed version
* or set its key. Initial keying will incorporate pool0 and
* bump the brng_generation (seed version).
*
* After initial keying, we do directly mix in first-time
* entropy sources. We use the root BRNG to generate 32 bytes
* and use fxrng_hash to mix it with the new entropy source and
* re-key with the first 256 bits of hash output.
*/
FXENT_LOCK();
FXRNG_BRNG_LOCK(&fxrng_root);
if (__predict_true(fxrng_root.brng_generation > 0)) {
/* Bypass the pools: */
FXENT_UNLOCK();
fxrng_brng_src_reseed(event);
FXRNG_BRNG_ASSERT_NOT(&fxrng_root);
return;
}
/*
* Keying the root PRNG requires both FXENT_LOCK and the PRNG's
* lock, so we only need to hold on to the pool lock to prevent
* initial keying without this entropy.
*/
FXRNG_BRNG_UNLOCK(&fxrng_root);
/* Root PRNG hasn't been keyed yet, just accumulate event. */
fxrng_hash_update(&fxent_pool[0], &event->he_somecounter,
sizeof(event->he_somecounter));
fxrng_hash_update(&fxent_pool[0], event->he_entropy,
event->he_size);
if (fxrng_hi_source(src)) {
/* Prevent overflow. */
if (fxrng_preseed_ent <= ULONG_MAX - event->he_size)
fxrng_preseed_ent += event->he_size;
}
FXENT_UNLOCK();
return;
}
/* !first_time */
/*
* "The first 32 bytes produced by a high entropy source after a reseed
* from the pools is always put in pool 0." (§ 3.4)
*
* The first-32-byte tracking data in fxrng_reseed_seen is reset in
* fxent_timer_reseed_npools() below.
*/
first_32 = event->he_size > 0 &&
fxrng_hi_source(src) &&
atomic_load_acq_int(&fxent_nactpools) > 1 &&
fxrng_hi_pool0_eligible_racy(src);
if (__predict_false(first_32)) {
unsigned rem, seen;
FXENT_LOCK();
seen = fxrng_reseed_seen[src];
if (seen == FXENT_HI_SRC_POOL0_BYTES)
goto round_robin;
rem = FXENT_HI_SRC_POOL0_BYTES - seen;
rem = MIN(rem, event->he_size);
fxrng_reseed_seen[src] = seen + rem;
/*
* We put 'rem' bytes in pool0, and any remaining bytes are
* round-robin'd across other pools.
*/
fxrng_hash_update(&fxent_pool[0],
((uint8_t *)event->he_entropy) + event->he_size - rem,
rem);
if (rem == event->he_size) {
fxrng_hash_update(&fxent_pool[0], &event->he_somecounter,
sizeof(event->he_somecounter));
FXENT_UNLOCK();
return;
}
/*
* If fewer bytes were needed than this even provied, We only
* take the last rem bytes of the entropy buffer and leave the
* timecounter to be round-robin'd with the remaining entropy.
*/
event->he_size -= rem;
goto round_robin;
}
/* !first_32 */
FXENT_LOCK();
round_robin:
FXENT_ASSERT();
pool = event->he_destination % fxent_nactpools;
fxrng_hash_update(&fxent_pool[pool], event->he_entropy,
event->he_size);
fxrng_hash_update(&fxent_pool[pool], &event->he_somecounter,
sizeof(event->he_somecounter));
if (__predict_false(fxrng_hi_source(src) &&
atomic_load_acq_64(&fxrng_root_generation) == 0)) {
/* Prevent overflow. */
if (fxrng_preseed_ent <= ULONG_MAX - event->he_size)
fxrng_preseed_ent += event->he_size;
}
FXENT_UNLOCK();
}
/*
* Top level "seeded" API/signal from randomdev.
*
* This is our warning that a request is coming: we need to be seeded. In
* fenestrasX, a request for random bytes _never_ fails. "We (ed: ditto) have
* observed that there are many callers that never check for the error code,
* even if they are generating cryptographic key material." (§ 1.6)
*
* If we returned 'false', both read_random(9) and chacha20_randomstir()
* (arc4random(9)) will blindly charge on with something almost certainly worse
* than what we've got, or are able to get quickly enough.
*/
bool
fxrng_alg_seeded(void)
{
uint8_t hash[FXRNG_HASH_SZ];
sbintime_t sbt;
/* The vast majority of the time, we expect to already be seeded. */
if (__predict_true(atomic_load_acq_64(&fxrng_root_generation) != 0))
return (true);
/*
* Take the lock and recheck; only one thread needs to do the initial
* seeding work.
*/
FXENT_LOCK();
if (atomic_load_acq_64(&fxrng_root_generation) != 0) {
FXENT_UNLOCK();
return (true);
}
/* XXX Any one-off initial seeding goes here. */
fxrng_hash_finish(&fxent_pool[0], hash, sizeof(hash));
fxrng_hash_init(&fxent_pool[0]);
fxrng_brng_reseed(hash, sizeof(hash));
FXENT_UNLOCK();
randomdev_unblock();
explicit_bzero(hash, sizeof(hash));
/*
* This may be called too early for taskqueue_thread to be initialized.
* fxent_pool_timer_init will detect if we've already unblocked and
* queue the first timer reseed at that point.
*/
if (atomic_load_acq_int(&fxent_timer_ready) != 0) {
sbt = SBT_1S;
taskqueue_enqueue_timeout_sbt(taskqueue_thread,
&fxent_reseed_timer, -sbt, (sbt / 3), C_PREL(2));
}
return (true);
}
/*
* Timer-based reseeds and pool expansion.
*/
static void
fxent_timer_reseed_npools(unsigned n)
{
/*
* 64 * 8 => moderately large 512 bytes. Could be static, as we are
* only used in a static context. On the other hand, this is in
* threadqueue TASK context and we're likely nearly at top of stack
* already.
*/
uint8_t hash[FXRNG_HASH_SZ * FXRNG_NPOOLS];
unsigned i;
ASSERT_DEBUG(n > 0 && n <= FXRNG_NPOOLS, "n:%u", n);
FXENT_ASSERT();
/*
* Collect entropy from pools 0..n-1 by concatenating the output hashes
* and then feeding them into fxrng_brng_reseed, which will hash the
* aggregate together with the current root PRNG keystate to produce a
* new key. It will also bump the global generation counter
* appropriately.
*/
for (i = 0; i < n; i++) {
fxrng_hash_finish(&fxent_pool[i], hash + i * FXRNG_HASH_SZ,
FXRNG_HASH_SZ);
fxrng_hash_init(&fxent_pool[i]);
}
fxrng_brng_reseed(hash, n * FXRNG_HASH_SZ);
explicit_bzero(hash, n * FXRNG_HASH_SZ);
/*
* "The first 32 bytes produced by a high entropy source after a reseed
* from the pools is always put in pool 0." (§ 3.4)
*
* So here we reset the tracking (somewhat naively given the majority
* of sources on most machines are not what we consider "high", but at
* 32 bytes it's smaller than a cache line), so the next 32 bytes are
* prioritized into pool0.
*
* See corresponding use of fxrng_reseed_seen in fxrng_event_processor.
*/
memset(fxrng_reseed_seen, 0, sizeof(fxrng_reseed_seen));
FXENT_ASSERT();
}
static void
fxent_timer_reseed(void *ctx __unused, int pending __unused)
{
static unsigned reseed_intvl_sec = 1;
/* Only reseeds after FXENT_RESEED_INTVL_MAX is achieved. */
static uint64_t reseed_number = 1;
unsigned next_ival, i, k;
sbintime_t sbt;
if (reseed_intvl_sec < FXENT_RESEED_INTVL_MAX) {
next_ival = FXENT_RESSED_INTVL_GFACT * reseed_intvl_sec;
if (next_ival > FXENT_RESEED_INTVL_MAX)
next_ival = FXENT_RESEED_INTVL_MAX;
FXENT_LOCK();
fxent_timer_reseed_npools(1);
FXENT_UNLOCK();
} else {
/*
* The creation of entropy pools beyond 0 is enabled when the
* reseed interval hits the maximum. (§ 3.3)
*/
next_ival = reseed_intvl_sec;
/*
* Pool 0 is used every reseed; pool 1..0 every 3rd reseed; and in
* general, pool n..0 every 3^n reseeds.
*/
k = reseed_number;
reseed_number++;
/* Count how many pools, from [0, i), to use for reseed. */
for (i = 1; i < MIN(fxent_nactpools + 1, FXRNG_NPOOLS); i++) {
if ((k % FXENT_RESEED_BASE) != 0)
break;
k /= FXENT_RESEED_BASE;
}
/*
* If we haven't activated pool i yet, activate it and only
* reseed from [0, i-1). (§ 3.3)
*/
FXENT_LOCK();
if (i == fxent_nactpools + 1) {
fxent_timer_reseed_npools(fxent_nactpools);
fxent_nactpools++;
} else {
/* Just reseed from [0, i). */
fxent_timer_reseed_npools(i);
}
FXENT_UNLOCK();
}
/* Schedule the next reseed. */
sbt = next_ival * SBT_1S;
taskqueue_enqueue_timeout_sbt(taskqueue_thread, &fxent_reseed_timer,
-sbt, (sbt / 3), C_PREL(2));
reseed_intvl_sec = next_ival;
}
static void
fxent_pool_timer_init(void *dummy __unused)
{
sbintime_t sbt;
TIMEOUT_TASK_INIT(taskqueue_thread, &fxent_reseed_timer, 0,
fxent_timer_reseed, NULL);
if (atomic_load_acq_64(&fxrng_root_generation) != 0) {
sbt = SBT_1S;
taskqueue_enqueue_timeout_sbt(taskqueue_thread,
&fxent_reseed_timer, -sbt, (sbt / 3), C_PREL(2));
}
atomic_store_rel_int(&fxent_timer_ready, 1);
}
/* After taskqueue_thread is initialized in SI_SUB_TASKQ:SI_ORDER_SECOND. */
SYSINIT(fxent_pool_timer_init, SI_SUB_TASKQ, SI_ORDER_ANY,
fxent_pool_timer_init, NULL);
Index: head/sys/dev/random/fenestrasX/fx_priv.h
===================================================================
--- head/sys/dev/random/fenestrasX/fx_priv.h (revision 366620)
+++ head/sys/dev/random/fenestrasX/fx_priv.h (revision 366621)
@@ -1,49 +1,48 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2019 Conrad Meyer <cem@FreeBSD.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#pragma once
#define ASSERT(x, fmt, ...) do { \
if (__predict_true(x)) \
break; \
panic("%s:%d: Assertion '" #x "' in function %s failed: " fmt, \
__FILE__, __LINE__, __func__, ## __VA_ARGS__); \
} while (false)
#ifdef INVARIANTS
#define ASSERT_DEBUG(x, fmt, ...) do { \
if (__predict_true(x)) \
break; \
panic("%s:%d: Assertion '" #x "' in function %s failed: " fmt, \
__FILE__, __LINE__, __func__, ## __VA_ARGS__); \
} while (false)
#else
#define ASSERT_DEBUG(...)
#endif
extern struct fxrng_buffered_rng fxrng_root;
-extern uint64_t __read_mostly fxrng_root_generation;
Index: head/sys/dev/random/fenestrasX/fx_pub.h
===================================================================
--- head/sys/dev/random/fenestrasX/fx_pub.h (nonexistent)
+++ head/sys/dev/random/fenestrasX/fx_pub.h (revision 366621)
@@ -0,0 +1,53 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2019 Conrad Meyer <cem@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+#pragma once
+
+#include <sys/systm.h>
+
+/*
+ * The root BRNG seed version, or generation.
+ *
+ * FenestrasX-aware downstream CSPRNGs (i.e., arc4random(9)) should track the
+ * generation number they seeded from, using the read_random_key(9) API below.
+ * If their current seed version is older than the root generation, they should
+ * reseed before producing output.
+ *
+ * The variable is read-only outside of the fenestrasX implementation and
+ * should be accessed using 'atomic_load_acq_64(&fxrng_root_generation)'.
+ * Reseeds are extremely infrequent, so callers may wish to hint to the
+ * compiler that a matching generation is the expected case, with
+ * __predict_true() or __predict_false().
+ */
+extern uint64_t __read_mostly fxrng_root_generation;
+
+/*
+ * A routine for generating seed/key material
+ * Bypasses random(4) for now, but conceivably could be incorporated into that.
+ */
+void read_random_key(void *buf, size_t nbytes, uint64_t *seed_version_out);
Property changes on: head/sys/dev/random/fenestrasX/fx_pub.h
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+FreeBSD=%H
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Index: head/sys/dev/random/randomdev.c
===================================================================
--- head/sys/dev/random/randomdev.c (revision 366620)
+++ head/sys/dev/random/randomdev.c (revision 366621)
@@ -1,433 +1,435 @@
/*-
* Copyright (c) 2017 Oliver Pinter
* Copyright (c) 2000-2015 Mark R V Murray
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer
* in this position and unchanged.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <sys/filio.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/lock.h>
#include <sys/module.h>
#include <sys/malloc.h>
#include <sys/poll.h>
#include <sys/proc.h>
#include <sys/random.h>
#include <sys/sbuf.h>
#include <sys/selinfo.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/uio.h>
#include <sys/unistd.h>
#include <crypto/rijndael/rijndael-api-fst.h>
#include <crypto/sha2/sha256.h>
#include <dev/random/hash.h>
#include <dev/random/randomdev.h>
#include <dev/random/random_harvestq.h>
#define RANDOM_UNIT 0
/*
* In loadable random, the core randomdev.c / random(9) routines have static
* visibility and an alternative name to avoid conflicting with the function
* pointers of the real names in the core kernel. random_alg_context_init
* installs pointers to the loadable static names into the core kernel's
* function pointers at SI_SUB_RANDOM:SI_ORDER_SECOND.
*/
#if defined(RANDOM_LOADABLE)
static int (read_random_uio)(struct uio *, bool);
static void (read_random)(void *, u_int);
static bool (is_random_seeded)(void);
#endif
static d_read_t randomdev_read;
static d_write_t randomdev_write;
static d_poll_t randomdev_poll;
static d_ioctl_t randomdev_ioctl;
static struct cdevsw random_cdevsw = {
.d_name = "random",
.d_version = D_VERSION,
.d_read = randomdev_read,
.d_write = randomdev_write,
.d_poll = randomdev_poll,
.d_ioctl = randomdev_ioctl,
};
/* For use with make_dev(9)/destroy_dev(9). */
static struct cdev *random_dev;
#if defined(RANDOM_LOADABLE)
static void
random_alg_context_init(void *dummy __unused)
{
_read_random_uio = (read_random_uio);
_read_random = (read_random);
_is_random_seeded = (is_random_seeded);
}
SYSINIT(random_device, SI_SUB_RANDOM, SI_ORDER_SECOND, random_alg_context_init,
NULL);
#endif
static struct selinfo rsel;
/*
* This is the read uio(9) interface for random(4).
*/
/* ARGSUSED */
static int
randomdev_read(struct cdev *dev __unused, struct uio *uio, int flags)
{
return ((read_random_uio)(uio, (flags & O_NONBLOCK) != 0));
}
/*
* If the random device is not seeded, blocks until it is seeded.
*
* Returns zero when the random device is seeded.
*
* If the 'interruptible' parameter is true, and the device is unseeded, this
* routine may be interrupted. If interrupted, it will return either ERESTART
* or EINTR.
*/
#define SEEDWAIT_INTERRUPTIBLE true
#define SEEDWAIT_UNINTERRUPTIBLE false
static int
randomdev_wait_until_seeded(bool interruptible)
{
int error, spamcount, slpflags;
slpflags = interruptible ? PCATCH : 0;
error = 0;
spamcount = 0;
while (!p_random_alg_context->ra_seeded()) {
/* keep tapping away at the pre-read until we seed/unblock. */
p_random_alg_context->ra_pre_read();
/* Only bother the console every 10 seconds or so */
if (spamcount == 0)
printf("random: %s unblock wait\n", __func__);
spamcount = (spamcount + 1) % 100;
error = tsleep(p_random_alg_context, slpflags, "randseed",
hz / 10);
if (error == ERESTART || error == EINTR) {
KASSERT(interruptible,
("unexpected wake of non-interruptible sleep"));
break;
}
/* Squash tsleep timeout condition */
if (error == EWOULDBLOCK)
error = 0;
KASSERT(error == 0, ("unexpected tsleep error %d", error));
}
return (error);
}
int
(read_random_uio)(struct uio *uio, bool nonblock)
{
/* 16 MiB takes about 0.08 s CPU time on my 2017 AMD Zen CPU */
#define SIGCHK_PERIOD (16 * 1024 * 1024)
const size_t sigchk_period = SIGCHK_PERIOD;
CTASSERT(SIGCHK_PERIOD % PAGE_SIZE == 0);
#undef SIGCHK_PERIOD
uint8_t *random_buf;
size_t total_read, read_len;
ssize_t bufsize;
int error;
KASSERT(uio->uio_rw == UIO_READ, ("%s: bogus write", __func__));
KASSERT(uio->uio_resid >= 0, ("%s: bogus negative resid", __func__));
p_random_alg_context->ra_pre_read();
error = 0;
/* (Un)Blocking logic */
if (!p_random_alg_context->ra_seeded()) {
if (nonblock)
error = EWOULDBLOCK;
else
error = randomdev_wait_until_seeded(
SEEDWAIT_INTERRUPTIBLE);
}
if (error != 0)
return (error);
read_rate_increment(howmany(uio->uio_resid + 1, sizeof(uint32_t)));
total_read = 0;
/* Easy to deal with the trivial 0 byte case. */
if (__predict_false(uio->uio_resid == 0))
return (0);
/*
* If memory is plentiful, use maximally sized requests to avoid
* per-call algorithm overhead. But fall back to a single page
* allocation if the full request isn't immediately available.
*/
bufsize = MIN(sigchk_period, (size_t)uio->uio_resid);
random_buf = malloc(bufsize, M_ENTROPY, M_NOWAIT);
if (random_buf == NULL) {
bufsize = PAGE_SIZE;
random_buf = malloc(bufsize, M_ENTROPY, M_WAITOK);
}
error = 0;
while (uio->uio_resid > 0 && error == 0) {
read_len = MIN((size_t)uio->uio_resid, bufsize);
p_random_alg_context->ra_read(random_buf, read_len);
/*
* uiomove() may yield the CPU before each 'read_len' bytes (up
* to bufsize) are copied out.
*/
error = uiomove(random_buf, read_len, uio);
total_read += read_len;
/*
* Poll for signals every few MBs to avoid very long
* uninterruptible syscalls.
*/
if (error == 0 && uio->uio_resid != 0 &&
total_read % sigchk_period == 0) {
error = tsleep_sbt(p_random_alg_context, PCATCH,
"randrd", SBT_1NS, 0, C_HARDCLOCK);
/* Squash tsleep timeout condition */
if (error == EWOULDBLOCK)
error = 0;
}
}
/*
* Short reads due to signal interrupt should not indicate error.
* Instead, the uio will reflect that the read was shorter than
* requested.
*/
if (error == ERESTART || error == EINTR)
error = 0;
zfree(random_buf, M_ENTROPY);
return (error);
}
/*-
* Kernel API version of read_random(). This is similar to read_random_uio(),
* except it doesn't interface with uio(9). It cannot assumed that random_buf
* is a multiple of RANDOM_BLOCKSIZE bytes.
*
* If the tunable 'kern.random.initial_seeding.bypass_before_seeding' is set
* non-zero, silently fail to emit random data (matching the pre-r346250
* behavior). If read_random is called prior to seeding and bypassed because
* of this tunable, the condition is reported in the read-only sysctl
* 'kern.random.initial_seeding.read_random_bypassed_before_seeding'.
*/
void
(read_random)(void *random_buf, u_int len)
{
KASSERT(random_buf != NULL, ("No suitable random buffer in %s", __func__));
p_random_alg_context->ra_pre_read();
if (len == 0)
return;
/* (Un)Blocking logic */
if (__predict_false(!p_random_alg_context->ra_seeded())) {
if (random_bypass_before_seeding) {
if (!read_random_bypassed_before_seeding) {
if (!random_bypass_disable_warnings)
printf("read_random: WARNING: bypassing"
" request for random data because "
"the random device is not yet "
"seeded and the knob "
"'bypass_before_seeding' was "
"enabled.\n");
read_random_bypassed_before_seeding = true;
}
/* Avoid potentially leaking stack garbage */
memset(random_buf, 0, len);
return;
}
(void)randomdev_wait_until_seeded(SEEDWAIT_UNINTERRUPTIBLE);
}
read_rate_increment(roundup2(len, sizeof(uint32_t)));
p_random_alg_context->ra_read(random_buf, len);
}
bool
(is_random_seeded)(void)
{
return (p_random_alg_context->ra_seeded());
}
static __inline void
randomdev_accumulate(uint8_t *buf, u_int count)
{
static u_int destination = 0;
static struct harvest_event event;
static struct randomdev_hash hash;
static uint32_t entropy_data[RANDOM_KEYSIZE_WORDS];
uint32_t timestamp;
int i;
/* Extra timing here is helpful to scrape scheduler jitter entropy */
randomdev_hash_init(&hash);
timestamp = (uint32_t)get_cyclecount();
randomdev_hash_iterate(&hash, ×tamp, sizeof(timestamp));
randomdev_hash_iterate(&hash, buf, count);
timestamp = (uint32_t)get_cyclecount();
randomdev_hash_iterate(&hash, ×tamp, sizeof(timestamp));
randomdev_hash_finish(&hash, entropy_data);
for (i = 0; i < RANDOM_KEYSIZE_WORDS; i += sizeof(event.he_entropy)/sizeof(event.he_entropy[0])) {
event.he_somecounter = (uint32_t)get_cyclecount();
event.he_size = sizeof(event.he_entropy);
event.he_source = RANDOM_CACHED;
event.he_destination = destination++; /* Harmless cheating */
memcpy(event.he_entropy, entropy_data + i, sizeof(event.he_entropy));
p_random_alg_context->ra_event_processor(&event);
}
explicit_bzero(&event, sizeof(event));
explicit_bzero(entropy_data, sizeof(entropy_data));
}
/* ARGSUSED */
static int
randomdev_write(struct cdev *dev __unused, struct uio *uio, int flags __unused)
{
uint8_t *random_buf;
int c, error = 0;
ssize_t nbytes;
random_buf = malloc(PAGE_SIZE, M_ENTROPY, M_WAITOK);
nbytes = uio->uio_resid;
while (uio->uio_resid > 0 && error == 0) {
c = MIN(uio->uio_resid, PAGE_SIZE);
error = uiomove(random_buf, c, uio);
if (error)
break;
randomdev_accumulate(random_buf, c);
tsleep(p_random_alg_context, 0, "randwr", hz/10);
}
if (nbytes != uio->uio_resid && (error == ERESTART || error == EINTR))
/* Partial write, not error. */
error = 0;
free(random_buf, M_ENTROPY);
return (error);
}
/* ARGSUSED */
static int
randomdev_poll(struct cdev *dev __unused, int events, struct thread *td __unused)
{
if (events & (POLLIN | POLLRDNORM)) {
if (p_random_alg_context->ra_seeded())
events &= (POLLIN | POLLRDNORM);
else
selrecord(td, &rsel);
}
return (events);
}
/* This will be called by the entropy processor when it seeds itself and becomes secure */
void
randomdev_unblock(void)
{
selwakeuppri(&rsel, PUSER);
wakeup(p_random_alg_context);
printf("random: unblocking device.\n");
+#ifndef RANDOM_FENESTRASX
/* Do random(9) a favour while we are about it. */
(void)atomic_cmpset_int(&arc4rand_iniseed_state, ARC4_ENTR_NONE, ARC4_ENTR_HAVE);
+#endif
}
/* ARGSUSED */
static int
randomdev_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t addr __unused,
int flags __unused, struct thread *td __unused)
{
int error = 0;
switch (cmd) {
/* Really handled in upper layer */
case FIOASYNC:
case FIONBIO:
break;
default:
error = ENOTTY;
}
return (error);
}
/* ARGSUSED */
static int
randomdev_modevent(module_t mod __unused, int type, void *data __unused)
{
int error = 0;
switch (type) {
case MOD_LOAD:
printf("random: entropy device external interface\n");
random_dev = make_dev_credf(MAKEDEV_ETERNAL_KLD, &random_cdevsw,
RANDOM_UNIT, NULL, UID_ROOT, GID_WHEEL, 0644, "random");
make_dev_alias(random_dev, "urandom"); /* compatibility */
break;
case MOD_UNLOAD:
error = EBUSY;
break;
case MOD_SHUTDOWN:
break;
default:
error = EOPNOTSUPP;
break;
}
return (error);
}
static moduledata_t randomdev_mod = {
"random_device",
randomdev_modevent,
0
};
DECLARE_MODULE(random_device, randomdev_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
MODULE_VERSION(random_device, 1);
MODULE_DEPEND(random_device, crypto, 1, 1, 1);
MODULE_DEPEND(random_device, random_harvestq, 1, 1, 1);
Index: head/sys/libkern/arc4random.c
===================================================================
--- head/sys/libkern/arc4random.c (revision 366620)
+++ head/sys/libkern/arc4random.c (revision 366621)
@@ -1,219 +1,253 @@
/*-
* Copyright (c) 2017 The FreeBSD Foundation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer
* in this position and unchanged.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/types.h>
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/libkern.h>
#include <sys/linker.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/random.h>
#include <sys/smp.h>
#include <sys/time.h>
+#include <machine/cpu.h>
+
#include <crypto/chacha20/chacha.h>
#include <crypto/sha2/sha256.h>
#include <dev/random/randomdev.h>
-#include <machine/cpu.h>
+#ifdef RANDOM_FENESTRASX
+#include <dev/random/fenestrasX/fx_pub.h>
+#endif
#define CHACHA20_RESEED_BYTES 65536
#define CHACHA20_RESEED_SECONDS 300
#define CHACHA20_KEYBYTES 32
#define CHACHA20_BUFFER_SIZE 64
CTASSERT(CHACHA20_KEYBYTES*8 >= CHACHA_MINKEYLEN);
+#ifndef RANDOM_FENESTRASX
int arc4rand_iniseed_state = ARC4_ENTR_NONE;
+#endif
MALLOC_DEFINE(M_CHACHA20RANDOM, "chacha20random", "chacha20random structures");
struct chacha20_s {
struct mtx mtx;
int numbytes;
time_t t_reseed;
u_int8_t m_buffer[CHACHA20_BUFFER_SIZE];
struct chacha_ctx ctx;
+#ifdef RANDOM_FENESTRASX
+ uint64_t seed_version;
+#endif
} __aligned(CACHE_LINE_SIZE);
static struct chacha20_s *chacha20inst = NULL;
#define CHACHA20_FOREACH(_chacha20) \
for (_chacha20 = &chacha20inst[0]; \
_chacha20 <= &chacha20inst[mp_maxid]; \
_chacha20++)
/*
* Mix up the current context.
*/
static void
chacha20_randomstir(struct chacha20_s *chacha20)
{
struct timeval tv_now;
u_int8_t key[CHACHA20_KEYBYTES];
+#ifdef RANDOM_FENESTRASX
+ uint64_t seed_version;
+#else
if (__predict_false(random_bypass_before_seeding && !is_random_seeded())) {
SHA256_CTX ctx;
uint64_t cc;
uint32_t fver;
if (!arc4random_bypassed_before_seeding) {
arc4random_bypassed_before_seeding = true;
if (!random_bypass_disable_warnings)
printf("arc4random: WARNING: initial seeding "
"bypassed the cryptographic random device "
"because it was not yet seeded and the "
"knob 'bypass_before_seeding' was "
"enabled.\n");
}
/* Last ditch effort to inject something in a bad condition. */
cc = get_cyclecount();
SHA256_Init(&ctx);
SHA256_Update(&ctx, key, sizeof(key));
SHA256_Update(&ctx, &cc, sizeof(cc));
fver = __FreeBSD_version;
SHA256_Update(&ctx, &fver, sizeof(fver));
_Static_assert(sizeof(key) == SHA256_DIGEST_LENGTH,
"make sure 256 bits is still 256 bits");
SHA256_Final(key, &ctx);
} else {
+#endif
+#ifdef RANDOM_FENESTRASX
+ read_random_key(key, CHACHA20_KEYBYTES, &seed_version);
+#else
/*
* If the loader(8) did not have an entropy stash from the
* previous shutdown to load, then we will block. The answer is
* to make sure there is an entropy stash at shutdown time.
*
* On the other hand, if the random_bypass_before_seeding knob
* was set and we landed in this branch, we know this won't
* block because we know the random device is seeded.
*/
read_random(key, CHACHA20_KEYBYTES);
}
+#endif
getmicrouptime(&tv_now);
mtx_lock(&chacha20->mtx);
chacha_keysetup(&chacha20->ctx, key, CHACHA20_KEYBYTES*8);
chacha_ivsetup(&chacha20->ctx, (u_char *)&tv_now.tv_sec, (u_char *)&tv_now.tv_usec);
/* Reset for next reseed cycle. */
chacha20->t_reseed = tv_now.tv_sec + CHACHA20_RESEED_SECONDS;
chacha20->numbytes = 0;
+#ifdef RANDOM_FENESTRASX
+ chacha20->seed_version = seed_version;
+#endif
mtx_unlock(&chacha20->mtx);
}
/*
* Initialize the contexts.
*/
static void
chacha20_init(void)
{
struct chacha20_s *chacha20;
chacha20inst = malloc((mp_maxid + 1) * sizeof(struct chacha20_s),
M_CHACHA20RANDOM, M_NOWAIT | M_ZERO);
KASSERT(chacha20inst != NULL, ("chacha20_init: memory allocation error"));
CHACHA20_FOREACH(chacha20) {
mtx_init(&chacha20->mtx, "chacha20_mtx", NULL, MTX_DEF);
chacha20->t_reseed = -1;
chacha20->numbytes = 0;
explicit_bzero(chacha20->m_buffer, CHACHA20_BUFFER_SIZE);
explicit_bzero(&chacha20->ctx, sizeof(chacha20->ctx));
}
}
SYSINIT(chacha20, SI_SUB_LOCK, SI_ORDER_ANY, chacha20_init, NULL);
static void
chacha20_uninit(void)
{
struct chacha20_s *chacha20;
CHACHA20_FOREACH(chacha20)
mtx_destroy(&chacha20->mtx);
free(chacha20inst, M_CHACHA20RANDOM);
}
SYSUNINIT(chacha20, SI_SUB_LOCK, SI_ORDER_ANY, chacha20_uninit, NULL);
/*
* MPSAFE
*/
void
arc4rand(void *ptr, u_int len, int reseed)
{
struct chacha20_s *chacha20;
struct timeval tv;
u_int length;
u_int8_t *p;
+#ifdef RANDOM_FENESTRASX
+ if (__predict_false(reseed))
+#else
if (__predict_false(reseed ||
(arc4rand_iniseed_state == ARC4_ENTR_HAVE &&
atomic_cmpset_int(&arc4rand_iniseed_state, ARC4_ENTR_HAVE, ARC4_ENTR_SEED))))
+#endif
CHACHA20_FOREACH(chacha20)
chacha20_randomstir(chacha20);
getmicrouptime(&tv);
chacha20 = &chacha20inst[curcpu];
/* We may get unlucky and be migrated off this CPU, but that is expected to be infrequent */
if ((chacha20->numbytes > CHACHA20_RESEED_BYTES) || (tv.tv_sec > chacha20->t_reseed))
chacha20_randomstir(chacha20);
- p = ptr;
mtx_lock(&chacha20->mtx);
+#ifdef RANDOM_FENESTRASX
+ if (__predict_false(
+ atomic_load_acq_64(&fxrng_root_generation) != chacha20->seed_version
+ )) {
+ mtx_unlock(&chacha20->mtx);
+ chacha20_randomstir(chacha20);
+ mtx_lock(&chacha20->mtx);
+ }
+#endif
+
+ p = ptr;
while (len) {
length = MIN(CHACHA20_BUFFER_SIZE, len);
chacha_encrypt_bytes(&chacha20->ctx, chacha20->m_buffer, p, length);
p += length;
len -= length;
chacha20->numbytes += length;
if (chacha20->numbytes > CHACHA20_RESEED_BYTES) {
mtx_unlock(&chacha20->mtx);
chacha20_randomstir(chacha20);
mtx_lock(&chacha20->mtx);
}
}
mtx_unlock(&chacha20->mtx);
}
uint32_t
arc4random(void)
{
uint32_t ret;
arc4rand(&ret, sizeof(ret), 0);
return ret;
}
void
arc4random_buf(void *ptr, size_t len)
{
arc4rand(ptr, len, 0);
}
Index: head/sys/sys/libkern.h
===================================================================
--- head/sys/sys/libkern.h (revision 366620)
+++ head/sys/sys/libkern.h (revision 366621)
@@ -1,227 +1,229 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1992, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)libkern.h 8.1 (Berkeley) 6/10/93
* $FreeBSD$
*/
#ifndef _SYS_LIBKERN_H_
#define _SYS_LIBKERN_H_
#include <sys/cdefs.h>
#include <sys/types.h>
#ifdef _KERNEL
#include <sys/systm.h>
#endif
#ifndef LIBKERN_INLINE
#define LIBKERN_INLINE static __inline
#define LIBKERN_BODY
#endif
/* BCD conversions. */
extern u_char const bcd2bin_data[];
extern u_char const bin2bcd_data[];
extern char const hex2ascii_data[];
#define LIBKERN_LEN_BCD2BIN 154
#define LIBKERN_LEN_BIN2BCD 100
#define LIBKERN_LEN_HEX2ASCII 36
static inline u_char
bcd2bin(int bcd)
{
KASSERT(bcd >= 0 && bcd < LIBKERN_LEN_BCD2BIN,
("invalid bcd %d", bcd));
return (bcd2bin_data[bcd]);
}
static inline u_char
bin2bcd(int bin)
{
KASSERT(bin >= 0 && bin < LIBKERN_LEN_BIN2BCD,
("invalid bin %d", bin));
return (bin2bcd_data[bin]);
}
static inline char
hex2ascii(int hex)
{
KASSERT(hex >= 0 && hex < LIBKERN_LEN_HEX2ASCII,
("invalid hex %d", hex));
return (hex2ascii_data[hex]);
}
static inline bool
validbcd(int bcd)
{
return (bcd == 0 || (bcd > 0 && bcd <= 0x99 && bcd2bin_data[bcd] != 0));
}
static __inline int imax(int a, int b) { return (a > b ? a : b); }
static __inline int imin(int a, int b) { return (a < b ? a : b); }
static __inline long lmax(long a, long b) { return (a > b ? a : b); }
static __inline long lmin(long a, long b) { return (a < b ? a : b); }
static __inline u_int max(u_int a, u_int b) { return (a > b ? a : b); }
static __inline u_int min(u_int a, u_int b) { return (a < b ? a : b); }
static __inline quad_t qmax(quad_t a, quad_t b) { return (a > b ? a : b); }
static __inline quad_t qmin(quad_t a, quad_t b) { return (a < b ? a : b); }
static __inline u_quad_t uqmax(u_quad_t a, u_quad_t b) { return (a > b ? a : b); }
static __inline u_quad_t uqmin(u_quad_t a, u_quad_t b) { return (a < b ? a : b); }
static __inline u_long ulmax(u_long a, u_long b) { return (a > b ? a : b); }
static __inline u_long ulmin(u_long a, u_long b) { return (a < b ? a : b); }
static __inline __uintmax_t ummax(__uintmax_t a, __uintmax_t b)
{
return (a > b ? a : b);
}
static __inline __uintmax_t ummin(__uintmax_t a, __uintmax_t b)
{
return (a < b ? a : b);
}
static __inline off_t omax(off_t a, off_t b) { return (a > b ? a : b); }
static __inline off_t omin(off_t a, off_t b) { return (a < b ? a : b); }
static __inline int abs(int a) { return (a < 0 ? -a : a); }
static __inline long labs(long a) { return (a < 0 ? -a : a); }
static __inline quad_t qabs(quad_t a) { return (a < 0 ? -a : a); }
+#ifndef RANDOM_FENESTRASX
#define ARC4_ENTR_NONE 0 /* Don't have entropy yet. */
#define ARC4_ENTR_HAVE 1 /* Have entropy. */
#define ARC4_ENTR_SEED 2 /* Reseeding. */
extern int arc4rand_iniseed_state;
+#endif
/* Prototypes for non-quad routines. */
struct malloc_type;
uint32_t arc4random(void);
void arc4random_buf(void *, size_t);
uint32_t arc4random_uniform(uint32_t);
void arc4rand(void *, u_int, int);
int timingsafe_bcmp(const void *, const void *, size_t);
void *bsearch(const void *, const void *, size_t,
size_t, int (*)(const void *, const void *));
#ifndef HAVE_INLINE_FFS
int ffs(int);
#endif
#ifndef HAVE_INLINE_FFSL
int ffsl(long);
#endif
#ifndef HAVE_INLINE_FFSLL
int ffsll(long long);
#endif
#ifndef HAVE_INLINE_FLS
int fls(int);
#endif
#ifndef HAVE_INLINE_FLSL
int flsl(long);
#endif
#ifndef HAVE_INLINE_FLSLL
int flsll(long long);
#endif
#define bitcount64(x) __bitcount64((uint64_t)(x))
#define bitcount32(x) __bitcount32((uint32_t)(x))
#define bitcount16(x) __bitcount16((uint16_t)(x))
#define bitcountl(x) __bitcountl((u_long)(x))
#define bitcount(x) __bitcount((u_int)(x))
int fnmatch(const char *, const char *, int);
int locc(int, char *, u_int);
void *memchr(const void *s, int c, size_t n);
void *memcchr(const void *s, int c, size_t n);
void *memmem(const void *l, size_t l_len, const void *s, size_t s_len);
void qsort(void *base, size_t nmemb, size_t size,
int (*compar)(const void *, const void *));
void qsort_r(void *base, size_t nmemb, size_t size, void *thunk,
int (*compar)(void *, const void *, const void *));
u_long random(void);
int scanc(u_int, const u_char *, const u_char *, int);
int strcasecmp(const char *, const char *);
char *strcat(char * __restrict, const char * __restrict);
char *strchr(const char *, int);
char *strchrnul(const char *, int);
int strcmp(const char *, const char *);
char *strcpy(char * __restrict, const char * __restrict);
size_t strcspn(const char * __restrict, const char * __restrict) __pure;
char *strdup_flags(const char *__restrict, struct malloc_type *, int);
char *strdup(const char *__restrict, struct malloc_type *);
char *strncat(char *, const char *, size_t);
char *strndup(const char *__restrict, size_t, struct malloc_type *);
size_t strlcat(char *, const char *, size_t);
size_t strlcpy(char *, const char *, size_t);
size_t strlen(const char *);
int strncasecmp(const char *, const char *, size_t);
int strncmp(const char *, const char *, size_t);
char *strncpy(char * __restrict, const char * __restrict, size_t);
size_t strnlen(const char *, size_t);
char *strrchr(const char *, int);
char *strsep(char **, const char *delim);
size_t strspn(const char *, const char *);
char *strstr(const char *, const char *);
int strvalid(const char *, size_t);
#ifdef KCSAN
char *kcsan_strcpy(char *, const char *);
int kcsan_strcmp(const char *, const char *);
size_t kcsan_strlen(const char *);
#define strcpy(d, s) kcsan_strcpy((d), (s))
#define strcmp(s1, s2) kcsan_strcmp((s1), (s2))
#define strlen(s) kcsan_strlen((s))
#endif
static __inline char *
index(const char *p, int ch)
{
return (strchr(p, ch));
}
static __inline char *
rindex(const char *p, int ch)
{
return (strrchr(p, ch));
}
/* fnmatch() return values. */
#define FNM_NOMATCH 1 /* Match failed. */
/* fnmatch() flags. */
#define FNM_NOESCAPE 0x01 /* Disable backslash escaping. */
#define FNM_PATHNAME 0x02 /* Slash must be matched by slash. */
#define FNM_PERIOD 0x04 /* Period must be matched by period. */
#define FNM_LEADING_DIR 0x08 /* Ignore /<tail> after Imatch. */
#define FNM_CASEFOLD 0x10 /* Case insensitive search. */
#define FNM_IGNORECASE FNM_CASEFOLD
#define FNM_FILE_NAME FNM_PATHNAME
#endif /* !_SYS_LIBKERN_H_ */