Page MenuHomeFreeBSD
Files F110601038

D22586.id67538.diff
No OneTemporary
Actions

D22586.id67538.diff
View Options

Index: head/sys/conf/files
===================================================================
--- head/sys/conf/files
+++ head/sys/conf/files
@@ -3848,6 +3848,7 @@
kern/subr_sglist.c standard
kern/subr_sleepqueue.c standard
kern/subr_smp.c standard
+kern/subr_smr.c standard
kern/subr_stack.c optional ddb | stack | ktr
kern/subr_stats.c optional stats
kern/subr_taskqueue.c standard
Index: head/sys/kern/subr_smr.c
===================================================================
--- head/sys/kern/subr_smr.c
+++ head/sys/kern/subr_smr.c
@@ -0,0 +1,387 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2019 Jeffrey Roberson <jeff@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice unmodified, this list of conditions, and the following
+ * disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/limits.h>
+#include <sys/kernel.h>
+#include <sys/proc.h>
+#include <sys/smp.h>
+#include <sys/smr.h>
+
+#include <vm/uma.h>
+
+/*
+ * This is a novel safe memory reclamation technique inspired by
+ * epoch based reclamation from Samy Al Bahra's concurrency kit which
+ * in turn was based on work described in:
+ * Fraser, K. 2004. Practical Lock-Freedom. PhD Thesis, University
+ * of Cambridge Computing Laboratory.
+ * And shares some similarities with:
+ * Wang, Stamler, Parmer. 2016 Parallel Sections: Scaling System-Level
+ * Data-Structures
+ *
+ * This is not an implementation of hazard pointers or related
+ * techniques. The term safe memory reclamation is used as a
+ * generic descriptor for algorithms that defer frees to avoid
+ * use-after-free errors with lockless datastructures.
+ *
+ * The basic approach is to maintain a monotonic write sequence
+ * number that is updated on some application defined granularity.
+ * Readers record the most recent write sequence number they have
+ * observed. A shared read sequence number records the lowest
+ * sequence number observed by any reader as of the last poll. Any
+ * write older than this value has been observed by all readers
+ * and memory can be reclaimed. Like Epoch we also detect idle
+ * readers by storing an invalid sequence number in the per-cpu
+ * state when the read section exits. Like Parsec we establish
+ * a global write clock that is used to mark memory on free.
+ *
+ * The write and read sequence numbers can be thought of as a two
+ * handed clock with readers always advancing towards writers. SMR
+ * maintains the invariant that all readers can safely access memory
+ * that was visible at the time they loaded their copy of the sequence
+ * number. Periodically the read sequence or hand is polled and
+ * advanced as far towards the write sequence as active readers allow.
+ * Memory which was freed between the old and new global read sequence
+ * number can now be reclaimed. When the system is idle the two hands
+ * meet and no deferred memory is outstanding. Readers never advance
+ * any sequence number, they only observe them. The shared read
+ * sequence number is consequently never higher than the write sequence.
+ * A stored sequence number that falls outside of this range has expired
+ * and needs no scan to reclaim.
+ *
+ * A notable distinction between this SMR and Epoch, qsbr, rcu, etc. is
+ * that advancing the sequence number is decoupled from detecting its
+ * observation. This results in a more granular assignment of sequence
+ * numbers even as read latencies prohibit all or some expiration.
+ * It also allows writers to advance the sequence number and save the
+ * poll for expiration until a later time when it is likely to
+ * complete without waiting. The batch granularity and free-to-use
+ * latency is dynamic and can be significantly smaller than in more
+ * strict systems.
+ *
+ * This mechanism is primarily intended to be used in coordination with
+ * UMA. By integrating with the allocator we avoid all of the callout
+ * queue machinery and are provided with an efficient way to batch
+ * sequence advancement and waiting. The allocator accumulates a full
+ * per-cpu cache of memory before advancing the sequence. It then
+ * delays waiting for this sequence to expire until the memory is
+ * selected for reuse. In this way we only increment the sequence
+ * value once for n=cache-size frees and the waits are done long
+ * after the sequence has been expired so they need only be verified
+ * to account for pathological conditions and to advance the read
+ * sequence. Tying the sequence number to the bucket size has the
+ * nice property that as the zone gets busier the buckets get larger
+ * and the sequence writes become fewer. If the coherency of advancing
+ * the write sequence number becomes too costly we can advance
+ * it for every N buckets in exchange for higher free-to-use
+ * latency and consequently higher memory consumption.
+ *
+ * If the read overhead of accessing the shared cacheline becomes
+ * especially burdensome an invariant TSC could be used in place of the
+ * sequence. The algorithm would then only need to maintain the minimum
+ * observed tsc. This would trade potential cache synchronization
+ * overhead for local serialization and cpu timestamp overhead.
+ */
+
+/*
+ * A simplified diagram:
+ *
+ * 0 UINT_MAX
+ * | -------------------- sequence number space -------------------- |
+ * ^ rd seq ^ wr seq
+ * | ----- valid sequence numbers ---- |
+ * ^cpuA ^cpuC
+ * | -- free -- | --------- deferred frees -------- | ---- free ---- |
+ *
+ *
+ * In this example cpuA has the lowest sequence number and poll can
+ * advance rd seq. cpuB is not running and is considered to observe
+ * wr seq.
+ *
+ * Freed memory that is tagged with a sequence number between rd seq and
+ * wr seq can not be safely reclaimed because cpuA may hold a reference to
+ * it. Any other memory is guaranteed to be unreferenced.
+ *
+ * Any writer is free to advance wr seq at any time however it may busy
+ * poll in pathological cases.
+ */
+
+static uma_zone_t smr_shared_zone;
+static uma_zone_t smr_zone;
+
+#ifndef INVARIANTS
+#define SMR_SEQ_INIT 1 /* All valid sequence numbers are odd. */
+#define SMR_SEQ_INCR 2
+
+/*
+ * SMR_SEQ_MAX_DELTA is the maximum distance allowed between rd_seq and
+ * wr_seq. For the modular arithmetic to work a value of UNIT_MAX / 2
+ * would be possible but it is checked after we increment the wr_seq so
+ * a safety margin is left to prevent overflow.
+ *
+ * We will block until SMR_SEQ_MAX_ADVANCE sequence numbers have progressed
+ * to prevent integer wrapping. See smr_advance() for more details.
+ */
+#define SMR_SEQ_MAX_DELTA (UINT_MAX / 4)
+#define SMR_SEQ_MAX_ADVANCE (SMR_SEQ_MAX_DELTA - 1024)
+#else
+/* We want to test the wrapping feature in invariants kernels. */
+#define SMR_SEQ_INCR (UINT_MAX / 10000)
+#define SMR_SEQ_INIT (UINT_MAX - 100000)
+/* Force extra polls to test the integer overflow detection. */
+#define SMR_SEQ_MAX_DELTA (1000)
+#define SMR_SEQ_MAX_ADVANCE SMR_SEQ_MAX_DELTA / 2
+#endif
+
+/*
+ * Advance the write sequence and return the new value for use as the
+ * wait goal. This guarantees that any changes made by the calling
+ * thread prior to this call will be visible to all threads after
+ * rd_seq meets or exceeds the return value.
+ *
+ * This function may busy loop if the readers are roughly 1 billion
+ * sequence numbers behind the writers.
+ */
+smr_seq_t
+smr_advance(smr_t smr)
+{
+ smr_shared_t s;
+ smr_seq_t goal;
+
+ /*
+ * It is illegal to enter while in an smr section.
+ */
+ KASSERT(curthread->td_critnest == 0,
+ ("smr_advance: Not allowed in a critical section."));
+
+ /*
+ * Modifications not done in a smr section need to be visible
+ * before advancing the seq.
+ */
+ atomic_thread_fence_rel();
+
+ /*
+ * Increment the shared write sequence by 2. Since it is
+ * initialized to 1 this means the only valid values are
+ * odd and an observed value of 0 in a particular CPU means
+ * it is not currently in a read section.
+ */
+ s = smr->c_shared;
+ goal = atomic_fetchadd_int(&s->s_wr_seq, SMR_SEQ_INCR) + SMR_SEQ_INCR;
+
+ /*
+ * Force a synchronization here if the goal is getting too
+ * far ahead of the read sequence number. This keeps the
+ * wrap detecting arithmetic working in pathological cases.
+ */
+ if (goal - atomic_load_int(&s->s_rd_seq) >= SMR_SEQ_MAX_DELTA)
+ smr_wait(smr, goal - SMR_SEQ_MAX_ADVANCE);
+
+ return (goal);
+}
+
+/*
+ * Poll to determine whether all readers have observed the 'goal' write
+ * sequence number.
+ *
+ * If wait is true this will spin until the goal is met.
+ *
+ * This routine will updated the minimum observed read sequence number in
+ * s_rd_seq if it does a scan. It may not do a scan if another call has
+ * advanced s_rd_seq beyond the callers goal already.
+ *
+ * Returns true if the goal is met and false if not.
+ */
+bool
+smr_poll(smr_t smr, smr_seq_t goal, bool wait)
+{
+ smr_shared_t s;
+ smr_t c;
+ smr_seq_t s_wr_seq, s_rd_seq, rd_seq, c_seq;
+ int i;
+ bool success;
+
+ /*
+ * It is illegal to enter while in an smr section.
+ */
+ KASSERT(!wait || curthread->td_critnest == 0,
+ ("smr_poll: Blocking not allowed in a critical section."));
+
+ /*
+ * Use a critical section so that we can avoid ABA races
+ * caused by long preemption sleeps.
+ */
+ success = true;
+ critical_enter();
+ s = smr->c_shared;
+
+ /*
+ * Acquire barrier loads s_wr_seq after s_rd_seq so that we can not
+ * observe an updated read sequence that is larger than write.
+ */
+ s_rd_seq = atomic_load_acq_int(&s->s_rd_seq);
+ s_wr_seq = smr_current(smr);
+
+ /*
+ * Detect whether the goal is valid and has already been observed.
+ *
+ * The goal must be in the range of s_wr_seq >= goal >= s_rd_seq for
+ * it to be valid. If it is not then the caller held on to it and
+ * the integer wrapped. If we wrapped back within range the caller
+ * will harmlessly scan.
+ *
+ * A valid goal must be greater than s_rd_seq or we have not verified
+ * that it has been observed and must fall through to polling.
+ */
+ if (SMR_SEQ_GEQ(s_rd_seq, goal) || SMR_SEQ_LT(s_wr_seq, goal))
+ goto out;
+
+ /*
+ * Loop until all cores have observed the goal sequence or have
+ * gone inactive. Keep track of the oldest sequence currently
+ * active as rd_seq.
+ */
+ rd_seq = s_wr_seq;
+ CPU_FOREACH(i) {
+ c = zpcpu_get_cpu(smr, i);
+ c_seq = SMR_SEQ_INVALID;
+ for (;;) {
+ c_seq = atomic_load_int(&c->c_seq);
+ if (c_seq == SMR_SEQ_INVALID)
+ break;
+
+ /*
+ * There is a race described in smr.h:smr_enter that
+ * can lead to a stale seq value but not stale data
+ * access. If we find a value out of range here we
+ * pin it to the current min to prevent it from
+ * advancing until that stale section has expired.
+ *
+ * The race is created when a cpu loads the s_wr_seq
+ * value in a local register and then another thread
+ * advances s_wr_seq and calls smr_poll() which will
+ * oberve no value yet in c_seq and advance s_rd_seq
+ * up to s_wr_seq which is beyond the register
+ * cached value. This is only likely to happen on
+ * hypervisor or with a system management interrupt.
+ */
+ if (SMR_SEQ_LT(c_seq, s_rd_seq))
+ c_seq = s_rd_seq;
+
+ /*
+ * If the sequence number meets the goal we are
+ * done with this cpu.
+ */
+ if (SMR_SEQ_GEQ(c_seq, goal))
+ break;
+
+ /*
+ * If we're not waiting we will still scan the rest
+ * of the cpus and update s_rd_seq before returning
+ * an error.
+ */
+ if (!wait) {
+ success = false;
+ break;
+ }
+ cpu_spinwait();
+ }
+
+ /*
+ * Limit the minimum observed rd_seq whether we met the goal
+ * or not.
+ */
+ if (c_seq != SMR_SEQ_INVALID && SMR_SEQ_GT(rd_seq, c_seq))
+ rd_seq = c_seq;
+ }
+
+ /*
+ * Advance the rd_seq as long as we observed the most recent one.
+ */
+ s_rd_seq = atomic_load_int(&s->s_rd_seq);
+ do {
+ if (SMR_SEQ_LEQ(rd_seq, s_rd_seq))
+ break;
+ } while (atomic_fcmpset_int(&s->s_rd_seq, &s_rd_seq, rd_seq) == 0);
+
+out:
+ critical_exit();
+
+ return (success);
+}
+
+smr_t
+smr_create(const char *name)
+{
+ smr_t smr, c;
+ smr_shared_t s;
+ int i;
+
+ s = uma_zalloc(smr_shared_zone, M_WAITOK);
+ smr = uma_zalloc(smr_zone, M_WAITOK);
+
+ s->s_name = name;
+ s->s_rd_seq = s->s_wr_seq = SMR_SEQ_INIT;
+
+ /* Initialize all CPUS, not just those running. */
+ for (i = 0; i <= mp_maxid; i++) {
+ c = zpcpu_get_cpu(smr, i);
+ c->c_seq = SMR_SEQ_INVALID;
+ c->c_shared = s;
+ }
+ atomic_thread_fence_seq_cst();
+
+ return (smr);
+}
+
+void
+smr_destroy(smr_t smr)
+{
+
+ smr_synchronize(smr);
+ uma_zfree(smr_shared_zone, smr->c_shared);
+ uma_zfree(smr_zone, smr);
+}
+
+/*
+ * Initialize the UMA slab zone.
+ */
+void
+smr_init(void)
+{
+
+ smr_shared_zone = uma_zcreate("SMR SHARED", sizeof(struct smr_shared),
+ NULL, NULL, NULL, NULL, (CACHE_LINE_SIZE * 2) - 1, 0);
+ smr_zone = uma_zcreate("SMR CPU", sizeof(struct smr),
+ NULL, NULL, NULL, NULL, (CACHE_LINE_SIZE * 2) - 1, UMA_ZONE_PCPU);
+}
Index: head/sys/sys/_smr.h
===================================================================
--- head/sys/sys/_smr.h
+++ head/sys/sys/_smr.h
@@ -0,0 +1,38 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2019,2020 Jeffrey Roberson <jeff@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice unmodified, this list of conditions, and the following
+ * disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#ifndef _SYS__SMR_H_
+#define _SYS__SMR_H_
+
+typedef uint32_t smr_seq_t;
+typedef struct smr *smr_t;
+
+#endif /* __SYS_SMR_H_ */
Index: head/sys/sys/smr.h
===================================================================
--- head/sys/sys/smr.h
+++ head/sys/sys/smr.h
@@ -0,0 +1,180 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2019,2020 Jeffrey Roberson <jeff@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice unmodified, this list of conditions, and the following
+ * disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+#ifndef _SYS_SMR_H_
+#define _SYS_SMR_H_
+
+#include <sys/_smr.h>
+
+/*
+ * Safe memory reclamation. See subr_smr.c for a description of the
+ * algorithm.
+ *
+ * Readers synchronize with smr_enter()/exit() and writers may either
+ * free directly to a SMR UMA zone or use smr_synchronize or wait.
+ */
+
+/*
+ * Modular arithmetic for comparing sequence numbers that have
+ * potentially wrapped. Copied from tcp_seq.h.
+ */
+#define SMR_SEQ_LT(a, b) ((int32_t)((a)-(b)) < 0)
+#define SMR_SEQ_LEQ(a, b) ((int32_t)((a)-(b)) <= 0)
+#define SMR_SEQ_GT(a, b) ((int32_t)((a)-(b)) > 0)
+#define SMR_SEQ_GEQ(a, b) ((int32_t)((a)-(b)) >= 0)
+
+#define SMR_SEQ_INVALID 0
+
+/* Shared SMR state. */
+struct smr_shared {
+ const char *s_name; /* Name for debugging/reporting. */
+ smr_seq_t s_wr_seq; /* Current write sequence #. */
+ smr_seq_t s_rd_seq; /* Minimum observed read sequence. */
+};
+typedef struct smr_shared *smr_shared_t;
+
+/* Per-cpu SMR state. */
+struct smr {
+ smr_seq_t c_seq; /* Current observed sequence. */
+ smr_shared_t c_shared; /* Shared SMR state. */
+};
+
+/*
+ * Return the current write sequence number.
+ */
+static inline smr_seq_t
+smr_current(smr_t smr)
+{
+
+ return (atomic_load_int(&smr->c_shared->s_wr_seq));
+}
+
+/*
+ * Enter a read section.
+ */
+static inline void
+smr_enter(smr_t smr)
+{
+
+ critical_enter();
+ smr = zpcpu_get(smr);
+ KASSERT(smr->c_seq == 0,
+ ("smr_enter(%s) does not support recursion.",
+ smr->c_shared->s_name));
+
+ /*
+ * Store the current observed write sequence number in our
+ * per-cpu state so that it can be queried via smr_poll().
+ * Frees that are newer than this stored value will be
+ * deferred until we call smr_exit().
+ *
+ * An acquire barrier is used to synchronize with smr_exit()
+ * and smr_poll().
+ *
+ * It is possible that a long delay between loading the wr_seq
+ * and storing the c_seq could create a situation where the
+ * rd_seq advances beyond our stored c_seq. In this situation
+ * only the observed wr_seq is stale, the fence still orders
+ * the load. See smr_poll() for details on how this condition
+ * is detected and handled there.
+ */
+ /* This is an add because we do not have atomic_store_acq_int */
+ atomic_add_acq_int(&smr->c_seq, smr_current(smr));
+}
+
+/*
+ * Exit a read section.
+ */
+static inline void
+smr_exit(smr_t smr)
+{
+
+ smr = zpcpu_get(smr);
+ CRITICAL_ASSERT(curthread);
+ KASSERT(smr->c_seq != SMR_SEQ_INVALID,
+ ("smr_exit(%s) not in a smr section.", smr->c_shared->s_name));
+
+ /*
+ * Clear the recorded sequence number. This allows poll() to
+ * detect CPUs not in read sections.
+ *
+ * Use release semantics to retire any stores before the sequence
+ * number is cleared.
+ */
+ atomic_store_rel_int(&smr->c_seq, SMR_SEQ_INVALID);
+ critical_exit();
+}
+
+/*
+ * Advances the write sequence number. Returns the sequence number
+ * required to ensure that all modifications are visible to readers.
+ */
+smr_seq_t smr_advance(smr_t smr);
+
+/*
+ * Returns true if a goal sequence has been reached. If
+ * wait is true this will busy loop until success.
+ */
+bool smr_poll(smr_t smr, smr_seq_t goal, bool wait);
+
+/* Create a new SMR context. */
+smr_t smr_create(const char *name);
+void smr_destroy(smr_t smr);
+
+/*
+ * Blocking wait for all readers to observe 'goal'.
+ */
+static inline bool
+smr_wait(smr_t smr, smr_seq_t goal)
+{
+
+ return (smr_poll(smr, goal, true));
+}
+
+/*
+ * Synchronize advances the write sequence and returns when all
+ * readers have observed it.
+ *
+ * If your application can cache a sequence number returned from
+ * smr_advance() and poll or wait at a later time there will
+ * be less chance of busy looping while waiting for readers.
+ */
+static inline void
+smr_synchronize(smr_t smr)
+{
+
+ smr_wait(smr, smr_advance(smr));
+}
+
+/* Only at startup. */
+void smr_init(void);
+
+#endif /* _SYS_SMR_H_ */
Index: head/sys/vm/uma.h
===================================================================
--- head/sys/vm/uma.h
+++ head/sys/vm/uma.h
@@ -40,6 +40,7 @@
#include <sys/param.h> /* For NULL */
#include <sys/malloc.h> /* For M_* */
+#include <sys/_smr.h>
/* User visible parameters */
#define UMA_SMALLEST_UNIT 8 /* Smallest item allocated */
@@ -259,16 +260,27 @@
* mini-dumps.
*/
#define UMA_ZONE_PCPU 0x8000 /*
- * Allocates mp_maxid + 1 slabs of PAGE_SIZE
+ * Allocates mp_maxid + 1 slabs of
+ * PAGE_SIZE
*/
#define UMA_ZONE_FIRSTTOUCH 0x10000 /* First touch NUMA policy */
#define UMA_ZONE_ROUNDROBIN 0x20000 /* Round-robin NUMA policy. */
+#define UMA_ZONE_SMR 0x40000 /*
+ * Safe memory reclamation defers
+ * frees until all read sections
+ * have exited. This flag creates
+ * a unique SMR context for this
+ * zone. To share contexts see
+ * uma_zone_set_smr() below.
+ *
+ * See sys/smr.h for more details.
+ */
/* In use by UMA_ZFLAGs: 0xffe00000 */
/*
- * These flags are shared between the keg and zone. In zones wishing to add
- * new kegs these flags must be compatible. Some are determined based on
- * physical parameters of the request and may not be provided by the consumer.
+ * These flags are shared between the keg and zone. Some are determined
+ * based on physical parameters of the request and may not be provided by
+ * the consumer.
*/
#define UMA_ZONE_INHERIT \
(UMA_ZONE_NOTOUCH | UMA_ZONE_MALLOC | UMA_ZONE_NOFREE | \
@@ -310,8 +322,13 @@
*/
void *uma_zalloc_arg(uma_zone_t zone, void *arg, int flags);
+
+/* Allocate per-cpu data. Access the correct data with zpcpu_get(). */
void *uma_zalloc_pcpu_arg(uma_zone_t zone, void *arg, int flags);
+/* Use with SMR zones. */
+void *uma_zalloc_smr(uma_zone_t zone, int flags);
+
/*
* Allocate an item from a specific NUMA domain. This uses a slow path in
* the allocator but is guaranteed to allocate memory from the requested
@@ -359,8 +376,13 @@
*/
void uma_zfree_arg(uma_zone_t zone, void *item, void *arg);
+
+/* Use with PCPU zones. */
void uma_zfree_pcpu_arg(uma_zone_t zone, void *item, void *arg);
+/* Use with SMR zones. */
+void uma_zfree_smr(uma_zone_t zone, void *item);
+
/*
* Frees an item back to the specified zone's domain specific pool.
*
@@ -599,6 +621,17 @@
*/
void uma_zone_set_freef(uma_zone_t zone, uma_free freef);
+
+/*
+ * Associate a zone with a smr context that is allocated after creation
+ * so that multiple zones may share the same context.
+ */
+void uma_zone_set_smr(uma_zone_t zone, smr_t smr);
+
+/*
+ * Fetch the smr context that was set or made in uma_zcreate().
+ */
+smr_t uma_zone_get_smr(uma_zone_t zone);
/*
* These flags are setable in the allocf and visible in the freef.
Index: head/sys/vm/uma_core.c
===================================================================
--- head/sys/vm/uma_core.c
+++ head/sys/vm/uma_core.c
@@ -77,6 +77,7 @@
#include <sys/sched.h>
#include <sys/sleepqueue.h>
#include <sys/smp.h>
+#include <sys/smr.h>
#include <sys/taskqueue.h>
#include <sys/vmmeter.h>
@@ -103,6 +104,12 @@
#include <machine/md_var.h>
+#ifdef INVARIANTS
+#define UMA_ALWAYS_CTORDTOR 1
+#else
+#define UMA_ALWAYS_CTORDTOR 0
+#endif
+
/*
* This is the zone and keg from which all zones are spawned.
*/
@@ -273,6 +280,8 @@
static void keg_dtor(void *, int, void *);
static int zone_ctor(void *, int, void *, int);
static void zone_dtor(void *, int, void *);
+static inline void item_dtor(uma_zone_t zone, void *item, int size,
+ void *udata, enum zfreeskip skip);
static int zero_init(void *, int, int);
static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *);
static void zone_foreach_unlocked(void (*zfunc)(uma_zone_t, void *), void *);
@@ -454,9 +463,9 @@
uma_bucket_t bucket;
/*
- * Don't allocate buckets in low memory situations.
+ * Don't allocate buckets early in boot.
*/
- if (bucketdisable)
+ if (__predict_false(booted < BOOT_KVA))
return (NULL);
/*
@@ -488,6 +497,9 @@
#endif
bucket->ub_cnt = 0;
bucket->ub_entries = ubz->ubz_entries;
+ bucket->ub_seq = SMR_SEQ_INVALID;
+ CTR3(KTR_UMA, "bucket_alloc: zone %s(%p) allocated bucket %p",
+ zone->uz_name, zone, bucket);
}
return (bucket);
@@ -500,6 +512,8 @@
KASSERT(bucket->ub_cnt == 0,
("bucket_free: Freeing a non free bucket."));
+ KASSERT(bucket->ub_seq == SMR_SEQ_INVALID,
+ ("bucket_free: Freeing an SMR bucket."));
if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0)
udata = (void *)(uintptr_t)zone->uz_flags;
ubz = bucket_zone_lookup(bucket->ub_entries);
@@ -517,23 +531,39 @@
/*
* Attempt to satisfy an allocation by retrieving a full bucket from one of the
- * zone's caches.
+ * zone's caches. If a bucket is found the zone is not locked on return.
*/
static uma_bucket_t
zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom)
{
uma_bucket_t bucket;
+ int i;
+ bool dtor = false;
ZONE_LOCK_ASSERT(zone);
- if ((bucket = TAILQ_FIRST(&zdom->uzd_buckets)) != NULL) {
- MPASS(zdom->uzd_nitems >= bucket->ub_cnt);
- TAILQ_REMOVE(&zdom->uzd_buckets, bucket, ub_link);
- zdom->uzd_nitems -= bucket->ub_cnt;
- if (zdom->uzd_imin > zdom->uzd_nitems)
- zdom->uzd_imin = zdom->uzd_nitems;
- zone->uz_bkt_count -= bucket->ub_cnt;
+ if ((bucket = TAILQ_FIRST(&zdom->uzd_buckets)) == NULL)
+ return (NULL);
+
+ if ((zone->uz_flags & UMA_ZONE_SMR) != 0 &&
+ bucket->ub_seq != SMR_SEQ_INVALID) {
+ if (!smr_poll(zone->uz_smr, bucket->ub_seq, false))
+ return (NULL);
+ bucket->ub_seq = SMR_SEQ_INVALID;
+ dtor = (zone->uz_dtor != NULL) | UMA_ALWAYS_CTORDTOR;
}
+ MPASS(zdom->uzd_nitems >= bucket->ub_cnt);
+ TAILQ_REMOVE(&zdom->uzd_buckets, bucket, ub_link);
+ zdom->uzd_nitems -= bucket->ub_cnt;
+ if (zdom->uzd_imin > zdom->uzd_nitems)
+ zdom->uzd_imin = zdom->uzd_nitems;
+ zone->uz_bkt_count -= bucket->ub_cnt;
+ ZONE_UNLOCK(zone);
+ if (dtor)
+ for (i = 0; i < bucket->ub_cnt; i++)
+ item_dtor(zone, bucket->ub_bucket[i], zone->uz_size,
+ NULL, SKIP_NONE);
+
return (bucket);
}
@@ -551,7 +581,7 @@
KASSERT(!ws || zone->uz_bkt_count < zone->uz_bkt_max,
("%s: zone %p overflow", __func__, zone));
- if (ws)
+ if (ws && bucket->ub_seq == SMR_SEQ_INVALID)
TAILQ_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link);
else
TAILQ_INSERT_TAIL(&zdom->uzd_buckets, bucket, ub_link);
@@ -941,12 +971,23 @@
if (bucket == NULL || bucket->ub_cnt == 0)
return;
+ if ((zone->uz_flags & UMA_ZONE_SMR) != 0 &&
+ bucket->ub_seq != SMR_SEQ_INVALID) {
+ smr_wait(zone->uz_smr, bucket->ub_seq);
+ for (i = 0; i < bucket->ub_cnt; i++)
+ item_dtor(zone, bucket->ub_bucket[i],
+ zone->uz_size, NULL, SKIP_NONE);
+ bucket->ub_seq = SMR_SEQ_INVALID;
+ }
if (zone->uz_fini)
for (i = 0; i < bucket->ub_cnt; i++)
zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
if (zone->uz_max_items > 0)
zone_free_limit(zone, bucket->ub_cnt);
+#ifdef INVARIANTS
+ bzero(bucket->ub_bucket, sizeof(void *) * bucket->ub_cnt);
+#endif
bucket->ub_cnt = 0;
}
@@ -1035,12 +1076,21 @@
zone_put_bucket(zone, &zone->uz_domain[domain], b1, false);
b1 = NULL;
}
+
+ /*
+ * Don't flush SMR zone buckets. This leaves the zone without a
+ * bucket and forces every free to synchronize().
+ */
+ if ((zone->uz_flags & UMA_ZONE_SMR) != 0)
+ goto out;
b2 = cache_bucket_unload_free(cache);
if (b2 != NULL && b2->ub_cnt != 0) {
zone_put_bucket(zone, &zone->uz_domain[domain], b2, false);
b2 = NULL;
}
b3 = cache_bucket_unload_cross(cache);
+
+out:
critical_exit();
ZONE_UNLOCK(zone);
if (b1)
@@ -1135,7 +1185,7 @@
target = drain ? 0 : lmax(zdom->uzd_wss, zdom->uzd_nitems -
zdom->uzd_imin);
while (zdom->uzd_nitems > target) {
- bucket = TAILQ_LAST(&zdom->uzd_buckets, uma_bucketlist);
+ bucket = TAILQ_FIRST(&zdom->uzd_buckets);
if (bucket == NULL)
break;
tofree = bucket->ub_cnt;
@@ -2294,7 +2344,7 @@
zone->uz_bucket_size = 0;
zone->uz_bucket_size_min = 0;
zone->uz_bucket_size_max = BUCKET_MAX;
- zone->uz_flags = 0;
+ zone->uz_flags = (arg->flags & UMA_ZONE_SMR);
zone->uz_warning = NULL;
/* The domain structures follow the cpu structures. */
zone->uz_domain =
@@ -2375,7 +2425,7 @@
karg.uminit = arg->uminit;
karg.fini = arg->fini;
karg.align = arg->align;
- karg.flags = arg->flags;
+ karg.flags = (arg->flags & ~UMA_ZONE_SMR);
karg.zone = zone;
error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg,
flags);
@@ -2399,6 +2449,10 @@
zone->uz_fails = EARLY_COUNTER;
}
+ /* Caller requests a private SMR context. */
+ if ((zone->uz_flags & UMA_ZONE_SMR) != 0)
+ zone->uz_smr = smr_create(zone->uz_name);
+
KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) !=
(UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET),
("Invalid zone flag combination"));
@@ -2600,6 +2654,7 @@
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
bucket_init();
+ smr_init();
}
#ifndef UMA_MD_SMALL_ALLOC
@@ -2844,14 +2899,9 @@
uma_zfree_arg(zone, item, udata);
}
-#ifdef INVARIANTS
-#define UMA_ALWAYS_CTORDTOR 1
-#else
-#define UMA_ALWAYS_CTORDTOR 0
-#endif
-
-static void *
-item_ctor(uma_zone_t zone, int size, void *udata, int flags, void *item)
+static inline void *
+item_ctor(uma_zone_t zone, int uz_flags, int size, void *udata, int flags,
+ void *item)
{
#ifdef INVARIANTS
bool skipdbg;
@@ -2861,7 +2911,9 @@
zone->uz_ctor != trash_ctor)
trash_ctor(item, size, udata, flags);
#endif
- if (__predict_false(zone->uz_ctor != NULL) &&
+ /* Check flags before loading ctor pointer. */
+ if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0) &&
+ __predict_false(zone->uz_ctor != NULL) &&
zone->uz_ctor(item, size, udata, flags) != 0) {
counter_u64_add(zone->uz_fails, 1);
zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT);
@@ -2903,58 +2955,151 @@
}
}
-/* See uma.h */
-void *
-uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
+#if defined(INVARIANTS) || defined(DEBUG_MEMGUARD) || defined(WITNESS)
+#define UMA_ZALLOC_DEBUG
+static int
+uma_zalloc_debug(uma_zone_t zone, void **itemp, void *udata, int flags)
{
- uma_cache_bucket_t bucket;
- uma_cache_t cache;
- void *item;
- int domain, size, uz_flags;
+ int error;
- /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
- random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA);
-
- /* This is the fast path allocation */
- CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name,
- zone, flags);
-
+ error = 0;
#ifdef WITNESS
if (flags & M_WAITOK) {
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
- "uma_zalloc_arg: zone \"%s\"", zone->uz_name);
+ "uma_zalloc_debug: zone \"%s\"", zone->uz_name);
}
#endif
#ifdef INVARIANTS
- KASSERT((flags & M_EXEC) == 0, ("uma_zalloc_arg: called with M_EXEC"));
+ KASSERT((flags & M_EXEC) == 0,
+ ("uma_zalloc_debug: called with M_EXEC"));
KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
- ("uma_zalloc_arg: called with spinlock or critical section held"));
- if (zone->uz_flags & UMA_ZONE_PCPU)
- KASSERT((flags & M_ZERO) == 0, ("allocating from a pcpu zone "
- "with M_ZERO passed"));
+ ("uma_zalloc_debug: called within spinlock or critical section"));
+ KASSERT((zone->uz_flags & UMA_ZONE_PCPU) == 0 || (flags & M_ZERO) == 0,
+ ("uma_zalloc_debug: allocating from a pcpu zone with M_ZERO"));
#endif
#ifdef DEBUG_MEMGUARD
- if (memguard_cmp_zone(zone)) {
+ if ((zone->uz_flags & UMA_ZONE_SMR == 0) && memguard_cmp_zone(zone)) {
+ void *item;
item = memguard_alloc(zone->uz_size, flags);
if (item != NULL) {
+ error = EJUSTRETURN;
if (zone->uz_init != NULL &&
- zone->uz_init(item, zone->uz_size, flags) != 0)
- return (NULL);
+ zone->uz_init(item, zone->uz_size, flags) != 0) {
+ *itemp = NULL;
+ return (error);
+ }
if (zone->uz_ctor != NULL &&
zone->uz_ctor(item, zone->uz_size, udata,
flags) != 0) {
counter_u64_add(zone->uz_fails, 1);
zone->uz_fini(item, zone->uz_size);
- return (NULL);
+ *itemp = NULL;
+ return (error);
}
- return (item);
+ *itemp = item;
+ return (error);
}
/* This is unfortunate but should not be fatal. */
}
#endif
+ return (error);
+}
+
+static int
+uma_zfree_debug(uma_zone_t zone, void *item, void *udata)
+{
+ KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
+ ("uma_zfree_debug: called with spinlock or critical section held"));
+
+#ifdef DEBUG_MEMGUARD
+ if ((zone->uz_flags & UMA_ZONE_SMR == 0) && is_memguard_addr(item)) {
+ if (zone->uz_dtor != NULL)
+ zone->uz_dtor(item, zone->uz_size, udata);
+ if (zone->uz_fini != NULL)
+ zone->uz_fini(item, zone->uz_size);
+ memguard_free(item);
+ return (EJUSTRETURN);
+ }
+#endif
+ return (0);
+}
+#endif
+
+static __noinline void *
+uma_zalloc_single(uma_zone_t zone, void *udata, int flags)
+{
+ int domain;
+
/*
+ * We can not get a bucket so try to return a single item.
+ */
+ if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH)
+ domain = PCPU_GET(domain);
+ else
+ domain = UMA_ANYDOMAIN;
+ return (zone_alloc_item(zone, udata, domain, flags));
+}
+
+/* See uma.h */
+void *
+uma_zalloc_smr(uma_zone_t zone, int flags)
+{
+ uma_cache_bucket_t bucket;
+ uma_cache_t cache;
+ void *item;
+ int size, uz_flags;
+
+#ifdef UMA_ZALLOC_DEBUG
+ KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0,
+ ("uma_zalloc_arg: called with non-SMR zone.\n"));
+ if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN)
+ return (item);
+#endif
+
+ critical_enter();
+ do {
+ cache = &zone->uz_cpu[curcpu];
+ bucket = &cache->uc_allocbucket;
+ size = cache_uz_size(cache);
+ uz_flags = cache_uz_flags(cache);
+ if (__predict_true(bucket->ucb_cnt != 0)) {
+ item = cache_bucket_pop(cache, bucket);
+ critical_exit();
+ return (item_ctor(zone, uz_flags, size, NULL, flags,
+ item));
+ }
+ } while (cache_alloc(zone, cache, NULL, flags));
+ critical_exit();
+
+ return (uma_zalloc_single(zone, NULL, flags));
+}
+
+/* See uma.h */
+void *
+uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
+{
+ uma_cache_bucket_t bucket;
+ uma_cache_t cache;
+ void *item;
+ int size, uz_flags;
+
+ /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
+ random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA);
+
+ /* This is the fast path allocation */
+ CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name,
+ zone, flags);
+
+#ifdef UMA_ZALLOC_DEBUG
+ KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0,
+ ("uma_zalloc_arg: called with SMR zone.\n"));
+ if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN)
+ return (item);
+#endif
+
+ /*
* If possible, allocate from the per-CPU cache. There are two
* requirements for safe access to the per-CPU cache: (1) the thread
* accessing the cache must not be preempted or yield during access,
@@ -2974,24 +3119,13 @@
if (__predict_true(bucket->ucb_cnt != 0)) {
item = cache_bucket_pop(cache, bucket);
critical_exit();
- if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0 ||
- UMA_ALWAYS_CTORDTOR))
- return (item_ctor(zone, size, udata, flags, item));
- if (flags & M_ZERO)
- bzero(item, size);
- return (item);
+ return (item_ctor(zone, uz_flags, size, udata, flags,
+ item));
}
} while (cache_alloc(zone, cache, udata, flags));
critical_exit();
- /*
- * We can not get a bucket so try to return a single item.
- */
- if (uz_flags & UMA_ZONE_FIRSTTOUCH)
- domain = PCPU_GET(domain);
- else
- domain = UMA_ANYDOMAIN;
- return (zone_alloc_item(zone, udata, domain, flags));
+ return (uma_zalloc_single(zone, udata, flags));
}
/*
@@ -3014,9 +3148,14 @@
/*
* If we have run out of items in our alloc bucket see
* if we can switch with the free bucket.
+ *
+ * SMR Zones can't re-use the free bucket until the sequence has
+ * expired.
*/
- if (cache->uc_freebucket.ucb_cnt != 0) {
- cache_bucket_swap(&cache->uc_freebucket, &cache->uc_allocbucket);
+ if ((zone->uz_flags & UMA_ZONE_SMR) == 0 &&
+ cache->uc_freebucket.ucb_cnt != 0) {
+ cache_bucket_swap(&cache->uc_freebucket,
+ &cache->uc_allocbucket);
return (true);
}
@@ -3070,7 +3209,6 @@
}
if ((bucket = zone_fetch_bucket(zone, zdom)) != NULL) {
- ZONE_UNLOCK(zone);
KASSERT(bucket->ub_cnt != 0,
("uma_zalloc_arg: Returning an empty bucket."));
cache_bucket_load_alloc(cache, bucket);
@@ -3607,7 +3745,8 @@
goto fail_cnt;
}
}
- item = item_ctor(zone, zone->uz_size, udata, flags, item);
+ item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, flags,
+ item);
if (item == NULL)
goto fail;
@@ -3630,6 +3769,54 @@
/* See uma.h */
void
+uma_zfree_smr(uma_zone_t zone, void *item)
+{
+ uma_cache_t cache;
+ uma_cache_bucket_t bucket;
+ int domain, itemdomain, uz_flags;
+
+#ifdef UMA_ZALLOC_DEBUG
+ KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0,
+ ("uma_zfree_smr: called with non-SMR zone.\n"));
+ KASSERT(item != NULL, ("uma_zfree_smr: Called with NULL pointer."));
+ if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN)
+ return;
+#endif
+ cache = &zone->uz_cpu[curcpu];
+ uz_flags = cache_uz_flags(cache);
+ domain = itemdomain = 0;
+#ifdef NUMA
+ if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0)
+ itemdomain = _vm_phys_domain(pmap_kextract((vm_offset_t)item));
+#endif
+ critical_enter();
+ do {
+ cache = &zone->uz_cpu[curcpu];
+ /* SMR Zones must free to the free bucket. */
+ bucket = &cache->uc_freebucket;
+#ifdef NUMA
+ domain = PCPU_GET(domain);
+ if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 &&
+ domain != itemdomain) {
+ bucket = &cache->uc_crossbucket;
+ }
+#endif
+ if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) {
+ cache_bucket_push(cache, bucket, item);
+ critical_exit();
+ return;
+ }
+ } while (cache_free(zone, cache, NULL, item, itemdomain));
+ critical_exit();
+
+ /*
+ * If nothing else caught this, we'll just do an internal free.
+ */
+ zone_free_item(zone, item, NULL, SKIP_NONE);
+}
+
+/* See uma.h */
+void
uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
{
uma_cache_t cache;
@@ -3641,22 +3828,15 @@
CTR2(KTR_UMA, "uma_zfree_arg zone %s(%p)", zone->uz_name, zone);
- KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(),
- ("uma_zfree_arg: called with spinlock or critical section held"));
-
+#ifdef UMA_ZALLOC_DEBUG
+ KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0,
+ ("uma_zfree_arg: called with SMR zone.\n"));
+ if (uma_zfree_debug(zone, item, udata) == EJUSTRETURN)
+ return;
+#endif
/* uma_zfree(..., NULL) does nothing, to match free(9). */
if (item == NULL)
return;
-#ifdef DEBUG_MEMGUARD
- if (is_memguard_addr(item)) {
- if (zone->uz_dtor != NULL)
- zone->uz_dtor(item, zone->uz_size, udata);
- if (zone->uz_fini != NULL)
- zone->uz_fini(item, zone->uz_size);
- memguard_free(item);
- return;
- }
-#endif
/*
* We are accessing the per-cpu cache without a critical section to
@@ -3665,8 +3845,8 @@
*/
cache = &zone->uz_cpu[curcpu];
uz_flags = cache_uz_flags(cache);
- if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0 ||
- UMA_ALWAYS_CTORDTOR))
+ if (UMA_ALWAYS_CTORDTOR ||
+ __predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0))
item_dtor(zone, item, cache_uz_size(cache), udata, SKIP_NONE);
/*
@@ -3697,6 +3877,13 @@
critical_enter();
do {
cache = &zone->uz_cpu[curcpu];
+ /*
+ * Try to free into the allocbucket first to give LIFO
+ * ordering for cache-hot datastructures. Spill over
+ * into the freebucket if necessary. Alloc will swap
+ * them if one runs dry.
+ */
+ bucket = &cache->uc_allocbucket;
#ifdef NUMA
domain = PCPU_GET(domain);
if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 &&
@@ -3704,18 +3891,8 @@
bucket = &cache->uc_crossbucket;
} else
#endif
- {
- /*
- * Try to free into the allocbucket first to give LIFO
- * ordering for cache-hot datastructures. Spill over
- * into the freebucket if necessary. Alloc will swap
- * them if one runs dry.
- */
- bucket = &cache->uc_allocbucket;
- if (__predict_false(bucket->ucb_cnt >=
- bucket->ucb_entries))
- bucket = &cache->uc_freebucket;
- }
+ if (bucket->ucb_cnt >= bucket->ucb_entries)
+ bucket = &cache->uc_freebucket;
if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) {
cache_bucket_push(cache, bucket, item);
critical_exit();
@@ -3778,6 +3955,8 @@
if (!TAILQ_EMPTY(&fullbuckets)) {
ZONE_LOCK(zone);
while ((b = TAILQ_FIRST(&fullbuckets)) != NULL) {
+ if ((zone->uz_flags & UMA_ZONE_SMR) != 0)
+ bucket->ub_seq = smr_current(zone->uz_smr);
TAILQ_REMOVE(&fullbuckets, b, ub_link);
if (zone->uz_bkt_count >= zone->uz_bkt_max) {
ZONE_UNLOCK(zone);
@@ -3796,6 +3975,7 @@
}
if (bucket->ub_cnt != 0)
bucket_drain(zone, bucket);
+ bucket->ub_seq = SMR_SEQ_INVALID;
bucket_free(zone, bucket, udata);
}
#endif
@@ -3862,15 +4042,16 @@
int itemdomain)
{
uma_cache_bucket_t cbucket;
- uma_bucket_t bucket;
+ uma_bucket_t newbucket, bucket;
int domain;
CRITICAL_ASSERT(curthread);
- if (zone->uz_bucket_size == 0 || bucketdisable)
+ if (zone->uz_bucket_size == 0)
return false;
cache = &zone->uz_cpu[curcpu];
+ newbucket = NULL;
/*
* FIRSTTOUCH domains need to free to the correct zdom. When
@@ -3895,14 +4076,29 @@
/* We are no longer associated with this CPU. */
critical_exit();
+ /*
+ * Don't let SMR zones operate without a free bucket. Force
+ * a synchronize and re-use this one. We will only degrade
+ * to a synchronize every bucket_size items rather than every
+ * item if we fail to allocate a bucket.
+ */
+ if ((zone->uz_flags & UMA_ZONE_SMR) != 0) {
+ if (bucket != NULL)
+ bucket->ub_seq = smr_advance(zone->uz_smr);
+ newbucket = bucket_alloc(zone, udata, M_NOWAIT);
+ if (newbucket == NULL && bucket != NULL) {
+ bucket_drain(zone, bucket);
+ newbucket = bucket;
+ bucket = NULL;
+ }
+ } else if (!bucketdisable)
+ newbucket = bucket_alloc(zone, udata, M_NOWAIT);
+
if (bucket != NULL)
zone_free_bucket(zone, bucket, udata, domain, itemdomain);
- bucket = bucket_alloc(zone, udata, M_NOWAIT);
- CTR3(KTR_UMA, "uma_zfree: zone %s(%p) allocated bucket %p",
- zone->uz_name, zone, bucket);
critical_enter();
- if (bucket == NULL)
+ if ((bucket = newbucket) == NULL)
return (false);
cache = &zone->uz_cpu[curcpu];
#ifdef NUMA
@@ -4031,6 +4227,15 @@
zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
{
+ /*
+ * If a free is sent directly to an SMR zone we have to
+ * synchronize immediately because the item can instantly
+ * be reallocated. This should only happen in degenerate
+ * cases when no memory is available for per-cpu caches.
+ */
+ if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && skip == SKIP_NONE)
+ smr_synchronize(zone->uz_smr);
+
item_dtor(zone, item, zone->uz_size, udata, skip);
if (skip < SKIP_FINI && zone->uz_fini)
@@ -4253,6 +4458,25 @@
KEG_GET(zone, keg);
KEG_ASSERT_COLD(keg);
keg->uk_allocf = allocf;
+}
+
+/* See uma.h */
+void
+uma_zone_set_smr(uma_zone_t zone, smr_t smr)
+{
+
+ ZONE_ASSERT_COLD(zone);
+
+ zone->uz_flags |= UMA_ZONE_SMR;
+ zone->uz_smr = smr;
+ zone_update_caches(zone);
+}
+
+smr_t
+uma_zone_get_smr(uma_zone_t zone)
+{
+
+ return (zone->uz_smr);
}
/* See uma.h */
Index: head/sys/vm/uma_int.h
===================================================================
--- head/sys/vm/uma_int.h
+++ head/sys/vm/uma_int.h
@@ -184,6 +184,7 @@
"\30VTOSLAB" \
"\27HASH" \
"\26OFFPAGE" \
+ "\23SMR" \
"\22ROUNDROBIN" \
"\21FIRSTTOUCH" \
"\20PCPU" \
@@ -245,9 +246,10 @@
*/
struct uma_bucket {
TAILQ_ENTRY(uma_bucket) ub_link; /* Link into the zone */
- int16_t ub_cnt; /* Count of items in bucket. */
- int16_t ub_entries; /* Max items. */
- void *ub_bucket[]; /* actual allocation storage */
+ int16_t ub_cnt; /* Count of items in bucket. */
+ int16_t ub_entries; /* Max items. */
+ smr_seq_t ub_seq; /* SMR sequence number. */
+ void *ub_bucket[]; /* actual allocation storage */
};
typedef struct uma_bucket * uma_bucket_t;
@@ -484,7 +486,7 @@
uint32_t uz_size; /* Size inherited from kegs */
uma_ctor uz_ctor; /* Constructor for each allocation */
uma_dtor uz_dtor; /* Destructor */
- uint64_t uz_spare0;
+ smr_t uz_smr; /* Safe memory reclaim context. */
uint64_t uz_max_items; /* Maximum number of items to alloc */
uint32_t uz_sleepers; /* Threads sleeping on limit */
uint16_t uz_bucket_size; /* Number of items in full bucket */

File Metadata

Mime Type
text/plain
Expires
Fri, Feb 21, 5:58 PM (11 m, 45 s)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
16759308
Default Alt Text
D22586.id67538.diff (45 KB)

Event Timeline