Index: head/sys/conf/files =================================================================== --- head/sys/conf/files +++ head/sys/conf/files @@ -3848,6 +3848,7 @@ kern/subr_sglist.c standard kern/subr_sleepqueue.c standard kern/subr_smp.c standard +kern/subr_smr.c standard kern/subr_stack.c optional ddb | stack | ktr kern/subr_stats.c optional stats kern/subr_taskqueue.c standard Index: head/sys/kern/subr_smr.c =================================================================== --- head/sys/kern/subr_smr.c +++ head/sys/kern/subr_smr.c @@ -0,0 +1,387 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 Jeffrey Roberson + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * This is a novel safe memory reclamation technique inspired by + * epoch based reclamation from Samy Al Bahra's concurrency kit which + * in turn was based on work described in: + * Fraser, K. 2004. Practical Lock-Freedom. PhD Thesis, University + * of Cambridge Computing Laboratory. + * And shares some similarities with: + * Wang, Stamler, Parmer. 2016 Parallel Sections: Scaling System-Level + * Data-Structures + * + * This is not an implementation of hazard pointers or related + * techniques. The term safe memory reclamation is used as a + * generic descriptor for algorithms that defer frees to avoid + * use-after-free errors with lockless datastructures. + * + * The basic approach is to maintain a monotonic write sequence + * number that is updated on some application defined granularity. + * Readers record the most recent write sequence number they have + * observed. A shared read sequence number records the lowest + * sequence number observed by any reader as of the last poll. Any + * write older than this value has been observed by all readers + * and memory can be reclaimed. Like Epoch we also detect idle + * readers by storing an invalid sequence number in the per-cpu + * state when the read section exits. Like Parsec we establish + * a global write clock that is used to mark memory on free. + * + * The write and read sequence numbers can be thought of as a two + * handed clock with readers always advancing towards writers. SMR + * maintains the invariant that all readers can safely access memory + * that was visible at the time they loaded their copy of the sequence + * number. Periodically the read sequence or hand is polled and + * advanced as far towards the write sequence as active readers allow. + * Memory which was freed between the old and new global read sequence + * number can now be reclaimed. When the system is idle the two hands + * meet and no deferred memory is outstanding. Readers never advance + * any sequence number, they only observe them. The shared read + * sequence number is consequently never higher than the write sequence. + * A stored sequence number that falls outside of this range has expired + * and needs no scan to reclaim. + * + * A notable distinction between this SMR and Epoch, qsbr, rcu, etc. is + * that advancing the sequence number is decoupled from detecting its + * observation. This results in a more granular assignment of sequence + * numbers even as read latencies prohibit all or some expiration. + * It also allows writers to advance the sequence number and save the + * poll for expiration until a later time when it is likely to + * complete without waiting. The batch granularity and free-to-use + * latency is dynamic and can be significantly smaller than in more + * strict systems. + * + * This mechanism is primarily intended to be used in coordination with + * UMA. By integrating with the allocator we avoid all of the callout + * queue machinery and are provided with an efficient way to batch + * sequence advancement and waiting. The allocator accumulates a full + * per-cpu cache of memory before advancing the sequence. It then + * delays waiting for this sequence to expire until the memory is + * selected for reuse. In this way we only increment the sequence + * value once for n=cache-size frees and the waits are done long + * after the sequence has been expired so they need only be verified + * to account for pathological conditions and to advance the read + * sequence. Tying the sequence number to the bucket size has the + * nice property that as the zone gets busier the buckets get larger + * and the sequence writes become fewer. If the coherency of advancing + * the write sequence number becomes too costly we can advance + * it for every N buckets in exchange for higher free-to-use + * latency and consequently higher memory consumption. + * + * If the read overhead of accessing the shared cacheline becomes + * especially burdensome an invariant TSC could be used in place of the + * sequence. The algorithm would then only need to maintain the minimum + * observed tsc. This would trade potential cache synchronization + * overhead for local serialization and cpu timestamp overhead. + */ + +/* + * A simplified diagram: + * + * 0 UINT_MAX + * | -------------------- sequence number space -------------------- | + * ^ rd seq ^ wr seq + * | ----- valid sequence numbers ---- | + * ^cpuA ^cpuC + * | -- free -- | --------- deferred frees -------- | ---- free ---- | + * + * + * In this example cpuA has the lowest sequence number and poll can + * advance rd seq. cpuB is not running and is considered to observe + * wr seq. + * + * Freed memory that is tagged with a sequence number between rd seq and + * wr seq can not be safely reclaimed because cpuA may hold a reference to + * it. Any other memory is guaranteed to be unreferenced. + * + * Any writer is free to advance wr seq at any time however it may busy + * poll in pathological cases. + */ + +static uma_zone_t smr_shared_zone; +static uma_zone_t smr_zone; + +#ifndef INVARIANTS +#define SMR_SEQ_INIT 1 /* All valid sequence numbers are odd. */ +#define SMR_SEQ_INCR 2 + +/* + * SMR_SEQ_MAX_DELTA is the maximum distance allowed between rd_seq and + * wr_seq. For the modular arithmetic to work a value of UNIT_MAX / 2 + * would be possible but it is checked after we increment the wr_seq so + * a safety margin is left to prevent overflow. + * + * We will block until SMR_SEQ_MAX_ADVANCE sequence numbers have progressed + * to prevent integer wrapping. See smr_advance() for more details. + */ +#define SMR_SEQ_MAX_DELTA (UINT_MAX / 4) +#define SMR_SEQ_MAX_ADVANCE (SMR_SEQ_MAX_DELTA - 1024) +#else +/* We want to test the wrapping feature in invariants kernels. */ +#define SMR_SEQ_INCR (UINT_MAX / 10000) +#define SMR_SEQ_INIT (UINT_MAX - 100000) +/* Force extra polls to test the integer overflow detection. */ +#define SMR_SEQ_MAX_DELTA (1000) +#define SMR_SEQ_MAX_ADVANCE SMR_SEQ_MAX_DELTA / 2 +#endif + +/* + * Advance the write sequence and return the new value for use as the + * wait goal. This guarantees that any changes made by the calling + * thread prior to this call will be visible to all threads after + * rd_seq meets or exceeds the return value. + * + * This function may busy loop if the readers are roughly 1 billion + * sequence numbers behind the writers. + */ +smr_seq_t +smr_advance(smr_t smr) +{ + smr_shared_t s; + smr_seq_t goal; + + /* + * It is illegal to enter while in an smr section. + */ + KASSERT(curthread->td_critnest == 0, + ("smr_advance: Not allowed in a critical section.")); + + /* + * Modifications not done in a smr section need to be visible + * before advancing the seq. + */ + atomic_thread_fence_rel(); + + /* + * Increment the shared write sequence by 2. Since it is + * initialized to 1 this means the only valid values are + * odd and an observed value of 0 in a particular CPU means + * it is not currently in a read section. + */ + s = smr->c_shared; + goal = atomic_fetchadd_int(&s->s_wr_seq, SMR_SEQ_INCR) + SMR_SEQ_INCR; + + /* + * Force a synchronization here if the goal is getting too + * far ahead of the read sequence number. This keeps the + * wrap detecting arithmetic working in pathological cases. + */ + if (goal - atomic_load_int(&s->s_rd_seq) >= SMR_SEQ_MAX_DELTA) + smr_wait(smr, goal - SMR_SEQ_MAX_ADVANCE); + + return (goal); +} + +/* + * Poll to determine whether all readers have observed the 'goal' write + * sequence number. + * + * If wait is true this will spin until the goal is met. + * + * This routine will updated the minimum observed read sequence number in + * s_rd_seq if it does a scan. It may not do a scan if another call has + * advanced s_rd_seq beyond the callers goal already. + * + * Returns true if the goal is met and false if not. + */ +bool +smr_poll(smr_t smr, smr_seq_t goal, bool wait) +{ + smr_shared_t s; + smr_t c; + smr_seq_t s_wr_seq, s_rd_seq, rd_seq, c_seq; + int i; + bool success; + + /* + * It is illegal to enter while in an smr section. + */ + KASSERT(!wait || curthread->td_critnest == 0, + ("smr_poll: Blocking not allowed in a critical section.")); + + /* + * Use a critical section so that we can avoid ABA races + * caused by long preemption sleeps. + */ + success = true; + critical_enter(); + s = smr->c_shared; + + /* + * Acquire barrier loads s_wr_seq after s_rd_seq so that we can not + * observe an updated read sequence that is larger than write. + */ + s_rd_seq = atomic_load_acq_int(&s->s_rd_seq); + s_wr_seq = smr_current(smr); + + /* + * Detect whether the goal is valid and has already been observed. + * + * The goal must be in the range of s_wr_seq >= goal >= s_rd_seq for + * it to be valid. If it is not then the caller held on to it and + * the integer wrapped. If we wrapped back within range the caller + * will harmlessly scan. + * + * A valid goal must be greater than s_rd_seq or we have not verified + * that it has been observed and must fall through to polling. + */ + if (SMR_SEQ_GEQ(s_rd_seq, goal) || SMR_SEQ_LT(s_wr_seq, goal)) + goto out; + + /* + * Loop until all cores have observed the goal sequence or have + * gone inactive. Keep track of the oldest sequence currently + * active as rd_seq. + */ + rd_seq = s_wr_seq; + CPU_FOREACH(i) { + c = zpcpu_get_cpu(smr, i); + c_seq = SMR_SEQ_INVALID; + for (;;) { + c_seq = atomic_load_int(&c->c_seq); + if (c_seq == SMR_SEQ_INVALID) + break; + + /* + * There is a race described in smr.h:smr_enter that + * can lead to a stale seq value but not stale data + * access. If we find a value out of range here we + * pin it to the current min to prevent it from + * advancing until that stale section has expired. + * + * The race is created when a cpu loads the s_wr_seq + * value in a local register and then another thread + * advances s_wr_seq and calls smr_poll() which will + * oberve no value yet in c_seq and advance s_rd_seq + * up to s_wr_seq which is beyond the register + * cached value. This is only likely to happen on + * hypervisor or with a system management interrupt. + */ + if (SMR_SEQ_LT(c_seq, s_rd_seq)) + c_seq = s_rd_seq; + + /* + * If the sequence number meets the goal we are + * done with this cpu. + */ + if (SMR_SEQ_GEQ(c_seq, goal)) + break; + + /* + * If we're not waiting we will still scan the rest + * of the cpus and update s_rd_seq before returning + * an error. + */ + if (!wait) { + success = false; + break; + } + cpu_spinwait(); + } + + /* + * Limit the minimum observed rd_seq whether we met the goal + * or not. + */ + if (c_seq != SMR_SEQ_INVALID && SMR_SEQ_GT(rd_seq, c_seq)) + rd_seq = c_seq; + } + + /* + * Advance the rd_seq as long as we observed the most recent one. + */ + s_rd_seq = atomic_load_int(&s->s_rd_seq); + do { + if (SMR_SEQ_LEQ(rd_seq, s_rd_seq)) + break; + } while (atomic_fcmpset_int(&s->s_rd_seq, &s_rd_seq, rd_seq) == 0); + +out: + critical_exit(); + + return (success); +} + +smr_t +smr_create(const char *name) +{ + smr_t smr, c; + smr_shared_t s; + int i; + + s = uma_zalloc(smr_shared_zone, M_WAITOK); + smr = uma_zalloc(smr_zone, M_WAITOK); + + s->s_name = name; + s->s_rd_seq = s->s_wr_seq = SMR_SEQ_INIT; + + /* Initialize all CPUS, not just those running. */ + for (i = 0; i <= mp_maxid; i++) { + c = zpcpu_get_cpu(smr, i); + c->c_seq = SMR_SEQ_INVALID; + c->c_shared = s; + } + atomic_thread_fence_seq_cst(); + + return (smr); +} + +void +smr_destroy(smr_t smr) +{ + + smr_synchronize(smr); + uma_zfree(smr_shared_zone, smr->c_shared); + uma_zfree(smr_zone, smr); +} + +/* + * Initialize the UMA slab zone. + */ +void +smr_init(void) +{ + + smr_shared_zone = uma_zcreate("SMR SHARED", sizeof(struct smr_shared), + NULL, NULL, NULL, NULL, (CACHE_LINE_SIZE * 2) - 1, 0); + smr_zone = uma_zcreate("SMR CPU", sizeof(struct smr), + NULL, NULL, NULL, NULL, (CACHE_LINE_SIZE * 2) - 1, UMA_ZONE_PCPU); +} Index: head/sys/sys/_smr.h =================================================================== --- head/sys/sys/_smr.h +++ head/sys/sys/_smr.h @@ -0,0 +1,38 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019,2020 Jeffrey Roberson + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifndef _SYS__SMR_H_ +#define _SYS__SMR_H_ + +typedef uint32_t smr_seq_t; +typedef struct smr *smr_t; + +#endif /* __SYS_SMR_H_ */ Index: head/sys/sys/smr.h =================================================================== --- head/sys/sys/smr.h +++ head/sys/sys/smr.h @@ -0,0 +1,180 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019,2020 Jeffrey Roberson + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifndef _SYS_SMR_H_ +#define _SYS_SMR_H_ + +#include + +/* + * Safe memory reclamation. See subr_smr.c for a description of the + * algorithm. + * + * Readers synchronize with smr_enter()/exit() and writers may either + * free directly to a SMR UMA zone or use smr_synchronize or wait. + */ + +/* + * Modular arithmetic for comparing sequence numbers that have + * potentially wrapped. Copied from tcp_seq.h. + */ +#define SMR_SEQ_LT(a, b) ((int32_t)((a)-(b)) < 0) +#define SMR_SEQ_LEQ(a, b) ((int32_t)((a)-(b)) <= 0) +#define SMR_SEQ_GT(a, b) ((int32_t)((a)-(b)) > 0) +#define SMR_SEQ_GEQ(a, b) ((int32_t)((a)-(b)) >= 0) + +#define SMR_SEQ_INVALID 0 + +/* Shared SMR state. */ +struct smr_shared { + const char *s_name; /* Name for debugging/reporting. */ + smr_seq_t s_wr_seq; /* Current write sequence #. */ + smr_seq_t s_rd_seq; /* Minimum observed read sequence. */ +}; +typedef struct smr_shared *smr_shared_t; + +/* Per-cpu SMR state. */ +struct smr { + smr_seq_t c_seq; /* Current observed sequence. */ + smr_shared_t c_shared; /* Shared SMR state. */ +}; + +/* + * Return the current write sequence number. + */ +static inline smr_seq_t +smr_current(smr_t smr) +{ + + return (atomic_load_int(&smr->c_shared->s_wr_seq)); +} + +/* + * Enter a read section. + */ +static inline void +smr_enter(smr_t smr) +{ + + critical_enter(); + smr = zpcpu_get(smr); + KASSERT(smr->c_seq == 0, + ("smr_enter(%s) does not support recursion.", + smr->c_shared->s_name)); + + /* + * Store the current observed write sequence number in our + * per-cpu state so that it can be queried via smr_poll(). + * Frees that are newer than this stored value will be + * deferred until we call smr_exit(). + * + * An acquire barrier is used to synchronize with smr_exit() + * and smr_poll(). + * + * It is possible that a long delay between loading the wr_seq + * and storing the c_seq could create a situation where the + * rd_seq advances beyond our stored c_seq. In this situation + * only the observed wr_seq is stale, the fence still orders + * the load. See smr_poll() for details on how this condition + * is detected and handled there. + */ + /* This is an add because we do not have atomic_store_acq_int */ + atomic_add_acq_int(&smr->c_seq, smr_current(smr)); +} + +/* + * Exit a read section. + */ +static inline void +smr_exit(smr_t smr) +{ + + smr = zpcpu_get(smr); + CRITICAL_ASSERT(curthread); + KASSERT(smr->c_seq != SMR_SEQ_INVALID, + ("smr_exit(%s) not in a smr section.", smr->c_shared->s_name)); + + /* + * Clear the recorded sequence number. This allows poll() to + * detect CPUs not in read sections. + * + * Use release semantics to retire any stores before the sequence + * number is cleared. + */ + atomic_store_rel_int(&smr->c_seq, SMR_SEQ_INVALID); + critical_exit(); +} + +/* + * Advances the write sequence number. Returns the sequence number + * required to ensure that all modifications are visible to readers. + */ +smr_seq_t smr_advance(smr_t smr); + +/* + * Returns true if a goal sequence has been reached. If + * wait is true this will busy loop until success. + */ +bool smr_poll(smr_t smr, smr_seq_t goal, bool wait); + +/* Create a new SMR context. */ +smr_t smr_create(const char *name); +void smr_destroy(smr_t smr); + +/* + * Blocking wait for all readers to observe 'goal'. + */ +static inline bool +smr_wait(smr_t smr, smr_seq_t goal) +{ + + return (smr_poll(smr, goal, true)); +} + +/* + * Synchronize advances the write sequence and returns when all + * readers have observed it. + * + * If your application can cache a sequence number returned from + * smr_advance() and poll or wait at a later time there will + * be less chance of busy looping while waiting for readers. + */ +static inline void +smr_synchronize(smr_t smr) +{ + + smr_wait(smr, smr_advance(smr)); +} + +/* Only at startup. */ +void smr_init(void); + +#endif /* _SYS_SMR_H_ */ Index: head/sys/vm/uma.h =================================================================== --- head/sys/vm/uma.h +++ head/sys/vm/uma.h @@ -40,6 +40,7 @@ #include /* For NULL */ #include /* For M_* */ +#include /* User visible parameters */ #define UMA_SMALLEST_UNIT 8 /* Smallest item allocated */ @@ -259,16 +260,27 @@ * mini-dumps. */ #define UMA_ZONE_PCPU 0x8000 /* - * Allocates mp_maxid + 1 slabs of PAGE_SIZE + * Allocates mp_maxid + 1 slabs of + * PAGE_SIZE */ #define UMA_ZONE_FIRSTTOUCH 0x10000 /* First touch NUMA policy */ #define UMA_ZONE_ROUNDROBIN 0x20000 /* Round-robin NUMA policy. */ +#define UMA_ZONE_SMR 0x40000 /* + * Safe memory reclamation defers + * frees until all read sections + * have exited. This flag creates + * a unique SMR context for this + * zone. To share contexts see + * uma_zone_set_smr() below. + * + * See sys/smr.h for more details. + */ /* In use by UMA_ZFLAGs: 0xffe00000 */ /* - * These flags are shared between the keg and zone. In zones wishing to add - * new kegs these flags must be compatible. Some are determined based on - * physical parameters of the request and may not be provided by the consumer. + * These flags are shared between the keg and zone. Some are determined + * based on physical parameters of the request and may not be provided by + * the consumer. */ #define UMA_ZONE_INHERIT \ (UMA_ZONE_NOTOUCH | UMA_ZONE_MALLOC | UMA_ZONE_NOFREE | \ @@ -310,8 +322,13 @@ */ void *uma_zalloc_arg(uma_zone_t zone, void *arg, int flags); + +/* Allocate per-cpu data. Access the correct data with zpcpu_get(). */ void *uma_zalloc_pcpu_arg(uma_zone_t zone, void *arg, int flags); +/* Use with SMR zones. */ +void *uma_zalloc_smr(uma_zone_t zone, int flags); + /* * Allocate an item from a specific NUMA domain. This uses a slow path in * the allocator but is guaranteed to allocate memory from the requested @@ -359,8 +376,13 @@ */ void uma_zfree_arg(uma_zone_t zone, void *item, void *arg); + +/* Use with PCPU zones. */ void uma_zfree_pcpu_arg(uma_zone_t zone, void *item, void *arg); +/* Use with SMR zones. */ +void uma_zfree_smr(uma_zone_t zone, void *item); + /* * Frees an item back to the specified zone's domain specific pool. * @@ -599,6 +621,17 @@ */ void uma_zone_set_freef(uma_zone_t zone, uma_free freef); + +/* + * Associate a zone with a smr context that is allocated after creation + * so that multiple zones may share the same context. + */ +void uma_zone_set_smr(uma_zone_t zone, smr_t smr); + +/* + * Fetch the smr context that was set or made in uma_zcreate(). + */ +smr_t uma_zone_get_smr(uma_zone_t zone); /* * These flags are setable in the allocf and visible in the freef. Index: head/sys/vm/uma_core.c =================================================================== --- head/sys/vm/uma_core.c +++ head/sys/vm/uma_core.c @@ -77,6 +77,7 @@ #include #include #include +#include #include #include @@ -103,6 +104,12 @@ #include +#ifdef INVARIANTS +#define UMA_ALWAYS_CTORDTOR 1 +#else +#define UMA_ALWAYS_CTORDTOR 0 +#endif + /* * This is the zone and keg from which all zones are spawned. */ @@ -273,6 +280,8 @@ static void keg_dtor(void *, int, void *); static int zone_ctor(void *, int, void *, int); static void zone_dtor(void *, int, void *); +static inline void item_dtor(uma_zone_t zone, void *item, int size, + void *udata, enum zfreeskip skip); static int zero_init(void *, int, int); static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *); static void zone_foreach_unlocked(void (*zfunc)(uma_zone_t, void *), void *); @@ -454,9 +463,9 @@ uma_bucket_t bucket; /* - * Don't allocate buckets in low memory situations. + * Don't allocate buckets early in boot. */ - if (bucketdisable) + if (__predict_false(booted < BOOT_KVA)) return (NULL); /* @@ -488,6 +497,9 @@ #endif bucket->ub_cnt = 0; bucket->ub_entries = ubz->ubz_entries; + bucket->ub_seq = SMR_SEQ_INVALID; + CTR3(KTR_UMA, "bucket_alloc: zone %s(%p) allocated bucket %p", + zone->uz_name, zone, bucket); } return (bucket); @@ -500,6 +512,8 @@ KASSERT(bucket->ub_cnt == 0, ("bucket_free: Freeing a non free bucket.")); + KASSERT(bucket->ub_seq == SMR_SEQ_INVALID, + ("bucket_free: Freeing an SMR bucket.")); if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) udata = (void *)(uintptr_t)zone->uz_flags; ubz = bucket_zone_lookup(bucket->ub_entries); @@ -517,23 +531,39 @@ /* * Attempt to satisfy an allocation by retrieving a full bucket from one of the - * zone's caches. + * zone's caches. If a bucket is found the zone is not locked on return. */ static uma_bucket_t zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom) { uma_bucket_t bucket; + int i; + bool dtor = false; ZONE_LOCK_ASSERT(zone); - if ((bucket = TAILQ_FIRST(&zdom->uzd_buckets)) != NULL) { - MPASS(zdom->uzd_nitems >= bucket->ub_cnt); - TAILQ_REMOVE(&zdom->uzd_buckets, bucket, ub_link); - zdom->uzd_nitems -= bucket->ub_cnt; - if (zdom->uzd_imin > zdom->uzd_nitems) - zdom->uzd_imin = zdom->uzd_nitems; - zone->uz_bkt_count -= bucket->ub_cnt; + if ((bucket = TAILQ_FIRST(&zdom->uzd_buckets)) == NULL) + return (NULL); + + if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && + bucket->ub_seq != SMR_SEQ_INVALID) { + if (!smr_poll(zone->uz_smr, bucket->ub_seq, false)) + return (NULL); + bucket->ub_seq = SMR_SEQ_INVALID; + dtor = (zone->uz_dtor != NULL) | UMA_ALWAYS_CTORDTOR; } + MPASS(zdom->uzd_nitems >= bucket->ub_cnt); + TAILQ_REMOVE(&zdom->uzd_buckets, bucket, ub_link); + zdom->uzd_nitems -= bucket->ub_cnt; + if (zdom->uzd_imin > zdom->uzd_nitems) + zdom->uzd_imin = zdom->uzd_nitems; + zone->uz_bkt_count -= bucket->ub_cnt; + ZONE_UNLOCK(zone); + if (dtor) + for (i = 0; i < bucket->ub_cnt; i++) + item_dtor(zone, bucket->ub_bucket[i], zone->uz_size, + NULL, SKIP_NONE); + return (bucket); } @@ -551,7 +581,7 @@ KASSERT(!ws || zone->uz_bkt_count < zone->uz_bkt_max, ("%s: zone %p overflow", __func__, zone)); - if (ws) + if (ws && bucket->ub_seq == SMR_SEQ_INVALID) TAILQ_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link); else TAILQ_INSERT_TAIL(&zdom->uzd_buckets, bucket, ub_link); @@ -941,12 +971,23 @@ if (bucket == NULL || bucket->ub_cnt == 0) return; + if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && + bucket->ub_seq != SMR_SEQ_INVALID) { + smr_wait(zone->uz_smr, bucket->ub_seq); + for (i = 0; i < bucket->ub_cnt; i++) + item_dtor(zone, bucket->ub_bucket[i], + zone->uz_size, NULL, SKIP_NONE); + bucket->ub_seq = SMR_SEQ_INVALID; + } if (zone->uz_fini) for (i = 0; i < bucket->ub_cnt; i++) zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); if (zone->uz_max_items > 0) zone_free_limit(zone, bucket->ub_cnt); +#ifdef INVARIANTS + bzero(bucket->ub_bucket, sizeof(void *) * bucket->ub_cnt); +#endif bucket->ub_cnt = 0; } @@ -1035,12 +1076,21 @@ zone_put_bucket(zone, &zone->uz_domain[domain], b1, false); b1 = NULL; } + + /* + * Don't flush SMR zone buckets. This leaves the zone without a + * bucket and forces every free to synchronize(). + */ + if ((zone->uz_flags & UMA_ZONE_SMR) != 0) + goto out; b2 = cache_bucket_unload_free(cache); if (b2 != NULL && b2->ub_cnt != 0) { zone_put_bucket(zone, &zone->uz_domain[domain], b2, false); b2 = NULL; } b3 = cache_bucket_unload_cross(cache); + +out: critical_exit(); ZONE_UNLOCK(zone); if (b1) @@ -1135,7 +1185,7 @@ target = drain ? 0 : lmax(zdom->uzd_wss, zdom->uzd_nitems - zdom->uzd_imin); while (zdom->uzd_nitems > target) { - bucket = TAILQ_LAST(&zdom->uzd_buckets, uma_bucketlist); + bucket = TAILQ_FIRST(&zdom->uzd_buckets); if (bucket == NULL) break; tofree = bucket->ub_cnt; @@ -2294,7 +2344,7 @@ zone->uz_bucket_size = 0; zone->uz_bucket_size_min = 0; zone->uz_bucket_size_max = BUCKET_MAX; - zone->uz_flags = 0; + zone->uz_flags = (arg->flags & UMA_ZONE_SMR); zone->uz_warning = NULL; /* The domain structures follow the cpu structures. */ zone->uz_domain = @@ -2375,7 +2425,7 @@ karg.uminit = arg->uminit; karg.fini = arg->fini; karg.align = arg->align; - karg.flags = arg->flags; + karg.flags = (arg->flags & ~UMA_ZONE_SMR); karg.zone = zone; error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, flags); @@ -2399,6 +2449,10 @@ zone->uz_fails = EARLY_COUNTER; } + /* Caller requests a private SMR context. */ + if ((zone->uz_flags & UMA_ZONE_SMR) != 0) + zone->uz_smr = smr_create(zone->uz_name); + KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) != (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET), ("Invalid zone flag combination")); @@ -2600,6 +2654,7 @@ NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); bucket_init(); + smr_init(); } #ifndef UMA_MD_SMALL_ALLOC @@ -2844,14 +2899,9 @@ uma_zfree_arg(zone, item, udata); } -#ifdef INVARIANTS -#define UMA_ALWAYS_CTORDTOR 1 -#else -#define UMA_ALWAYS_CTORDTOR 0 -#endif - -static void * -item_ctor(uma_zone_t zone, int size, void *udata, int flags, void *item) +static inline void * +item_ctor(uma_zone_t zone, int uz_flags, int size, void *udata, int flags, + void *item) { #ifdef INVARIANTS bool skipdbg; @@ -2861,7 +2911,9 @@ zone->uz_ctor != trash_ctor) trash_ctor(item, size, udata, flags); #endif - if (__predict_false(zone->uz_ctor != NULL) && + /* Check flags before loading ctor pointer. */ + if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0) && + __predict_false(zone->uz_ctor != NULL) && zone->uz_ctor(item, size, udata, flags) != 0) { counter_u64_add(zone->uz_fails, 1); zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); @@ -2903,58 +2955,151 @@ } } -/* See uma.h */ -void * -uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) +#if defined(INVARIANTS) || defined(DEBUG_MEMGUARD) || defined(WITNESS) +#define UMA_ZALLOC_DEBUG +static int +uma_zalloc_debug(uma_zone_t zone, void **itemp, void *udata, int flags) { - uma_cache_bucket_t bucket; - uma_cache_t cache; - void *item; - int domain, size, uz_flags; + int error; - /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ - random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); - - /* This is the fast path allocation */ - CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name, - zone, flags); - + error = 0; #ifdef WITNESS if (flags & M_WAITOK) { WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, - "uma_zalloc_arg: zone \"%s\"", zone->uz_name); + "uma_zalloc_debug: zone \"%s\"", zone->uz_name); } #endif #ifdef INVARIANTS - KASSERT((flags & M_EXEC) == 0, ("uma_zalloc_arg: called with M_EXEC")); + KASSERT((flags & M_EXEC) == 0, + ("uma_zalloc_debug: called with M_EXEC")); KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), - ("uma_zalloc_arg: called with spinlock or critical section held")); - if (zone->uz_flags & UMA_ZONE_PCPU) - KASSERT((flags & M_ZERO) == 0, ("allocating from a pcpu zone " - "with M_ZERO passed")); + ("uma_zalloc_debug: called within spinlock or critical section")); + KASSERT((zone->uz_flags & UMA_ZONE_PCPU) == 0 || (flags & M_ZERO) == 0, + ("uma_zalloc_debug: allocating from a pcpu zone with M_ZERO")); #endif #ifdef DEBUG_MEMGUARD - if (memguard_cmp_zone(zone)) { + if ((zone->uz_flags & UMA_ZONE_SMR == 0) && memguard_cmp_zone(zone)) { + void *item; item = memguard_alloc(zone->uz_size, flags); if (item != NULL) { + error = EJUSTRETURN; if (zone->uz_init != NULL && - zone->uz_init(item, zone->uz_size, flags) != 0) - return (NULL); + zone->uz_init(item, zone->uz_size, flags) != 0) { + *itemp = NULL; + return (error); + } if (zone->uz_ctor != NULL && zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { counter_u64_add(zone->uz_fails, 1); zone->uz_fini(item, zone->uz_size); - return (NULL); + *itemp = NULL; + return (error); } - return (item); + *itemp = item; + return (error); } /* This is unfortunate but should not be fatal. */ } #endif + return (error); +} + +static int +uma_zfree_debug(uma_zone_t zone, void *item, void *udata) +{ + KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), + ("uma_zfree_debug: called with spinlock or critical section held")); + +#ifdef DEBUG_MEMGUARD + if ((zone->uz_flags & UMA_ZONE_SMR == 0) && is_memguard_addr(item)) { + if (zone->uz_dtor != NULL) + zone->uz_dtor(item, zone->uz_size, udata); + if (zone->uz_fini != NULL) + zone->uz_fini(item, zone->uz_size); + memguard_free(item); + return (EJUSTRETURN); + } +#endif + return (0); +} +#endif + +static __noinline void * +uma_zalloc_single(uma_zone_t zone, void *udata, int flags) +{ + int domain; + /* + * We can not get a bucket so try to return a single item. + */ + if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH) + domain = PCPU_GET(domain); + else + domain = UMA_ANYDOMAIN; + return (zone_alloc_item(zone, udata, domain, flags)); +} + +/* See uma.h */ +void * +uma_zalloc_smr(uma_zone_t zone, int flags) +{ + uma_cache_bucket_t bucket; + uma_cache_t cache; + void *item; + int size, uz_flags; + +#ifdef UMA_ZALLOC_DEBUG + KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, + ("uma_zalloc_arg: called with non-SMR zone.\n")); + if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) + return (item); +#endif + + critical_enter(); + do { + cache = &zone->uz_cpu[curcpu]; + bucket = &cache->uc_allocbucket; + size = cache_uz_size(cache); + uz_flags = cache_uz_flags(cache); + if (__predict_true(bucket->ucb_cnt != 0)) { + item = cache_bucket_pop(cache, bucket); + critical_exit(); + return (item_ctor(zone, uz_flags, size, NULL, flags, + item)); + } + } while (cache_alloc(zone, cache, NULL, flags)); + critical_exit(); + + return (uma_zalloc_single(zone, NULL, flags)); +} + +/* See uma.h */ +void * +uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) +{ + uma_cache_bucket_t bucket; + uma_cache_t cache; + void *item; + int size, uz_flags; + + /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ + random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); + + /* This is the fast path allocation */ + CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name, + zone, flags); + +#ifdef UMA_ZALLOC_DEBUG + KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, + ("uma_zalloc_arg: called with SMR zone.\n")); + if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) + return (item); +#endif + + /* * If possible, allocate from the per-CPU cache. There are two * requirements for safe access to the per-CPU cache: (1) the thread * accessing the cache must not be preempted or yield during access, @@ -2974,24 +3119,13 @@ if (__predict_true(bucket->ucb_cnt != 0)) { item = cache_bucket_pop(cache, bucket); critical_exit(); - if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0 || - UMA_ALWAYS_CTORDTOR)) - return (item_ctor(zone, size, udata, flags, item)); - if (flags & M_ZERO) - bzero(item, size); - return (item); + return (item_ctor(zone, uz_flags, size, udata, flags, + item)); } } while (cache_alloc(zone, cache, udata, flags)); critical_exit(); - /* - * We can not get a bucket so try to return a single item. - */ - if (uz_flags & UMA_ZONE_FIRSTTOUCH) - domain = PCPU_GET(domain); - else - domain = UMA_ANYDOMAIN; - return (zone_alloc_item(zone, udata, domain, flags)); + return (uma_zalloc_single(zone, udata, flags)); } /* @@ -3014,9 +3148,14 @@ /* * If we have run out of items in our alloc bucket see * if we can switch with the free bucket. + * + * SMR Zones can't re-use the free bucket until the sequence has + * expired. */ - if (cache->uc_freebucket.ucb_cnt != 0) { - cache_bucket_swap(&cache->uc_freebucket, &cache->uc_allocbucket); + if ((zone->uz_flags & UMA_ZONE_SMR) == 0 && + cache->uc_freebucket.ucb_cnt != 0) { + cache_bucket_swap(&cache->uc_freebucket, + &cache->uc_allocbucket); return (true); } @@ -3070,7 +3209,6 @@ } if ((bucket = zone_fetch_bucket(zone, zdom)) != NULL) { - ZONE_UNLOCK(zone); KASSERT(bucket->ub_cnt != 0, ("uma_zalloc_arg: Returning an empty bucket.")); cache_bucket_load_alloc(cache, bucket); @@ -3607,7 +3745,8 @@ goto fail_cnt; } } - item = item_ctor(zone, zone->uz_size, udata, flags, item); + item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, flags, + item); if (item == NULL) goto fail; @@ -3630,6 +3769,54 @@ /* See uma.h */ void +uma_zfree_smr(uma_zone_t zone, void *item) +{ + uma_cache_t cache; + uma_cache_bucket_t bucket; + int domain, itemdomain, uz_flags; + +#ifdef UMA_ZALLOC_DEBUG + KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, + ("uma_zfree_smr: called with non-SMR zone.\n")); + KASSERT(item != NULL, ("uma_zfree_smr: Called with NULL pointer.")); + if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN) + return; +#endif + cache = &zone->uz_cpu[curcpu]; + uz_flags = cache_uz_flags(cache); + domain = itemdomain = 0; +#ifdef NUMA + if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) + itemdomain = _vm_phys_domain(pmap_kextract((vm_offset_t)item)); +#endif + critical_enter(); + do { + cache = &zone->uz_cpu[curcpu]; + /* SMR Zones must free to the free bucket. */ + bucket = &cache->uc_freebucket; +#ifdef NUMA + domain = PCPU_GET(domain); + if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && + domain != itemdomain) { + bucket = &cache->uc_crossbucket; + } +#endif + if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { + cache_bucket_push(cache, bucket, item); + critical_exit(); + return; + } + } while (cache_free(zone, cache, NULL, item, itemdomain)); + critical_exit(); + + /* + * If nothing else caught this, we'll just do an internal free. + */ + zone_free_item(zone, item, NULL, SKIP_NONE); +} + +/* See uma.h */ +void uma_zfree_arg(uma_zone_t zone, void *item, void *udata) { uma_cache_t cache; @@ -3641,22 +3828,15 @@ CTR2(KTR_UMA, "uma_zfree_arg zone %s(%p)", zone->uz_name, zone); - KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), - ("uma_zfree_arg: called with spinlock or critical section held")); - +#ifdef UMA_ZALLOC_DEBUG + KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, + ("uma_zfree_arg: called with SMR zone.\n")); + if (uma_zfree_debug(zone, item, udata) == EJUSTRETURN) + return; +#endif /* uma_zfree(..., NULL) does nothing, to match free(9). */ if (item == NULL) return; -#ifdef DEBUG_MEMGUARD - if (is_memguard_addr(item)) { - if (zone->uz_dtor != NULL) - zone->uz_dtor(item, zone->uz_size, udata); - if (zone->uz_fini != NULL) - zone->uz_fini(item, zone->uz_size); - memguard_free(item); - return; - } -#endif /* * We are accessing the per-cpu cache without a critical section to @@ -3665,8 +3845,8 @@ */ cache = &zone->uz_cpu[curcpu]; uz_flags = cache_uz_flags(cache); - if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0 || - UMA_ALWAYS_CTORDTOR)) + if (UMA_ALWAYS_CTORDTOR || + __predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0)) item_dtor(zone, item, cache_uz_size(cache), udata, SKIP_NONE); /* @@ -3697,6 +3877,13 @@ critical_enter(); do { cache = &zone->uz_cpu[curcpu]; + /* + * Try to free into the allocbucket first to give LIFO + * ordering for cache-hot datastructures. Spill over + * into the freebucket if necessary. Alloc will swap + * them if one runs dry. + */ + bucket = &cache->uc_allocbucket; #ifdef NUMA domain = PCPU_GET(domain); if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && @@ -3704,18 +3891,8 @@ bucket = &cache->uc_crossbucket; } else #endif - { - /* - * Try to free into the allocbucket first to give LIFO - * ordering for cache-hot datastructures. Spill over - * into the freebucket if necessary. Alloc will swap - * them if one runs dry. - */ - bucket = &cache->uc_allocbucket; - if (__predict_false(bucket->ucb_cnt >= - bucket->ucb_entries)) - bucket = &cache->uc_freebucket; - } + if (bucket->ucb_cnt >= bucket->ucb_entries) + bucket = &cache->uc_freebucket; if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { cache_bucket_push(cache, bucket, item); critical_exit(); @@ -3778,6 +3955,8 @@ if (!TAILQ_EMPTY(&fullbuckets)) { ZONE_LOCK(zone); while ((b = TAILQ_FIRST(&fullbuckets)) != NULL) { + if ((zone->uz_flags & UMA_ZONE_SMR) != 0) + bucket->ub_seq = smr_current(zone->uz_smr); TAILQ_REMOVE(&fullbuckets, b, ub_link); if (zone->uz_bkt_count >= zone->uz_bkt_max) { ZONE_UNLOCK(zone); @@ -3796,6 +3975,7 @@ } if (bucket->ub_cnt != 0) bucket_drain(zone, bucket); + bucket->ub_seq = SMR_SEQ_INVALID; bucket_free(zone, bucket, udata); } #endif @@ -3862,15 +4042,16 @@ int itemdomain) { uma_cache_bucket_t cbucket; - uma_bucket_t bucket; + uma_bucket_t newbucket, bucket; int domain; CRITICAL_ASSERT(curthread); - if (zone->uz_bucket_size == 0 || bucketdisable) + if (zone->uz_bucket_size == 0) return false; cache = &zone->uz_cpu[curcpu]; + newbucket = NULL; /* * FIRSTTOUCH domains need to free to the correct zdom. When @@ -3895,14 +4076,29 @@ /* We are no longer associated with this CPU. */ critical_exit(); + /* + * Don't let SMR zones operate without a free bucket. Force + * a synchronize and re-use this one. We will only degrade + * to a synchronize every bucket_size items rather than every + * item if we fail to allocate a bucket. + */ + if ((zone->uz_flags & UMA_ZONE_SMR) != 0) { + if (bucket != NULL) + bucket->ub_seq = smr_advance(zone->uz_smr); + newbucket = bucket_alloc(zone, udata, M_NOWAIT); + if (newbucket == NULL && bucket != NULL) { + bucket_drain(zone, bucket); + newbucket = bucket; + bucket = NULL; + } + } else if (!bucketdisable) + newbucket = bucket_alloc(zone, udata, M_NOWAIT); + if (bucket != NULL) zone_free_bucket(zone, bucket, udata, domain, itemdomain); - bucket = bucket_alloc(zone, udata, M_NOWAIT); - CTR3(KTR_UMA, "uma_zfree: zone %s(%p) allocated bucket %p", - zone->uz_name, zone, bucket); critical_enter(); - if (bucket == NULL) + if ((bucket = newbucket) == NULL) return (false); cache = &zone->uz_cpu[curcpu]; #ifdef NUMA @@ -4031,6 +4227,15 @@ zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) { + /* + * If a free is sent directly to an SMR zone we have to + * synchronize immediately because the item can instantly + * be reallocated. This should only happen in degenerate + * cases when no memory is available for per-cpu caches. + */ + if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && skip == SKIP_NONE) + smr_synchronize(zone->uz_smr); + item_dtor(zone, item, zone->uz_size, udata, skip); if (skip < SKIP_FINI && zone->uz_fini) @@ -4253,6 +4458,25 @@ KEG_GET(zone, keg); KEG_ASSERT_COLD(keg); keg->uk_allocf = allocf; +} + +/* See uma.h */ +void +uma_zone_set_smr(uma_zone_t zone, smr_t smr) +{ + + ZONE_ASSERT_COLD(zone); + + zone->uz_flags |= UMA_ZONE_SMR; + zone->uz_smr = smr; + zone_update_caches(zone); +} + +smr_t +uma_zone_get_smr(uma_zone_t zone) +{ + + return (zone->uz_smr); } /* See uma.h */ Index: head/sys/vm/uma_int.h =================================================================== --- head/sys/vm/uma_int.h +++ head/sys/vm/uma_int.h @@ -184,6 +184,7 @@ "\30VTOSLAB" \ "\27HASH" \ "\26OFFPAGE" \ + "\23SMR" \ "\22ROUNDROBIN" \ "\21FIRSTTOUCH" \ "\20PCPU" \ @@ -245,9 +246,10 @@ */ struct uma_bucket { TAILQ_ENTRY(uma_bucket) ub_link; /* Link into the zone */ - int16_t ub_cnt; /* Count of items in bucket. */ - int16_t ub_entries; /* Max items. */ - void *ub_bucket[]; /* actual allocation storage */ + int16_t ub_cnt; /* Count of items in bucket. */ + int16_t ub_entries; /* Max items. */ + smr_seq_t ub_seq; /* SMR sequence number. */ + void *ub_bucket[]; /* actual allocation storage */ }; typedef struct uma_bucket * uma_bucket_t; @@ -484,7 +486,7 @@ uint32_t uz_size; /* Size inherited from kegs */ uma_ctor uz_ctor; /* Constructor for each allocation */ uma_dtor uz_dtor; /* Destructor */ - uint64_t uz_spare0; + smr_t uz_smr; /* Safe memory reclaim context. */ uint64_t uz_max_items; /* Maximum number of items to alloc */ uint32_t uz_sleepers; /* Threads sleeping on limit */ uint16_t uz_bucket_size; /* Number of items in full bucket */