Changeset View
Standalone View
sys/vm/uma_core.c
Show First 20 Lines • Show All 71 Lines • ▼ Show 20 Lines | |||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/random.h> | #include <sys/random.h> | ||||
#include <sys/rwlock.h> | #include <sys/rwlock.h> | ||||
#include <sys/sbuf.h> | #include <sys/sbuf.h> | ||||
#include <sys/sched.h> | #include <sys/sched.h> | ||||
#include <sys/sleepqueue.h> | #include <sys/sleepqueue.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/smr.h> | |||||
#include <sys/taskqueue.h> | #include <sys/taskqueue.h> | ||||
#include <sys/vmmeter.h> | #include <sys/vmmeter.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/vm_domainset.h> | #include <vm/vm_domainset.h> | ||||
#include <vm/vm_object.h> | #include <vm/vm_object.h> | ||||
#include <vm/vm_page.h> | #include <vm/vm_page.h> | ||||
#include <vm/vm_pageout.h> | #include <vm/vm_pageout.h> | ||||
▲ Show 20 Lines • Show All 133 Lines • ▼ Show 20 Lines | struct uma_bucket_zone { | ||||
int ubz_maxsize; /* Maximum allocation size per-item. */ | int ubz_maxsize; /* Maximum allocation size per-item. */ | ||||
}; | }; | ||||
/* | /* | ||||
* Compute the actual number of bucket entries to pack them in power | * Compute the actual number of bucket entries to pack them in power | ||||
* of two sizes for more efficient space utilization. | * of two sizes for more efficient space utilization. | ||||
*/ | */ | ||||
#define BUCKET_SIZE(n) \ | #define BUCKET_SIZE(n) \ | ||||
(((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *)) | (((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *)) | ||||
rlibby: More parens for hygiene, although obviously safe in practice below. | |||||
Done Inline ActionsI was experimenting with a 64bit value so I could use tsc instead of a synthetic clock and this was necessary. It is likely valuable on its own and should be committed separately. jeff: I was experimenting with a 64bit value so I could use tsc instead of a synthetic clock and this… | |||||
#define BUCKET_MAX BUCKET_SIZE(256) | #define BUCKET_MAX BUCKET_SIZE(256) | ||||
#define BUCKET_MIN BUCKET_SIZE(4) | #define BUCKET_MIN BUCKET_SIZE(4) | ||||
struct uma_bucket_zone bucket_zones[] = { | struct uma_bucket_zone bucket_zones[] = { | ||||
{ NULL, "4 Bucket", BUCKET_SIZE(4), 4096 }, | { NULL, "4 Bucket", BUCKET_SIZE(4), 4096 }, | ||||
{ NULL, "6 Bucket", BUCKET_SIZE(6), 3072 }, | { NULL, "6 Bucket", BUCKET_SIZE(6), 3072 }, | ||||
{ NULL, "8 Bucket", BUCKET_SIZE(8), 2048 }, | { NULL, "8 Bucket", BUCKET_SIZE(8), 2048 }, | ||||
{ NULL, "12 Bucket", BUCKET_SIZE(12), 1536 }, | { NULL, "12 Bucket", BUCKET_SIZE(12), 1536 }, | ||||
Show All 29 Lines | |||||
static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int, int); | static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int, int); | ||||
static void cache_drain(uma_zone_t); | static void cache_drain(uma_zone_t); | ||||
static void bucket_drain(uma_zone_t, uma_bucket_t); | static void bucket_drain(uma_zone_t, uma_bucket_t); | ||||
static void bucket_cache_reclaim(uma_zone_t zone, bool); | static void bucket_cache_reclaim(uma_zone_t zone, bool); | ||||
static int keg_ctor(void *, int, void *, int); | static int keg_ctor(void *, int, void *, int); | ||||
static void keg_dtor(void *, int, void *); | static void keg_dtor(void *, int, void *); | ||||
static int zone_ctor(void *, int, void *, int); | static int zone_ctor(void *, int, void *, int); | ||||
static void zone_dtor(void *, int, void *); | static void zone_dtor(void *, int, void *); | ||||
static inline void item_dtor(uma_zone_t zone, void *item, int size, | |||||
void *udata, enum zfreeskip skip); | |||||
static int zero_init(void *, int, int); | static int zero_init(void *, int, int); | ||||
static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *); | static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *); | ||||
static void zone_foreach_unlocked(void (*zfunc)(uma_zone_t, void *), void *); | static void zone_foreach_unlocked(void (*zfunc)(uma_zone_t, void *), void *); | ||||
static void zone_timeout(uma_zone_t zone, void *); | static void zone_timeout(uma_zone_t zone, void *); | ||||
static int hash_alloc(struct uma_hash *, u_int); | static int hash_alloc(struct uma_hash *, u_int); | ||||
static int hash_expand(struct uma_hash *, struct uma_hash *); | static int hash_expand(struct uma_hash *, struct uma_hash *); | ||||
static void hash_free(struct uma_hash *hash); | static void hash_free(struct uma_hash *hash); | ||||
static void uma_timeout(void *); | static void uma_timeout(void *); | ||||
▲ Show 20 Lines • Show All 165 Lines • ▼ Show 20 Lines | |||||
static uma_bucket_t | static uma_bucket_t | ||||
bucket_alloc(uma_zone_t zone, void *udata, int flags) | bucket_alloc(uma_zone_t zone, void *udata, int flags) | ||||
{ | { | ||||
struct uma_bucket_zone *ubz; | struct uma_bucket_zone *ubz; | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
/* | /* | ||||
* Don't allocate buckets in low memory situations. | * Don't allocate buckets early in boot. | ||||
*/ | */ | ||||
if (bucketdisable) | if (__predict_false(booted < BOOT_KVA)) | ||||
Not Done Inline ActionsThis is fine because callers of zone_alloc_bucket() and zone_free_bucket() already check bucketdisable, right? markj: This is fine because callers of zone_alloc_bucket() and zone_free_bucket() already check… | |||||
Done Inline ActionsYes. The one wrinkle with UMA integration is that you more or less can't run without a per-cpu bucket in a SMR zone. If you can't get a bucket it means you're going to serialize on every free. So I changed the bucketdisable logic to ignore SMR zones and added the bit that preserves the existing bucket if you can't allocate so that you get at least N frees per-synchronization event. You could push the epoch # into the slab layer and force it to handle smr as well and get rid of the constraints on buckets. But it's a lot more complicated for questionable gain. jeff: Yes. The one wrinkle with UMA integration is that you more or less can't run without a per-cpu… | |||||
return (NULL); | return (NULL); | ||||
/* | /* | ||||
* To limit bucket recursion we store the original zone flags | * To limit bucket recursion we store the original zone flags | ||||
* in a cookie passed via zalloc_arg/zfree_arg. This allows the | * in a cookie passed via zalloc_arg/zfree_arg. This allows the | ||||
* NOVM flag to persist even through deep recursions. We also | * NOVM flag to persist even through deep recursions. We also | ||||
* store ZFLAG_BUCKET once we have recursed attempting to allocate | * store ZFLAG_BUCKET once we have recursed attempting to allocate | ||||
* a bucket for a bucket zone so we do not allow infinite bucket | * a bucket for a bucket zone so we do not allow infinite bucket | ||||
Show All 15 Lines | if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0) | ||||
ubz++; | ubz++; | ||||
bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags); | bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags); | ||||
if (bucket) { | if (bucket) { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); | bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); | ||||
#endif | #endif | ||||
bucket->ub_cnt = 0; | bucket->ub_cnt = 0; | ||||
bucket->ub_entries = ubz->ubz_entries; | bucket->ub_entries = ubz->ubz_entries; | ||||
bucket->ub_seq = 0; | |||||
markjUnsubmitted Not Done Inline ActionsShouldn't we be using the named constant SMR_SEQ_INVALID here and elsewhere in UMA? markj: Shouldn't we be using the named constant SMR_SEQ_INVALID here and elsewhere in UMA? | |||||
CTR3(KTR_UMA, "bucket_alloc: zone %s(%p) allocated bucket %p", | |||||
zone->uz_name, zone, bucket); | |||||
} | } | ||||
return (bucket); | return (bucket); | ||||
} | } | ||||
static void | static void | ||||
bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata) | bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata) | ||||
{ | { | ||||
struct uma_bucket_zone *ubz; | struct uma_bucket_zone *ubz; | ||||
KASSERT(bucket->ub_cnt == 0, | KASSERT(bucket->ub_cnt == 0, | ||||
("bucket_free: Freeing a non free bucket.")); | ("bucket_free: Freeing a non free bucket.")); | ||||
KASSERT(bucket->ub_seq == 0, | |||||
("bucket_free: Freeing an SMR bucket.")); | |||||
if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) | if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) | ||||
udata = (void *)(uintptr_t)zone->uz_flags; | udata = (void *)(uintptr_t)zone->uz_flags; | ||||
ubz = bucket_zone_lookup(bucket->ub_entries); | ubz = bucket_zone_lookup(bucket->ub_entries); | ||||
uma_zfree_arg(ubz->ubz_zone, bucket, udata); | uma_zfree_arg(ubz->ubz_zone, bucket, udata); | ||||
} | } | ||||
static void | static void | ||||
bucket_zone_drain(void) | bucket_zone_drain(void) | ||||
{ | { | ||||
struct uma_bucket_zone *ubz; | struct uma_bucket_zone *ubz; | ||||
for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) | for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) | ||||
uma_zone_reclaim(ubz->ubz_zone, UMA_RECLAIM_DRAIN); | uma_zone_reclaim(ubz->ubz_zone, UMA_RECLAIM_DRAIN); | ||||
} | } | ||||
/* | /* | ||||
* Attempt to satisfy an allocation by retrieving a full bucket from one of the | * Attempt to satisfy an allocation by retrieving a full bucket from one of the | ||||
* zone's caches. | * zone's caches. If a bucket is found the zone is not locked on return. | ||||
*/ | */ | ||||
static uma_bucket_t | static uma_bucket_t | ||||
zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom) | zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom) | ||||
{ | { | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
int i; | |||||
bool dtor = false; | |||||
ZONE_LOCK_ASSERT(zone); | ZONE_LOCK_ASSERT(zone); | ||||
if ((bucket = TAILQ_FIRST(&zdom->uzd_buckets)) != NULL) { | if ((bucket = TAILQ_FIRST(&zdom->uzd_buckets)) == NULL) | ||||
return (NULL); | |||||
if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && bucket->ub_seq) { | |||||
if (!smr_poll(zone->uz_smr, bucket->ub_seq, false)) | |||||
return (NULL); | |||||
markjUnsubmitted Not Done Inline ActionsIt would be nice to have a counter for this case. markj: It would be nice to have a counter for this case. | |||||
bucket->ub_seq = 0; | |||||
dtor = true; | |||||
markjUnsubmitted Not Done Inline ActionsCan't we optimize this a bit by writing dtor = (uz_flags & UMA_ZFLAG_CTORDTOR) != 0 || UMA_ALWAYS_CTORDTOR? markj: Can't we optimize this a bit by writing dtor = (uz_flags & UMA_ZFLAG_CTORDTOR) != 0 ||… | |||||
jeffAuthorUnsubmitted Done Inline ActionsYes I can make the test an inline since it would be repeated. jeff: Yes I can make the test an inline since it would be repeated. | |||||
markjUnsubmitted Not Done Inline ActionsSorry, I don't quite follow what you mean. I'm just pointing out that we may be unnecessarily iterating over the bucket entries, same in bucket_drain(). markj: Sorry, I don't quite follow what you mean. I'm just pointing out that we may be unnecessarily… | |||||
} | |||||
MPASS(zdom->uzd_nitems >= bucket->ub_cnt); | MPASS(zdom->uzd_nitems >= bucket->ub_cnt); | ||||
TAILQ_REMOVE(&zdom->uzd_buckets, bucket, ub_link); | TAILQ_REMOVE(&zdom->uzd_buckets, bucket, ub_link); | ||||
zdom->uzd_nitems -= bucket->ub_cnt; | zdom->uzd_nitems -= bucket->ub_cnt; | ||||
if (zdom->uzd_imin > zdom->uzd_nitems) | if (zdom->uzd_imin > zdom->uzd_nitems) | ||||
zdom->uzd_imin = zdom->uzd_nitems; | zdom->uzd_imin = zdom->uzd_nitems; | ||||
zone->uz_bkt_count -= bucket->ub_cnt; | zone->uz_bkt_count -= bucket->ub_cnt; | ||||
} | ZONE_UNLOCK(zone); | ||||
if (dtor) | |||||
for (i = 0; i < bucket->ub_cnt; i++) | |||||
item_dtor(zone, bucket->ub_bucket[i], zone->uz_size, | |||||
NULL, SKIP_NONE); | |||||
return (bucket); | return (bucket); | ||||
} | } | ||||
/* | /* | ||||
* Insert a full bucket into the specified cache. The "ws" parameter indicates | * Insert a full bucket into the specified cache. The "ws" parameter indicates | ||||
* whether the bucket's contents should be counted as part of the zone's working | * whether the bucket's contents should be counted as part of the zone's working | ||||
* set. | * set. | ||||
*/ | */ | ||||
static void | static void | ||||
zone_put_bucket(uma_zone_t zone, uma_zone_domain_t zdom, uma_bucket_t bucket, | zone_put_bucket(uma_zone_t zone, uma_zone_domain_t zdom, uma_bucket_t bucket, | ||||
const bool ws) | const bool ws) | ||||
{ | { | ||||
ZONE_LOCK_ASSERT(zone); | ZONE_LOCK_ASSERT(zone); | ||||
KASSERT(!ws || zone->uz_bkt_count < zone->uz_bkt_max, | KASSERT(!ws || zone->uz_bkt_count < zone->uz_bkt_max, | ||||
("%s: zone %p overflow", __func__, zone)); | ("%s: zone %p overflow", __func__, zone)); | ||||
if (ws) | |||||
TAILQ_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link); | |||||
else | |||||
TAILQ_INSERT_TAIL(&zdom->uzd_buckets, bucket, ub_link); | TAILQ_INSERT_TAIL(&zdom->uzd_buckets, bucket, ub_link); | ||||
markjUnsubmitted Not Done Inline ActionsThis means that the bucket cache is now FIFO for all zones, and zone_fetch_bucket() is getting the least recently used bucket. Can't we use LRU only for SMR zones? It is also kind of weird that bucket_cache_reclaim() still fetches buckets from the tail of the queue. For SMR zones this increases the likelihood that a draining thread will have to poll while waiting for the read sequence number to advance. markj: This means that the bucket cache is now FIFO for all zones, and zone_fetch_bucket() is getting… | |||||
jeffAuthorUnsubmitted Done Inline ActionsYes I can restore this. Your observation about reclaim is important. I will fix both. jeff: Yes I can restore this. Your observation about reclaim is important. I will fix both. | |||||
jeffAuthorUnsubmitted Done Inline ActionsI have my doubts about the efficacy of this given that we have two buckets already between alloc/free. So data in uzd_buckets is going to be stale and going to a different cpu. However, I have restored it. I'm just going to have reclaim pop the head though. Even if there is some minor improvement in cache effect it is rare enough that it won't matter and I'm sensitive to adding tons of special cases to support this. jeff: I have my doubts about the efficacy of this given that we have two buckets already between… | |||||
zdom->uzd_nitems += bucket->ub_cnt; | zdom->uzd_nitems += bucket->ub_cnt; | ||||
if (ws && zdom->uzd_imax < zdom->uzd_nitems) | if (ws && zdom->uzd_imax < zdom->uzd_nitems) | ||||
zdom->uzd_imax = zdom->uzd_nitems; | zdom->uzd_imax = zdom->uzd_nitems; | ||||
zone->uz_bkt_count += bucket->ub_cnt; | zone->uz_bkt_count += bucket->ub_cnt; | ||||
} | } | ||||
/* Pops an item out of a per-cpu cache bucket. */ | /* Pops an item out of a per-cpu cache bucket. */ | ||||
static inline void * | static inline void * | ||||
▲ Show 20 Lines • Show All 370 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
bucket_drain(uma_zone_t zone, uma_bucket_t bucket) | bucket_drain(uma_zone_t zone, uma_bucket_t bucket) | ||||
{ | { | ||||
int i; | int i; | ||||
if (bucket == NULL || bucket->ub_cnt == 0) | if (bucket == NULL || bucket->ub_cnt == 0) | ||||
return; | return; | ||||
if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && bucket->ub_seq != 0) { | |||||
smr_wait(zone->uz_smr, bucket->ub_seq); | |||||
Not Done Inline Actions&& should come on the previous line. markj: && should come on the previous line. | |||||
for (i = 0; i < bucket->ub_cnt; i++) | |||||
item_dtor(zone, bucket->ub_bucket[i], | |||||
zone->uz_size, NULL, SKIP_NONE); | |||||
bucket->ub_seq = 0; | |||||
} | |||||
if (zone->uz_fini) | if (zone->uz_fini) | ||||
for (i = 0; i < bucket->ub_cnt; i++) | for (i = 0; i < bucket->ub_cnt; i++) | ||||
zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); | zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); | ||||
zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); | zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); | ||||
if (zone->uz_max_items > 0) | if (zone->uz_max_items > 0) | ||||
zone_free_limit(zone, bucket->ub_cnt); | zone_free_limit(zone, bucket->ub_cnt); | ||||
#ifdef INVARIANTS | |||||
bzero(bucket->ub_bucket, sizeof(void *) * bucket->ub_cnt); | |||||
#endif | |||||
bucket->ub_cnt = 0; | bucket->ub_cnt = 0; | ||||
} | } | ||||
/* | /* | ||||
* Drains the per cpu caches for a zone. | * Drains the per cpu caches for a zone. | ||||
* | * | ||||
* NOTE: This may only be called while the zone is being torn down, and not | * NOTE: This may only be called while the zone is being torn down, and not | ||||
* during normal operation. This is necessary in order that we do not have | * during normal operation. This is necessary in order that we do not have | ||||
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | cache_drain_safe_cpu(uma_zone_t zone, void *unused) | ||||
else | else | ||||
domain = 0; | domain = 0; | ||||
cache = &zone->uz_cpu[curcpu]; | cache = &zone->uz_cpu[curcpu]; | ||||
b1 = cache_bucket_unload_alloc(cache); | b1 = cache_bucket_unload_alloc(cache); | ||||
if (b1 != NULL && b1->ub_cnt != 0) { | if (b1 != NULL && b1->ub_cnt != 0) { | ||||
zone_put_bucket(zone, &zone->uz_domain[domain], b1, false); | zone_put_bucket(zone, &zone->uz_domain[domain], b1, false); | ||||
b1 = NULL; | b1 = NULL; | ||||
} | } | ||||
/* | |||||
* Don't flush SMR zone buckets. This leaves the zone without a | |||||
* bucket and forces every free to synchronize(). | |||||
*/ | |||||
if ((zone->uz_flags & UMA_ZONE_SMR) != 0) | |||||
goto out; | |||||
b2 = cache_bucket_unload_free(cache); | b2 = cache_bucket_unload_free(cache); | ||||
if (b2 != NULL && b2->ub_cnt != 0) { | if (b2 != NULL && b2->ub_cnt != 0) { | ||||
zone_put_bucket(zone, &zone->uz_domain[domain], b2, false); | zone_put_bucket(zone, &zone->uz_domain[domain], b2, false); | ||||
b2 = NULL; | b2 = NULL; | ||||
} | } | ||||
b3 = cache_bucket_unload_cross(cache); | b3 = cache_bucket_unload_cross(cache); | ||||
out: | |||||
critical_exit(); | critical_exit(); | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
if (b1) | if (b1) | ||||
bucket_free(zone, b1, NULL); | bucket_free(zone, b1, NULL); | ||||
if (b2) | if (b2) | ||||
bucket_free(zone, b2, NULL); | bucket_free(zone, b2, NULL); | ||||
if (b3) { | if (b3) { | ||||
bucket_drain(zone, b3); | bucket_drain(zone, b3); | ||||
▲ Show 20 Lines • Show All 1,237 Lines • ▼ Show 20 Lines | zone_ctor(void *mem, int size, void *udata, int flags) | ||||
zone->uz_dtor = arg->dtor; | zone->uz_dtor = arg->dtor; | ||||
zone->uz_init = NULL; | zone->uz_init = NULL; | ||||
zone->uz_fini = NULL; | zone->uz_fini = NULL; | ||||
zone->uz_sleeps = 0; | zone->uz_sleeps = 0; | ||||
zone->uz_xdomain = 0; | zone->uz_xdomain = 0; | ||||
zone->uz_bucket_size = 0; | zone->uz_bucket_size = 0; | ||||
zone->uz_bucket_size_min = 0; | zone->uz_bucket_size_min = 0; | ||||
zone->uz_bucket_size_max = BUCKET_MAX; | zone->uz_bucket_size_max = BUCKET_MAX; | ||||
zone->uz_flags = 0; | zone->uz_flags = (arg->flags & UMA_ZONE_SMR); | ||||
zone->uz_warning = NULL; | zone->uz_warning = NULL; | ||||
/* The domain structures follow the cpu structures. */ | /* The domain structures follow the cpu structures. */ | ||||
zone->uz_domain = | zone->uz_domain = | ||||
(struct uma_zone_domain *)&zone->uz_cpu[mp_maxid + 1]; | (struct uma_zone_domain *)&zone->uz_cpu[mp_maxid + 1]; | ||||
zone->uz_bkt_max = ULONG_MAX; | zone->uz_bkt_max = ULONG_MAX; | ||||
timevalclear(&zone->uz_ratecheck); | timevalclear(&zone->uz_ratecheck); | ||||
/* Count the number of duplicate names. */ | /* Count the number of duplicate names. */ | ||||
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines | if (arg->flags & UMA_ZONE_SECONDARY) { | ||||
struct uma_kctor_args karg; | struct uma_kctor_args karg; | ||||
int error; | int error; | ||||
/* We should only be here from uma_startup() */ | /* We should only be here from uma_startup() */ | ||||
karg.size = arg->size; | karg.size = arg->size; | ||||
karg.uminit = arg->uminit; | karg.uminit = arg->uminit; | ||||
karg.fini = arg->fini; | karg.fini = arg->fini; | ||||
karg.align = arg->align; | karg.align = arg->align; | ||||
karg.flags = arg->flags; | karg.flags = (arg->flags & ~UMA_ZONE_SMR); | ||||
karg.zone = zone; | karg.zone = zone; | ||||
error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, | error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, | ||||
flags); | flags); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
} | } | ||||
/* Inherit properties from the keg. */ | /* Inherit properties from the keg. */ | ||||
zone->uz_keg = keg; | zone->uz_keg = keg; | ||||
zone->uz_size = keg->uk_size; | zone->uz_size = keg->uk_size; | ||||
zone->uz_flags |= (keg->uk_flags & | zone->uz_flags |= (keg->uk_flags & | ||||
(UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT)); | (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT)); | ||||
out: | out: | ||||
if (__predict_true(booted >= BOOT_RUNNING)) { | if (__predict_true(booted >= BOOT_RUNNING)) { | ||||
zone_alloc_counters(zone, NULL); | zone_alloc_counters(zone, NULL); | ||||
zone_alloc_sysctl(zone, NULL); | zone_alloc_sysctl(zone, NULL); | ||||
} else { | } else { | ||||
zone->uz_allocs = EARLY_COUNTER; | zone->uz_allocs = EARLY_COUNTER; | ||||
zone->uz_frees = EARLY_COUNTER; | zone->uz_frees = EARLY_COUNTER; | ||||
zone->uz_fails = EARLY_COUNTER; | zone->uz_fails = EARLY_COUNTER; | ||||
} | } | ||||
/* Caller requests a private SMR context. */ | |||||
if ((zone->uz_flags & UMA_ZONE_SMR) != 0) | |||||
zone->uz_smr = smr_create(zone->uz_name); | |||||
KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) != | KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) != | ||||
(UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET), | (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET), | ||||
("Invalid zone flag combination")); | ("Invalid zone flag combination")); | ||||
if (arg->flags & UMA_ZFLAG_INTERNAL) | if (arg->flags & UMA_ZFLAG_INTERNAL) | ||||
zone->uz_bucket_size_max = zone->uz_bucket_size = 0; | zone->uz_bucket_size_max = zone->uz_bucket_size = 0; | ||||
if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) | if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) | ||||
zone->uz_bucket_size = BUCKET_MAX; | zone->uz_bucket_size = BUCKET_MAX; | ||||
else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0) | else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0) | ||||
▲ Show 20 Lines • Show All 185 Lines • ▼ Show 20 Lines | uma_startup1(vm_offset_t virtual_avail) | ||||
slabzones[1] = uma_zcreate("UMA Slabs 1", SLABZONE1_SIZE, | slabzones[1] = uma_zcreate("UMA Slabs 1", SLABZONE1_SIZE, | ||||
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | ||||
hashzone = uma_zcreate("UMA Hash", | hashzone = uma_zcreate("UMA Hash", | ||||
sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, | sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, | ||||
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | ||||
bucket_init(); | bucket_init(); | ||||
smr_init(); | |||||
rlibbyUnsubmitted Not Done Inline ActionsThis placement is a little surprising with smr having moved out from under uma. You might consider if you think it would read better in vm_init() now. rlibby: This placement is a little surprising with smr having moved out from under uma. You might… | |||||
} | } | ||||
#ifndef UMA_MD_SMALL_ALLOC | #ifndef UMA_MD_SMALL_ALLOC | ||||
extern void vm_radix_reserve_kva(void); | extern void vm_radix_reserve_kva(void); | ||||
#endif | #endif | ||||
/* | /* | ||||
* Advertise the availability of normal kva allocations and switch to | * Advertise the availability of normal kva allocations and switch to | ||||
▲ Show 20 Lines • Show All 234 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
#define UMA_ALWAYS_CTORDTOR 1 | #define UMA_ALWAYS_CTORDTOR 1 | ||||
#else | #else | ||||
#define UMA_ALWAYS_CTORDTOR 0 | #define UMA_ALWAYS_CTORDTOR 0 | ||||
#endif | #endif | ||||
static void * | static void * | ||||
item_ctor(uma_zone_t zone, int size, void *udata, int flags, void *item) | item_ctor(uma_zone_t zone, int size, void *udata, int flags, void *item) | ||||
Not Done Inline Actions"static inline" now? I think the compiler may do it anyway, but it may better capture the intent. rlibby: "static inline" now? I think the compiler may do it anyway, but it may better capture the… | |||||
{ | { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
bool skipdbg; | bool skipdbg; | ||||
skipdbg = uma_dbg_zskip(zone, item); | skipdbg = uma_dbg_zskip(zone, item); | ||||
if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && | if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && | ||||
zone->uz_ctor != trash_ctor) | zone->uz_ctor != trash_ctor) | ||||
trash_ctor(item, size, udata, flags); | trash_ctor(item, size, udata, flags); | ||||
▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines | if (item != NULL) { | ||||
zone->uz_fini(item, zone->uz_size); | zone->uz_fini(item, zone->uz_size); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
return (item); | return (item); | ||||
} | } | ||||
/* This is unfortunate but should not be fatal. */ | /* This is unfortunate but should not be fatal. */ | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* If possible, allocate from the per-CPU cache. There are two | * If possible, allocate from the per-CPU cache. There are two | ||||
* requirements for safe access to the per-CPU cache: (1) the thread | * requirements for safe access to the per-CPU cache: (1) the thread | ||||
* accessing the cache must not be preempted or yield during access, | * accessing the cache must not be preempted or yield during access, | ||||
* and (2) the thread must not migrate CPUs without switching which | * and (2) the thread must not migrate CPUs without switching which | ||||
* cache it accesses. We rely on a critical section to prevent | * cache it accesses. We rely on a critical section to prevent | ||||
* preemption and migration. We release the critical section in | * preemption and migration. We release the critical section in | ||||
* order to acquire the zone mutex if we are unable to allocate from | * order to acquire the zone mutex if we are unable to allocate from | ||||
* the current cache; when we re-acquire the critical section, we | * the current cache; when we re-acquire the critical section, we | ||||
* must detect and handle migration if it has occurred. | * must detect and handle migration if it has occurred. | ||||
*/ | */ | ||||
critical_enter(); | critical_enter(); | ||||
do { | do { | ||||
cache = &zone->uz_cpu[curcpu]; | cache = &zone->uz_cpu[curcpu]; | ||||
bucket = &cache->uc_allocbucket; | bucket = &cache->uc_allocbucket; | ||||
size = cache_uz_size(cache); | size = cache_uz_size(cache); | ||||
uz_flags = cache_uz_flags(cache); | uz_flags = cache_uz_flags(cache); | ||||
if (__predict_true(bucket->ucb_cnt != 0)) { | if (__predict_true(bucket->ucb_cnt != 0)) { | ||||
item = cache_bucket_pop(cache, bucket); | item = cache_bucket_pop(cache, bucket); | ||||
critical_exit(); | critical_exit(); | ||||
if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0 || | if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0 || | ||||
UMA_ALWAYS_CTORDTOR)) | UMA_ALWAYS_CTORDTOR)) | ||||
return (item_ctor(zone, size, udata, flags, item)); | return (item_ctor(zone, size, udata, flags, item)); | ||||
if (flags & M_ZERO) | if (flags & M_ZERO) | ||||
bzero(item, size); | bzero(item, size); | ||||
return (item); | return (item); | ||||
} | } | ||||
} while (cache_alloc(zone, cache, udata, flags)); | } while (cache_alloc(zone, cache, udata, flags)); | ||||
critical_exit(); | critical_exit(); | ||||
/* | /* | ||||
* We can not get a bucket so try to return a single item. | * We can not get a bucket so try to return a single item. | ||||
*/ | */ | ||||
if (uz_flags & UMA_ZONE_FIRSTTOUCH) | if (uz_flags & UMA_ZONE_FIRSTTOUCH) | ||||
domain = PCPU_GET(domain); | domain = PCPU_GET(domain); | ||||
else | else | ||||
domain = UMA_ANYDOMAIN; | domain = UMA_ANYDOMAIN; | ||||
return (zone_alloc_item(zone, udata, domain, flags)); | return (zone_alloc_item(zone, udata, domain, flags)); | ||||
Not Done Inline ActionsCould consider moving this to a static inline and just passing udata=NULL from SMR instead of c&p, or do you see this and the SMR path diverging further in the future? rlibby: Could consider moving this to a static inline and just passing udata=NULL from SMR instead of… | |||||
Not Done Inline ActionsMissing parens. markj: Missing parens. | |||||
} | } | ||||
/* | /* | ||||
* Replenish an alloc bucket and possibly restore an old one. Called in | * Replenish an alloc bucket and possibly restore an old one. Called in | ||||
* a critical section. Returns in a critical section. | * a critical section. Returns in a critical section. | ||||
* | * | ||||
* A false return value indicates an allocation failure. | * A false return value indicates an allocation failure. | ||||
* A true return value indicates success and the caller should retry. | * A true return value indicates success and the caller should retry. | ||||
*/ | */ | ||||
static __noinline bool | static __noinline bool | ||||
cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) | cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) | ||||
{ | { | ||||
uma_zone_domain_t zdom; | uma_zone_domain_t zdom; | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
int domain; | int domain; | ||||
bool lockfail; | bool lockfail; | ||||
CRITICAL_ASSERT(curthread); | CRITICAL_ASSERT(curthread); | ||||
/* | /* | ||||
* If we have run out of items in our alloc bucket see | * If we have run out of items in our alloc bucket see | ||||
* if we can switch with the free bucket. | * if we can switch with the free bucket. | ||||
* | |||||
* SMR Zones can't re-use the free bucket until the sequence has | |||||
* expired. | |||||
*/ | */ | ||||
if (cache->uc_freebucket.ucb_cnt != 0) { | if ((zone->uz_flags & UMA_ZONE_SMR) == 0 && | ||||
cache_bucket_swap(&cache->uc_freebucket, &cache->uc_allocbucket); | cache->uc_freebucket.ucb_cnt != 0) { | ||||
cache_bucket_swap(&cache->uc_freebucket, | |||||
&cache->uc_allocbucket); | |||||
return (true); | return (true); | ||||
} | } | ||||
/* | /* | ||||
* Discard any empty allocation bucket while we hold no locks. | * Discard any empty allocation bucket while we hold no locks. | ||||
*/ | */ | ||||
bucket = cache_bucket_unload_alloc(cache); | bucket = cache_bucket_unload_alloc(cache); | ||||
critical_exit(); | critical_exit(); | ||||
Show All 37 Lines | if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH) { | ||||
domain = PCPU_GET(domain); | domain = PCPU_GET(domain); | ||||
zdom = &zone->uz_domain[domain]; | zdom = &zone->uz_domain[domain]; | ||||
} else { | } else { | ||||
domain = UMA_ANYDOMAIN; | domain = UMA_ANYDOMAIN; | ||||
zdom = &zone->uz_domain[0]; | zdom = &zone->uz_domain[0]; | ||||
} | } | ||||
if ((bucket = zone_fetch_bucket(zone, zdom)) != NULL) { | if ((bucket = zone_fetch_bucket(zone, zdom)) != NULL) { | ||||
ZONE_UNLOCK(zone); | |||||
KASSERT(bucket->ub_cnt != 0, | KASSERT(bucket->ub_cnt != 0, | ||||
("uma_zalloc_arg: Returning an empty bucket.")); | ("uma_zalloc_arg: Returning an empty bucket.")); | ||||
cache_bucket_load_alloc(cache, bucket); | cache_bucket_load_alloc(cache, bucket); | ||||
return (true); | return (true); | ||||
} | } | ||||
/* We are no longer associated with this CPU. */ | /* We are no longer associated with this CPU. */ | ||||
critical_exit(); | critical_exit(); | ||||
▲ Show 20 Lines • Show All 557 Lines • ▼ Show 20 Lines | uma_zfree_arg(uma_zone_t zone, void *item, void *udata) | ||||
/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | ||||
random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | ||||
CTR2(KTR_UMA, "uma_zfree_arg zone %s(%p)", zone->uz_name, zone); | CTR2(KTR_UMA, "uma_zfree_arg zone %s(%p)", zone->uz_name, zone); | ||||
KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | ||||
("uma_zfree_arg: called with spinlock or critical section held")); | ("uma_zfree_arg: called with spinlock or critical section held")); | ||||
/* uma_zfree(..., NULL) does nothing, to match free(9). */ | /* uma_zfree(..., NULL) does nothing, to match free(9). */ | ||||
if (item == NULL) | if (item == NULL) | ||||
return; | return; | ||||
Not Done Inline ActionsIs this deliberately missing from uma_zfree_smr? It seems like a potential point of confusion, being different from free() and uma_zfree/_arg(). If so, maybe KASSERT(item != NULL) there instead of crashing in pmap_kextract so that misuse may be more easily diagnosed? rlibby: Is this deliberately missing from uma_zfree_smr? It seems like a potential point of confusion… | |||||
#ifdef DEBUG_MEMGUARD | #ifdef DEBUG_MEMGUARD | ||||
if (is_memguard_addr(item)) { | if (is_memguard_addr(item)) { | ||||
Done Inline ActionsBetter to do domain = itemdomain = 0 here too, like in uma_zfree_arg. Else if !NUMA then itemdomain is uninitialized, and a compiler may get angry when it is passed to cache_free() (although it is harmless). rlibby: Better to do `domain = itemdomain = 0` here too, like in uma_zfree_arg. Else if !NUMA then… | |||||
if (zone->uz_dtor != NULL) | if (zone->uz_dtor != NULL) | ||||
zone->uz_dtor(item, zone->uz_size, udata); | zone->uz_dtor(item, zone->uz_size, udata); | ||||
if (zone->uz_fini != NULL) | if (zone->uz_fini != NULL) | ||||
zone->uz_fini(item, zone->uz_size); | zone->uz_fini(item, zone->uz_size); | ||||
memguard_free(item); | memguard_free(item); | ||||
return; | return; | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* We are accessing the per-cpu cache without a critical section to | * We are accessing the per-cpu cache without a critical section to | ||||
* fetch size and flags. This is acceptable, if we are preempted we | * fetch size and flags. This is acceptable, if we are preempted we | ||||
* will simply read another cpu's line. | * will simply read another cpu's line. | ||||
*/ | */ | ||||
cache = &zone->uz_cpu[curcpu]; | cache = &zone->uz_cpu[curcpu]; | ||||
uz_flags = cache_uz_flags(cache); | uz_flags = cache_uz_flags(cache); | ||||
if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0 || | if (__predict_false(((uz_flags & UMA_ZFLAG_CTORDTOR) != 0 || | ||||
UMA_ALWAYS_CTORDTOR)) | UMA_ALWAYS_CTORDTOR) && (uz_flags & UMA_ZONE_SMR) == 0)) | ||||
item_dtor(zone, item, cache_uz_size(cache), udata, SKIP_NONE); | item_dtor(zone, item, cache_uz_size(cache), udata, SKIP_NONE); | ||||
Not Done Inline ActionsI just noticed that this means that uma_zfree_arg() can't pass non-NULL udata when freeing to an SMR zone. It's probably not a big deal (consumers can embed a pointer in the structure itself if desired), but we should probably have an assert to catch that. markj: I just noticed that this means that uma_zfree_arg() can't pass non-NULL udata when freeing to… | |||||
/* | /* | ||||
* The race here is acceptable. If we miss it we'll just have to wait | * The race here is acceptable. If we miss it we'll just have to wait | ||||
* a little longer for the limits to be reset. | * a little longer for the limits to be reset. | ||||
*/ | */ | ||||
if (__predict_false(uz_flags & UMA_ZFLAG_LIMIT)) { | if (__predict_false(uz_flags & UMA_ZFLAG_LIMIT)) { | ||||
if (zone->uz_sleepers > 0) | if (zone->uz_sleepers > 0) | ||||
goto zfree_item; | goto zfree_item; | ||||
Show All 13 Lines | #endif | ||||
domain = itemdomain = 0; | domain = itemdomain = 0; | ||||
#ifdef NUMA | #ifdef NUMA | ||||
if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) | if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) | ||||
itemdomain = _vm_phys_domain(pmap_kextract((vm_offset_t)item)); | itemdomain = _vm_phys_domain(pmap_kextract((vm_offset_t)item)); | ||||
#endif | #endif | ||||
critical_enter(); | critical_enter(); | ||||
do { | do { | ||||
cache = &zone->uz_cpu[curcpu]; | cache = &zone->uz_cpu[curcpu]; | ||||
/* | |||||
* Try to free into the allocbucket first to give LIFO | |||||
* ordering for cache-hot datastructures. Spill over | |||||
* into the freebucket if necessary. Alloc will swap | |||||
* them if one runs dry. | |||||
*/ | |||||
bucket = &cache->uc_allocbucket; | |||||
#ifdef NUMA | #ifdef NUMA | ||||
domain = PCPU_GET(domain); | domain = PCPU_GET(domain); | ||||
if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && | if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && | ||||
domain != itemdomain) { | domain != itemdomain) { | ||||
bucket = &cache->uc_crossbucket; | bucket = &cache->uc_crossbucket; | ||||
} else | } else | ||||
#endif | #endif | ||||
{ | |||||
/* | /* | ||||
* Try to free into the allocbucket first to give LIFO | * If this is a SMR zone or the allocbucket is entry we use | ||||
* ordering for cache-hot datastructures. Spill over | * the free bucket. | ||||
* into the freebucket if necessary. Alloc will swap | |||||
* them if one runs dry. | |||||
*/ | */ | ||||
bucket = &cache->uc_allocbucket; | if (__predict_false((uz_flags & UMA_ZONE_SMR) != 0 || | ||||
if (__predict_false(bucket->ucb_cnt >= | bucket->ucb_cnt >= bucket->ucb_entries)) | ||||
bucket->ucb_entries)) | |||||
bucket = &cache->uc_freebucket; | bucket = &cache->uc_freebucket; | ||||
} | |||||
if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { | if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { | ||||
cache_bucket_push(cache, bucket, item); | cache_bucket_push(cache, bucket, item); | ||||
critical_exit(); | critical_exit(); | ||||
return; | return; | ||||
} | } | ||||
} while (cache_free(zone, cache, udata, item, itemdomain)); | } while (cache_free(zone, cache, udata, item, itemdomain)); | ||||
critical_exit(); | critical_exit(); | ||||
/* | /* | ||||
* If nothing else caught this, we'll just do an internal free. | * If nothing else caught this, we'll just do an internal free. | ||||
*/ | */ | ||||
zfree_item: | zfree_item: | ||||
zone_free_item(zone, item, udata, SKIP_DTOR); | zone_free_item(zone, item, udata, | ||||
(uz_flags & UMA_ZONE_SMR) == 0 ? SKIP_DTOR : SKIP_NONE); | |||||
} | } | ||||
#ifdef NUMA | #ifdef NUMA | ||||
/* | /* | ||||
* sort crossdomain free buckets to domain correct buckets and cache | * sort crossdomain free buckets to domain correct buckets and cache | ||||
* them. | * them. | ||||
*/ | */ | ||||
static void | static void | ||||
Show All 33 Lines | if (zdom->uzd_cross->ub_cnt == zdom->uzd_cross->ub_entries) { | ||||
zdom->uzd_cross = NULL; | zdom->uzd_cross = NULL; | ||||
} | } | ||||
bucket->ub_cnt--; | bucket->ub_cnt--; | ||||
} | } | ||||
ZONE_CROSS_UNLOCK(zone); | ZONE_CROSS_UNLOCK(zone); | ||||
if (!TAILQ_EMPTY(&fullbuckets)) { | if (!TAILQ_EMPTY(&fullbuckets)) { | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
while ((b = TAILQ_FIRST(&fullbuckets)) != NULL) { | while ((b = TAILQ_FIRST(&fullbuckets)) != NULL) { | ||||
if ((zone->uz_flags & UMA_ZONE_SMR) != 0) | |||||
bucket->ub_seq = smr_current(zone->uz_smr); | |||||
TAILQ_REMOVE(&fullbuckets, b, ub_link); | TAILQ_REMOVE(&fullbuckets, b, ub_link); | ||||
if (zone->uz_bkt_count >= zone->uz_bkt_max) { | if (zone->uz_bkt_count >= zone->uz_bkt_max) { | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
bucket_drain(zone, b); | bucket_drain(zone, b); | ||||
bucket_free(zone, b, udata); | bucket_free(zone, b, udata); | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
} else { | } else { | ||||
domain = _vm_phys_domain( | domain = _vm_phys_domain( | ||||
pmap_kextract( | pmap_kextract( | ||||
(vm_offset_t)b->ub_bucket[0])); | (vm_offset_t)b->ub_bucket[0])); | ||||
zdom = &zone->uz_domain[domain]; | zdom = &zone->uz_domain[domain]; | ||||
zone_put_bucket(zone, zdom, b, true); | zone_put_bucket(zone, zdom, b, true); | ||||
} | } | ||||
} | } | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
} | } | ||||
if (bucket->ub_cnt != 0) | if (bucket->ub_cnt != 0) | ||||
bucket_drain(zone, bucket); | bucket_drain(zone, bucket); | ||||
bucket->ub_seq = 0; | |||||
bucket_free(zone, bucket, udata); | bucket_free(zone, bucket, udata); | ||||
} | } | ||||
#endif | #endif | ||||
static void | static void | ||||
zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata, | zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata, | ||||
int domain, int itemdomain) | int domain, int itemdomain) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | |||||
* we can not satisfy this free in the cache layer. true indicates that | * we can not satisfy this free in the cache layer. true indicates that | ||||
* the caller should retry. | * the caller should retry. | ||||
*/ | */ | ||||
static __noinline bool | static __noinline bool | ||||
cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, void *item, | cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, void *item, | ||||
int itemdomain) | int itemdomain) | ||||
{ | { | ||||
uma_cache_bucket_t cbucket; | uma_cache_bucket_t cbucket; | ||||
uma_bucket_t bucket; | uma_bucket_t newbucket, bucket; | ||||
int domain; | int domain; | ||||
CRITICAL_ASSERT(curthread); | CRITICAL_ASSERT(curthread); | ||||
if (zone->uz_bucket_size == 0 || bucketdisable) | if (zone->uz_bucket_size == 0) | ||||
return false; | return false; | ||||
cache = &zone->uz_cpu[curcpu]; | cache = &zone->uz_cpu[curcpu]; | ||||
newbucket = NULL; | |||||
/* | /* | ||||
* FIRSTTOUCH domains need to free to the correct zdom. When | * FIRSTTOUCH domains need to free to the correct zdom. When | ||||
* enabled this is the zdom of the item. The bucket is the | * enabled this is the zdom of the item. The bucket is the | ||||
* cross bucket if the current domain and itemdomain do not match. | * cross bucket if the current domain and itemdomain do not match. | ||||
*/ | */ | ||||
cbucket = &cache->uc_freebucket; | cbucket = &cache->uc_freebucket; | ||||
#ifdef NUMA | #ifdef NUMA | ||||
if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) { | if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) { | ||||
domain = PCPU_GET(domain); | domain = PCPU_GET(domain); | ||||
if (domain != itemdomain) { | if (domain != itemdomain) { | ||||
cbucket = &cache->uc_crossbucket; | cbucket = &cache->uc_crossbucket; | ||||
if (cbucket->ucb_cnt != 0) | if (cbucket->ucb_cnt != 0) | ||||
atomic_add_64(&zone->uz_xdomain, | atomic_add_64(&zone->uz_xdomain, | ||||
cbucket->ucb_cnt); | cbucket->ucb_cnt); | ||||
} | } | ||||
} else | } else | ||||
#endif | #endif | ||||
itemdomain = domain = 0; | itemdomain = domain = 0; | ||||
bucket = cache_bucket_unload(cbucket); | bucket = cache_bucket_unload(cbucket); | ||||
/* We are no longer associated with this CPU. */ | /* We are no longer associated with this CPU. */ | ||||
critical_exit(); | critical_exit(); | ||||
/* | |||||
* Don't let SMR zones operate without a free bucket. Force | |||||
* a synchronize and re-use this one. We will only degrade | |||||
* to a synchronize every bucket_size items rather than every | |||||
* item if we fail to allocate a bucket. | |||||
*/ | |||||
if ((zone->uz_flags & UMA_ZONE_SMR) != 0) { | |||||
if (bucket != NULL) | if (bucket != NULL) | ||||
bucket->ub_seq = smr_advance(zone->uz_smr); | |||||
newbucket = bucket_alloc(zone, udata, M_NOWAIT); | |||||
if (newbucket == NULL && bucket != NULL) { | |||||
bucket_drain(zone, bucket); | |||||
newbucket = bucket; | |||||
bucket = NULL; | |||||
} | |||||
} else if (!bucketdisable) | |||||
newbucket = bucket_alloc(zone, udata, M_NOWAIT); | |||||
if (bucket != NULL) | |||||
zone_free_bucket(zone, bucket, udata, domain, itemdomain); | zone_free_bucket(zone, bucket, udata, domain, itemdomain); | ||||
bucket = bucket_alloc(zone, udata, M_NOWAIT); | |||||
CTR3(KTR_UMA, "uma_zfree: zone %s(%p) allocated bucket %p", | |||||
zone->uz_name, zone, bucket); | |||||
critical_enter(); | critical_enter(); | ||||
if (bucket == NULL) | if ((bucket = newbucket) == NULL) | ||||
return (false); | return (false); | ||||
cache = &zone->uz_cpu[curcpu]; | cache = &zone->uz_cpu[curcpu]; | ||||
#ifdef NUMA | #ifdef NUMA | ||||
/* | /* | ||||
* Check to see if we should be populating the cross bucket. If it | * Check to see if we should be populating the cross bucket. If it | ||||
* is already populated we will fall through and attempt to populate | * is already populated we will fall through and attempt to populate | ||||
* the free bucket. | * the free bucket. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 112 Lines • ▼ Show 20 Lines | |||||
* item The item we're freeing | * item The item we're freeing | ||||
* udata User supplied data for the dtor | * udata User supplied data for the dtor | ||||
* skip Skip dtors and finis | * skip Skip dtors and finis | ||||
*/ | */ | ||||
static void | static void | ||||
zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) | zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) | ||||
{ | { | ||||
/* | |||||
* If a free is sent directly to an SMR zone we have to | |||||
* synchronize immediately because the item can instantly | |||||
* be reallocated. This should only happen in degenerate | |||||
* cases when no memory is available for per-cpu caches. | |||||
*/ | |||||
if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && skip == SKIP_DTOR) | |||||
smr_synchronize(zone->uz_smr); | |||||
Not Done Inline ActionsWhat case is this covering? If it is the uma_zfree_arg() path, it looks wrong (we pass SKIP_NONE for SMR), and I don't see any other callers that pass SKIP_DTOR. Also comparison with == feels more fragile than a comparison with </<= (e.g. because of SKIP_CNT). Should we add a new SKIP_SYNC? rlibby: What case is this covering? If it is the `uma_zfree_arg()` path, it looks wrong (we pass… | |||||
item_dtor(zone, item, zone->uz_size, udata, skip); | item_dtor(zone, item, zone->uz_size, udata, skip); | ||||
if (skip < SKIP_FINI && zone->uz_fini) | if (skip < SKIP_FINI && zone->uz_fini) | ||||
zone->uz_fini(item, zone->uz_size); | zone->uz_fini(item, zone->uz_size); | ||||
zone->uz_release(zone->uz_arg, &item, 1); | zone->uz_release(zone->uz_arg, &item, 1); | ||||
if (skip & SKIP_CNT) | if (skip & SKIP_CNT) | ||||
▲ Show 20 Lines • Show All 206 Lines • ▼ Show 20 Lines | |||||
void | void | ||||
uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) | uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) | ||||
{ | { | ||||
uma_keg_t keg; | uma_keg_t keg; | ||||
KEG_GET(zone, keg); | KEG_GET(zone, keg); | ||||
KEG_ASSERT_COLD(keg); | KEG_ASSERT_COLD(keg); | ||||
keg->uk_allocf = allocf; | keg->uk_allocf = allocf; | ||||
} | |||||
/* See uma.h */ | |||||
void | |||||
uma_zone_set_smr(uma_zone_t zone, smr_t smr) | |||||
{ | |||||
ZONE_ASSERT_COLD(zone); | |||||
zone->uz_flags |= UMA_ZONE_SMR; | |||||
zone->uz_smr = smr; | |||||
zone_update_caches(zone); | |||||
} | |||||
Done Inline ActionsI just noticed that the keg does not need the SMR flag. jeff: I just noticed that the keg does not need the SMR flag. | |||||
smr_t | |||||
uma_zone_get_smr(uma_zone_t zone) | |||||
{ | |||||
return (zone->uz_smr); | |||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
void | void | ||||
uma_zone_reserve(uma_zone_t zone, int items) | uma_zone_reserve(uma_zone_t zone, int items) | ||||
{ | { | ||||
uma_keg_t keg; | uma_keg_t keg; | ||||
▲ Show 20 Lines • Show All 754 Lines • Show Last 20 Lines |
More parens for hygiene, although obviously safe in practice below.