Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_core.c
Show First 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | |||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/random.h> | #include <sys/random.h> | ||||
#include <sys/rwlock.h> | #include <sys/rwlock.h> | ||||
#include <sys/sbuf.h> | #include <sys/sbuf.h> | ||||
#include <sys/sched.h> | #include <sys/sched.h> | ||||
#include <sys/sleepqueue.h> | |||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/taskqueue.h> | #include <sys/taskqueue.h> | ||||
#include <sys/vmmeter.h> | #include <sys/vmmeter.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/vm_domainset.h> | #include <vm/vm_domainset.h> | ||||
#include <vm/vm_object.h> | #include <vm/vm_object.h> | ||||
#include <vm/vm_page.h> | #include <vm/vm_page.h> | ||||
▲ Show 20 Lines • Show All 176 Lines • ▼ Show 20 Lines | |||||
static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *); | static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *); | ||||
static void zone_timeout(uma_zone_t zone, void *); | static void zone_timeout(uma_zone_t zone, void *); | ||||
static int hash_alloc(struct uma_hash *, u_int); | static int hash_alloc(struct uma_hash *, u_int); | ||||
static int hash_expand(struct uma_hash *, struct uma_hash *); | static int hash_expand(struct uma_hash *, struct uma_hash *); | ||||
static void hash_free(struct uma_hash *hash); | static void hash_free(struct uma_hash *hash); | ||||
static void uma_timeout(void *); | static void uma_timeout(void *); | ||||
static void uma_startup3(void); | static void uma_startup3(void); | ||||
static void *zone_alloc_item(uma_zone_t, void *, int, int); | static void *zone_alloc_item(uma_zone_t, void *, int, int); | ||||
static void *zone_alloc_item_locked(uma_zone_t, void *, int, int); | |||||
static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip); | static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip); | ||||
static int zone_alloc_limit(uma_zone_t zone, int count, int flags); | |||||
static void zone_free_limit(uma_zone_t zone, int count); | |||||
static void bucket_enable(void); | static void bucket_enable(void); | ||||
static void bucket_init(void); | static void bucket_init(void); | ||||
static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int); | static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int); | ||||
static void bucket_free(uma_zone_t zone, uma_bucket_t, void *); | static void bucket_free(uma_zone_t zone, uma_bucket_t, void *); | ||||
static void bucket_zone_drain(void); | static void bucket_zone_drain(void); | ||||
static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int); | static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int); | ||||
static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab); | static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab); | ||||
static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item); | static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item); | ||||
static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, | static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, | ||||
uma_fini fini, int align, uint32_t flags); | uma_fini fini, int align, uint32_t flags); | ||||
static int zone_import(void *, void **, int, int, int); | static int zone_import(void *, void **, int, int, int); | ||||
static void zone_release(void *, void **, int); | static void zone_release(void *, void **, int); | ||||
static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int); | static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int); | ||||
static bool cache_free(uma_zone_t, uma_cache_t, void *, void *, int); | static bool cache_free(uma_zone_t, uma_cache_t, void *, void *, int); | ||||
static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); | static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); | ||||
static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); | static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); | ||||
static int sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS); | static int sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS); | ||||
static int sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS); | static int sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS); | ||||
static int sysctl_handle_uma_zone_flags(SYSCTL_HANDLER_ARGS); | static int sysctl_handle_uma_zone_flags(SYSCTL_HANDLER_ARGS); | ||||
static int sysctl_handle_uma_slab_efficiency(SYSCTL_HANDLER_ARGS); | static int sysctl_handle_uma_slab_efficiency(SYSCTL_HANDLER_ARGS); | ||||
static int sysctl_handle_uma_zone_items(SYSCTL_HANDLER_ARGS); | |||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
static inline struct noslabbits *slab_dbg_bits(uma_slab_t slab, uma_keg_t keg); | static inline struct noslabbits *slab_dbg_bits(uma_slab_t slab, uma_keg_t keg); | ||||
static bool uma_dbg_kskip(uma_keg_t keg, void *mem); | static bool uma_dbg_kskip(uma_keg_t keg, void *mem); | ||||
static bool uma_dbg_zskip(uma_zone_t zone, void *mem); | static bool uma_dbg_zskip(uma_zone_t zone, void *mem); | ||||
static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item); | static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item); | ||||
static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item); | static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item); | ||||
▲ Show 20 Lines • Show All 587 Lines • ▼ Show 20 Lines | else | ||||
free(hash->uh_slab_hash, M_UMAHASH); | free(hash->uh_slab_hash, M_UMAHASH); | ||||
} | } | ||||
/* | /* | ||||
* Frees all outstanding items in a bucket | * Frees all outstanding items in a bucket | ||||
* | * | ||||
* Arguments: | * Arguments: | ||||
* zone The zone to free to, must be unlocked. | * zone The zone to free to, must be unlocked. | ||||
* bucket The free/alloc bucket with items, cpu queue must be locked. | * bucket The free/alloc bucket with items. | ||||
* | * | ||||
* Returns: | * Returns: | ||||
* Nothing | * Nothing | ||||
*/ | */ | ||||
static void | static void | ||||
bucket_drain(uma_zone_t zone, uma_bucket_t bucket) | bucket_drain(uma_zone_t zone, uma_bucket_t bucket) | ||||
{ | { | ||||
int i; | int i; | ||||
if (bucket == NULL) | if (bucket == NULL || bucket->ub_cnt == 0) | ||||
return; | return; | ||||
if (zone->uz_fini) | if (zone->uz_fini) | ||||
for (i = 0; i < bucket->ub_cnt; i++) | for (i = 0; i < bucket->ub_cnt; i++) | ||||
zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); | zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); | ||||
zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); | zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); | ||||
if (zone->uz_max_items > 0) { | if (zone->uz_max_items > 0) | ||||
ZONE_LOCK(zone); | zone_free_limit(zone, bucket->ub_cnt); | ||||
zone->uz_items -= bucket->ub_cnt; | |||||
if (zone->uz_sleepers && zone->uz_items < zone->uz_max_items) | |||||
wakeup_one(zone); | |||||
ZONE_UNLOCK(zone); | |||||
} | |||||
bucket->ub_cnt = 0; | bucket->ub_cnt = 0; | ||||
} | } | ||||
/* | /* | ||||
* Drains the per cpu caches for a zone. | * Drains the per cpu caches for a zone. | ||||
* | * | ||||
* NOTE: This may only be called while the zone is being turn down, and not | * NOTE: This may only be called while the zone is being turn down, and not | ||||
* during normal operation. This is necessary in order that we do not have | * during normal operation. This is necessary in order that we do not have | ||||
▲ Show 20 Lines • Show All 1,162 Lines • ▼ Show 20 Lines | if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) { | ||||
SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"name", CTLFLAG_RD, nokeg, "Keg name"); | "name", CTLFLAG_RD, nokeg, "Keg name"); | ||||
/* | /* | ||||
* Information about zone limits. | * Information about zone limits. | ||||
*/ | */ | ||||
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, | oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, | ||||
"limit", CTLFLAG_RD, NULL, ""); | "limit", CTLFLAG_RD, NULL, ""); | ||||
SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | |||||
"items", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE, | |||||
zone, 0, sysctl_handle_uma_zone_items, "QU", | |||||
"current number of allocated items if limit is set"); | |||||
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"items", CTLFLAG_RD, &zone->uz_items, 0, | |||||
"current number of cached items"); | |||||
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | |||||
"max_items", CTLFLAG_RD, &zone->uz_max_items, 0, | "max_items", CTLFLAG_RD, &zone->uz_max_items, 0, | ||||
"Maximum number of cached items"); | "Maximum number of cached items"); | ||||
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0, | "sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0, | ||||
"Number of threads sleeping at limit"); | "Number of threads sleeping at limit"); | ||||
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0, | "sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0, | ||||
"Total zone limit sleeps"); | "Total zone limit sleeps"); | ||||
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | |||||
"bucket_max", CTLFLAG_RD, &zone->uz_bkt_max, 0, | |||||
"Maximum number of items in the bucket cache"); | |||||
SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | |||||
"bucket_cnt", CTLFLAG_RD, &zone->uz_bkt_count, 0, | |||||
"Number of items in the bucket cache"); | |||||
/* | /* | ||||
* Per-domain information. | * Per-domain information. | ||||
*/ | */ | ||||
if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) | if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) | ||||
domains = vm_ndomains; | domains = vm_ndomains; | ||||
else | else | ||||
domains = 1; | domains = 1; | ||||
▲ Show 20 Lines • Show All 837 Lines • ▼ Show 20 Lines | #endif | ||||
/* | /* | ||||
* We can not get a bucket so try to return a single item. | * We can not get a bucket so try to return a single item. | ||||
*/ | */ | ||||
if (uz_flags & UMA_ZONE_NUMA) | if (uz_flags & UMA_ZONE_NUMA) | ||||
domain = PCPU_GET(domain); | domain = PCPU_GET(domain); | ||||
else | else | ||||
domain = UMA_ANYDOMAIN; | domain = UMA_ANYDOMAIN; | ||||
return (zone_alloc_item_locked(zone, udata, domain, flags)); | return (zone_alloc_item(zone, udata, domain, flags)); | ||||
} | } | ||||
/* | /* | ||||
* Replenish an alloc bucket and possibly restore an old one. Called in | * Replenish an alloc bucket and possibly restore an old one. Called in | ||||
* a critical section. Returns in a critical section. | * a critical section. Returns in a critical section. | ||||
* | * | ||||
* A false return value indicates failure and returns with the zone lock | * A false return value indicates an allocation failure. | ||||
* held. A true return value indicates success and the caller should retry. | * A true return value indicates success and the caller should retry. | ||||
*/ | */ | ||||
static __noinline bool | static __noinline bool | ||||
cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) | cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) | ||||
{ | { | ||||
uma_zone_domain_t zdom; | uma_zone_domain_t zdom; | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
int domain; | int domain; | ||||
bool lockfail; | bool lockfail; | ||||
Show All 12 Lines | cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) | ||||
/* | /* | ||||
* Discard any empty allocation bucket while we hold no locks. | * Discard any empty allocation bucket while we hold no locks. | ||||
*/ | */ | ||||
bucket = cache_bucket_unload_alloc(cache); | bucket = cache_bucket_unload_alloc(cache); | ||||
critical_exit(); | critical_exit(); | ||||
if (bucket != NULL) | if (bucket != NULL) | ||||
bucket_free(zone, bucket, udata); | bucket_free(zone, bucket, udata); | ||||
/* Short-circuit for zones without buckets and low memory. */ | |||||
if (zone->uz_bucket_size == 0 || bucketdisable) { | |||||
critical_enter(); | |||||
return (false); | |||||
} | |||||
/* | /* | ||||
* Attempt to retrieve the item from the per-CPU cache has failed, so | * Attempt to retrieve the item from the per-CPU cache has failed, so | ||||
* we must go back to the zone. This requires the zone lock, so we | * we must go back to the zone. This requires the zone lock, so we | ||||
* must drop the critical section, then re-acquire it when we go back | * must drop the critical section, then re-acquire it when we go back | ||||
* to the cache. Since the critical section is released, we may be | * to the cache. Since the critical section is released, we may be | ||||
* preempted or migrate. As such, make sure not to maintain any | * preempted or migrate. As such, make sure not to maintain any | ||||
* thread-local state specific to the cache from prior to releasing | * thread-local state specific to the cache from prior to releasing | ||||
* the critical section. | * the critical section. | ||||
*/ | */ | ||||
lockfail = 0; | lockfail = 0; | ||||
if (ZONE_TRYLOCK(zone) == 0) { | if (ZONE_TRYLOCK(zone) == 0) { | ||||
/* Record contention to size the buckets. */ | /* Record contention to size the buckets. */ | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
lockfail = 1; | lockfail = 1; | ||||
} | } | ||||
/* See if we lost the race to fill the cache. */ | |||||
critical_enter(); | critical_enter(); | ||||
/* Short-circuit for zones without buckets and low memory. */ | |||||
if (zone->uz_bucket_size == 0 || bucketdisable) | |||||
return (false); | |||||
cache = &zone->uz_cpu[curcpu]; | cache = &zone->uz_cpu[curcpu]; | ||||
/* See if we lost the race to fill the cache. */ | |||||
if (cache->uc_allocbucket.ucb_bucket != NULL) { | if (cache->uc_allocbucket.ucb_bucket != NULL) { | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
return (true); | return (true); | ||||
} | } | ||||
/* | /* | ||||
* Check the zone's cache of buckets. | * Check the zone's cache of buckets. | ||||
*/ | */ | ||||
Show All 16 Lines | cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) | ||||
critical_exit(); | critical_exit(); | ||||
/* | /* | ||||
* We bump the uz count when the cache size is insufficient to | * We bump the uz count when the cache size is insufficient to | ||||
* handle the working set. | * handle the working set. | ||||
*/ | */ | ||||
if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max) | if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max) | ||||
zone->uz_bucket_size++; | zone->uz_bucket_size++; | ||||
ZONE_UNLOCK(zone); | |||||
/* | /* | ||||
* Fill a bucket and attempt to use it as the alloc bucket. | * Fill a bucket and attempt to use it as the alloc bucket. | ||||
*/ | */ | ||||
bucket = zone_alloc_bucket(zone, udata, domain, flags); | bucket = zone_alloc_bucket(zone, udata, domain, flags); | ||||
CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p", | CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p", | ||||
zone->uz_name, zone, bucket); | zone->uz_name, zone, bucket); | ||||
if (bucket == NULL) { | |||||
critical_enter(); | critical_enter(); | ||||
if (bucket == NULL) | |||||
return (false); | return (false); | ||||
} | |||||
/* | /* | ||||
* See if we lost the race or were migrated. Cache the | * See if we lost the race or were migrated. Cache the | ||||
* initialized bucket to make this less likely or claim | * initialized bucket to make this less likely or claim | ||||
* the memory directly. | * the memory directly. | ||||
*/ | */ | ||||
ZONE_LOCK(zone); | |||||
critical_enter(); | |||||
cache = &zone->uz_cpu[curcpu]; | cache = &zone->uz_cpu[curcpu]; | ||||
if (cache->uc_allocbucket.ucb_bucket == NULL && | if (cache->uc_allocbucket.ucb_bucket == NULL && | ||||
((zone->uz_flags & UMA_ZONE_NUMA) == 0 || | ((zone->uz_flags & UMA_ZONE_NUMA) == 0 || | ||||
domain == PCPU_GET(domain))) { | domain == PCPU_GET(domain))) { | ||||
cache_bucket_load_alloc(cache, bucket); | cache_bucket_load_alloc(cache, bucket); | ||||
zdom->uzd_imax += bucket->ub_cnt; | zdom->uzd_imax += bucket->ub_cnt; | ||||
} else if (zone->uz_bkt_count >= zone->uz_bkt_max) { | } else if (zone->uz_bkt_count >= zone->uz_bkt_max) { | ||||
critical_exit(); | critical_exit(); | ||||
▲ Show 20 Lines • Show All 116 Lines • ▼ Show 20 Lines | if (slab != NULL) | ||||
return (slab); | return (slab); | ||||
/* | /* | ||||
* M_NOVM means don't ask at all! | * M_NOVM means don't ask at all! | ||||
*/ | */ | ||||
if (flags & M_NOVM) | if (flags & M_NOVM) | ||||
break; | break; | ||||
KASSERT(zone->uz_max_items == 0 || | |||||
zone->uz_items <= zone->uz_max_items, | |||||
("%s: zone %p overflow", __func__, zone)); | |||||
slab = keg_alloc_slab(keg, zone, domain, flags, aflags); | slab = keg_alloc_slab(keg, zone, domain, flags, aflags); | ||||
/* | /* | ||||
* If we got a slab here it's safe to mark it partially used | * If we got a slab here it's safe to mark it partially used | ||||
* and return. We assume that the caller is going to remove | * and return. We assume that the caller is going to remove | ||||
* at least one item. | * at least one item. | ||||
*/ | */ | ||||
if (slab) { | if (slab) { | ||||
dom = &keg->uk_domain[slab->us_domain]; | dom = &keg->uk_domain[slab->us_domain]; | ||||
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines | #endif | ||||
flags &= ~M_WAITOK; | flags &= ~M_WAITOK; | ||||
flags |= M_NOWAIT; | flags |= M_NOWAIT; | ||||
} | } | ||||
KEG_UNLOCK(keg); | KEG_UNLOCK(keg); | ||||
return i; | return i; | ||||
} | } | ||||
static int | |||||
zone_alloc_limit_hard(uma_zone_t zone, int count, int flags) | |||||
{ | |||||
uint64_t old, new, total, max; | |||||
/* | |||||
* The hard case. We're going to sleep because there were existing | |||||
* sleepers or because we ran out of items. This routine enforces | |||||
* fairness by keeping fifo order. | |||||
* | |||||
* First release our ill gotten gains and make some noise. | |||||
*/ | |||||
for (;;) { | |||||
zone_free_limit(zone, count); | |||||
zone_log_warning(zone); | |||||
zone_maxaction(zone); | |||||
if (flags & M_NOWAIT) | |||||
return (0); | |||||
/* | |||||
* We need to allocate an item or set ourself as a sleeper | |||||
* while the sleepq lock is held to avoid wakeup races. This | |||||
* is essentially a home rolled semaphore. | |||||
*/ | |||||
sleepq_lock(&zone->uz_max_items); | |||||
old = zone->uz_items; | |||||
do { | |||||
MPASS(UZ_ITEMS_SLEEPERS(old) < UZ_ITEMS_SLEEPERS_MAX); | |||||
/* Cache the max since we will evaluate twice. */ | |||||
max = zone->uz_max_items; | |||||
if (UZ_ITEMS_SLEEPERS(old) != 0 || | |||||
UZ_ITEMS_COUNT(old) >= max) | |||||
new = old + UZ_ITEMS_SLEEPER; | |||||
else | |||||
new = old + MIN(count, max - old); | |||||
} while (atomic_fcmpset_64(&zone->uz_items, &old, new) == 0); | |||||
/* We may have successfully allocated under the sleepq lock. */ | |||||
if (UZ_ITEMS_SLEEPERS(new) == 0) { | |||||
sleepq_release(&zone->uz_max_items); | |||||
return (new - old); | |||||
} | |||||
/* | |||||
* This is in a different cacheline from uz_items so that we | |||||
* don't constantly invalidate the fastpath cacheline when we | |||||
* adjust item counts. This could be limited to toggling on | |||||
* transitions. | |||||
*/ | |||||
atomic_add_32(&zone->uz_sleepers, 1); | |||||
atomic_add_64(&zone->uz_sleeps, 1); | |||||
/* | |||||
* We have added ourselves as a sleeper. The sleepq lock | |||||
* protects us from wakeup races. Sleep now and then retry. | |||||
*/ | |||||
sleepq_add(&zone->uz_max_items, NULL, "zonelimit", 0, 0); | |||||
sleepq_wait(&zone->uz_max_items, PVM); | |||||
/* | |||||
* After wakeup, remove ourselves as a sleeper and try | |||||
* again. We no longer have the sleepq lock for protection. | |||||
* | |||||
* Subract ourselves as a sleeper while attempting to add | |||||
* our count. | |||||
*/ | |||||
atomic_subtract_32(&zone->uz_sleepers, 1); | |||||
old = atomic_fetchadd_64(&zone->uz_items, | |||||
-(UZ_ITEMS_SLEEPER - count)); | |||||
/* We're no longer a sleeper. */ | |||||
old -= UZ_ITEMS_SLEEPER; | |||||
/* | |||||
* If we're still at the limit, restart. Notably do not | |||||
* block on other sleepers. Cache the max value to protect | |||||
* against changes via sysctl. | |||||
*/ | |||||
total = UZ_ITEMS_COUNT(old); | |||||
max = zone->uz_max_items; | |||||
if (total >= max) | |||||
continue; | |||||
/* Truncate if necessary, otherwise wake other sleepers. */ | |||||
if (total + count > max) { | |||||
zone_free_limit(zone, total + count - max); | |||||
count = max - total; | |||||
} else if (total + count < max && UZ_ITEMS_SLEEPERS(old) != 0) | |||||
wakeup_one(&zone->uz_max_items); | |||||
return (count); | |||||
} | |||||
} | |||||
/* | |||||
* Allocate 'count' items from our max_items limit. Returns the number | |||||
* available. If M_NOWAIT is not specified it will sleep until at least | |||||
* one item can be allocated. | |||||
*/ | |||||
static int | |||||
zone_alloc_limit(uma_zone_t zone, int count, int flags) | |||||
{ | |||||
uint64_t old; | |||||
uint64_t max; | |||||
max = zone->uz_max_items; | |||||
MPASS(max > 0); | |||||
/* | |||||
* We expect normal allocations to succeed with a simple | |||||
* fetchadd. | |||||
*/ | |||||
old = atomic_fetchadd_64(&zone->uz_items, count); | |||||
if (__predict_true(old + count <= max)) | |||||
return (count); | |||||
/* | |||||
* If we had some items and no sleepers just return the | |||||
* truncated value. We have to release the excess space | |||||
* though because that may wake sleepers who weren't woken | |||||
* because we were temporarily over the limit. | |||||
*/ | |||||
if (old < max) { | |||||
zone_free_limit(zone, (old + count) - max); | |||||
return (max - old); | |||||
} | |||||
return (zone_alloc_limit_hard(zone, count, flags)); | |||||
} | |||||
/* | |||||
* Free a number of items back to the limit. | |||||
*/ | |||||
static void | |||||
zone_free_limit(uma_zone_t zone, int count) | |||||
{ | |||||
uint64_t old; | |||||
MPASS(count > 0); | |||||
/* | |||||
* In the common case we either have no sleepers or | |||||
* are still over the limit and can just return. | |||||
*/ | |||||
old = atomic_fetchadd_64(&zone->uz_items, -count); | |||||
if (__predict_true(UZ_ITEMS_SLEEPERS(old) == 0 || | |||||
UZ_ITEMS_COUNT(old) - count >= zone->uz_max_items)) | |||||
return; | |||||
/* | |||||
* Moderate the rate of wakeups. Sleepers will continue | |||||
* to generate wakeups if necessary. | |||||
*/ | |||||
wakeup_one(&zone->uz_max_items); | |||||
} | |||||
static uma_bucket_t | static uma_bucket_t | ||||
zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) | zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) | ||||
{ | { | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
int maxbucket, cnt; | int maxbucket, cnt; | ||||
CTR1(KTR_UMA, "zone_alloc:_bucket domain %d)", domain); | CTR1(KTR_UMA, "zone_alloc:_bucket domain %d)", domain); | ||||
/* Avoid allocs targeting empty domains. */ | /* Avoid allocs targeting empty domains. */ | ||||
if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain)) | if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain)) | ||||
domain = UMA_ANYDOMAIN; | domain = UMA_ANYDOMAIN; | ||||
if (zone->uz_max_items > 0) { | if (zone->uz_max_items > 0) | ||||
if (zone->uz_items >= zone->uz_max_items) | maxbucket = zone_alloc_limit(zone, zone->uz_bucket_size, | ||||
return (false); | M_NOWAIT); | ||||
maxbucket = MIN(zone->uz_bucket_size, | else | ||||
zone->uz_max_items - zone->uz_items); | |||||
zone->uz_items += maxbucket; | |||||
} else | |||||
maxbucket = zone->uz_bucket_size; | maxbucket = zone->uz_bucket_size; | ||||
ZONE_UNLOCK(zone); | if (maxbucket == 0) | ||||
return (false); | |||||
/* Don't wait for buckets, preserve caller's NOVM setting. */ | /* Don't wait for buckets, preserve caller's NOVM setting. */ | ||||
bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM)); | bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM)); | ||||
if (bucket == NULL) { | if (bucket == NULL) { | ||||
cnt = 0; | cnt = 0; | ||||
goto out; | goto out; | ||||
} | } | ||||
Show All 27 Lines | #endif | ||||
cnt = bucket->ub_cnt; | cnt = bucket->ub_cnt; | ||||
if (bucket->ub_cnt == 0) { | if (bucket->ub_cnt == 0) { | ||||
bucket_free(zone, bucket, udata); | bucket_free(zone, bucket, udata); | ||||
counter_u64_add(zone->uz_fails, 1); | counter_u64_add(zone->uz_fails, 1); | ||||
bucket = NULL; | bucket = NULL; | ||||
} | } | ||||
out: | out: | ||||
ZONE_LOCK(zone); | if (zone->uz_max_items > 0 && cnt < maxbucket) | ||||
if (zone->uz_max_items > 0 && cnt < maxbucket) { | zone_free_limit(zone, maxbucket - cnt); | ||||
MPASS(zone->uz_items >= maxbucket - cnt); | |||||
zone->uz_items -= maxbucket - cnt; | |||||
if (zone->uz_sleepers > 0 && | |||||
(cnt == 0 ? zone->uz_items + 1 : zone->uz_items) < | |||||
zone->uz_max_items) | |||||
wakeup_one(zone); | |||||
} | |||||
return (bucket); | return (bucket); | ||||
} | } | ||||
/* | /* | ||||
* Allocates a single item from a zone. | * Allocates a single item from a zone. | ||||
* | * | ||||
* Arguments | * Arguments | ||||
* zone The zone to alloc for. | * zone The zone to alloc for. | ||||
* udata The data to be passed to the constructor. | * udata The data to be passed to the constructor. | ||||
* domain The domain to allocate from or UMA_ANYDOMAIN. | * domain The domain to allocate from or UMA_ANYDOMAIN. | ||||
* flags M_WAITOK, M_NOWAIT, M_ZERO. | * flags M_WAITOK, M_NOWAIT, M_ZERO. | ||||
* | * | ||||
* Returns | * Returns | ||||
* NULL if there is no memory and M_NOWAIT is set | * NULL if there is no memory and M_NOWAIT is set | ||||
* An item if successful | * An item if successful | ||||
*/ | */ | ||||
static void * | static void * | ||||
zone_alloc_item(uma_zone_t zone, void *udata, int domain, int flags) | zone_alloc_item(uma_zone_t zone, void *udata, int domain, int flags) | ||||
{ | { | ||||
ZONE_LOCK(zone); | |||||
return (zone_alloc_item_locked(zone, udata, domain, flags)); | |||||
} | |||||
/* | |||||
* Returns with zone unlocked. | |||||
*/ | |||||
static void * | |||||
zone_alloc_item_locked(uma_zone_t zone, void *udata, int domain, int flags) | |||||
{ | |||||
void *item; | void *item; | ||||
ZONE_LOCK_ASSERT(zone); | if (zone->uz_max_items > 0 && zone_alloc_limit(zone, 1, flags) == 0) | ||||
if (zone->uz_max_items > 0) { | |||||
if (zone->uz_items >= zone->uz_max_items) { | |||||
zone_log_warning(zone); | |||||
zone_maxaction(zone); | |||||
if (flags & M_NOWAIT) { | |||||
ZONE_UNLOCK(zone); | |||||
return (NULL); | return (NULL); | ||||
} | |||||
zone->uz_sleeps++; | |||||
zone->uz_sleepers++; | |||||
while (zone->uz_items >= zone->uz_max_items) | |||||
mtx_sleep(zone, zone->uz_lockptr, PVM, | |||||
"zonelimit", 0); | |||||
zone->uz_sleepers--; | |||||
if (zone->uz_sleepers > 0 && | |||||
zone->uz_items + 1 < zone->uz_max_items) | |||||
wakeup_one(zone); | |||||
} | |||||
zone->uz_items++; | |||||
} | |||||
ZONE_UNLOCK(zone); | |||||
/* Avoid allocs targeting empty domains. */ | /* Avoid allocs targeting empty domains. */ | ||||
if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain)) | if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain)) | ||||
domain = UMA_ANYDOMAIN; | domain = UMA_ANYDOMAIN; | ||||
if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1) | if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1) | ||||
goto fail_cnt; | goto fail_cnt; | ||||
Show All 17 Lines | zone_alloc_item(uma_zone_t zone, void *udata, int domain, int flags) | ||||
CTR3(KTR_UMA, "zone_alloc_item item %p from %s(%p)", item, | CTR3(KTR_UMA, "zone_alloc_item item %p from %s(%p)", item, | ||||
zone->uz_name, zone); | zone->uz_name, zone); | ||||
return (item); | return (item); | ||||
fail_cnt: | fail_cnt: | ||||
counter_u64_add(zone->uz_fails, 1); | counter_u64_add(zone->uz_fails, 1); | ||||
fail: | fail: | ||||
if (zone->uz_max_items > 0) { | if (zone->uz_max_items > 0) | ||||
ZONE_LOCK(zone); | zone_free_limit(zone, 1); | ||||
/* XXX Decrement without wakeup */ | |||||
zone->uz_items--; | |||||
ZONE_UNLOCK(zone); | |||||
} | |||||
CTR2(KTR_UMA, "zone_alloc_item failed from %s(%p)", | CTR2(KTR_UMA, "zone_alloc_item failed from %s(%p)", | ||||
zone->uz_name, zone); | zone->uz_name, zone); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
void | void | ||||
uma_zfree_arg(uma_zone_t zone, void *item, void *udata) | uma_zfree_arg(uma_zone_t zone, void *item, void *udata) | ||||
{ | { | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
▲ Show 20 Lines • Show All 329 Lines • ▼ Show 20 Lines | zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) | ||||
zone->uz_release(zone->uz_arg, &item, 1); | zone->uz_release(zone->uz_arg, &item, 1); | ||||
if (skip & SKIP_CNT) | if (skip & SKIP_CNT) | ||||
return; | return; | ||||
counter_u64_add(zone->uz_frees, 1); | counter_u64_add(zone->uz_frees, 1); | ||||
if (zone->uz_max_items > 0) { | if (zone->uz_max_items > 0) | ||||
ZONE_LOCK(zone); | zone_free_limit(zone, 1); | ||||
zone->uz_items--; | |||||
if (zone->uz_sleepers > 0 && | |||||
zone->uz_items < zone->uz_max_items) | |||||
wakeup_one(zone); | |||||
ZONE_UNLOCK(zone); | |||||
} | } | ||||
} | |||||
/* See uma.h */ | /* See uma.h */ | ||||
int | int | ||||
uma_zone_set_max(uma_zone_t zone, int nitems) | uma_zone_set_max(uma_zone_t zone, int nitems) | ||||
{ | { | ||||
struct uma_bucket_zone *ubz; | struct uma_bucket_zone *ubz; | ||||
int count; | int count; | ||||
/* | |||||
* XXX This can misbehave if the zone has any allocations with | |||||
* no limit and a limit is imposed. There is currently no | |||||
* way to clear a limit. | |||||
*/ | |||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
ubz = bucket_zone_max(zone, nitems); | ubz = bucket_zone_max(zone, nitems); | ||||
count = ubz != NULL ? ubz->ubz_entries : 0; | count = ubz != NULL ? ubz->ubz_entries : 0; | ||||
zone->uz_bucket_size_max = zone->uz_bucket_size = count; | zone->uz_bucket_size_max = zone->uz_bucket_size = count; | ||||
if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) | if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) | ||||
zone->uz_bucket_size_min = zone->uz_bucket_size_max; | zone->uz_bucket_size_min = zone->uz_bucket_size_max; | ||||
zone->uz_max_items = nitems; | zone->uz_max_items = nitems; | ||||
zone->uz_flags |= UMA_ZFLAG_LIMIT; | zone->uz_flags |= UMA_ZFLAG_LIMIT; | ||||
zone_update_caches(zone); | zone_update_caches(zone); | ||||
/* We may need to wake waiters. */ | |||||
wakeup(&zone->uz_max_items); | |||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
return (nitems); | return (nitems); | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
void | void | ||||
uma_zone_set_maxcache(uma_zone_t zone, int nitems) | uma_zone_set_maxcache(uma_zone_t zone, int nitems) | ||||
▲ Show 20 Lines • Show All 542 Lines • ▼ Show 20 Lines | |||||
sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) | sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
struct uma_stream_header ush; | struct uma_stream_header ush; | ||||
struct uma_type_header uth; | struct uma_type_header uth; | ||||
struct uma_percpu_stat *ups; | struct uma_percpu_stat *ups; | ||||
struct sbuf sbuf; | struct sbuf sbuf; | ||||
uma_keg_t kz; | uma_keg_t kz; | ||||
uma_zone_t z; | uma_zone_t z; | ||||
uint64_t items; | |||||
int count, error, i; | int count, error, i; | ||||
error = sysctl_wire_old_buffer(req, 0); | error = sysctl_wire_old_buffer(req, 0); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
sbuf_new_for_sysctl(&sbuf, NULL, 128, req); | sbuf_new_for_sysctl(&sbuf, NULL, 128, req); | ||||
sbuf_clear_flags(&sbuf, SBUF_INCLUDENUL); | sbuf_clear_flags(&sbuf, SBUF_INCLUDENUL); | ||||
ups = malloc((mp_maxid + 1) * sizeof(*ups), M_TEMP, M_WAITOK); | ups = malloc((mp_maxid + 1) * sizeof(*ups), M_TEMP, M_WAITOK); | ||||
Show All 20 Lines | sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) | ||||
LIST_FOREACH(kz, &uma_kegs, uk_link) { | LIST_FOREACH(kz, &uma_kegs, uk_link) { | ||||
LIST_FOREACH(z, &kz->uk_zones, uz_link) { | LIST_FOREACH(z, &kz->uk_zones, uz_link) { | ||||
bzero(&uth, sizeof(uth)); | bzero(&uth, sizeof(uth)); | ||||
ZONE_LOCK(z); | ZONE_LOCK(z); | ||||
strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); | strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); | ||||
uth.uth_align = kz->uk_align; | uth.uth_align = kz->uk_align; | ||||
uth.uth_size = kz->uk_size; | uth.uth_size = kz->uk_size; | ||||
uth.uth_rsize = kz->uk_rsize; | uth.uth_rsize = kz->uk_rsize; | ||||
if (z->uz_max_items > 0) | if (z->uz_max_items > 0) { | ||||
uth.uth_pages = (z->uz_items / kz->uk_ipers) * | items = UZ_ITEMS_COUNT(z->uz_items); | ||||
uth.uth_pages = (items / kz->uk_ipers) * | |||||
kz->uk_ppera; | kz->uk_ppera; | ||||
else | } else | ||||
uth.uth_pages = kz->uk_pages; | uth.uth_pages = kz->uk_pages; | ||||
uth.uth_maxpages = (z->uz_max_items / kz->uk_ipers) * | uth.uth_maxpages = (z->uz_max_items / kz->uk_ipers) * | ||||
kz->uk_ppera; | kz->uk_ppera; | ||||
uth.uth_limit = z->uz_max_items; | uth.uth_limit = z->uz_max_items; | ||||
uth.uth_keg_free = z->uz_keg->uk_free; | uth.uth_keg_free = z->uz_keg->uk_free; | ||||
/* | /* | ||||
* A zone is secondary is it is not the first entry | * A zone is secondary is it is not the first entry | ||||
▲ Show 20 Lines • Show All 115 Lines • ▼ Show 20 Lines | sysctl_handle_uma_slab_efficiency(SYSCTL_HANDLER_ARGS) | ||||
* real size determination uk_rsize, because we also adjust the real | * real size determination uk_rsize, because we also adjust the real | ||||
* size for internal implementation reasons (max bitset size). | * size for internal implementation reasons (max bitset size). | ||||
*/ | */ | ||||
avail = keg->uk_ipers * roundup2(keg->uk_size, keg->uk_align + 1); | avail = keg->uk_ipers * roundup2(keg->uk_size, keg->uk_align + 1); | ||||
if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) | if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) | ||||
avail *= mp_maxid + 1; | avail *= mp_maxid + 1; | ||||
effpct = 100 * avail / total; | effpct = 100 * avail / total; | ||||
return (sysctl_handle_int(oidp, &effpct, 0, req)); | return (sysctl_handle_int(oidp, &effpct, 0, req)); | ||||
} | |||||
static int | |||||
sysctl_handle_uma_zone_items(SYSCTL_HANDLER_ARGS) | |||||
{ | |||||
uma_zone_t zone = arg1; | |||||
uint64_t cur; | |||||
cur = UZ_ITEMS_COUNT(atomic_load_64(&zone->uz_items)); | |||||
return (sysctl_handle_64(oidp, &cur, 0, req)); | |||||
} | } | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
static uma_slab_t | static uma_slab_t | ||||
uma_dbg_getslab(uma_zone_t zone, void *item) | uma_dbg_getslab(uma_zone_t zone, void *item) | ||||
{ | { | ||||
uma_slab_t slab; | uma_slab_t slab; | ||||
uma_keg_t keg; | uma_keg_t keg; | ||||
▲ Show 20 Lines • Show All 243 Lines • Show Last 20 Lines |