Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_core.c
Show First 20 Lines • Show All 779 Lines • ▼ Show 20 Lines | cache_drain(uma_zone_t zone) | ||||
* | * | ||||
* XXX: We lock the zone before passing into bucket_cache_drain() as | * XXX: We lock the zone before passing into bucket_cache_drain() as | ||||
* it is used elsewhere. Should the tear-down path be made special | * it is used elsewhere. Should the tear-down path be made special | ||||
* there in some form? | * there in some form? | ||||
*/ | */ | ||||
CPU_FOREACH(cpu) { | CPU_FOREACH(cpu) { | ||||
cache = &zone->uz_cpu[cpu]; | cache = &zone->uz_cpu[cpu]; | ||||
bucket_drain(zone, cache->uc_allocbucket); | bucket_drain(zone, cache->uc_allocbucket); | ||||
bucket_drain(zone, cache->uc_freebucket); | |||||
if (cache->uc_allocbucket != NULL) | if (cache->uc_allocbucket != NULL) | ||||
bucket_free(zone, cache->uc_allocbucket, NULL); | bucket_free(zone, cache->uc_allocbucket, NULL); | ||||
cache->uc_allocbucket = NULL; | |||||
bucket_drain(zone, cache->uc_freebucket); | |||||
if (cache->uc_freebucket != NULL) | if (cache->uc_freebucket != NULL) | ||||
bucket_free(zone, cache->uc_freebucket, NULL); | bucket_free(zone, cache->uc_freebucket, NULL); | ||||
cache->uc_allocbucket = cache->uc_freebucket = NULL; | cache->uc_freebucket = NULL; | ||||
bucket_drain(zone, cache->uc_crossbucket); | |||||
if (cache->uc_crossbucket != NULL) | |||||
bucket_free(zone, cache->uc_crossbucket, NULL); | |||||
cache->uc_crossbucket = NULL; | |||||
} | } | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
bucket_cache_drain(zone); | bucket_cache_drain(zone); | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
} | } | ||||
static void | static void | ||||
cache_shrink(uma_zone_t zone) | cache_shrink(uma_zone_t zone) | ||||
{ | { | ||||
if (zone->uz_flags & UMA_ZFLAG_INTERNAL) | if (zone->uz_flags & UMA_ZFLAG_INTERNAL) | ||||
return; | return; | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2; | zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2; | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
} | } | ||||
static void | static void | ||||
cache_drain_safe_cpu(uma_zone_t zone) | cache_drain_safe_cpu(uma_zone_t zone) | ||||
{ | { | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
uma_bucket_t b1, b2; | uma_bucket_t b1, b2, b3; | ||||
int domain; | int domain; | ||||
if (zone->uz_flags & UMA_ZFLAG_INTERNAL) | if (zone->uz_flags & UMA_ZFLAG_INTERNAL) | ||||
return; | return; | ||||
b1 = b2 = NULL; | b1 = b2 = b3 = NULL; | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
critical_enter(); | critical_enter(); | ||||
if (zone->uz_flags & UMA_ZONE_NUMA) | if (zone->uz_flags & UMA_ZONE_NUMA) | ||||
domain = PCPU_GET(domain); | domain = PCPU_GET(domain); | ||||
else | else | ||||
domain = 0; | domain = 0; | ||||
cache = &zone->uz_cpu[curcpu]; | cache = &zone->uz_cpu[curcpu]; | ||||
if (cache->uc_allocbucket) { | if (cache->uc_allocbucket) { | ||||
if (cache->uc_allocbucket->ub_cnt != 0) | if (cache->uc_allocbucket->ub_cnt != 0) | ||||
zone_put_bucket(zone, &zone->uz_domain[domain], | zone_put_bucket(zone, &zone->uz_domain[domain], | ||||
cache->uc_allocbucket, false); | cache->uc_allocbucket, false); | ||||
else | else | ||||
b1 = cache->uc_allocbucket; | b1 = cache->uc_allocbucket; | ||||
cache->uc_allocbucket = NULL; | cache->uc_allocbucket = NULL; | ||||
} | } | ||||
if (cache->uc_freebucket) { | if (cache->uc_freebucket) { | ||||
if (cache->uc_freebucket->ub_cnt != 0) | if (cache->uc_freebucket->ub_cnt != 0) | ||||
zone_put_bucket(zone, &zone->uz_domain[domain], | zone_put_bucket(zone, &zone->uz_domain[domain], | ||||
cache->uc_freebucket, false); | cache->uc_freebucket, false); | ||||
else | else | ||||
b2 = cache->uc_freebucket; | b2 = cache->uc_freebucket; | ||||
cache->uc_freebucket = NULL; | cache->uc_freebucket = NULL; | ||||
} | } | ||||
b3 = cache->uc_crossbucket; | |||||
cache->uc_crossbucket = NULL; | |||||
critical_exit(); | critical_exit(); | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
if (b1) | if (b1) | ||||
bucket_free(zone, b1, NULL); | bucket_free(zone, b1, NULL); | ||||
if (b2) | if (b2) | ||||
bucket_free(zone, b2, NULL); | bucket_free(zone, b2, NULL); | ||||
if (b3) { | |||||
bucket_drain(zone, b3); | |||||
bucket_free(zone, b3, NULL); | |||||
} | } | ||||
} | |||||
/* | /* | ||||
* Safely drain per-CPU caches of a zone(s) to alloc bucket. | * Safely drain per-CPU caches of a zone(s) to alloc bucket. | ||||
* This is an expensive call because it needs to bind to all CPUs | * This is an expensive call because it needs to bind to all CPUs | ||||
* one by one and enter a critical section on each of them in order | * one by one and enter a critical section on each of them in order | ||||
* to safely access their cache buckets. | * to safely access their cache buckets. | ||||
* Zone lock must not be held on call this function. | * Zone lock must not be held on call this function. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 888 Lines • ▼ Show 20 Lines | zone_ctor(void *mem, int size, void *udata, int flags) | ||||
bzero(zone, size); | bzero(zone, size); | ||||
zone->uz_name = arg->name; | zone->uz_name = arg->name; | ||||
zone->uz_ctor = arg->ctor; | zone->uz_ctor = arg->ctor; | ||||
zone->uz_dtor = arg->dtor; | zone->uz_dtor = arg->dtor; | ||||
zone->uz_init = NULL; | zone->uz_init = NULL; | ||||
zone->uz_fini = NULL; | zone->uz_fini = NULL; | ||||
zone->uz_sleeps = 0; | zone->uz_sleeps = 0; | ||||
zone->uz_xdomain = 0; | |||||
zone->uz_count = 0; | zone->uz_count = 0; | ||||
zone->uz_count_min = 0; | zone->uz_count_min = 0; | ||||
zone->uz_count_max = BUCKET_MAX; | zone->uz_count_max = BUCKET_MAX; | ||||
zone->uz_flags = 0; | zone->uz_flags = 0; | ||||
zone->uz_warning = NULL; | zone->uz_warning = NULL; | ||||
/* The domain structures follow the cpu structures. */ | /* The domain structures follow the cpu structures. */ | ||||
zone->uz_domain = (struct uma_zone_domain *)&zone->uz_cpu[mp_ncpus]; | zone->uz_domain = (struct uma_zone_domain *)&zone->uz_cpu[mp_ncpus]; | ||||
zone->uz_bkt_max = ULONG_MAX; | zone->uz_bkt_max = ULONG_MAX; | ||||
▲ Show 20 Lines • Show All 415 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
struct uma_zctor_args args; | struct uma_zctor_args args; | ||||
uma_zone_t res; | uma_zone_t res; | ||||
bool locked; | bool locked; | ||||
KASSERT(powerof2(align + 1), ("invalid zone alignment %d for \"%s\"", | KASSERT(powerof2(align + 1), ("invalid zone alignment %d for \"%s\"", | ||||
align, name)); | align, name)); | ||||
/* Sets all zones to a first-touch domain policy. */ | |||||
#ifdef UMA_FIRSTTOUCH | |||||
flags |= UMA_ZONE_NUMA; | |||||
#endif | |||||
/* This stuff is essential for the zone ctor */ | /* This stuff is essential for the zone ctor */ | ||||
memset(&args, 0, sizeof(args)); | memset(&args, 0, sizeof(args)); | ||||
args.name = name; | args.name = name; | ||||
args.size = size; | args.size = size; | ||||
args.ctor = ctor; | args.ctor = ctor; | ||||
args.dtor = dtor; | args.dtor = dtor; | ||||
args.uminit = uminit; | args.uminit = uminit; | ||||
args.fini = fini; | args.fini = fini; | ||||
▲ Show 20 Lines • Show All 255 Lines • ▼ Show 20 Lines | #endif | ||||
* Discard any empty allocation bucket while we hold no locks. | * Discard any empty allocation bucket while we hold no locks. | ||||
*/ | */ | ||||
bucket = cache->uc_allocbucket; | bucket = cache->uc_allocbucket; | ||||
cache->uc_allocbucket = NULL; | cache->uc_allocbucket = NULL; | ||||
critical_exit(); | critical_exit(); | ||||
if (bucket != NULL) | if (bucket != NULL) | ||||
bucket_free(zone, bucket, udata); | bucket_free(zone, bucket, udata); | ||||
if (zone->uz_flags & UMA_ZONE_NUMA) { | |||||
domain = PCPU_GET(domain); | |||||
if (VM_DOMAIN_EMPTY(domain)) | |||||
domain = UMA_ANYDOMAIN; | |||||
} else | |||||
domain = UMA_ANYDOMAIN; | |||||
/* Short-circuit for zones without buckets and low memory. */ | /* Short-circuit for zones without buckets and low memory. */ | ||||
if (zone->uz_count == 0 || bucketdisable) { | if (zone->uz_count == 0 || bucketdisable) { | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
if (zone->uz_flags & UMA_ZONE_NUMA) | |||||
domain = PCPU_GET(domain); | |||||
else | |||||
domain = UMA_ANYDOMAIN; | |||||
goto zalloc_item; | goto zalloc_item; | ||||
} | } | ||||
/* | /* | ||||
* Attempt to retrieve the item from the per-CPU cache has failed, so | * Attempt to retrieve the item from the per-CPU cache has failed, so | ||||
* we must go back to the zone. This requires the zone lock, so we | * we must go back to the zone. This requires the zone lock, so we | ||||
* must drop the critical section, then re-acquire it when we go back | * must drop the critical section, then re-acquire it when we go back | ||||
* to the cache. Since the critical section is released, we may be | * to the cache. Since the critical section is released, we may be | ||||
Show All 15 Lines | #endif | ||||
if (cache->uc_allocbucket != NULL) { | if (cache->uc_allocbucket != NULL) { | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
goto zalloc_start; | goto zalloc_start; | ||||
} | } | ||||
/* | /* | ||||
* Check the zone's cache of buckets. | * Check the zone's cache of buckets. | ||||
*/ | */ | ||||
if (domain == UMA_ANYDOMAIN) | if (zone->uz_flags & UMA_ZONE_NUMA) { | ||||
zdom = &zone->uz_domain[0]; | domain = PCPU_GET(domain); | ||||
else | |||||
zdom = &zone->uz_domain[domain]; | zdom = &zone->uz_domain[domain]; | ||||
} else { | |||||
domain = UMA_ANYDOMAIN; | |||||
zdom = &zone->uz_domain[0]; | |||||
} | |||||
if ((bucket = zone_try_fetch_bucket(zone, zdom, true)) != NULL) { | if ((bucket = zone_try_fetch_bucket(zone, zdom, true)) != NULL) { | ||||
KASSERT(bucket->ub_cnt != 0, | KASSERT(bucket->ub_cnt != 0, | ||||
("uma_zalloc_arg: Returning an empty bucket.")); | ("uma_zalloc_arg: Returning an empty bucket.")); | ||||
cache->uc_allocbucket = bucket; | cache->uc_allocbucket = bucket; | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
goto zalloc_start; | goto zalloc_start; | ||||
} | } | ||||
/* We are no longer associated with this CPU. */ | /* We are no longer associated with this CPU. */ | ||||
▲ Show 20 Lines • Show All 325 Lines • ▼ Show 20 Lines | |||||
static uma_bucket_t | static uma_bucket_t | ||||
zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags, int max) | zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags, int max) | ||||
{ | { | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
CTR1(KTR_UMA, "zone_alloc:_bucket domain %d)", domain); | CTR1(KTR_UMA, "zone_alloc:_bucket domain %d)", domain); | ||||
/* Avoid allocs targeting empty domains. */ | |||||
if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain)) | |||||
domain = UMA_ANYDOMAIN; | |||||
/* Don't wait for buckets, preserve caller's NOVM setting. */ | /* Don't wait for buckets, preserve caller's NOVM setting. */ | ||||
bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM)); | bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM)); | ||||
if (bucket == NULL) | if (bucket == NULL) | ||||
return (NULL); | return (NULL); | ||||
bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket, | bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket, | ||||
MIN(max, bucket->ub_entries), domain, flags); | MIN(max, bucket->ub_entries), domain, flags); | ||||
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines | if (zone->uz_items >= zone->uz_max_items) { | ||||
if (zone->uz_sleepers > 0 && | if (zone->uz_sleepers > 0 && | ||||
zone->uz_items + 1 < zone->uz_max_items) | zone->uz_items + 1 < zone->uz_max_items) | ||||
wakeup_one(zone); | wakeup_one(zone); | ||||
} | } | ||||
zone->uz_items++; | zone->uz_items++; | ||||
} | } | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
if (domain != UMA_ANYDOMAIN) { | /* Avoid allocs targeting empty domains. */ | ||||
/* avoid allocs targeting empty domains */ | if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain)) | ||||
if (VM_DOMAIN_EMPTY(domain)) | |||||
domain = UMA_ANYDOMAIN; | domain = UMA_ANYDOMAIN; | ||||
} | |||||
if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1) | if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1) | ||||
goto fail; | goto fail; | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
skipdbg = uma_dbg_zskip(zone, item); | skipdbg = uma_dbg_zskip(zone, item); | ||||
#endif | #endif | ||||
/* | /* | ||||
* We have to call both the zone's init (not the keg's init) | * We have to call both the zone's init (not the keg's init) | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | |||||
/* See uma.h */ | /* See uma.h */ | ||||
void | void | ||||
uma_zfree_arg(uma_zone_t zone, void *item, void *udata) | uma_zfree_arg(uma_zone_t zone, void *item, void *udata) | ||||
{ | { | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
uma_zone_domain_t zdom; | uma_zone_domain_t zdom; | ||||
int cpu, domain; | int cpu, domain; | ||||
#ifdef UMA_XDOMAIN | |||||
int itemdomain; | |||||
#endif | |||||
bool lockfail; | bool lockfail; | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
bool skipdbg; | bool skipdbg; | ||||
#endif | #endif | ||||
/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | ||||
random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | ||||
Show All 33 Lines | #endif | ||||
/* | /* | ||||
* The race here is acceptable. If we miss it we'll just have to wait | * The race here is acceptable. If we miss it we'll just have to wait | ||||
* a little longer for the limits to be reset. | * a little longer for the limits to be reset. | ||||
*/ | */ | ||||
if (zone->uz_sleepers > 0) | if (zone->uz_sleepers > 0) | ||||
goto zfree_item; | goto zfree_item; | ||||
#ifdef UMA_XDOMAIN | |||||
if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) | |||||
itemdomain = _vm_phys_domain(pmap_kextract((vm_offset_t)item)); | |||||
#endif | |||||
/* | /* | ||||
* If possible, free to the per-CPU cache. There are two | * If possible, free to the per-CPU cache. There are two | ||||
* requirements for safe access to the per-CPU cache: (1) the thread | * requirements for safe access to the per-CPU cache: (1) the thread | ||||
* accessing the cache must not be preempted or yield during access, | * accessing the cache must not be preempted or yield during access, | ||||
* and (2) the thread must not migrate CPUs without switching which | * and (2) the thread must not migrate CPUs without switching which | ||||
* cache it accesses. We rely on a critical section to prevent | * cache it accesses. We rely on a critical section to prevent | ||||
* preemption and migration. We release the critical section in | * preemption and migration. We release the critical section in | ||||
* order to acquire the zone mutex if we are unable to free to the | * order to acquire the zone mutex if we are unable to free to the | ||||
* current cache; when we re-acquire the critical section, we must | * current cache; when we re-acquire the critical section, we must | ||||
* detect and handle migration if it has occurred. | * detect and handle migration if it has occurred. | ||||
*/ | */ | ||||
zfree_restart: | zfree_restart: | ||||
critical_enter(); | critical_enter(); | ||||
cpu = curcpu; | cpu = curcpu; | ||||
cache = &zone->uz_cpu[cpu]; | cache = &zone->uz_cpu[cpu]; | ||||
zfree_start: | zfree_start: | ||||
domain = PCPU_GET(domain); | |||||
#ifdef UMA_XDOMAIN | |||||
if ((zone->uz_flags & UMA_ZONE_NUMA) == 0) | |||||
itemdomain = domain; | |||||
#endif | |||||
/* | /* | ||||
* Try to free into the allocbucket first to give LIFO ordering | * Try to free into the allocbucket first to give LIFO ordering | ||||
* for cache-hot datastructures. Spill over into the freebucket | * for cache-hot datastructures. Spill over into the freebucket | ||||
* if necessary. Alloc will swap them if one runs dry. | * if necessary. Alloc will swap them if one runs dry. | ||||
*/ | */ | ||||
#ifdef UMA_XDOMAIN | |||||
if (domain != itemdomain) { | |||||
bucket = cache->uc_crossbucket; | |||||
} else | |||||
#endif | |||||
{ | |||||
bucket = cache->uc_allocbucket; | bucket = cache->uc_allocbucket; | ||||
if (bucket == NULL || bucket->ub_cnt >= bucket->ub_entries) | if (bucket == NULL || bucket->ub_cnt >= bucket->ub_entries) | ||||
bucket = cache->uc_freebucket; | bucket = cache->uc_freebucket; | ||||
} | |||||
if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) { | if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) { | ||||
KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL, | KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL, | ||||
("uma_zfree: Freeing to non free bucket index.")); | ("uma_zfree: Freeing to non free bucket index.")); | ||||
bucket->ub_bucket[bucket->ub_cnt] = item; | bucket->ub_bucket[bucket->ub_cnt] = item; | ||||
bucket->ub_cnt++; | bucket->ub_cnt++; | ||||
cache->uc_frees++; | cache->uc_frees++; | ||||
critical_exit(); | critical_exit(); | ||||
return; | return; | ||||
Show All 14 Lines | #endif | ||||
lockfail = false; | lockfail = false; | ||||
if (ZONE_TRYLOCK(zone) == 0) { | if (ZONE_TRYLOCK(zone) == 0) { | ||||
/* Record contention to size the buckets. */ | /* Record contention to size the buckets. */ | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
lockfail = true; | lockfail = true; | ||||
} | } | ||||
critical_enter(); | critical_enter(); | ||||
cpu = curcpu; | cpu = curcpu; | ||||
domain = PCPU_GET(domain); | |||||
cache = &zone->uz_cpu[cpu]; | cache = &zone->uz_cpu[cpu]; | ||||
#ifdef UMA_XDOMAIN | |||||
if (domain != itemdomain) | |||||
bucket = cache->uc_crossbucket; | |||||
else | |||||
#endif | |||||
bucket = cache->uc_freebucket; | bucket = cache->uc_freebucket; | ||||
if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) { | if (bucket != NULL && bucket->ub_cnt < bucket->ub_entries) { | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
goto zfree_start; | goto zfree_start; | ||||
} | } | ||||
#ifdef UMA_XDOMAIN | |||||
if (domain != itemdomain) | |||||
cache->uc_crossbucket = NULL; | |||||
else | |||||
#endif | |||||
cache->uc_freebucket = NULL; | cache->uc_freebucket = NULL; | ||||
/* We are no longer associated with this CPU. */ | /* We are no longer associated with this CPU. */ | ||||
critical_exit(); | critical_exit(); | ||||
if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) { | #ifdef UMA_XDOMAIN | ||||
domain = PCPU_GET(domain); | if (domain != itemdomain) { | ||||
if (VM_DOMAIN_EMPTY(domain)) | if (bucket != NULL) { | ||||
domain = UMA_ANYDOMAIN; | zone->uz_xdomain += bucket->ub_cnt; | ||||
if (vm_ndomains > 2 || | |||||
zone->uz_bkt_count >= zone->uz_bkt_max) { | |||||
ZONE_UNLOCK(zone); | |||||
bucket_drain(zone, bucket); | |||||
bucket_free(zone, bucket, udata); | |||||
} else { | |||||
zdom = &zone->uz_domain[itemdomain]; | |||||
zone_put_bucket(zone, zdom, bucket, true); | |||||
ZONE_UNLOCK(zone); | |||||
} | |||||
} else | } else | ||||
ZONE_UNLOCK(zone); | |||||
bucket = bucket_alloc(zone, udata, M_NOWAIT); | |||||
if (bucket == NULL) | |||||
goto zfree_item; | |||||
critical_enter(); | |||||
cpu = curcpu; | |||||
cache = &zone->uz_cpu[cpu]; | |||||
if (cache->uc_crossbucket == NULL) { | |||||
cache->uc_crossbucket = bucket; | |||||
goto zfree_start; | |||||
} | |||||
critical_exit(); | |||||
bucket_free(zone, bucket, udata); | |||||
goto zfree_restart; | |||||
} | |||||
#endif | |||||
if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) { | |||||
zdom = &zone->uz_domain[domain]; | |||||
} else { | |||||
domain = 0; | domain = 0; | ||||
zdom = &zone->uz_domain[0]; | zdom = &zone->uz_domain[0]; | ||||
} | |||||
/* Can we throw this on the zone full list? */ | /* Can we throw this on the zone full list? */ | ||||
if (bucket != NULL) { | if (bucket != NULL) { | ||||
CTR3(KTR_UMA, | CTR3(KTR_UMA, | ||||
"uma_zfree: zone %s(%p) putting bucket %p on free list", | "uma_zfree: zone %s(%p) putting bucket %p on free list", | ||||
zone->uz_name, zone, bucket); | zone->uz_name, zone, bucket); | ||||
/* ub_cnt is pointing to the last free item */ | /* ub_cnt is pointing to the last free item */ | ||||
KASSERT(bucket->ub_cnt == bucket->ub_entries, | KASSERT(bucket->ub_cnt == bucket->ub_entries, | ||||
▲ Show 20 Lines • Show All 624 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
printf("slab: keg %p, data %p, freecount %d\n", | printf("slab: keg %p, data %p, freecount %d\n", | ||||
slab->us_keg, slab->us_data, slab->us_freecount); | slab->us_keg, slab->us_data, slab->us_freecount); | ||||
} | } | ||||
static void | static void | ||||
cache_print(uma_cache_t cache) | cache_print(uma_cache_t cache) | ||||
{ | { | ||||
printf("alloc: %p(%d), free: %p(%d)\n", | printf("alloc: %p(%d), free: %p(%d), cross: %p(%d)j\n", | ||||
cache->uc_allocbucket, | cache->uc_allocbucket, | ||||
cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0, | cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0, | ||||
cache->uc_freebucket, | cache->uc_freebucket, | ||||
cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0); | cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0, | ||||
cache->uc_crossbucket, | |||||
cache->uc_crossbucket?cache->uc_crossbucket->ub_cnt:0); | |||||
} | } | ||||
static void | static void | ||||
uma_print_keg(uma_keg_t keg) | uma_print_keg(uma_keg_t keg) | ||||
{ | { | ||||
uma_domain_t dom; | uma_domain_t dom; | ||||
uma_slab_t slab; | uma_slab_t slab; | ||||
int i; | int i; | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | |||||
* per-CPU cache statistic. | * per-CPU cache statistic. | ||||
* | * | ||||
* XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't | * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't | ||||
* safe from off-CPU; we should modify the caches to track this information | * safe from off-CPU; we should modify the caches to track this information | ||||
* directly so that we don't have to. | * directly so that we don't have to. | ||||
*/ | */ | ||||
static void | static void | ||||
uma_zone_sumstat(uma_zone_t z, long *cachefreep, uint64_t *allocsp, | uma_zone_sumstat(uma_zone_t z, long *cachefreep, uint64_t *allocsp, | ||||
uint64_t *freesp, uint64_t *sleepsp) | uint64_t *freesp, uint64_t *sleepsp, uint64_t *xdomainp) | ||||
{ | { | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
uint64_t allocs, frees, sleeps; | uint64_t allocs, frees, sleeps, xdomain; | ||||
int cachefree, cpu; | int cachefree, cpu; | ||||
allocs = frees = sleeps = 0; | allocs = frees = sleeps = xdomain = 0; | ||||
cachefree = 0; | cachefree = 0; | ||||
CPU_FOREACH(cpu) { | CPU_FOREACH(cpu) { | ||||
cache = &z->uz_cpu[cpu]; | cache = &z->uz_cpu[cpu]; | ||||
if (cache->uc_allocbucket != NULL) | if (cache->uc_allocbucket != NULL) | ||||
cachefree += cache->uc_allocbucket->ub_cnt; | cachefree += cache->uc_allocbucket->ub_cnt; | ||||
if (cache->uc_freebucket != NULL) | if (cache->uc_freebucket != NULL) | ||||
cachefree += cache->uc_freebucket->ub_cnt; | cachefree += cache->uc_freebucket->ub_cnt; | ||||
if (cache->uc_crossbucket != NULL) { | |||||
xdomain += cache->uc_crossbucket->ub_cnt; | |||||
cachefree += cache->uc_crossbucket->ub_cnt; | |||||
} | |||||
allocs += cache->uc_allocs; | allocs += cache->uc_allocs; | ||||
frees += cache->uc_frees; | frees += cache->uc_frees; | ||||
} | } | ||||
allocs += counter_u64_fetch(z->uz_allocs); | allocs += counter_u64_fetch(z->uz_allocs); | ||||
frees += counter_u64_fetch(z->uz_frees); | frees += counter_u64_fetch(z->uz_frees); | ||||
sleeps += z->uz_sleeps; | sleeps += z->uz_sleeps; | ||||
xdomain += z->uz_xdomain; | |||||
if (cachefreep != NULL) | if (cachefreep != NULL) | ||||
*cachefreep = cachefree; | *cachefreep = cachefree; | ||||
if (allocsp != NULL) | if (allocsp != NULL) | ||||
*allocsp = allocs; | *allocsp = allocs; | ||||
if (freesp != NULL) | if (freesp != NULL) | ||||
*freesp = frees; | *freesp = frees; | ||||
if (sleepsp != NULL) | if (sleepsp != NULL) | ||||
*sleepsp = sleeps; | *sleepsp = sleeps; | ||||
if (xdomainp != NULL) | |||||
*xdomainp = xdomain; | |||||
} | } | ||||
#endif /* DDB */ | #endif /* DDB */ | ||||
static int | static int | ||||
sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS) | sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
uma_keg_t kz; | uma_keg_t kz; | ||||
uma_zone_t z; | uma_zone_t z; | ||||
Show All 24 Lines | uma_vm_zone_stats(struct uma_type_header *uth, uma_zone_t z, struct sbuf *sbuf, | ||||
for (i = 0; i < vm_ndomains; i++) { | for (i = 0; i < vm_ndomains; i++) { | ||||
zdom = &z->uz_domain[i]; | zdom = &z->uz_domain[i]; | ||||
uth->uth_zone_free += zdom->uzd_nitems; | uth->uth_zone_free += zdom->uzd_nitems; | ||||
} | } | ||||
uth->uth_allocs = counter_u64_fetch(z->uz_allocs); | uth->uth_allocs = counter_u64_fetch(z->uz_allocs); | ||||
uth->uth_frees = counter_u64_fetch(z->uz_frees); | uth->uth_frees = counter_u64_fetch(z->uz_frees); | ||||
uth->uth_fails = counter_u64_fetch(z->uz_fails); | uth->uth_fails = counter_u64_fetch(z->uz_fails); | ||||
uth->uth_sleeps = z->uz_sleeps; | uth->uth_sleeps = z->uz_sleeps; | ||||
uth->uth_xdomain = z->uz_xdomain; | |||||
/* | /* | ||||
* While it is not normally safe to access the cache | * While it is not normally safe to access the cache | ||||
* bucket pointers while not on the CPU that owns the | * bucket pointers while not on the CPU that owns the | ||||
* cache, we only allow the pointers to be exchanged | * cache, we only allow the pointers to be exchanged | ||||
* without the zone lock held, not invalidated, so | * without the zone lock held, not invalidated, so | ||||
* accept the possible race associated with bucket | * accept the possible race associated with bucket | ||||
* exchange during monitoring. | * exchange during monitoring. | ||||
*/ | */ | ||||
for (i = 0; i < mp_maxid + 1; i++) { | for (i = 0; i < mp_maxid + 1; i++) { | ||||
bzero(&ups[i], sizeof(*ups)); | bzero(&ups[i], sizeof(*ups)); | ||||
if (internal || CPU_ABSENT(i)) | if (internal || CPU_ABSENT(i)) | ||||
continue; | continue; | ||||
cache = &z->uz_cpu[i]; | cache = &z->uz_cpu[i]; | ||||
if (cache->uc_allocbucket != NULL) | if (cache->uc_allocbucket != NULL) | ||||
ups[i].ups_cache_free += | ups[i].ups_cache_free += | ||||
cache->uc_allocbucket->ub_cnt; | cache->uc_allocbucket->ub_cnt; | ||||
if (cache->uc_freebucket != NULL) | if (cache->uc_freebucket != NULL) | ||||
ups[i].ups_cache_free += | ups[i].ups_cache_free += | ||||
cache->uc_freebucket->ub_cnt; | cache->uc_freebucket->ub_cnt; | ||||
if (cache->uc_crossbucket != NULL) | |||||
ups[i].ups_cache_free += | |||||
cache->uc_crossbucket->ub_cnt; | |||||
ups[i].ups_allocs = cache->uc_allocs; | ups[i].ups_allocs = cache->uc_allocs; | ||||
ups[i].ups_frees = cache->uc_frees; | ups[i].ups_frees = cache->uc_frees; | ||||
} | } | ||||
} | } | ||||
static int | static int | ||||
sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) | sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 239 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
#endif /* INVARIANTS */ | #endif /* INVARIANTS */ | ||||
#ifdef DDB | #ifdef DDB | ||||
DB_SHOW_COMMAND(uma, db_show_uma) | DB_SHOW_COMMAND(uma, db_show_uma) | ||||
{ | { | ||||
uma_keg_t kz; | uma_keg_t kz; | ||||
uma_zone_t z; | uma_zone_t z; | ||||
uint64_t allocs, frees, sleeps; | uint64_t allocs, frees, sleeps, xdomain; | ||||
long cachefree; | long cachefree; | ||||
int i; | int i; | ||||
db_printf("%18s %8s %8s %8s %12s %8s %8s\n", "Zone", "Size", "Used", | db_printf("%18s %8s %8s %8s %12s %8s %8s %8s\n", "Zone", "Size", "Used", | ||||
"Free", "Requests", "Sleeps", "Bucket"); | "Free", "Requests", "Sleeps", "Bucket", "XFree"); | ||||
LIST_FOREACH(kz, &uma_kegs, uk_link) { | LIST_FOREACH(kz, &uma_kegs, uk_link) { | ||||
LIST_FOREACH(z, &kz->uk_zones, uz_link) { | LIST_FOREACH(z, &kz->uk_zones, uz_link) { | ||||
if (kz->uk_flags & UMA_ZFLAG_INTERNAL) { | if (kz->uk_flags & UMA_ZFLAG_INTERNAL) { | ||||
allocs = counter_u64_fetch(z->uz_allocs); | allocs = counter_u64_fetch(z->uz_allocs); | ||||
frees = counter_u64_fetch(z->uz_frees); | frees = counter_u64_fetch(z->uz_frees); | ||||
sleeps = z->uz_sleeps; | sleeps = z->uz_sleeps; | ||||
cachefree = 0; | cachefree = 0; | ||||
} else | } else | ||||
uma_zone_sumstat(z, &cachefree, &allocs, | uma_zone_sumstat(z, &cachefree, &allocs, | ||||
&frees, &sleeps); | &frees, &sleeps, &xdomain); | ||||
if (!((z->uz_flags & UMA_ZONE_SECONDARY) && | if (!((z->uz_flags & UMA_ZONE_SECONDARY) && | ||||
(LIST_FIRST(&kz->uk_zones) != z))) | (LIST_FIRST(&kz->uk_zones) != z))) | ||||
cachefree += kz->uk_free; | cachefree += kz->uk_free; | ||||
for (i = 0; i < vm_ndomains; i++) | for (i = 0; i < vm_ndomains; i++) | ||||
cachefree += z->uz_domain[i].uzd_nitems; | cachefree += z->uz_domain[i].uzd_nitems; | ||||
db_printf("%18s %8ju %8jd %8ld %12ju %8ju %8u\n", | db_printf("%18s %8ju %8jd %8ld %12ju %8ju %8u %8ju\n", | ||||
z->uz_name, (uintmax_t)kz->uk_size, | z->uz_name, (uintmax_t)kz->uk_size, | ||||
(intmax_t)(allocs - frees), cachefree, | (intmax_t)(allocs - frees), cachefree, | ||||
(uintmax_t)allocs, sleeps, z->uz_count); | (uintmax_t)allocs, sleeps, z->uz_count, xdomain); | ||||
if (db_pager_quit) | if (db_pager_quit) | ||||
return; | return; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
DB_SHOW_COMMAND(umacache, db_show_umacache) | DB_SHOW_COMMAND(umacache, db_show_umacache) | ||||
{ | { | ||||
uma_zone_t z; | uma_zone_t z; | ||||
uint64_t allocs, frees; | uint64_t allocs, frees; | ||||
long cachefree; | long cachefree; | ||||
int i; | int i; | ||||
db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free", | db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free", | ||||
"Requests", "Bucket"); | "Requests", "Bucket"); | ||||
LIST_FOREACH(z, &uma_cachezones, uz_link) { | LIST_FOREACH(z, &uma_cachezones, uz_link) { | ||||
uma_zone_sumstat(z, &cachefree, &allocs, &frees, NULL); | uma_zone_sumstat(z, &cachefree, &allocs, &frees, NULL, NULL); | ||||
for (i = 0; i < vm_ndomains; i++) | for (i = 0; i < vm_ndomains; i++) | ||||
cachefree += z->uz_domain[i].uzd_nitems; | cachefree += z->uz_domain[i].uzd_nitems; | ||||
db_printf("%18s %8ju %8jd %8ld %12ju %8u\n", | db_printf("%18s %8ju %8jd %8ld %12ju %8u\n", | ||||
z->uz_name, (uintmax_t)z->uz_size, | z->uz_name, (uintmax_t)z->uz_size, | ||||
(intmax_t)(allocs - frees), cachefree, | (intmax_t)(allocs - frees), cachefree, | ||||
(uintmax_t)allocs, z->uz_count); | (uintmax_t)allocs, z->uz_count); | ||||
if (db_pager_quit) | if (db_pager_quit) | ||||
return; | return; | ||||
} | } | ||||
} | } | ||||
#endif /* DDB */ | #endif /* DDB */ |