Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_core.c
Show First 20 Lines • Show All 275 Lines • ▼ Show 20 Lines | |||||
static void bucket_zone_drain(void); | static void bucket_zone_drain(void); | ||||
static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int); | static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int); | ||||
static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab); | static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab); | ||||
static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item); | static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item); | ||||
static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, | static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, | ||||
uma_fini fini, int align, uint32_t flags); | uma_fini fini, int align, uint32_t flags); | ||||
static int zone_import(void *, void **, int, int, int); | static int zone_import(void *, void **, int, int, int); | ||||
static void zone_release(void *, void **, int); | static void zone_release(void *, void **, int); | ||||
static void uma_zero_item(void *, uma_zone_t); | |||||
static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int); | static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int); | ||||
static bool cache_free(uma_zone_t, uma_cache_t, void *, void *, int); | static bool cache_free(uma_zone_t, uma_cache_t, void *, void *, int); | ||||
static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); | static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); | ||||
static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); | static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); | ||||
static int sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS); | static int sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS); | ||||
static int sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS); | static int sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS); | ||||
static int sysctl_handle_uma_zone_flags(SYSCTL_HANDLER_ARGS); | static int sysctl_handle_uma_zone_flags(SYSCTL_HANDLER_ARGS); | ||||
▲ Show 20 Lines • Show All 1,885 Lines • ▼ Show 20 Lines | zone_count(uma_zone_t zone, void *arg) | ||||
* destroyed out of order. This can lead to gaps in the count. | * destroyed out of order. This can lead to gaps in the count. | ||||
* Use one greater than the maximum observed for this name. | * Use one greater than the maximum observed for this name. | ||||
*/ | */ | ||||
if (strcmp(zone->uz_name, cnt->name) == 0) | if (strcmp(zone->uz_name, cnt->name) == 0) | ||||
cnt->count = MAX(cnt->count, | cnt->count = MAX(cnt->count, | ||||
zone->uz_namecnt + 1); | zone->uz_namecnt + 1); | ||||
} | } | ||||
static void | |||||
zone_update_caches(uma_zone_t zone) | |||||
{ | |||||
int i; | |||||
for (i = 0; i <= mp_maxid; i++) { | |||||
cache_set_uz_size(&zone->uz_cpu[i], zone->uz_size); | |||||
cache_set_uz_flags(&zone->uz_cpu[i], zone->uz_flags); | |||||
} | |||||
} | |||||
/* | /* | ||||
* Zone header ctor. This initializes all fields, locks, etc. | * Zone header ctor. This initializes all fields, locks, etc. | ||||
* | * | ||||
* Arguments/Returns follow uma_ctor specifications | * Arguments/Returns follow uma_ctor specifications | ||||
* udata Actually uma_zctor_args | * udata Actually uma_zctor_args | ||||
*/ | */ | ||||
static int | static int | ||||
zone_ctor(void *mem, int size, void *udata, int flags) | zone_ctor(void *mem, int size, void *udata, int flags) | ||||
Show All 29 Lines | zone_ctor(void *mem, int size, void *udata, int flags) | ||||
zone_foreach(zone_count, &cnt); | zone_foreach(zone_count, &cnt); | ||||
zone->uz_namecnt = cnt.count; | zone->uz_namecnt = cnt.count; | ||||
for (i = 0; i < vm_ndomains; i++) | for (i = 0; i < vm_ndomains; i++) | ||||
TAILQ_INIT(&zone->uz_domain[i].uzd_buckets); | TAILQ_INIT(&zone->uz_domain[i].uzd_buckets); | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
if (arg->uminit == trash_init && arg->fini == trash_fini) | if (arg->uminit == trash_init && arg->fini == trash_fini) | ||||
zone->uz_flags |= UMA_ZFLAG_TRASH; | zone->uz_flags |= UMA_ZFLAG_TRASH | UMA_ZFLAG_CTORDTOR; | ||||
#endif | #endif | ||||
/* | /* | ||||
* This is a pure cache zone, no kegs. | * This is a pure cache zone, no kegs. | ||||
*/ | */ | ||||
if (arg->import) { | if (arg->import) { | ||||
if (arg->flags & UMA_ZONE_VM) | if (arg->flags & UMA_ZONE_VM) | ||||
arg->flags |= UMA_ZFLAG_CACHEONLY; | arg->flags |= UMA_ZFLAG_CACHEONLY; | ||||
▲ Show 20 Lines • Show All 82 Lines • ▼ Show 20 Lines | if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) | ||||
zone->uz_bucket_size = BUCKET_MAX; | zone->uz_bucket_size = BUCKET_MAX; | ||||
else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0) | else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0) | ||||
zone->uz_bucket_size_max = zone->uz_bucket_size = BUCKET_MIN; | zone->uz_bucket_size_max = zone->uz_bucket_size = BUCKET_MIN; | ||||
else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0) | else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0) | ||||
zone->uz_bucket_size = 0; | zone->uz_bucket_size = 0; | ||||
else | else | ||||
zone->uz_bucket_size = bucket_select(zone->uz_size); | zone->uz_bucket_size = bucket_select(zone->uz_size); | ||||
zone->uz_bucket_size_min = zone->uz_bucket_size; | zone->uz_bucket_size_min = zone->uz_bucket_size; | ||||
if (zone->uz_dtor != NULL || zone->uz_ctor != NULL) | |||||
zone->uz_flags |= UMA_ZFLAG_CTORDTOR; | |||||
zone_update_caches(zone); | |||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Keg header dtor. This frees all data, destroys locks, frees the hash | * Keg header dtor. This frees all data, destroys locks, frees the hash | ||||
* table and removes the keg from the global list. | * table and removes the keg from the global list. | ||||
* | * | ||||
▲ Show 20 Lines • Show All 458 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
#ifdef SMP | #ifdef SMP | ||||
MPASS(zone->uz_flags & UMA_ZONE_PCPU); | MPASS(zone->uz_flags & UMA_ZONE_PCPU); | ||||
#endif | #endif | ||||
uma_zfree_arg(zone, item, udata); | uma_zfree_arg(zone, item, udata); | ||||
} | } | ||||
#ifdef INVARIANTS | |||||
#define UMA_ALWAYS_CTORDTOR 1 | |||||
#else | |||||
#define UMA_ALWAYS_CTORDTOR 0 | |||||
#endif | |||||
static void * | static void * | ||||
item_ctor(uma_zone_t zone, void *udata, int flags, void *item) | item_ctor(uma_zone_t zone, int size, void *udata, int flags, void *item) | ||||
{ | { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
bool skipdbg; | bool skipdbg; | ||||
skipdbg = uma_dbg_zskip(zone, item); | skipdbg = uma_dbg_zskip(zone, item); | ||||
if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && | if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && | ||||
zone->uz_ctor != trash_ctor) | zone->uz_ctor != trash_ctor) | ||||
trash_ctor(item, zone->uz_size, udata, flags); | trash_ctor(item, size, udata, flags); | ||||
#endif | #endif | ||||
if (__predict_false(zone->uz_ctor != NULL) && | if (__predict_false(zone->uz_ctor != NULL) && | ||||
zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { | zone->uz_ctor(item, size, udata, flags) != 0) { | ||||
counter_u64_add(zone->uz_fails, 1); | counter_u64_add(zone->uz_fails, 1); | ||||
zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); | zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
if (!skipdbg) | if (!skipdbg) | ||||
uma_dbg_alloc(zone, NULL, item); | uma_dbg_alloc(zone, NULL, item); | ||||
#endif | #endif | ||||
if (flags & M_ZERO) | if (flags & M_ZERO) | ||||
uma_zero_item(item, zone); | bzero(item, size); | ||||
return (item); | return (item); | ||||
} | } | ||||
static inline void | static inline void | ||||
item_dtor(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) | item_dtor(uma_zone_t zone, void *item, int size, void *udata, | ||||
enum zfreeskip skip) | |||||
{ | { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
bool skipdbg; | bool skipdbg; | ||||
skipdbg = uma_dbg_zskip(zone, item); | skipdbg = uma_dbg_zskip(zone, item); | ||||
if (skip == SKIP_NONE && !skipdbg) { | if (skip == SKIP_NONE && !skipdbg) { | ||||
if ((zone->uz_flags & UMA_ZONE_MALLOC) != 0) | if ((zone->uz_flags & UMA_ZONE_MALLOC) != 0) | ||||
uma_dbg_free(zone, udata, item); | uma_dbg_free(zone, udata, item); | ||||
else | else | ||||
uma_dbg_free(zone, NULL, item); | uma_dbg_free(zone, NULL, item); | ||||
} | } | ||||
#endif | #endif | ||||
if (skip < SKIP_DTOR) { | if (__predict_true(skip < SKIP_DTOR)) { | ||||
if (zone->uz_dtor != NULL) | if (zone->uz_dtor != NULL) | ||||
zone->uz_dtor(item, zone->uz_size, udata); | zone->uz_dtor(item, size, udata); | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && | if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && | ||||
zone->uz_dtor != trash_dtor) | zone->uz_dtor != trash_dtor) | ||||
trash_dtor(item, zone->uz_size, udata); | trash_dtor(item, size, udata); | ||||
#endif | #endif | ||||
} | } | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
void * | void * | ||||
uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) | uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) | ||||
{ | { | ||||
uma_cache_bucket_t bucket; | uma_cache_bucket_t bucket; | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
void *item; | void *item; | ||||
int cpu, domain; | int domain, size, uz_flags; | ||||
/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | ||||
random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | ||||
/* This is the fast path allocation */ | /* This is the fast path allocation */ | ||||
CTR4(KTR_UMA, "uma_zalloc_arg thread %x zone %s(%p) flags %d", | CTR4(KTR_UMA, "uma_zalloc_arg thread %x zone %s(%p) flags %d", | ||||
curthread, zone->uz_name, zone, flags); | curthread, zone->uz_name, zone, flags); | ||||
#ifdef WITNESS | |||||
if (flags & M_WAITOK) { | if (flags & M_WAITOK) { | ||||
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, | WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, | ||||
"uma_zalloc_arg: zone \"%s\"", zone->uz_name); | "uma_zalloc_arg: zone \"%s\"", zone->uz_name); | ||||
} | } | ||||
#endif | |||||
#ifdef INVARIANTS | |||||
KASSERT((flags & M_EXEC) == 0, ("uma_zalloc_arg: called with M_EXEC")); | KASSERT((flags & M_EXEC) == 0, ("uma_zalloc_arg: called with M_EXEC")); | ||||
KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | ||||
("uma_zalloc_arg: called with spinlock or critical section held")); | ("uma_zalloc_arg: called with spinlock or critical section held")); | ||||
if (zone->uz_flags & UMA_ZONE_PCPU) | if (zone->uz_flags & UMA_ZONE_PCPU) | ||||
KASSERT((flags & M_ZERO) == 0, ("allocating from a pcpu zone " | KASSERT((flags & M_ZERO) == 0, ("allocating from a pcpu zone " | ||||
"with M_ZERO passed")); | "with M_ZERO passed")); | ||||
#endif | |||||
#ifdef DEBUG_MEMGUARD | #ifdef DEBUG_MEMGUARD | ||||
if (memguard_cmp_zone(zone)) { | if (memguard_cmp_zone(zone)) { | ||||
item = memguard_alloc(zone->uz_size, flags); | item = memguard_alloc(zone->uz_size, flags); | ||||
if (item != NULL) { | if (item != NULL) { | ||||
if (zone->uz_init != NULL && | if (zone->uz_init != NULL && | ||||
zone->uz_init(item, zone->uz_size, flags) != 0) | zone->uz_init(item, zone->uz_size, flags) != 0) | ||||
return (NULL); | return (NULL); | ||||
Show All 17 Lines | #endif | ||||
* cache it accesses. We rely on a critical section to prevent | * cache it accesses. We rely on a critical section to prevent | ||||
* preemption and migration. We release the critical section in | * preemption and migration. We release the critical section in | ||||
* order to acquire the zone mutex if we are unable to allocate from | * order to acquire the zone mutex if we are unable to allocate from | ||||
* the current cache; when we re-acquire the critical section, we | * the current cache; when we re-acquire the critical section, we | ||||
* must detect and handle migration if it has occurred. | * must detect and handle migration if it has occurred. | ||||
*/ | */ | ||||
critical_enter(); | critical_enter(); | ||||
do { | do { | ||||
cpu = curcpu; | cache = &zone->uz_cpu[curcpu]; | ||||
cache = &zone->uz_cpu[cpu]; | |||||
bucket = &cache->uc_allocbucket; | bucket = &cache->uc_allocbucket; | ||||
size = cache_uz_size(cache); | |||||
uz_flags = cache_uz_flags(cache); | |||||
if (__predict_true(bucket->ucb_cnt != 0)) { | if (__predict_true(bucket->ucb_cnt != 0)) { | ||||
item = cache_bucket_pop(cache, bucket); | item = cache_bucket_pop(cache, bucket); | ||||
critical_exit(); | critical_exit(); | ||||
return (item_ctor(zone, udata, flags, item)); | if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0 || | ||||
UMA_ALWAYS_CTORDTOR)) | |||||
return (item_ctor(zone, size, udata, flags, item)); | |||||
if (flags & M_ZERO) | |||||
bzero(item, size); | |||||
return (item); | |||||
} | } | ||||
} while (cache_alloc(zone, cache, udata, flags)); | } while (cache_alloc(zone, cache, udata, flags)); | ||||
critical_exit(); | critical_exit(); | ||||
/* | /* | ||||
* We can not get a bucket so try to return a single item. | * We can not get a bucket so try to return a single item. | ||||
*/ | */ | ||||
if (zone->uz_flags & UMA_ZONE_NUMA) | if (uz_flags & UMA_ZONE_NUMA) | ||||
domain = PCPU_GET(domain); | domain = PCPU_GET(domain); | ||||
else | else | ||||
domain = UMA_ANYDOMAIN; | domain = UMA_ANYDOMAIN; | ||||
return (zone_alloc_item_locked(zone, udata, domain, flags)); | return (zone_alloc_item_locked(zone, udata, domain, flags)); | ||||
} | } | ||||
/* | /* | ||||
* Replenish an alloc bucket and possibly restore an old one. Called in | * Replenish an alloc bucket and possibly restore an old one. Called in | ||||
* a critical section. Returns in a critical section. | * a critical section. Returns in a critical section. | ||||
* | * | ||||
* A false return value indicates failure and returns with the zone lock | * A false return value indicates failure and returns with the zone lock | ||||
* held. A true return value indicates success and the caller should retry. | * held. A true return value indicates success and the caller should retry. | ||||
*/ | */ | ||||
static __noinline bool | static __noinline bool | ||||
cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) | cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) | ||||
{ | { | ||||
uma_zone_domain_t zdom; | uma_zone_domain_t zdom; | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
int cpu, domain; | int domain; | ||||
bool lockfail; | bool lockfail; | ||||
CRITICAL_ASSERT(curthread); | CRITICAL_ASSERT(curthread); | ||||
/* | /* | ||||
* If we have run out of items in our alloc bucket see | * If we have run out of items in our alloc bucket see | ||||
* if we can switch with the free bucket. | * if we can switch with the free bucket. | ||||
*/ | */ | ||||
Show All 26 Lines | if (ZONE_TRYLOCK(zone) == 0) { | ||||
lockfail = 1; | lockfail = 1; | ||||
} | } | ||||
critical_enter(); | critical_enter(); | ||||
/* Short-circuit for zones without buckets and low memory. */ | /* Short-circuit for zones without buckets and low memory. */ | ||||
if (zone->uz_bucket_size == 0 || bucketdisable) | if (zone->uz_bucket_size == 0 || bucketdisable) | ||||
return (false); | return (false); | ||||
cpu = curcpu; | cache = &zone->uz_cpu[curcpu]; | ||||
cache = &zone->uz_cpu[cpu]; | |||||
/* See if we lost the race to fill the cache. */ | /* See if we lost the race to fill the cache. */ | ||||
if (cache->uc_allocbucket.ucb_bucket != NULL) { | if (cache->uc_allocbucket.ucb_bucket != NULL) { | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
return (true); | return (true); | ||||
} | } | ||||
/* | /* | ||||
Show All 34 Lines | cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) | ||||
if (bucket == NULL) | if (bucket == NULL) | ||||
return (false); | return (false); | ||||
/* | /* | ||||
* See if we lost the race or were migrated. Cache the | * See if we lost the race or were migrated. Cache the | ||||
* initialized bucket to make this less likely or claim | * initialized bucket to make this less likely or claim | ||||
* the memory directly. | * the memory directly. | ||||
*/ | */ | ||||
cpu = curcpu; | cache = &zone->uz_cpu[curcpu]; | ||||
cache = &zone->uz_cpu[cpu]; | |||||
if (cache->uc_allocbucket.ucb_bucket == NULL && | if (cache->uc_allocbucket.ucb_bucket == NULL && | ||||
((zone->uz_flags & UMA_ZONE_NUMA) == 0 || | ((zone->uz_flags & UMA_ZONE_NUMA) == 0 || | ||||
domain == PCPU_GET(domain))) { | domain == PCPU_GET(domain))) { | ||||
cache_bucket_load_alloc(cache, bucket); | cache_bucket_load_alloc(cache, bucket); | ||||
zdom->uzd_imax += bucket->ub_cnt; | zdom->uzd_imax += bucket->ub_cnt; | ||||
} else if (zone->uz_bkt_count >= zone->uz_bkt_max) { | } else if (zone->uz_bkt_count >= zone->uz_bkt_max) { | ||||
critical_exit(); | critical_exit(); | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
▲ Show 20 Lines • Show All 379 Lines • ▼ Show 20 Lines | zone_alloc_item_locked(uma_zone_t zone, void *udata, int domain, int flags) | ||||
* to be both zone-init'd as well as zone-ctor'd. | * to be both zone-init'd as well as zone-ctor'd. | ||||
*/ | */ | ||||
if (zone->uz_init != NULL) { | if (zone->uz_init != NULL) { | ||||
if (zone->uz_init(item, zone->uz_size, flags) != 0) { | if (zone->uz_init(item, zone->uz_size, flags) != 0) { | ||||
zone_free_item(zone, item, udata, SKIP_FINI | SKIP_CNT); | zone_free_item(zone, item, udata, SKIP_FINI | SKIP_CNT); | ||||
goto fail_cnt; | goto fail_cnt; | ||||
} | } | ||||
} | } | ||||
item = item_ctor(zone, udata, flags, item); | item = item_ctor(zone, zone->uz_size, udata, flags, item); | ||||
if (item == NULL) | if (item == NULL) | ||||
goto fail; | goto fail; | ||||
counter_u64_add(zone->uz_allocs, 1); | counter_u64_add(zone->uz_allocs, 1); | ||||
CTR3(KTR_UMA, "zone_alloc_item item %p from %s(%p)", item, | CTR3(KTR_UMA, "zone_alloc_item item %p from %s(%p)", item, | ||||
zone->uz_name, zone); | zone->uz_name, zone); | ||||
return (item); | return (item); | ||||
Show All 13 Lines | |||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
void | void | ||||
uma_zfree_arg(uma_zone_t zone, void *item, void *udata) | uma_zfree_arg(uma_zone_t zone, void *item, void *udata) | ||||
{ | { | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
uma_cache_bucket_t bucket; | uma_cache_bucket_t bucket; | ||||
int cpu, domain, itemdomain; | int domain, itemdomain, uz_flags; | ||||
/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | ||||
random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | ||||
CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread, | CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread, | ||||
zone->uz_name); | zone->uz_name); | ||||
KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | ||||
("uma_zfree_arg: called with spinlock or critical section held")); | ("uma_zfree_arg: called with spinlock or critical section held")); | ||||
/* uma_zfree(..., NULL) does nothing, to match free(9). */ | /* uma_zfree(..., NULL) does nothing, to match free(9). */ | ||||
if (item == NULL) | if (item == NULL) | ||||
return; | return; | ||||
#ifdef DEBUG_MEMGUARD | #ifdef DEBUG_MEMGUARD | ||||
if (is_memguard_addr(item)) { | if (is_memguard_addr(item)) { | ||||
if (zone->uz_dtor != NULL) | if (zone->uz_dtor != NULL) | ||||
zone->uz_dtor(item, zone->uz_size, udata); | zone->uz_dtor(item, zone->uz_size, udata); | ||||
if (zone->uz_fini != NULL) | if (zone->uz_fini != NULL) | ||||
zone->uz_fini(item, zone->uz_size); | zone->uz_fini(item, zone->uz_size); | ||||
memguard_free(item); | memguard_free(item); | ||||
return; | return; | ||||
} | } | ||||
#endif | #endif | ||||
item_dtor(zone, item, udata, SKIP_NONE); | |||||
/* | /* | ||||
* We are accessing the per-cpu cache without a critical section to | |||||
* fetch size and flags. This is acceptable, if we are preempted we | |||||
* will simply read another cpu's line. | |||||
*/ | |||||
cache = &zone->uz_cpu[curcpu]; | |||||
uz_flags = cache_uz_flags(cache); | |||||
if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0 || | |||||
UMA_ALWAYS_CTORDTOR)) | |||||
item_dtor(zone, item, cache_uz_size(cache), udata, SKIP_NONE); | |||||
/* | |||||
* The race here is acceptable. If we miss it we'll just have to wait | * The race here is acceptable. If we miss it we'll just have to wait | ||||
* a little longer for the limits to be reset. | * a little longer for the limits to be reset. | ||||
*/ | */ | ||||
if (__predict_false(uz_flags & UMA_ZFLAG_LIMIT)) { | |||||
if (zone->uz_sleepers > 0) | if (zone->uz_sleepers > 0) | ||||
goto zfree_item; | goto zfree_item; | ||||
} | |||||
/* | /* | ||||
* If possible, free to the per-CPU cache. There are two | * If possible, free to the per-CPU cache. There are two | ||||
* requirements for safe access to the per-CPU cache: (1) the thread | * requirements for safe access to the per-CPU cache: (1) the thread | ||||
* accessing the cache must not be preempted or yield during access, | * accessing the cache must not be preempted or yield during access, | ||||
* and (2) the thread must not migrate CPUs without switching which | * and (2) the thread must not migrate CPUs without switching which | ||||
* cache it accesses. We rely on a critical section to prevent | * cache it accesses. We rely on a critical section to prevent | ||||
* preemption and migration. We release the critical section in | * preemption and migration. We release the critical section in | ||||
* order to acquire the zone mutex if we are unable to free to the | * order to acquire the zone mutex if we are unable to free to the | ||||
* current cache; when we re-acquire the critical section, we must | * current cache; when we re-acquire the critical section, we must | ||||
* detect and handle migration if it has occurred. | * detect and handle migration if it has occurred. | ||||
*/ | */ | ||||
domain = itemdomain = 0; | domain = itemdomain = 0; | ||||
critical_enter(); | critical_enter(); | ||||
do { | do { | ||||
cpu = curcpu; | cache = &zone->uz_cpu[curcpu]; | ||||
cache = &zone->uz_cpu[cpu]; | |||||
bucket = &cache->uc_allocbucket; | bucket = &cache->uc_allocbucket; | ||||
#ifdef UMA_XDOMAIN | #ifdef UMA_XDOMAIN | ||||
if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) { | if ((uz_flags & UMA_ZONE_NUMA) != 0) { | ||||
itemdomain = _vm_phys_domain(pmap_kextract((vm_offset_t)item)); | itemdomain = _vm_phys_domain(pmap_kextract((vm_offset_t)item)); | ||||
domain = PCPU_GET(domain); | domain = PCPU_GET(domain); | ||||
} | } | ||||
if ((zone->uz_flags & UMA_ZONE_NUMA) != 0 && | if ((uz_flags & UMA_ZONE_NUMA) != 0 && domain != itemdomain) { | ||||
domain != itemdomain) { | |||||
bucket = &cache->uc_crossbucket; | bucket = &cache->uc_crossbucket; | ||||
} else | } else | ||||
#endif | #endif | ||||
/* | /* | ||||
* Try to free into the allocbucket first to give LIFO ordering | * Try to free into the allocbucket first to give LIFO ordering | ||||
* for cache-hot datastructures. Spill over into the freebucket | * for cache-hot datastructures. Spill over into the freebucket | ||||
* if necessary. Alloc will swap them if one runs dry. | * if necessary. Alloc will swap them if one runs dry. | ||||
▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Lines | |||||
* we can not satisfy this free in the cache layer. true indicates that | * we can not satisfy this free in the cache layer. true indicates that | ||||
* the caller should retry. | * the caller should retry. | ||||
*/ | */ | ||||
static __noinline bool | static __noinline bool | ||||
cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, void *item, | cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, void *item, | ||||
int itemdomain) | int itemdomain) | ||||
{ | { | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
int cpu, domain; | int domain; | ||||
CRITICAL_ASSERT(curthread); | CRITICAL_ASSERT(curthread); | ||||
if (zone->uz_bucket_size == 0 || bucketdisable) | if (zone->uz_bucket_size == 0 || bucketdisable) | ||||
return false; | return false; | ||||
cpu = curcpu; | cache = &zone->uz_cpu[curcpu]; | ||||
cache = &zone->uz_cpu[cpu]; | |||||
/* | /* | ||||
* NUMA domains need to free to the correct zdom. When XDOMAIN | * NUMA domains need to free to the correct zdom. When XDOMAIN | ||||
* is enabled this is the zdom of the item and the bucket may be | * is enabled this is the zdom of the item and the bucket may be | ||||
* the cross bucket if they do not match. | * the cross bucket if they do not match. | ||||
*/ | */ | ||||
if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) | if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) | ||||
#ifdef UMA_XDOMAIN | #ifdef UMA_XDOMAIN | ||||
Show All 20 Lines | if (bucket != NULL) | ||||
zone_free_bucket(zone, bucket, udata, domain, itemdomain); | zone_free_bucket(zone, bucket, udata, domain, itemdomain); | ||||
bucket = bucket_alloc(zone, udata, M_NOWAIT); | bucket = bucket_alloc(zone, udata, M_NOWAIT); | ||||
CTR3(KTR_UMA, "uma_zfree: zone %s(%p) allocated bucket %p", | CTR3(KTR_UMA, "uma_zfree: zone %s(%p) allocated bucket %p", | ||||
zone->uz_name, zone, bucket); | zone->uz_name, zone, bucket); | ||||
critical_enter(); | critical_enter(); | ||||
if (bucket == NULL) | if (bucket == NULL) | ||||
return (false); | return (false); | ||||
cpu = curcpu; | cache = &zone->uz_cpu[curcpu]; | ||||
cache = &zone->uz_cpu[cpu]; | |||||
#ifdef UMA_XDOMAIN | #ifdef UMA_XDOMAIN | ||||
/* | /* | ||||
* Check to see if we should be populating the cross bucket. If it | * Check to see if we should be populating the cross bucket. If it | ||||
* is already populated we will fall through and attempt to populate | * is already populated we will fall through and attempt to populate | ||||
* the free bucket. | * the free bucket. | ||||
*/ | */ | ||||
if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) { | if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) { | ||||
domain = PCPU_GET(domain); | domain = PCPU_GET(domain); | ||||
▲ Show 20 Lines • Show All 105 Lines • ▼ Show 20 Lines | |||||
* item The item we're freeing | * item The item we're freeing | ||||
* udata User supplied data for the dtor | * udata User supplied data for the dtor | ||||
* skip Skip dtors and finis | * skip Skip dtors and finis | ||||
*/ | */ | ||||
static void | static void | ||||
zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) | zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) | ||||
{ | { | ||||
item_dtor(zone, item, udata, skip); | item_dtor(zone, item, zone->uz_size, udata, skip); | ||||
if (skip < SKIP_FINI && zone->uz_fini) | if (skip < SKIP_FINI && zone->uz_fini) | ||||
zone->uz_fini(item, zone->uz_size); | zone->uz_fini(item, zone->uz_size); | ||||
zone->uz_release(zone->uz_arg, &item, 1); | zone->uz_release(zone->uz_arg, &item, 1); | ||||
if (skip & SKIP_CNT) | if (skip & SKIP_CNT) | ||||
return; | return; | ||||
Show All 19 Lines | uma_zone_set_max(uma_zone_t zone, int nitems) | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
ubz = bucket_zone_max(zone, nitems); | ubz = bucket_zone_max(zone, nitems); | ||||
count = ubz != NULL ? ubz->ubz_entries : 0; | count = ubz != NULL ? ubz->ubz_entries : 0; | ||||
zone->uz_bucket_size_max = zone->uz_bucket_size = count; | zone->uz_bucket_size_max = zone->uz_bucket_size = count; | ||||
if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) | if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) | ||||
zone->uz_bucket_size_min = zone->uz_bucket_size_max; | zone->uz_bucket_size_min = zone->uz_bucket_size_max; | ||||
zone->uz_max_items = nitems; | zone->uz_max_items = nitems; | ||||
zone->uz_flags |= UMA_ZFLAG_LIMIT; | |||||
zone_update_caches(zone); | |||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
return (nitems); | return (nitems); | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
void | void | ||||
uma_zone_set_maxcache(uma_zone_t zone, int nitems) | uma_zone_set_maxcache(uma_zone_t zone, int nitems) | ||||
▲ Show 20 Lines • Show All 251 Lines • ▼ Show 20 Lines | #endif | ||||
keg->uk_kva = kva; | keg->uk_kva = kva; | ||||
keg->uk_offset = 0; | keg->uk_offset = 0; | ||||
zone->uz_max_items = pages * keg->uk_ipers; | zone->uz_max_items = pages * keg->uk_ipers; | ||||
#ifdef UMA_MD_SMALL_ALLOC | #ifdef UMA_MD_SMALL_ALLOC | ||||
keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc; | keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc; | ||||
#else | #else | ||||
keg->uk_allocf = noobj_alloc; | keg->uk_allocf = noobj_alloc; | ||||
#endif | #endif | ||||
keg->uk_flags |= UMA_ZONE_NOFREE; | keg->uk_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE; | ||||
zone->uz_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE; | |||||
zone_update_caches(zone); | |||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
return (1); | return (1); | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
void | void | ||||
uma_prealloc(uma_zone_t zone, int items) | uma_prealloc(uma_zone_t zone, int items) | ||||
▲ Show 20 Lines • Show All 128 Lines • ▼ Show 20 Lines | uma_zone_exhausted(uma_zone_t zone) | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
return (full); | return (full); | ||||
} | } | ||||
int | int | ||||
uma_zone_exhausted_nolock(uma_zone_t zone) | uma_zone_exhausted_nolock(uma_zone_t zone) | ||||
{ | { | ||||
return (zone->uz_sleepers > 0); | return (zone->uz_sleepers > 0); | ||||
} | |||||
static void | |||||
uma_zero_item(void *item, uma_zone_t zone) | |||||
{ | |||||
bzero(item, zone->uz_size); | |||||
} | } | ||||
unsigned long | unsigned long | ||||
uma_limit(void) | uma_limit(void) | ||||
{ | { | ||||
return (uma_kmem_limit); | return (uma_kmem_limit); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 558 Lines • Show Last 20 Lines |