Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/uma_core.c
Show First 20 Lines • Show All 3,110 Lines • ▼ Show 20 Lines | if ((zone->uz_flags & UMA_ZONE_SMR) == 0 && is_memguard_addr(item)) { | ||||
memguard_free(item); | memguard_free(item); | ||||
return (EJUSTRETURN); | return (EJUSTRETURN); | ||||
} | } | ||||
#endif | #endif | ||||
return (0); | return (0); | ||||
} | } | ||||
#endif | #endif | ||||
static inline void * | |||||
zone_alloc_fast(uma_zone_t zone, uma_cache_t cache, uma_cache_bucket_t bucket, | |||||
jeff: I guess this should be cache_alloc_fast() | |||||
void *udata, int flags) | |||||
{ | |||||
void *item; | |||||
int size, uz_flags; | |||||
item = cache_bucket_pop(cache, bucket); | |||||
size = cache_uz_size(cache); | |||||
uz_flags = cache_uz_flags(cache); | |||||
critical_exit(); | |||||
return (item_ctor(zone, uz_flags, size, udata, flags, item)); | |||||
} | |||||
static __noinline void * | static __noinline void * | ||||
uma_zalloc_single(uma_zone_t zone, void *udata, int flags) | zone_alloc_slow(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) | ||||
jeffAuthorUnsubmitted Done Inline ActionsMaybe cache_alloc_retry or cache_alloc_loop jeff: Maybe cache_alloc_retry or cache_alloc_loop | |||||
{ | { | ||||
uma_cache_bucket_t bucket; | |||||
int domain; | int domain; | ||||
while (cache_alloc(zone, cache, udata, flags)) { | |||||
cache = &zone->uz_cpu[curcpu]; | |||||
bucket = &cache->uc_allocbucket; | |||||
if (__predict_false(bucket->ucb_cnt == 0)) | |||||
continue; | |||||
return (zone_alloc_fast(zone, cache, bucket, udata, flags)); | |||||
} | |||||
critical_exit(); | |||||
/* | /* | ||||
* We can not get a bucket so try to return a single item. | * We can not get a bucket so try to return a single item. | ||||
*/ | */ | ||||
if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH) | if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH) | ||||
domain = PCPU_GET(domain); | domain = PCPU_GET(domain); | ||||
else | else | ||||
domain = UMA_ANYDOMAIN; | domain = UMA_ANYDOMAIN; | ||||
return (zone_alloc_item(zone, udata, domain, flags)); | return (zone_alloc_item(zone, udata, domain, flags)); | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
void * | void * | ||||
uma_zalloc_smr(uma_zone_t zone, int flags) | uma_zalloc_smr(uma_zone_t zone, int flags) | ||||
{ | { | ||||
uma_cache_bucket_t bucket; | uma_cache_bucket_t bucket; | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
void *item; | |||||
int size, uz_flags; | |||||
#ifdef UMA_ZALLOC_DEBUG | #ifdef UMA_ZALLOC_DEBUG | ||||
void *item; | |||||
KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, | KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, | ||||
("uma_zalloc_arg: called with non-SMR zone.\n")); | ("uma_zalloc_arg: called with non-SMR zone.\n")); | ||||
if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) | if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) | ||||
return (item); | return (item); | ||||
#endif | #endif | ||||
critical_enter(); | critical_enter(); | ||||
do { | |||||
cache = &zone->uz_cpu[curcpu]; | cache = &zone->uz_cpu[curcpu]; | ||||
bucket = &cache->uc_allocbucket; | bucket = &cache->uc_allocbucket; | ||||
size = cache_uz_size(cache); | if (__predict_false(bucket->ucb_cnt == 0)) | ||||
uz_flags = cache_uz_flags(cache); | return (zone_alloc_slow(zone, cache, NULL, flags)); | ||||
if (__predict_true(bucket->ucb_cnt != 0)) { | return (zone_alloc_fast(zone, cache, bucket, NULL, flags)); | ||||
item = cache_bucket_pop(cache, bucket); | |||||
critical_exit(); | |||||
return (item_ctor(zone, uz_flags, size, NULL, flags, | |||||
item)); | |||||
} | } | ||||
} while (cache_alloc(zone, cache, NULL, flags)); | |||||
critical_exit(); | |||||
return (uma_zalloc_single(zone, NULL, flags)); | |||||
} | |||||
/* See uma.h */ | /* See uma.h */ | ||||
void * | void * | ||||
uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) | uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) | ||||
{ | { | ||||
uma_cache_bucket_t bucket; | uma_cache_bucket_t bucket; | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
void *item; | |||||
int size, uz_flags; | |||||
/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | ||||
random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | ||||
/* This is the fast path allocation */ | /* This is the fast path allocation */ | ||||
CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name, | CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name, | ||||
zone, flags); | zone, flags); | ||||
#ifdef UMA_ZALLOC_DEBUG | #ifdef UMA_ZALLOC_DEBUG | ||||
void *item; | |||||
KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, | KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, | ||||
("uma_zalloc_arg: called with SMR zone.\n")); | ("uma_zalloc_arg: called with SMR zone.\n")); | ||||
if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) | if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) | ||||
return (item); | return (item); | ||||
#endif | #endif | ||||
/* | /* | ||||
* If possible, allocate from the per-CPU cache. There are two | * If possible, allocate from the per-CPU cache. There are two | ||||
* requirements for safe access to the per-CPU cache: (1) the thread | * requirements for safe access to the per-CPU cache: (1) the thread | ||||
* accessing the cache must not be preempted or yield during access, | * accessing the cache must not be preempted or yield during access, | ||||
* and (2) the thread must not migrate CPUs without switching which | * and (2) the thread must not migrate CPUs without switching which | ||||
* cache it accesses. We rely on a critical section to prevent | * cache it accesses. We rely on a critical section to prevent | ||||
* preemption and migration. We release the critical section in | * preemption and migration. We release the critical section in | ||||
* order to acquire the zone mutex if we are unable to allocate from | * order to acquire the zone mutex if we are unable to allocate from | ||||
* the current cache; when we re-acquire the critical section, we | * the current cache; when we re-acquire the critical section, we | ||||
* must detect and handle migration if it has occurred. | * must detect and handle migration if it has occurred. | ||||
*/ | */ | ||||
critical_enter(); | critical_enter(); | ||||
do { | |||||
cache = &zone->uz_cpu[curcpu]; | cache = &zone->uz_cpu[curcpu]; | ||||
bucket = &cache->uc_allocbucket; | bucket = &cache->uc_allocbucket; | ||||
size = cache_uz_size(cache); | if (__predict_false(bucket->ucb_cnt == 0)) | ||||
uz_flags = cache_uz_flags(cache); | return (zone_alloc_slow(zone, cache, udata, flags)); | ||||
if (__predict_true(bucket->ucb_cnt != 0)) { | return (zone_alloc_fast(zone, cache, bucket, udata, flags)); | ||||
item = cache_bucket_pop(cache, bucket); | |||||
critical_exit(); | |||||
return (item_ctor(zone, uz_flags, size, udata, flags, | |||||
item)); | |||||
} | } | ||||
} while (cache_alloc(zone, cache, udata, flags)); | |||||
critical_exit(); | |||||
return (uma_zalloc_single(zone, udata, flags)); | |||||
} | |||||
/* | /* | ||||
* Replenish an alloc bucket and possibly restore an old one. Called in | * Replenish an alloc bucket and possibly restore an old one. Called in | ||||
* a critical section. Returns in a critical section. | * a critical section. Returns in a critical section. | ||||
* | * | ||||
* A false return value indicates an allocation failure. | * A false return value indicates an allocation failure. | ||||
* A true return value indicates success and the caller should retry. | * A true return value indicates success and the caller should retry. | ||||
*/ | */ | ||||
static __noinline bool | static __noinline bool | ||||
▲ Show 20 Lines • Show All 1,092 Lines • ▼ Show 20 Lines | |||||
* Frees a single item to any zone. | * Frees a single item to any zone. | ||||
* | * | ||||
* Arguments: | * Arguments: | ||||
* zone The zone to free to | * zone The zone to free to | ||||
* item The item we're freeing | * item The item we're freeing | ||||
* udata User supplied data for the dtor | * udata User supplied data for the dtor | ||||
* skip Skip dtors and finis | * skip Skip dtors and finis | ||||
*/ | */ | ||||
static void | static __noinline void | ||||
zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) | zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) | ||||
jeffAuthorUnsubmitted Done Inline ActionsThis improves zfree which was pulling in the whole thing. jeff: This improves zfree which was pulling in the whole thing. | |||||
markjUnsubmitted Not Done Inline ActionsWhy does that matter? In zfree, isn't the zone_free_item() call out of the critical path? markj: Why does that matter? In zfree, isn't the zone_free_item() call out of the critical path? | |||||
jeffAuthorUnsubmitted Done Inline Actionsuma_zfree_arg calls it and uma was inlining it. jeff: uma_zfree_arg calls it and uma was inlining it. | |||||
markjUnsubmitted Not Done Inline ActionsRight, and that call follows the code which looks for an item in the per-CPU buckets and returns it if one is found. So I can't see why the inlining should have any effect on performance. markj: Right, and that call follows the code which looks for an item in the per-CPU buckets and… | |||||
{ | { | ||||
/* | /* | ||||
* If a free is sent directly to an SMR zone we have to | * If a free is sent directly to an SMR zone we have to | ||||
* synchronize immediately because the item can instantly | * synchronize immediately because the item can instantly | ||||
* be reallocated. This should only happen in degenerate | * be reallocated. This should only happen in degenerate | ||||
* cases when no memory is available for per-cpu caches. | * cases when no memory is available for per-cpu caches. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 1,011 Lines • Show Last 20 Lines |
I guess this should be cache_alloc_fast()