Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_core.c
Show First 20 Lines • Show All 242 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Compute the actual number of bucket entries to pack them in power | * Compute the actual number of bucket entries to pack them in power | ||||
* of two sizes for more efficient space utilization. | * of two sizes for more efficient space utilization. | ||||
*/ | */ | ||||
#define BUCKET_SIZE(n) \ | #define BUCKET_SIZE(n) \ | ||||
(((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *)) | (((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *)) | ||||
#define BUCKET_MAX BUCKET_SIZE(256) | #define BUCKET_MAX BUCKET_SIZE(256) | ||||
#define BUCKET_MIN 2 | |||||
struct uma_bucket_zone bucket_zones[] = { | struct uma_bucket_zone bucket_zones[] = { | ||||
/* Literal bucket sizes. */ | /* Literal bucket sizes. */ | ||||
{ NULL, "2 Bucket", 2, 4096 }, | { NULL, "2 Bucket", 2, 4096 }, | ||||
{ NULL, "4 Bucket", 4, 3072 }, | { NULL, "4 Bucket", 4, 3072 }, | ||||
{ NULL, "8 Bucket", 8, 2048 }, | { NULL, "8 Bucket", 8, 2048 }, | ||||
{ NULL, "16 Bucket", 16, 1024 }, | { NULL, "16 Bucket", 16, 1024 }, | ||||
/* Rounded down power of 2 sizes for efficiency. */ | /* Rounded down power of 2 sizes for efficiency. */ | ||||
▲ Show 20 Lines • Show All 242 Lines • ▼ Show 20 Lines | if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) | ||||
udata = (void *)(uintptr_t)zone->uz_flags; | udata = (void *)(uintptr_t)zone->uz_flags; | ||||
else { | else { | ||||
if ((uintptr_t)udata & UMA_ZFLAG_BUCKET) | if ((uintptr_t)udata & UMA_ZFLAG_BUCKET) | ||||
return (NULL); | return (NULL); | ||||
udata = (void *)((uintptr_t)udata | UMA_ZFLAG_BUCKET); | udata = (void *)((uintptr_t)udata | UMA_ZFLAG_BUCKET); | ||||
} | } | ||||
if (((uintptr_t)udata & UMA_ZONE_VM) != 0) | if (((uintptr_t)udata & UMA_ZONE_VM) != 0) | ||||
flags |= M_NOVM; | flags |= M_NOVM; | ||||
ubz = bucket_zone_lookup(zone->uz_bucket_size); | ubz = bucket_zone_lookup(atomic_load_16(&zone->uz_bucket_size)); | ||||
rlibby: This is the only atomic access to uz_bucket_size. Is it important? ... | |||||
markjAuthorUnsubmitted Done Inline ActionsHmm, right, I'd forgotten that we permit racy updates of uz_bucket_size now. I can't remember what led me to think that using an atomic load here was necessary. markj: Hmm, right, I'd forgotten that we permit racy updates of uz_bucket_size now. I can't remember… | |||||
if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0) | if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0) | ||||
ubz++; | ubz++; | ||||
bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags); | bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags); | ||||
if (bucket) { | if (bucket) { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); | bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); | ||||
#endif | #endif | ||||
bucket->ub_cnt = 0; | bucket->ub_cnt = 0; | ||||
bucket->ub_entries = ubz->ubz_entries; | bucket->ub_entries = min(ubz->ubz_entries, | ||||
zone->uz_bucket_size_max); | |||||
bucket->ub_seq = SMR_SEQ_INVALID; | bucket->ub_seq = SMR_SEQ_INVALID; | ||||
CTR3(KTR_UMA, "bucket_alloc: zone %s(%p) allocated bucket %p", | CTR3(KTR_UMA, "bucket_alloc: zone %s(%p) allocated bucket %p", | ||||
zone->uz_name, zone, bucket); | zone->uz_name, zone, bucket); | ||||
} | } | ||||
return (bucket); | return (bucket); | ||||
} | } | ||||
Show All 33 Lines | zone_domain_lock(uma_zone_t zone, int domain) | ||||
uma_zone_domain_t zdom; | uma_zone_domain_t zdom; | ||||
bool lockfail; | bool lockfail; | ||||
zdom = ZDOM_GET(zone, domain); | zdom = ZDOM_GET(zone, domain); | ||||
lockfail = false; | lockfail = false; | ||||
if (ZDOM_OWNED(zdom)) | if (ZDOM_OWNED(zdom)) | ||||
lockfail = true; | lockfail = true; | ||||
ZDOM_LOCK(zdom); | ZDOM_LOCK(zdom); | ||||
/* This is unsynchronized. The counter does not need to be precise. */ | /* This is unsynchronized. The counter does not need to be precise. */ | ||||
if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max) | if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max) | ||||
zone->uz_bucket_size++; | zone->uz_bucket_size++; | ||||
rlibbyUnsubmitted Not Done Inline Actions(uz_bucket_size accessed without atomic and an explicit comment saying we don't care) rlibby: (uz_bucket_size accessed without atomic and an explicit comment saying we don't care) | |||||
return (zdom); | return (zdom); | ||||
} | } | ||||
/* | /* | ||||
* Search for the domain with the least cached items and return it if it | * Search for the domain with the least cached items and return it if it | ||||
* is out of balance with the preferred domain. | * is out of balance with the preferred domain. | ||||
*/ | */ | ||||
static __noinline int | static __noinline int | ||||
▲ Show 20 Lines • Show All 2,139 Lines • ▼ Show 20 Lines | out: | ||||
KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) != | KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) != | ||||
(UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET), | (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET), | ||||
("Invalid zone flag combination")); | ("Invalid zone flag combination")); | ||||
if (arg->flags & UMA_ZFLAG_INTERNAL) | if (arg->flags & UMA_ZFLAG_INTERNAL) | ||||
zone->uz_bucket_size_max = zone->uz_bucket_size = 0; | zone->uz_bucket_size_max = zone->uz_bucket_size = 0; | ||||
if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) | if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) | ||||
zone->uz_bucket_size = BUCKET_MAX; | zone->uz_bucket_size = BUCKET_MAX; | ||||
else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0) | |||||
zone->uz_bucket_size_max = zone->uz_bucket_size = BUCKET_MIN; | |||||
else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0) | else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0) | ||||
zone->uz_bucket_size = 0; | zone->uz_bucket_size = 0; | ||||
else | else | ||||
zone->uz_bucket_size = bucket_select(zone->uz_size); | zone->uz_bucket_size = bucket_select(zone->uz_size); | ||||
zone->uz_bucket_size_min = zone->uz_bucket_size; | zone->uz_bucket_size_min = zone->uz_bucket_size; | ||||
if (zone->uz_dtor != NULL || zone->uz_ctor != NULL) | if (zone->uz_dtor != NULL || zone->uz_ctor != NULL) | ||||
zone->uz_flags |= UMA_ZFLAG_CTORDTOR; | zone->uz_flags |= UMA_ZFLAG_CTORDTOR; | ||||
zone_update_caches(zone); | zone_update_caches(zone); | ||||
▲ Show 20 Lines • Show All 2,835 Lines • Show Last 20 Lines |
This is the only atomic access to uz_bucket_size. Is it important? ...