Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_core.c
Show First 20 Lines • Show All 4,244 Lines • ▼ Show 20 Lines | |||||
#ifdef NUMA | #ifdef NUMA | ||||
/* | /* | ||||
* sort crossdomain free buckets to domain correct buckets and cache | * sort crossdomain free buckets to domain correct buckets and cache | ||||
* them. | * them. | ||||
*/ | */ | ||||
static void | static void | ||||
zone_free_cross(uma_zone_t zone, uma_bucket_t bucket, void *udata) | zone_free_cross(uma_zone_t zone, uma_bucket_t bucket, void *udata) | ||||
{ | { | ||||
struct uma_bucketlist fullbuckets; | struct uma_bucketlist emptybuckets, fullbuckets; | ||||
uma_zone_domain_t zdom; | uma_zone_domain_t zdom; | ||||
uma_bucket_t b; | uma_bucket_t b; | ||||
smr_seq_t seq; | smr_seq_t seq; | ||||
void *item; | void *item; | ||||
int domain; | int domain; | ||||
CTR3(KTR_UMA, | CTR3(KTR_UMA, | ||||
"uma_zfree: zone %s(%p) draining cross bucket %p", | "uma_zfree: zone %s(%p) draining cross bucket %p", | ||||
zone->uz_name, zone, bucket); | zone->uz_name, zone, bucket); | ||||
/* | /* | ||||
* It is possible for buckets to arrive here out of order so we fetch | * It is possible for buckets to arrive here out of order so we fetch | ||||
* the current smr seq rather than accepting the bucket's. | * the current smr seq rather than accepting the bucket's. | ||||
*/ | */ | ||||
seq = SMR_SEQ_INVALID; | seq = SMR_SEQ_INVALID; | ||||
if ((zone->uz_flags & UMA_ZONE_SMR) != 0) | if ((zone->uz_flags & UMA_ZONE_SMR) != 0) | ||||
seq = smr_advance(zone->uz_smr); | seq = smr_advance(zone->uz_smr); | ||||
/* | /* | ||||
* To avoid having ndomain * ndomain buckets for sorting we have a | * To avoid having ndomain * ndomain buckets for sorting we have a | ||||
* lock on the current crossfree bucket. A full matrix with | * lock on the current crossfree bucket. A full matrix with | ||||
* per-domain locking could be used if necessary. | * per-domain locking could be used if necessary. | ||||
*/ | */ | ||||
STAILQ_INIT(&emptybuckets); | |||||
STAILQ_INIT(&fullbuckets); | STAILQ_INIT(&fullbuckets); | ||||
ZONE_CROSS_LOCK(zone); | ZONE_CROSS_LOCK(zone); | ||||
while (bucket->ub_cnt > 0) { | for (; bucket->ub_cnt > 0; bucket->ub_cnt--) { | ||||
item = bucket->ub_bucket[bucket->ub_cnt - 1]; | item = bucket->ub_bucket[bucket->ub_cnt - 1]; | ||||
domain = item_domain(item); | domain = item_domain(item); | ||||
zdom = ZDOM_GET(zone, domain); | zdom = ZDOM_GET(zone, domain); | ||||
if (zdom->uzd_cross == NULL) { | if (zdom->uzd_cross == NULL) { | ||||
zdom->uzd_cross = bucket_alloc(zone, udata, M_NOWAIT); | if ((b = STAILQ_FIRST(&emptybuckets)) != NULL) { | ||||
if (zdom->uzd_cross == NULL) | STAILQ_REMOVE_HEAD(&emptybuckets, ub_link); | ||||
break; | zdom->uzd_cross = b; | ||||
} else { | |||||
/* | |||||
* Avoid allocating a bucket with the cross lock | |||||
* held, since allocation can trigger a | |||||
* cross-domain free and bucket zones may | |||||
* allocate from each other. | |||||
*/ | |||||
ZONE_CROSS_UNLOCK(zone); | |||||
b = bucket_alloc(zone, udata, M_NOWAIT); | |||||
if (b == NULL) | |||||
goto out; | |||||
ZONE_CROSS_LOCK(zone); | |||||
if (zdom->uzd_cross != NULL) { | |||||
STAILQ_INSERT_HEAD(&emptybuckets, b, | |||||
ub_link); | |||||
} else { | |||||
zdom->uzd_cross = b; | |||||
} | } | ||||
} | |||||
} | |||||
b = zdom->uzd_cross; | b = zdom->uzd_cross; | ||||
b->ub_bucket[b->ub_cnt++] = item; | b->ub_bucket[b->ub_cnt++] = item; | ||||
b->ub_seq = seq; | b->ub_seq = seq; | ||||
if (b->ub_cnt == b->ub_entries) { | if (b->ub_cnt == b->ub_entries) { | ||||
STAILQ_INSERT_HEAD(&fullbuckets, b, ub_link); | STAILQ_INSERT_HEAD(&fullbuckets, b, ub_link); | ||||
zdom->uzd_cross = NULL; | if ((b = STAILQ_FIRST(&emptybuckets)) != NULL) | ||||
STAILQ_REMOVE_HEAD(&emptybuckets, ub_link); | |||||
zdom->uzd_cross = b; | |||||
} | } | ||||
bucket->ub_cnt--; | |||||
} | } | ||||
ZONE_CROSS_UNLOCK(zone); | ZONE_CROSS_UNLOCK(zone); | ||||
out: | |||||
if (bucket->ub_cnt == 0) | if (bucket->ub_cnt == 0) | ||||
bucket->ub_seq = SMR_SEQ_INVALID; | bucket->ub_seq = SMR_SEQ_INVALID; | ||||
bucket_free(zone, bucket, udata); | bucket_free(zone, bucket, udata); | ||||
while ((b = STAILQ_FIRST(&emptybuckets)) != NULL) { | |||||
STAILQ_REMOVE_HEAD(&emptybuckets, ub_link); | |||||
bucket_free(zone, b, udata); | |||||
} | |||||
while ((b = STAILQ_FIRST(&fullbuckets)) != NULL) { | while ((b = STAILQ_FIRST(&fullbuckets)) != NULL) { | ||||
STAILQ_REMOVE_HEAD(&fullbuckets, ub_link); | STAILQ_REMOVE_HEAD(&fullbuckets, ub_link); | ||||
domain = item_domain(b->ub_bucket[0]); | domain = item_domain(b->ub_bucket[0]); | ||||
zone_put_bucket(zone, domain, b, udata, true); | zone_put_bucket(zone, domain, b, udata, true); | ||||
} | } | ||||
} | } | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 1,236 Lines • Show Last 20 Lines |