Differential D23738 Diff 68657 sys/vm/uma_core.c

Changeset View

Standalone View

sys/vm/uma_core.c

Show First 20 Lines • Show All 1,134 Lines • ▼ Show 20 Lines
*		*
* Arguments:		* Arguments:
* zone The zone to free to, must be unlocked.		* zone The zone to free to, must be unlocked.
* bucket The free/alloc bucket with items.		* bucket The free/alloc bucket with items.
*		*
* Returns:		* Returns:
* Nothing		* Nothing
*/		*/

static void		static void
bucket_drain(uma_zone_t zone, uma_bucket_t bucket)		bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
{		{
int i;		int i;

if (bucket->ub_cnt == 0)		if (bucket->ub_cnt == 0)
return;		return;

▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines	cache_drain(uma_zone_t zone)
* tearing down the zone anyway. I.e., there will be no further use		* tearing down the zone anyway. I.e., there will be no further use
* of the caches at this point.		* of the caches at this point.
*		*
* XXX: It would good to be able to assert that the zone is being		* XXX: It would good to be able to assert that the zone is being
* torn down to prevent improper use of cache_drain().		* torn down to prevent improper use of cache_drain().
*/		*/
seq = SMR_SEQ_INVALID;		seq = SMR_SEQ_INVALID;
if ((zone->uz_flags & UMA_ZONE_SMR) != 0)		if ((zone->uz_flags & UMA_ZONE_SMR) != 0)
seq = smr_current(zone->uz_smr);		seq = smr_advance(zone->uz_smr);
CPU_FOREACH(cpu) {		CPU_FOREACH(cpu) {
cache = &zone->uz_cpu[cpu];		cache = &zone->uz_cpu[cpu];
bucket = cache_bucket_unload_alloc(cache);		bucket = cache_bucket_unload_alloc(cache);
if (bucket != NULL)		if (bucket != NULL)
bucket_free(zone, bucket, NULL);		bucket_free(zone, bucket, NULL);
bucket = cache_bucket_unload_free(cache);		bucket = cache_bucket_unload_free(cache);
if (bucket != NULL) {		if (bucket != NULL) {
bucket->ub_seq = seq;		bucket->ub_seq = seq;
▲ Show 20 Lines • Show All 112 Lines • ▼ Show 20 Lines	if (zone->uz_bucket_size > zone->uz_bucket_size_min)
zone->uz_bucket_size--;		zone->uz_bucket_size--;

for (i = 0; i < vm_ndomains; i++) {		for (i = 0; i < vm_ndomains; i++) {
/*		/*
* The cross bucket is partially filled and not part of		* The cross bucket is partially filled and not part of
* the item count. Reclaim it individually here.		* the item count. Reclaim it individually here.
*/		*/
zdom = ZDOM_GET(zone, i);		zdom = ZDOM_GET(zone, i);
if ((zone->uz_flags & UMA_ZONE_SMR) == 0) {		if ((zone->uz_flags & UMA_ZONE_SMR) == 0 \|\| drain) {
ZONE_CROSS_LOCK(zone);		ZONE_CROSS_LOCK(zone);
bucket = zdom->uzd_cross;		bucket = zdom->uzd_cross;
zdom->uzd_cross = NULL;		zdom->uzd_cross = NULL;
ZONE_CROSS_UNLOCK(zone);		ZONE_CROSS_UNLOCK(zone);
if (bucket != NULL)		if (bucket != NULL)
bucket_free(zone, bucket, NULL);		bucket_free(zone, bucket, NULL);
}		}

▲ Show 20 Lines • Show All 1,333 Lines • ▼ Show 20 Lines	out:
} else {		} else {
zone->uz_allocs = EARLY_COUNTER;		zone->uz_allocs = EARLY_COUNTER;
zone->uz_frees = EARLY_COUNTER;		zone->uz_frees = EARLY_COUNTER;
zone->uz_fails = EARLY_COUNTER;		zone->uz_fails = EARLY_COUNTER;
}		}

/* Caller requests a private SMR context. */		/* Caller requests a private SMR context. */
if ((zone->uz_flags & UMA_ZONE_SMR) != 0)		if ((zone->uz_flags & UMA_ZONE_SMR) != 0)
zone->uz_smr = smr_create(zone->uz_name);		zone->uz_smr = smr_create(zone->uz_name, 0, 0);

KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET \| UMA_ZONE_NOBUCKET)) !=		KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET \| UMA_ZONE_NOBUCKET)) !=
(UMA_ZONE_MAXBUCKET \| UMA_ZONE_NOBUCKET),		(UMA_ZONE_MAXBUCKET \| UMA_ZONE_NOBUCKET),
("Invalid zone flag combination"));		("Invalid zone flag combination"));
if (arg->flags & UMA_ZFLAG_INTERNAL)		if (arg->flags & UMA_ZFLAG_INTERNAL)
zone->uz_bucket_size_max = zone->uz_bucket_size = 0;		zone->uz_bucket_size_max = zone->uz_bucket_size = 0;
if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0)		if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0)
zone->uz_bucket_size = BUCKET_MAX;		zone->uz_bucket_size = BUCKET_MAX;
▲ Show 20 Lines • Show All 1,441 Lines • ▼ Show 20 Lines	zone_free_cross(uma_zone_t zone, uma_bucket_t bucket, void *udata)
smr_seq_t seq;		smr_seq_t seq;
void *item;		void *item;
int domain;		int domain;

CTR3(KTR_UMA,		CTR3(KTR_UMA,
"uma_zfree: zone %s(%p) draining cross bucket %p",		"uma_zfree: zone %s(%p) draining cross bucket %p",
zone->uz_name, zone, bucket);		zone->uz_name, zone, bucket);

STAILQ_INIT(&fullbuckets);		/*
		* It is possible for buckets to arrive here out of order so we fetch
		* the current smr seq rather than accepting the bucket's.
		*/
		seq = SMR_SEQ_INVALID;
		if ((zone->uz_flags & UMA_ZONE_SMR) != 0)
		seq = smr_advance(zone->uz_smr);

/*		/*
* To avoid having ndomain * ndomain buckets for sorting we have a		* To avoid having ndomain * ndomain buckets for sorting we have a
* lock on the current crossfree bucket. A full matrix with		* lock on the current crossfree bucket. A full matrix with
* per-domain locking could be used if necessary.		* per-domain locking could be used if necessary.
*/		*/
		STAILQ_INIT(&fullbuckets);
ZONE_CROSS_LOCK(zone);		ZONE_CROSS_LOCK(zone);

/*
* It is possible for buckets to arrive here out of order so we fetch
* the current smr seq rather than accepting the bucket's.
*/
seq = SMR_SEQ_INVALID;
if ((zone->uz_flags & UMA_ZONE_SMR) != 0)
seq = smr_current(zone->uz_smr);
while (bucket->ub_cnt > 0) {		while (bucket->ub_cnt > 0) {
item = bucket->ub_bucket[bucket->ub_cnt - 1];		item = bucket->ub_bucket[bucket->ub_cnt - 1];
domain = _vm_phys_domain(pmap_kextract((vm_offset_t)item));		domain = _vm_phys_domain(pmap_kextract((vm_offset_t)item));
zdom = ZDOM_GET(zone, domain);		zdom = ZDOM_GET(zone, domain);
if (zdom->uzd_cross == NULL) {		if (zdom->uzd_cross == NULL) {
zdom->uzd_cross = bucket_alloc(zone, udata, M_NOWAIT);		zdom->uzd_cross = bucket_alloc(zone, udata, M_NOWAIT);
if (zdom->uzd_cross == NULL)		if (zdom->uzd_cross == NULL)
break;		break;
▲ Show 20 Lines • Show All 1,276 Lines • Show Last 20 Lines