D23673.id68553.diff
No OneTemporary
Actions

Size

38 KB

Referenced Files

None

Subscribers

None

D23673.id68553.diff
View Options

	Index: head/lib/libmemstat/memstat_uma.c
	===================================================================
	--- head/lib/libmemstat/memstat_uma.c
	+++ head/lib/libmemstat/memstat_uma.c
	@@ -425,12 +425,13 @@
	(unsigned long )uz.uz_frees);
	mtp->mt_failures = kvm_counter_u64_fetch(kvm,
	(unsigned long )uz.uz_fails);
	+ mtp->mt_xdomain = kvm_counter_u64_fetch(kvm,
	+ (unsigned long )uz.uz_xdomain);
	mtp->mt_sleeps = uz.uz_sleeps;
	/* See comment above in memstat_sysctl_uma(). */
	if (mtp->mt_numallocs < mtp->mt_numfrees)
	mtp->mt_numallocs = mtp->mt_numfrees;

	- mtp->mt_xdomain = uz.uz_xdomain;
	if (kz.uk_flags & UMA_ZFLAG_INTERNAL)
	goto skip_percpu;
	for (i = 0; i < mp_maxid + 1; i++) {
	@@ -454,8 +455,9 @@
	mtp->mt_byteslimit = mtp->mt_countlimit * mtp->mt_size;
	mtp->mt_count = mtp->mt_numallocs - mtp->mt_numfrees;
	for (i = 0; i < ndomains; i++) {
	- ret = kread(kvm, &uz.uz_domain[i], &uzd,
	- sizeof(uzd), 0);
	+ ret = kread(kvm,
	+ &uz.uz_cpu[mp_maxid + 1] + i * sizeof(uzd),
	+ &uzd, sizeof(uzd), 0);
	if (ret != 0)
	continue;
	for (ubp =
	Index: head/sys/vm/uma_core.c
	===================================================================
	--- head/sys/vm/uma_core.c
	+++ head/sys/vm/uma_core.c
	@@ -285,6 +285,8 @@
	static inline void item_dtor(uma_zone_t zone, void *item, int size,
	void *udata, enum zfreeskip skip);
	static int zero_init(void *, int, int);
	+static void zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata,
	+ int itemdomain, bool ws);
	static void zone_foreach(void (zfunc)(uma_zone_t, void ), void *);
	static void zone_foreach_unlocked(void (zfunc)(uma_zone_t, void ), void *);
	static void zone_timeout(uma_zone_t zone, void *);
	@@ -518,6 +520,9 @@
	{
	struct uma_bucket_zone *ubz;

	+ if (bucket->ub_cnt != 0)
	+ bucket_drain(zone, bucket);
	+
	KASSERT(bucket->ub_cnt == 0,
	("bucket_free: Freeing a non free bucket."));
	KASSERT(bucket->ub_seq == SMR_SEQ_INVALID,
	@@ -538,17 +543,122 @@
	}

	/*
	+ * Acquire the domain lock and record contention.
	+ */
	+static uma_zone_domain_t
	+zone_domain_lock(uma_zone_t zone, int domain)
	+{
	+ uma_zone_domain_t zdom;
	+ bool lockfail;
	+
	+ zdom = ZDOM_GET(zone, domain);
	+ lockfail = false;
	+ if (ZDOM_OWNED(zdom))
	+ lockfail = true;
	+ ZDOM_LOCK(zdom);
	+ /* This is unsynchronized. The counter does not need to be precise. */
	+ if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max)
	+ zone->uz_bucket_size++;
	+ return (zdom);
	+}
	+
	+/*
	+ * Search for the domain with the least cached items and return it, breaking
	+ * ties with a preferred domain by returning it.
	+ */
	+static __noinline int
	+zone_domain_lowest(uma_zone_t zone, int pref)
	+{
	+ long least, nitems;
	+ int domain;
	+ int i;
	+
	+ least = LONG_MAX;
	+ domain = 0;
	+ for (i = 0; i < vm_ndomains; i++) {
	+ nitems = ZDOM_GET(zone, i)->uzd_nitems;
	+ if (nitems < least) {
	+ domain = i;
	+ least = nitems;
	+ } else if (nitems == least && (i == pref \|\| domain == pref))
	+ domain = pref;
	+ }
	+
	+ return (domain);
	+}
	+
	+/*
	+ * Search for the domain with the most cached items and return it or the
	+ * preferred domain if it has enough to proceed.
	+ */
	+static __noinline int
	+zone_domain_highest(uma_zone_t zone, int pref)
	+{
	+ long most, nitems;
	+ int domain;
	+ int i;
	+
	+ if (ZDOM_GET(zone, pref)->uzd_nitems > BUCKET_MAX)
	+ return (pref);
	+
	+ most = 0;
	+ domain = 0;
	+ for (i = 0; i < vm_ndomains; i++) {
	+ nitems = ZDOM_GET(zone, i)->uzd_nitems;
	+ if (nitems > most) {
	+ domain = i;
	+ most = nitems;
	+ }
	+ }
	+
	+ return (domain);
	+}
	+
	+/*
	+ * Safely subtract cnt from imax.
	+ */
	+static void
	+zone_domain_imax_sub(uma_zone_domain_t zdom, int cnt)
	+{
	+ long new;
	+ long old;
	+
	+ old = zdom->uzd_imax;
	+ do {
	+ if (old <= cnt)
	+ new = 0;
	+ else
	+ new = old - cnt;
	+ } while (atomic_fcmpset_long(&zdom->uzd_imax, &old, new) == 0);
	+}
	+
	+/*
	+ * Set the maximum imax value.
	+ */
	+static void
	+zone_domain_imax_set(uma_zone_domain_t zdom, int nitems)
	+{
	+ long old;
	+
	+ old = zdom->uzd_imax;
	+ do {
	+ if (old >= nitems)
	+ break;
	+ } while (atomic_fcmpset_long(&zdom->uzd_imax, &old, nitems) == 0);
	+}
	+
	+/*
	* Attempt to satisfy an allocation by retrieving a full bucket from one of the
	* zone's caches. If a bucket is found the zone is not locked on return.
	*/
	static uma_bucket_t
	-zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom)
	+zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, bool reclaim)
	{
	uma_bucket_t bucket;
	int i;
	bool dtor = false;

	- ZONE_LOCK_ASSERT(zone);
	+ ZDOM_LOCK_ASSERT(zdom);

	if ((bucket = STAILQ_FIRST(&zdom->uzd_buckets)) == NULL)
	return (NULL);
	@@ -560,14 +670,24 @@
	return (NULL);
	bucket->ub_seq = SMR_SEQ_INVALID;
	dtor = (zone->uz_dtor != NULL) \|\| UMA_ALWAYS_CTORDTOR;
	+ if (STAILQ_NEXT(bucket, ub_link) != NULL)
	+ zdom->uzd_seq = STAILQ_NEXT(bucket, ub_link)->ub_seq;
	}
	MPASS(zdom->uzd_nitems >= bucket->ub_cnt);
	STAILQ_REMOVE_HEAD(&zdom->uzd_buckets, ub_link);
	zdom->uzd_nitems -= bucket->ub_cnt;
	- if (zdom->uzd_imin > zdom->uzd_nitems)
	+
	+ /*
	+ * Shift the bounds of the current WSS interval to avoid
	+ * perturbing the estimate.
	+ */
	+ if (reclaim) {
	+ zdom->uzd_imin -= lmin(zdom->uzd_imin, bucket->ub_cnt);
	+ zone_domain_imax_sub(zdom, bucket->ub_cnt);
	+ } else if (zdom->uzd_imin > zdom->uzd_nitems)
	zdom->uzd_imin = zdom->uzd_nitems;
	- zone->uz_bkt_count -= bucket->ub_cnt;
	- ZONE_UNLOCK(zone);
	+
	+ ZDOM_UNLOCK(zdom);
	if (dtor)
	for (i = 0; i < bucket->ub_cnt; i++)
	item_dtor(zone, bucket->ub_bucket[i], zone->uz_size,
	@@ -579,22 +699,39 @@
	/*
	* Insert a full bucket into the specified cache. The "ws" parameter indicates
	* whether the bucket's contents should be counted as part of the zone's working
	- * set.
	+ * set. The bucket may be freed if it exceeds the bucket limit.
	*/
	static void
	-zone_put_bucket(uma_zone_t zone, uma_zone_domain_t zdom, uma_bucket_t bucket,
	+zone_put_bucket(uma_zone_t zone, int domain, uma_bucket_t bucket, void *udata,
	const bool ws)
	{
	+ uma_zone_domain_t zdom;

	- ZONE_LOCK_ASSERT(zone);
	- KASSERT(!ws \|\| zone->uz_bkt_count < zone->uz_bkt_max,
	+ /* We don't cache empty buckets. This can happen after a reclaim. */
	+ if (bucket->ub_cnt == 0)
	+ goto out;
	+ zdom = zone_domain_lock(zone, domain);
	+
	+ KASSERT(!ws \|\| zdom->uzd_nitems < zone->uz_bucket_max,
	("%s: zone %p overflow", __func__, zone));

	- STAILQ_INSERT_TAIL(&zdom->uzd_buckets, bucket, ub_link);
	+ /*
	+ * Conditionally set the maximum number of items.
	+ */
	zdom->uzd_nitems += bucket->ub_cnt;
	- if (ws && zdom->uzd_imax < zdom->uzd_nitems)
	- zdom->uzd_imax = zdom->uzd_nitems;
	- zone->uz_bkt_count += bucket->ub_cnt;
	+ if (__predict_true(zdom->uzd_nitems < zone->uz_bucket_max)) {
	+ if (ws)
	+ zone_domain_imax_set(zdom, zdom->uzd_nitems);
	+ if (STAILQ_EMPTY(&zdom->uzd_buckets))
	+ zdom->uzd_seq = bucket->ub_seq;
	+ STAILQ_INSERT_TAIL(&zdom->uzd_buckets, bucket, ub_link);
	+ ZDOM_UNLOCK(zdom);
	+ return;
	+ }
	+ zdom->uzd_nitems -= bucket->ub_cnt;
	+ ZDOM_UNLOCK(zdom);
	+out:
	+ bucket_free(zone, bucket, udata);
	}

	/* Pops an item out of a per-cpu cache bucket. */
	@@ -736,6 +873,40 @@
	cache_bucket_copy(b2, &b3);
	}

	+/*
	+ * Attempt to fetch a bucket from a zone on behalf of the current cpu cache.
	+ */
	+static uma_bucket_t
	+cache_fetch_bucket(uma_zone_t zone, uma_cache_t cache, int domain)
	+{
	+ uma_zone_domain_t zdom;
	+ uma_bucket_t bucket;
	+
	+ /*
	+ * Avoid the lock if possible.
	+ */
	+ zdom = ZDOM_GET(zone, domain);
	+ if (zdom->uzd_nitems == 0)
	+ return (NULL);
	+
	+ if ((cache_uz_flags(cache) & UMA_ZONE_SMR) != 0 &&
	+ !smr_poll(zone->uz_smr, zdom->uzd_seq, false))
	+ return (NULL);
	+
	+ /*
	+ * Check the zone's cache of buckets.
	+ */
	+ zdom = zone_domain_lock(zone, domain);
	+ if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) {
	+ KASSERT(bucket->ub_cnt != 0,
	+ ("cache_fetch_bucket: Returning an empty bucket."));
	+ return (bucket);
	+ }
	+ ZDOM_UNLOCK(zdom);
	+
	+ return (NULL);
	+}
	+
	static void
	zone_log_warning(uma_zone_t zone)
	{
	@@ -787,10 +958,12 @@
	{
	long wss;

	+ ZDOM_LOCK(zdom);
	MPASS(zdom->uzd_imax >= zdom->uzd_imin);
	wss = zdom->uzd_imax - zdom->uzd_imin;
	zdom->uzd_imax = zdom->uzd_imin = zdom->uzd_nitems;
	zdom->uzd_wss = (4 * wss + zdom->uzd_wss) / 5;
	+ ZDOM_UNLOCK(zdom);
	}

	/*
	@@ -853,10 +1026,8 @@
	KEG_UNLOCK(keg, 0);

	update_wss:
	- ZONE_LOCK(zone);
	for (int i = 0; i < vm_ndomains; i++)
	- zone_domain_update_wss(&zone->uz_domain[i]);
	- ZONE_UNLOCK(zone);
	+ zone_domain_update_wss(ZDOM_GET(zone, i));
	}

	/*
	@@ -975,7 +1146,7 @@
	{
	int i;

	- if (bucket == NULL \|\| bucket->ub_cnt == 0)
	+ if (bucket->ub_cnt == 0)
	return;

	if ((zone->uz_flags & UMA_ZONE_SMR) != 0 &&
	@@ -1033,20 +1204,16 @@
	CPU_FOREACH(cpu) {
	cache = &zone->uz_cpu[cpu];
	bucket = cache_bucket_unload_alloc(cache);
	- if (bucket != NULL) {
	- bucket_drain(zone, bucket);
	+ if (bucket != NULL)
	bucket_free(zone, bucket, NULL);
	- }
	bucket = cache_bucket_unload_free(cache);
	if (bucket != NULL) {
	bucket->ub_seq = seq;
	- bucket_drain(zone, bucket);
	bucket_free(zone, bucket, NULL);
	}
	bucket = cache_bucket_unload_cross(cache);
	if (bucket != NULL) {
	bucket->ub_seq = seq;
	- bucket_drain(zone, bucket);
	bucket_free(zone, bucket, NULL);
	}
	}
	@@ -1060,10 +1227,8 @@
	if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
	return;

	- ZONE_LOCK(zone);
	zone->uz_bucket_size =
	(zone->uz_bucket_size_min + zone->uz_bucket_size) / 2;
	- ZONE_UNLOCK(zone);
	}

	static void
	@@ -1078,11 +1243,8 @@

	b1 = b2 = b3 = NULL;
	critical_enter();
	- if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH)
	- domain = PCPU_GET(domain);
	- else
	- domain = 0;
	cache = &zone->uz_cpu[curcpu];
	+ domain = PCPU_GET(domain);
	b1 = cache_bucket_unload_alloc(cache);

	/*
	@@ -1095,24 +1257,14 @@
	}
	critical_exit();

	- ZONE_LOCK(zone);
	- if (b1 != NULL && b1->ub_cnt != 0) {
	- zone_put_bucket(zone, &zone->uz_domain[domain], b1, false);
	- b1 = NULL;
	- }
	- if (b2 != NULL && b2->ub_cnt != 0) {
	- zone_put_bucket(zone, &zone->uz_domain[domain], b2, false);
	- b2 = NULL;
	- }
	- ZONE_UNLOCK(zone);
	-
	if (b1 != NULL)
	- bucket_free(zone, b1, NULL);
	+ zone_free_bucket(zone, b1, NULL, domain, false);
	if (b2 != NULL)
	- bucket_free(zone, b2, NULL);
	+ zone_free_bucket(zone, b2, NULL, domain, false);
	if (b3 != NULL) {
	- bucket_drain(zone, b3);
	- bucket_free(zone, b3, NULL);
	+ /* Adjust the domain so it goes to zone_free_cross. */
	+ domain = (domain + 1) % vm_ndomains;
	+ zone_free_bucket(zone, b3, NULL, domain, false);
	}
	}

	@@ -1161,33 +1313,32 @@
	{
	uma_zone_domain_t zdom;
	uma_bucket_t bucket;
	- long target, tofree;
	+ long target;
	int i;

	+ /*
	+ * Shrink the zone bucket size to ensure that the per-CPU caches
	+ * don't grow too large.
	+ */
	+ if (zone->uz_bucket_size > zone->uz_bucket_size_min)
	+ zone->uz_bucket_size--;
	+
	for (i = 0; i < vm_ndomains; i++) {
	/*
	* The cross bucket is partially filled and not part of
	* the item count. Reclaim it individually here.
	*/
	- zdom = &zone->uz_domain[i];
	- ZONE_CROSS_LOCK(zone);
	- bucket = zdom->uzd_cross;
	- zdom->uzd_cross = NULL;
	- ZONE_CROSS_UNLOCK(zone);
	- if (bucket != NULL) {
	- bucket_drain(zone, bucket);
	- bucket_free(zone, bucket, NULL);
	+ zdom = ZDOM_GET(zone, i);
	+ if ((zone->uz_flags & UMA_ZONE_SMR) == 0) {
	+ ZONE_CROSS_LOCK(zone);
	+ bucket = zdom->uzd_cross;
	+ zdom->uzd_cross = NULL;
	+ ZONE_CROSS_UNLOCK(zone);
	+ if (bucket != NULL)
	+ bucket_free(zone, bucket, NULL);
	}

	/*
	- * Shrink the zone bucket size to ensure that the per-CPU caches
	- * don't grow too large.
	- */
	- ZONE_LOCK(zone);
	- if (i == 0 && zone->uz_bucket_size > zone->uz_bucket_size_min)
	- zone->uz_bucket_size--;
	-
	- /*
	* If we were asked to drain the zone, we are done only once
	* this bucket cache is empty. Otherwise, we reclaim items in
	* excess of the zone's estimated working set size. If the
	@@ -1195,30 +1346,17 @@
	* then the estimate will grow at the end of this interval and
	* we ignore the historical average.
	*/
	+ ZDOM_LOCK(zdom);
	target = drain ? 0 : lmax(zdom->uzd_wss, zdom->uzd_nitems -
	zdom->uzd_imin);
	while (zdom->uzd_nitems > target) {
	- bucket = STAILQ_FIRST(&zdom->uzd_buckets);
	+ bucket = zone_fetch_bucket(zone, zdom, true);
	if (bucket == NULL)
	break;
	- tofree = bucket->ub_cnt;
	- STAILQ_REMOVE_HEAD(&zdom->uzd_buckets, ub_link);
	- zdom->uzd_nitems -= tofree;
	- zone->uz_bkt_count -= tofree;
	-
	- /*
	- * Shift the bounds of the current WSS interval to avoid
	- * perturbing the estimate.
	- */
	- zdom->uzd_imax -= lmin(zdom->uzd_imax, tofree);
	- zdom->uzd_imin -= lmin(zdom->uzd_imin, tofree);
	-
	- ZONE_UNLOCK(zone);
	- bucket_drain(zone, bucket);
	bucket_free(zone, bucket, NULL);
	- ZONE_LOCK(zone);
	+ ZDOM_LOCK(zdom);
	}
	- ZONE_UNLOCK(zone);
	+ ZDOM_UNLOCK(zdom);
	}
	}

	@@ -1312,7 +1450,8 @@
	while (zone->uz_flags & UMA_ZFLAG_RECLAIMING) {
	if (waitok == M_NOWAIT)
	goto out;
	- msleep(zone, &zone->uz_lock, PVM, "zonedrain", 1);
	+ msleep(zone, &ZDOM_GET(zone, 0)->uzd_lock, PVM, "zonedrain",
	+ 1);
	}
	zone->uz_flags \|= UMA_ZFLAG_RECLAIMING;
	ZONE_UNLOCK(zone);
	@@ -2089,15 +2228,14 @@
	keg_layout(keg);

	/*
	- * Use a first-touch NUMA policy for all kegs that pmap_extract()
	- * will work on with the exception of critical VM structures
	- * necessary for paging.
	+ * Use a first-touch NUMA policy for kegs that pmap_extract() will
	+ * work on. Use round-robin for everything else.
	*
	* Zones may override the default by specifying either.
	*/
	#ifdef NUMA
	if ((keg->uk_flags &
	- (UMA_ZFLAG_HASH \| UMA_ZONE_VM \| UMA_ZONE_ROUNDROBIN)) == 0)
	+ (UMA_ZONE_ROUNDROBIN \| UMA_ZFLAG_CACHE \| UMA_ZONE_NOTPAGE)) == 0)
	keg->uk_flags \|= UMA_ZONE_FIRSTTOUCH;
	else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0)
	keg->uk_flags \|= UMA_ZONE_ROUNDROBIN;
	@@ -2199,6 +2337,7 @@
	zone->uz_allocs = counter_u64_alloc(M_WAITOK);
	zone->uz_frees = counter_u64_alloc(M_WAITOK);
	zone->uz_fails = counter_u64_alloc(M_WAITOK);
	+ zone->uz_xdomain = counter_u64_alloc(M_WAITOK);
	}

	static void
	@@ -2316,21 +2455,16 @@
	"sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0,
	"Total zone limit sleeps");
	SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	- "bucket_max", CTLFLAG_RD, &zone->uz_bkt_max, 0,
	- "Maximum number of items in the bucket cache");
	- SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	- "bucket_cnt", CTLFLAG_RD, &zone->uz_bkt_count, 0,
	- "Number of items in the bucket cache");
	+ "bucket_max", CTLFLAG_RD, &zone->uz_bucket_max, 0,
	+ "Maximum number of items in each domain's bucket cache");

	/*
	* Per-domain zone information.
	*/
	domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid),
	OID_AUTO, "domain", CTLFLAG_RD, NULL, "");
	- if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) == 0)
	- domains = 1;
	for (i = 0; i < domains; i++) {
	- zdom = &zone->uz_domain[i];
	+ zdom = ZDOM_GET(zone, i);
	oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid),
	OID_AUTO, VM_DOMAIN(i)->vmd_name, CTLFLAG_RD, NULL, "");
	SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	@@ -2367,8 +2501,8 @@
	SYSCTL_ADD_COUNTER_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	"fails", CTLFLAG_RD, &zone->uz_fails,
	"Number of allocation failures");
	- SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	- "xdomain", CTLFLAG_RD, &zone->uz_xdomain, 0,
	+ SYSCTL_ADD_COUNTER_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "xdomain", CTLFLAG_RD, &zone->uz_xdomain,
	"Free calls from the wrong domain");
	}

	@@ -2415,6 +2549,7 @@
	{
	struct uma_zone_count cnt;
	struct uma_zctor_args *arg = udata;
	+ uma_zone_domain_t zdom;
	uma_zone_t zone = mem;
	uma_zone_t z;
	uma_keg_t keg;
	@@ -2427,16 +2562,13 @@
	zone->uz_init = NULL;
	zone->uz_fini = NULL;
	zone->uz_sleeps = 0;
	- zone->uz_xdomain = 0;
	zone->uz_bucket_size = 0;
	zone->uz_bucket_size_min = 0;
	zone->uz_bucket_size_max = BUCKET_MAX;
	zone->uz_flags = (arg->flags & UMA_ZONE_SMR);
	zone->uz_warning = NULL;
	/* The domain structures follow the cpu structures. */
	- zone->uz_domain =
	- (struct uma_zone_domain *)&zone->uz_cpu[mp_maxid + 1];
	- zone->uz_bkt_max = ULONG_MAX;
	+ zone->uz_bucket_max = ULONG_MAX;
	timevalclear(&zone->uz_ratecheck);

	/* Count the number of duplicate names. */
	@@ -2444,11 +2576,13 @@
	cnt.count = 0;
	zone_foreach(zone_count, &cnt);
	zone->uz_namecnt = cnt.count;
	- ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS));
	ZONE_CROSS_LOCK_INIT(zone);

	- for (i = 0; i < vm_ndomains; i++)
	- STAILQ_INIT(&zone->uz_domain[i].uzd_buckets);
	+ for (i = 0; i < vm_ndomains; i++) {
	+ zdom = ZDOM_GET(zone, i);
	+ ZDOM_LOCK_INIT(zone, zdom, (arg->flags & UMA_ZONE_MTXCLASS));
	+ STAILQ_INIT(&zdom->uzd_buckets);
	+ }

	#ifdef INVARIANTS
	if (arg->uminit == trash_init && arg->fini == trash_fini)
	@@ -2466,6 +2600,15 @@
	zone->uz_import = arg->import;
	zone->uz_release = arg->release;
	zone->uz_arg = arg->arg;
	+#ifdef NUMA
	+ /*
	+ * Cache zones are round-robin unless a policy is
	+ * specified because they may have incompatible
	+ * constraints.
	+ */
	+ if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) == 0)
	+ zone->uz_flags \|= UMA_ZONE_ROUNDROBIN;
	+#endif
	rw_wlock(&uma_rwlock);
	LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link);
	rw_wunlock(&uma_rwlock);
	@@ -2600,6 +2743,7 @@
	{
	uma_zone_t zone;
	uma_keg_t keg;
	+ int i;

	zone = (uma_zone_t)arg;

	@@ -2611,13 +2755,8 @@
	rw_wlock(&uma_rwlock);
	LIST_REMOVE(zone, uz_link);
	rw_wunlock(&uma_rwlock);
	- /*
	- * XXX there are some races here where
	- * the zone can be drained but zone lock
	- * released and then refilled before we
	- * remove it... we dont care for now
	- */
	zone_reclaim(zone, M_WAITOK, true);
	+
	/*
	* We only destroy kegs from non secondary/non cache zones.
	*/
	@@ -2631,8 +2770,10 @@
	counter_u64_free(zone->uz_allocs);
	counter_u64_free(zone->uz_frees);
	counter_u64_free(zone->uz_fails);
	+ counter_u64_free(zone->uz_xdomain);
	free(zone->uz_ctlname, M_UMA);
	- ZONE_LOCK_FINI(zone);
	+ for (i = 0; i < vm_ndomains; i++)
	+ ZDOM_LOCK_FINI(ZDOM_GET(zone, i));
	ZONE_CROSS_LOCK_FINI(zone);
	}

	@@ -3235,10 +3376,9 @@
	static __noinline bool
	cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags)
	{
	- uma_zone_domain_t zdom;
	uma_bucket_t bucket;
	int domain;
	- bool lockfail;
	+ bool new;

	CRITICAL_ASSERT(curthread);

	@@ -3249,7 +3389,7 @@
	* SMR Zones can't re-use the free bucket until the sequence has
	* expired.
	*/
	- if ((zone->uz_flags & UMA_ZONE_SMR) == 0 &&
	+ if ((cache_uz_flags(cache) & UMA_ZONE_SMR) == 0 &&
	cache->uc_freebucket.ucb_cnt != 0) {
	cache_bucket_swap(&cache->uc_freebucket,
	&cache->uc_allocbucket);
	@@ -3261,8 +3401,12 @@
	*/
	bucket = cache_bucket_unload_alloc(cache);
	critical_exit();
	- if (bucket != NULL)
	+
	+ if (bucket != NULL) {
	+ KASSERT(bucket->ub_cnt == 0,
	+ ("cache_alloc: Entered with non-empty alloc bucket."));
	bucket_free(zone, bucket, udata);
	+ }

	/* Short-circuit for zones without buckets and low memory. */
	if (zone->uz_bucket_size == 0 \|\| bucketdisable) {
	@@ -3272,60 +3416,23 @@

	/*
	* Attempt to retrieve the item from the per-CPU cache has failed, so
	- * we must go back to the zone. This requires the zone lock, so we
	+ * we must go back to the zone. This requires the zdom lock, so we
	* must drop the critical section, then re-acquire it when we go back
	* to the cache. Since the critical section is released, we may be
	* preempted or migrate. As such, make sure not to maintain any
	* thread-local state specific to the cache from prior to releasing
	* the critical section.
	*/
	- lockfail = 0;
	- if (ZONE_TRYLOCK(zone) == 0) {
	- /* Record contention to size the buckets. */
	- ZONE_LOCK(zone);
	- lockfail = 1;
	- }
	+ domain = PCPU_GET(domain);
	+ if ((cache_uz_flags(cache) & UMA_ZONE_ROUNDROBIN) != 0)
	+ domain = zone_domain_highest(zone, domain);
	+ bucket = cache_fetch_bucket(zone, cache, domain);
	+ if (bucket == NULL) {
	+ bucket = zone_alloc_bucket(zone, udata, domain, flags);
	+ new = true;
	+ } else
	+ new = false;

	- /* See if we lost the race to fill the cache. */
	- critical_enter();
	- cache = &zone->uz_cpu[curcpu];
	- if (cache->uc_allocbucket.ucb_bucket != NULL) {
	- ZONE_UNLOCK(zone);
	- return (true);
	- }
	-
	- /*
	- * Check the zone's cache of buckets.
	- */
	- if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH) {
	- domain = PCPU_GET(domain);
	- zdom = &zone->uz_domain[domain];
	- } else {
	- domain = UMA_ANYDOMAIN;
	- zdom = &zone->uz_domain[0];
	- }
	-
	- if ((bucket = zone_fetch_bucket(zone, zdom)) != NULL) {
	- KASSERT(bucket->ub_cnt != 0,
	- ("uma_zalloc_arg: Returning an empty bucket."));
	- cache_bucket_load_alloc(cache, bucket);
	- return (true);
	- }
	- /* We are no longer associated with this CPU. */
	- critical_exit();
	-
	- /*
	- * We bump the uz count when the cache size is insufficient to
	- * handle the working set.
	- */
	- if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max)
	- zone->uz_bucket_size++;
	- ZONE_UNLOCK(zone);
	-
	- /*
	- * Fill a bucket and attempt to use it as the alloc bucket.
	- */
	- bucket = zone_alloc_bucket(zone, udata, domain, flags);
	CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p",
	zone->uz_name, zone, bucket);
	if (bucket == NULL) {
	@@ -3338,24 +3445,25 @@
	* initialized bucket to make this less likely or claim
	* the memory directly.
	*/
	- ZONE_LOCK(zone);
	critical_enter();
	cache = &zone->uz_cpu[curcpu];
	if (cache->uc_allocbucket.ucb_bucket == NULL &&
	- ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) == 0 \|\|
	+ ((cache_uz_flags(cache) & UMA_ZONE_FIRSTTOUCH) == 0 \|\|
	domain == PCPU_GET(domain))) {
	+ if (new)
	+ atomic_add_long(&ZDOM_GET(zone, domain)->uzd_imax,
	+ bucket->ub_cnt);
	cache_bucket_load_alloc(cache, bucket);
	- zdom->uzd_imax += bucket->ub_cnt;
	- } else if (zone->uz_bkt_count >= zone->uz_bkt_max) {
	- critical_exit();
	- ZONE_UNLOCK(zone);
	- bucket_drain(zone, bucket);
	- bucket_free(zone, bucket, udata);
	- critical_enter();
	return (true);
	- } else
	- zone_put_bucket(zone, zdom, bucket, false);
	- ZONE_UNLOCK(zone);
	+ }
	+
	+ /*
	+ * We lost the race, release this bucket and start over.
	+ */
	+ critical_exit();
	+ zone_put_bucket(zone, domain, bucket, udata, false);
	+ critical_enter();
	+
	return (true);
	}

	@@ -3748,6 +3856,8 @@
	/* Avoid allocs targeting empty domains. */
	if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain))
	domain = UMA_ANYDOMAIN;
	+ if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0)
	+ domain = UMA_ANYDOMAIN;

	if (zone->uz_max_items > 0)
	maxbucket = zone_alloc_limit(zone, zone->uz_bucket_size,
	@@ -3874,18 +3984,19 @@
	{
	uma_cache_t cache;
	uma_cache_bucket_t bucket;
	- int domain, itemdomain, uz_flags;
	+ int itemdomain, uz_flags;

	#ifdef UMA_ZALLOC_DEBUG
	KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0,
	("uma_zfree_smr: called with non-SMR zone.\n"));
	KASSERT(item != NULL, ("uma_zfree_smr: Called with NULL pointer."));
	+ SMR_ASSERT_NOT_ENTERED(zone->uz_smr);
	if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN)
	return;
	#endif
	cache = &zone->uz_cpu[curcpu];
	uz_flags = cache_uz_flags(cache);
	- domain = itemdomain = 0;
	+ itemdomain = 0;
	#ifdef NUMA
	if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0)
	itemdomain = _vm_phys_domain(pmap_kextract((vm_offset_t)item));
	@@ -3896,9 +4007,8 @@
	/* SMR Zones must free to the free bucket. */
	bucket = &cache->uc_freebucket;
	#ifdef NUMA
	- domain = PCPU_GET(domain);
	if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 &&
	- domain != itemdomain) {
	+ PCPU_GET(domain) != itemdomain) {
	bucket = &cache->uc_crossbucket;
	}
	#endif
	@@ -3922,7 +4032,7 @@
	{
	uma_cache_t cache;
	uma_cache_bucket_t bucket;
	- int domain, itemdomain, uz_flags;
	+ int itemdomain, uz_flags;

	/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
	random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA);
	@@ -3970,7 +4080,7 @@
	* current cache; when we re-acquire the critical section, we must
	* detect and handle migration if it has occurred.
	*/
	- domain = itemdomain = 0;
	+ itemdomain = 0;
	#ifdef NUMA
	if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0)
	itemdomain = _vm_phys_domain(pmap_kextract((vm_offset_t)item));
	@@ -3986,9 +4096,8 @@
	*/
	bucket = &cache->uc_allocbucket;
	#ifdef NUMA
	- domain = PCPU_GET(domain);
	if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 &&
	- domain != itemdomain) {
	+ PCPU_GET(domain) != itemdomain) {
	bucket = &cache->uc_crossbucket;
	} else
	#endif
	@@ -4047,7 +4156,7 @@
	while (bucket->ub_cnt > 0) {
	item = bucket->ub_bucket[bucket->ub_cnt - 1];
	domain = _vm_phys_domain(pmap_kextract((vm_offset_t)item));
	- zdom = &zone->uz_domain[domain];
	+ zdom = ZDOM_GET(zone, domain);
	if (zdom->uzd_cross == NULL) {
	zdom->uzd_cross = bucket_alloc(zone, udata, M_NOWAIT);
	if (zdom->uzd_cross == NULL)
	@@ -4063,37 +4172,23 @@
	bucket->ub_cnt--;
	}
	ZONE_CROSS_UNLOCK(zone);
	- if (!STAILQ_EMPTY(&fullbuckets)) {
	- ZONE_LOCK(zone);
	- while ((b = STAILQ_FIRST(&fullbuckets)) != NULL) {
	- STAILQ_REMOVE_HEAD(&fullbuckets, ub_link);
	- if (zone->uz_bkt_count >= zone->uz_bkt_max) {
	- ZONE_UNLOCK(zone);
	- bucket_drain(zone, b);
	- bucket_free(zone, b, udata);
	- ZONE_LOCK(zone);
	- } else {
	- domain = _vm_phys_domain(
	- pmap_kextract(
	- (vm_offset_t)b->ub_bucket[0]));
	- zdom = &zone->uz_domain[domain];
	- zone_put_bucket(zone, zdom, b, true);
	- }
	- }
	- ZONE_UNLOCK(zone);
	- }
	- if (bucket->ub_cnt != 0)
	- bucket_drain(zone, bucket);
	- bucket->ub_seq = SMR_SEQ_INVALID;
	+ if (bucket->ub_cnt == 0)
	+ bucket->ub_seq = SMR_SEQ_INVALID;
	bucket_free(zone, bucket, udata);
	+
	+ while ((b = STAILQ_FIRST(&fullbuckets)) != NULL) {
	+ STAILQ_REMOVE_HEAD(&fullbuckets, ub_link);
	+ domain = _vm_phys_domain(pmap_kextract(
	+ (vm_offset_t)b->ub_bucket[0]));
	+ zone_put_bucket(zone, domain, b, udata, true);
	+ }
	}
	#endif

	static void
	zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata,
	- int domain, int itemdomain)
	+ int itemdomain, bool ws)
	{
	- uma_zone_domain_t zdom;

	#ifdef NUMA
	/*
	@@ -4102,7 +4197,8 @@
	* simply cache them. Otherwise we need to sort them back to
	* correct domains.
	*/
	- if (domain != itemdomain && vm_ndomains > 2) {
	+ if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 &&
	+ vm_ndomains > 2 && PCPU_GET(domain) != itemdomain) {
	zone_free_cross(zone, bucket, udata);
	return;
	}
	@@ -4110,32 +4206,14 @@

	/*
	* Attempt to save the bucket in the zone's domain bucket cache.
	- *
	- * We bump the uz count when the cache size is insufficient to
	- * handle the working set.
	*/
	- if (ZONE_TRYLOCK(zone) == 0) {
	- /* Record contention to size the buckets. */
	- ZONE_LOCK(zone);
	- if (zone->uz_bucket_size < zone->uz_bucket_size_max)
	- zone->uz_bucket_size++;
	- }
	-
	CTR3(KTR_UMA,
	"uma_zfree: zone %s(%p) putting bucket %p on free list",
	zone->uz_name, zone, bucket);
	/* ub_cnt is pointing to the last free item */
	- KASSERT(bucket->ub_cnt == bucket->ub_entries,
	- ("uma_zfree: Attempting to insert partial bucket onto the full list.\n"));
	- if (zone->uz_bkt_count >= zone->uz_bkt_max) {
	- ZONE_UNLOCK(zone);
	- bucket_drain(zone, bucket);
	- bucket_free(zone, bucket, udata);
	- } else {
	- zdom = &zone->uz_domain[itemdomain];
	- zone_put_bucket(zone, zdom, bucket, true);
	- ZONE_UNLOCK(zone);
	- }
	+ if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0)
	+ itemdomain = zone_domain_lowest(zone, itemdomain);
	+ zone_put_bucket(zone, itemdomain, bucket, udata, ws);
	}

	/*
	@@ -4152,7 +4230,6 @@
	{
	uma_cache_bucket_t cbucket;
	uma_bucket_t newbucket, bucket;
	- int domain;

	CRITICAL_ASSERT(curthread);

	@@ -4169,18 +4246,18 @@
	*/
	cbucket = &cache->uc_freebucket;
	#ifdef NUMA
	- if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) {
	- domain = PCPU_GET(domain);
	- if (domain != itemdomain) {
	+ if ((cache_uz_flags(cache) & UMA_ZONE_FIRSTTOUCH) != 0) {
	+ if (PCPU_GET(domain) != itemdomain) {
	cbucket = &cache->uc_crossbucket;
	if (cbucket->ucb_cnt != 0)
	- atomic_add_64(&zone->uz_xdomain,
	+ counter_u64_add(zone->uz_xdomain,
	cbucket->ucb_cnt);
	}
	- } else
	+ }
	#endif
	- itemdomain = domain = 0;
	bucket = cache_bucket_unload(cbucket);
	+ KASSERT(bucket == NULL \|\| bucket->ub_cnt == bucket->ub_entries,
	+ ("cache_free: Entered with non-full free bucket."));

	/* We are no longer associated with this CPU. */
	critical_exit();
	@@ -4204,7 +4281,7 @@
	newbucket = bucket_alloc(zone, udata, M_NOWAIT);

	if (bucket != NULL)
	- zone_free_bucket(zone, bucket, udata, domain, itemdomain);
	+ zone_free_bucket(zone, bucket, udata, itemdomain, true);

	critical_enter();
	if ((bucket = newbucket) == NULL)
	@@ -4216,9 +4293,8 @@
	* is already populated we will fall through and attempt to populate
	* the free bucket.
	*/
	- if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) {
	- domain = PCPU_GET(domain);
	- if (domain != itemdomain &&
	+ if ((cache_uz_flags(cache) & UMA_ZONE_FIRSTTOUCH) != 0) {
	+ if (PCPU_GET(domain) != itemdomain &&
	cache->uc_crossbucket.ucb_bucket == NULL) {
	cache_bucket_load_cross(cache, bucket);
	return (true);
	@@ -4411,7 +4487,7 @@
	}
	if (zone->uz_bucket_size_min > zone->uz_bucket_size_max)
	zone->uz_bucket_size_min = zone->uz_bucket_size_max;
	- zone->uz_bkt_max = nitems;
	+ zone->uz_bucket_max = nitems / vm_ndomains;
	ZONE_UNLOCK(zone);
	}

	@@ -4625,7 +4701,6 @@
	} else
	kva = 0;

	- ZONE_LOCK(zone);
	MPASS(keg->uk_kva == 0);
	keg->uk_kva = kva;
	keg->uk_offset = 0;
	@@ -4638,7 +4713,6 @@
	keg->uk_flags \|= UMA_ZFLAG_LIMIT \| UMA_ZONE_NOFREE;
	zone->uz_flags \|= UMA_ZFLAG_LIMIT \| UMA_ZONE_NOFREE;
	zone_update_caches(zone);
	- ZONE_UNLOCK(zone);

	return (1);
	}
	@@ -4693,7 +4767,7 @@
	sz = 0;
	if (zone->uz_flags & UMA_ZFLAG_CACHE) {
	for (i = 0; i < vm_ndomains; i++)
	- sz += zone->uz_domain[i].uzd_nitems;
	+ sz += ZDOM_GET(zone, i)->uzd_nitems;
	return (sz * zone->uz_size);
	}
	for (i = 0; i < vm_ndomains; i++)
	@@ -4853,8 +4927,8 @@
	}
	allocs += counter_u64_fetch(z->uz_allocs);
	frees += counter_u64_fetch(z->uz_frees);
	+ xdomain += counter_u64_fetch(z->uz_xdomain);
	sleeps += z->uz_sleeps;
	- xdomain += z->uz_xdomain;
	if (cachefreep != NULL)
	*cachefreep = cachefree;
	if (allocsp != NULL)
	@@ -4898,23 +4972,15 @@


	for (i = 0; i < vm_ndomains; i++) {
	- zdom = &z->uz_domain[i];
	+ zdom = ZDOM_GET(z, i);
	uth->uth_zone_free += zdom->uzd_nitems;
	}
	uth->uth_allocs = counter_u64_fetch(z->uz_allocs);
	uth->uth_frees = counter_u64_fetch(z->uz_frees);
	uth->uth_fails = counter_u64_fetch(z->uz_fails);
	+ uth->uth_xdomain = counter_u64_fetch(z->uz_xdomain);
	uth->uth_sleeps = z->uz_sleeps;
	- uth->uth_xdomain = z->uz_xdomain;

	- /*
	- * While it is not normally safe to access the cache bucket pointers
	- * while not on the CPU that owns the cache, we only allow the pointers
	- * to be exchanged without the zone lock held, not invalidated, so
	- * accept the possible race associated with bucket exchange during
	- * monitoring. Use atomic_load_ptr() to ensure that the bucket pointers
	- * are loaded only once.
	- */
	for (i = 0; i < mp_maxid + 1; i++) {
	bzero(&ups[i], sizeof(*ups));
	if (internal \|\| CPU_ABSENT(i))
	@@ -4975,7 +5041,6 @@
	}
	LIST_FOREACH(z, &kz->uk_zones, uz_link) {
	bzero(&uth, sizeof(uth));
	- ZONE_LOCK(z);
	strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME);
	uth.uth_align = kz->uk_align;
	uth.uth_size = kz->uk_size;
	@@ -5000,7 +5065,6 @@
	uth.uth_zone_flags = UTH_ZONE_SECONDARY;
	uma_vm_zone_stats(&uth, z, &sbuf, ups,
	kz->uk_flags & UMA_ZFLAG_INTERNAL);
	- ZONE_UNLOCK(z);
	(void)sbuf_bcat(&sbuf, &uth, sizeof(uth));
	for (i = 0; i < mp_maxid + 1; i++)
	(void)sbuf_bcat(&sbuf, &ups[i], sizeof(ups[i]));
	@@ -5008,11 +5072,9 @@
	}
	LIST_FOREACH(z, &uma_cachezones, uz_link) {
	bzero(&uth, sizeof(uth));
	- ZONE_LOCK(z);
	strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME);
	uth.uth_size = z->uz_size;
	uma_vm_zone_stats(&uth, z, &sbuf, ups, false);
	- ZONE_UNLOCK(z);
	(void)sbuf_bcat(&sbuf, &uth, sizeof(uth));
	for (i = 0; i < mp_maxid + 1; i++)
	(void)sbuf_bcat(&sbuf, &ups[i], sizeof(ups[i]));
	@@ -5271,7 +5333,7 @@
	uma_zone_sumstat(z, cachefree, allocs, &frees, sleeps,
	xdomain);
	for (i = 0; i < vm_ndomains; i++) {
	- *cachefree += z->uz_domain[i].uzd_nitems;
	+ *cachefree += ZDOM_GET(z, i)->uzd_nitems;
	if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
	(LIST_FIRST(&kz->uk_zones) != z)))
	*cachefree += kz->uk_domain[i].ud_free_items;
	@@ -5365,7 +5427,7 @@
	LIST_FOREACH(z, &uma_cachezones, uz_link) {
	uma_zone_sumstat(z, &cachefree, &allocs, &frees, NULL, NULL);
	for (i = 0; i < vm_ndomains; i++)
	- cachefree += z->uz_domain[i].uzd_nitems;
	+ cachefree += ZDOM_GET(z, i)->uzd_nitems;
	db_printf("%18s %8ju %8jd %8ld %12ju %8u\n",
	z->uz_name, (uintmax_t)z->uz_size,
	(intmax_t)(allocs - frees), cachefree,
	Index: head/sys/vm/uma_int.h
	===================================================================
	--- head/sys/vm/uma_int.h
	+++ head/sys/vm/uma_int.h
	@@ -471,6 +471,8 @@
	long uzd_imax; /* maximum item count this period */
	long uzd_imin; /* minimum item count this period */
	long uzd_wss; /* working set size estimate */
	+ smr_seq_t uzd_seq; /* Lowest queued seq. */
	+ struct mtx uzd_lock; /* Lock for the domain */
	} __aligned(CACHE_LINE_SIZE);

	typedef struct uma_zone_domain * uma_zone_domain_t;
	@@ -480,64 +482,55 @@
	*/
	struct uma_zone {
	/* Offset 0, used in alloc/free fast/medium fast path and const. */
	- uma_keg_t uz_keg; /* This zone's keg if !CACHE */
	- struct uma_zone_domain uz_domain; / per-domain buckets */
	uint32_t uz_flags; /* Flags inherited from kegs */
	uint32_t uz_size; /* Size inherited from kegs */
	uma_ctor uz_ctor; /* Constructor for each allocation */
	uma_dtor uz_dtor; /* Destructor */
	smr_t uz_smr; /* Safe memory reclaim context. */
	uint64_t uz_max_items; /* Maximum number of items to alloc */
	- uint32_t uz_sleepers; /* Threads sleeping on limit */
	+ uint64_t uz_bucket_max; /* Maximum bucket cache size */
	uint16_t uz_bucket_size; /* Number of items in full bucket */
	uint16_t uz_bucket_size_max; /* Maximum number of bucket items */
	+ uint32_t uz_sleepers; /* Threads sleeping on limit */
	+ counter_u64_t uz_xdomain; /* Total number of cross-domain frees */

	/* Offset 64, used in bucket replenish. */
	+ uma_keg_t uz_keg; /* This zone's keg if !CACHE */
	uma_import uz_import; /* Import new memory to cache. */
	uma_release uz_release; /* Release memory from cache. */
	void uz_arg; / Import/release argument. */
	uma_init uz_init; /* Initializer for each item */
	uma_fini uz_fini; /* Finalizer for each item. */
	- void *uz_spare1;
	- uint64_t uz_bkt_count; /* Items in bucket cache */
	- uint64_t uz_bkt_max; /* Maximum bucket cache size */
	+ volatile uint64_t uz_items; /* Total items count & sleepers */
	+ uint64_t uz_sleeps; /* Total number of alloc sleeps */

	- /* Offset 128 Rare. */
	- /*
	- * The lock is placed here to avoid adjacent line prefetcher
	- * in fast paths and to take up space near infrequently accessed
	- * members to reduce alignment overhead.
	- */
	- struct mtx uz_lock; /* Lock for the zone */
	+ /* Offset 128 Rare stats, misc read-only. */
	LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */
	- const char uz_name; / Text name of the zone */
	- /* The next two fields are used to print a rate-limited warnings. */
	- const char uz_warning; / Warning to print on failure */
	- struct timeval uz_ratecheck; /* Warnings rate-limiting */
	- struct task uz_maxaction; /* Task to run when at limit */
	- uint16_t uz_bucket_size_min; /* Min number of items in bucket */
	-
	- struct mtx_padalign uz_cross_lock; /* Cross domain free lock */
	-
	- /* Offset 256+, stats and misc. */
	counter_u64_t uz_allocs; /* Total number of allocations */
	counter_u64_t uz_frees; /* Total number of frees */
	counter_u64_t uz_fails; /* Total number of alloc failures */
	- uint64_t uz_sleeps; /* Total number of alloc sleeps */
	- uint64_t uz_xdomain; /* Total number of cross-domain frees */
	- volatile uint64_t uz_items; /* Total items count & sleepers */
	-
	+ const char uz_name; / Text name of the zone */
	char uz_ctlname; / sysctl safe name string. */
	- struct sysctl_oid uz_oid; / sysctl oid pointer. */
	int uz_namecnt; /* duplicate name count. */
	+ uint16_t uz_bucket_size_min; /* Min number of items in bucket */
	+ uint16_t uz_pad0;

	+ /* Offset 192, rare read-only. */
	+ struct sysctl_oid uz_oid; / sysctl oid pointer. */
	+ const char uz_warning; / Warning to print on failure */
	+ struct timeval uz_ratecheck; /* Warnings rate-limiting */
	+ struct task uz_maxaction; /* Task to run when at limit */
	+
	+ /* Offset 256. */
	+ struct mtx uz_cross_lock; /* Cross domain free lock */
	+
	/*
	* This HAS to be the last item because we adjust the zone size
	* based on NCPU and then allocate the space for the zones.
	*/
	struct uma_cache uz_cpu[]; /* Per cpu caches */

	- /* uz_domain follows here. */
	+ /* domains follow here. */
	};

	/*
	@@ -582,26 +575,33 @@

	#define KEG_GET(zone, keg) do { \
	(keg) = (zone)->uz_keg; \
	- KASSERT((void )(keg) != (void )&(zone)->uz_lock, \
	+ KASSERT((void *)(keg) != NULL, \
	("%s: Invalid zone %p type", __func__, (zone))); \
	} while (0)

	-#define ZONE_LOCK_INIT(z, lc) \
	- do { \
	- if ((lc)) \
	- mtx_init(&(z)->uz_lock, (z)->uz_name, \
	- (z)->uz_name, MTX_DEF \| MTX_DUPOK); \
	- else \
	- mtx_init(&(z)->uz_lock, (z)->uz_name, \
	- "UMA zone", MTX_DEF \| MTX_DUPOK); \
	+/* Domains are contiguous after the last CPU */
	+#define ZDOM_GET(z, n) \
	+ (&((uma_zone_domain_t)&(z)->uz_cpu[mp_maxid + 1])[n])
	+
	+#define ZDOM_LOCK_INIT(z, zdom, lc) \
	+ do { \
	+ if ((lc)) \
	+ mtx_init(&(zdom)->uzd_lock, (z)->uz_name, \
	+ (z)->uz_name, MTX_DEF \| MTX_DUPOK); \
	+ else \
	+ mtx_init(&(zdom)->uzd_lock, (z)->uz_name, \
	+ "UMA zone", MTX_DEF \| MTX_DUPOK); \
	} while (0)
	+#define ZDOM_LOCK_FINI(z) mtx_destroy(&(z)->uzd_lock)
	+#define ZDOM_LOCK_ASSERT(z) mtx_assert(&(z)->uzd_lock, MA_OWNED)

	-#define ZONE_LOCK(z) mtx_lock(&(z)->uz_lock)
	-#define ZONE_TRYLOCK(z) mtx_trylock(&(z)->uz_lock)
	-#define ZONE_UNLOCK(z) mtx_unlock(&(z)->uz_lock)
	-#define ZONE_LOCK_FINI(z) mtx_destroy(&(z)->uz_lock)
	-#define ZONE_LOCK_ASSERT(z) mtx_assert(&(z)->uz_lock, MA_OWNED)
	+#define ZDOM_LOCK(z) mtx_lock(&(z)->uzd_lock)
	+#define ZDOM_OWNED(z) (mtx_owner(&(z)->uzd_lock) != NULL)
	+#define ZDOM_UNLOCK(z) mtx_unlock(&(z)->uzd_lock)

	+#define ZONE_LOCK(z) ZDOM_LOCK(ZDOM_GET((z), 0))
	+#define ZONE_UNLOCK(z) ZDOM_UNLOCK(ZDOM_GET((z), 0))
	+
	#define ZONE_CROSS_LOCK_INIT(z) \
	mtx_init(&(z)->uz_cross_lock, "UMA Cross", NULL, MTX_DEF)
	#define ZONE_CROSS_LOCK(z) mtx_lock(&(z)->uz_cross_lock)
	@@ -693,6 +693,7 @@

	/* Set a global soft limit on UMA managed memory. */
	void uma_set_limit(unsigned long limit);
	+
	#endif /* _KERNEL */

	#endif /* VM_UMA_INT_H */

File Metadata

Mime Type: text/plain
Expires: Tue, Apr 28, 4:38 AM (3 h, 44 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 32251147
Default Alt Text: D23673.id68553.diff (38 KB)

D23673.id68553.diffNo OneTemporaryActions

D23673.id68553.diffView Options

File Metadata

Event Timeline

D23673.id68553.diff
No OneTemporary
Actions

D23673.id68553.diff
View Options