D22554.diff
No OneTemporary
Actions

Size

25 KB

Referenced Files

None

Subscribers

None

D22554.diff
View Options

	Index: head/sys/vm/uma_core.c
	===================================================================
	--- head/sys/vm/uma_core.c
	+++ head/sys/vm/uma_core.c
	@@ -119,6 +119,7 @@
	int uma_align_cache = 64 - 1;

	static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets");
	+static MALLOC_DEFINE(M_UMA, "UMA", "UMA Misc");

	/*
	* Are we allowed to allocate buckets?
	@@ -258,8 +259,8 @@
	static int zero_init(void *, int, int);
	static void keg_small_init(uma_keg_t keg);
	static void keg_large_init(uma_keg_t keg);
	-static void zone_foreach(void (*zfunc)(uma_zone_t));
	-static void zone_timeout(uma_zone_t zone);
	+static void zone_foreach(void (zfunc)(uma_zone_t, void ), void *);
	+static void zone_timeout(uma_zone_t zone, void *);
	static int hash_alloc(struct uma_hash *, u_int);
	static int hash_expand(struct uma_hash , struct uma_hash );
	static void hash_free(struct uma_hash *hash);
	@@ -285,10 +286,10 @@
	static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int);
	static bool cache_free(uma_zone_t, uma_cache_t, void , void , int);

	-void uma_print_zone(uma_zone_t);
	-void uma_print_stats(void);
	static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
	static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
	+static int sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS);
	+static int sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS);

	#ifdef INVARIANTS
	static bool uma_dbg_kskip(uma_keg_t keg, void *mem);
	@@ -296,6 +297,7 @@
	static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item);
	static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item);

	+SYSCTL_NODE(_vm, OID_AUTO, uma, CTLFLAG_RW, 0, "Universal Memory Allocator");
	static SYSCTL_NODE(_vm, OID_AUTO, debug, CTLFLAG_RD, 0,
	"Memory allocation debugging");

	@@ -458,7 +460,7 @@
	}
	if ((uintptr_t)udata & UMA_ZFLAG_CACHEONLY)
	flags \|= M_NOVM;
	- ubz = bucket_zone_lookup(zone->uz_count);
	+ ubz = bucket_zone_lookup(zone->uz_bucket_size);
	if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0)
	ubz++;
	bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags);
	@@ -575,7 +577,7 @@
	uma_timeout(void *unused)
	{
	bucket_enable();
	- zone_foreach(zone_timeout);
	+ zone_foreach(zone_timeout, NULL);

	/* Reschedule this event */
	callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
	@@ -605,7 +607,7 @@
	* Returns nothing.
	*/
	static void
	-zone_timeout(uma_zone_t zone)
	+zone_timeout(uma_zone_t zone, void *unused)
	{
	uma_keg_t keg;
	u_int slabs;
	@@ -845,19 +847,20 @@
	}

	static void
	-cache_shrink(uma_zone_t zone)
	+cache_shrink(uma_zone_t zone, void *unused)
	{

	if (zone->uz_flags & UMA_ZFLAG_INTERNAL)
	return;

	ZONE_LOCK(zone);
	- zone->uz_count = (zone->uz_count_min + zone->uz_count) / 2;
	+ zone->uz_bucket_size =
	+ (zone->uz_bucket_size_min + zone->uz_bucket_size) / 2;
	ZONE_UNLOCK(zone);
	}

	static void
	-cache_drain_safe_cpu(uma_zone_t zone)
	+cache_drain_safe_cpu(uma_zone_t zone, void *unused)
	{
	uma_cache_t cache;
	uma_bucket_t b1, b2, b3;
	@@ -920,9 +923,9 @@
	* Polite bucket sizes shrinking was not enouth, shrink aggressively.
	*/
	if (zone)
	- cache_shrink(zone);
	+ cache_shrink(zone, NULL);
	else
	- zone_foreach(cache_shrink);
	+ zone_foreach(cache_shrink, NULL);

	CPU_FOREACH(cpu) {
	thread_lock(curthread);
	@@ -930,9 +933,9 @@
	thread_unlock(curthread);

	if (zone)
	- cache_drain_safe_cpu(zone);
	+ cache_drain_safe_cpu(zone, NULL);
	else
	- zone_foreach(cache_drain_safe_cpu);
	+ zone_foreach(cache_drain_safe_cpu, NULL);
	}
	thread_lock(curthread);
	sched_unbind(curthread);
	@@ -991,8 +994,8 @@
	* Shrink the zone bucket size to ensure that the per-CPU caches
	* don't grow too large.
	*/
	- if (zone->uz_count > zone->uz_count_min)
	- zone->uz_count--;
	+ if (zone->uz_bucket_size > zone->uz_bucket_size_min)
	+ zone->uz_bucket_size--;
	}

	static void
	@@ -1121,14 +1124,14 @@
	}

	static void
	-zone_drain(uma_zone_t zone)
	+zone_drain(uma_zone_t zone, void *unused)
	{

	zone_reclaim(zone, M_NOWAIT, true);
	}

	static void
	-zone_trim(uma_zone_t zone)
	+zone_trim(uma_zone_t zone, void *unused)
	{

	zone_reclaim(zone, M_NOWAIT, false);
	@@ -1818,7 +1821,7 @@
	}

	static void
	-zone_alloc_counters(uma_zone_t zone)
	+zone_alloc_counters(uma_zone_t zone, void *unused)
	{

	zone->uz_allocs = counter_u64_alloc(M_WAITOK);
	@@ -1826,6 +1829,169 @@
	zone->uz_fails = counter_u64_alloc(M_WAITOK);
	}

	+#define UMA_MAX_DUP 999
	+static void
	+zone_alloc_sysctl(uma_zone_t zone, void *unused)
	+{
	+ uma_zone_domain_t zdom;
	+ uma_keg_t keg;
	+ struct sysctl_oid oid, domainoid;
	+ int domains, i;
	+ static const char *nokeg = "cache zone";
	+ char *c;
	+
	+ /*
	+ * Make a sysctl safe copy of the zone name by removing
	+ * any special characters and handling dups by appending
	+ * an index.
	+ */
	+ if (zone->uz_namecnt != 0) {
	+ if (zone->uz_namecnt > UMA_MAX_DUP)
	+ zone->uz_namecnt = UMA_MAX_DUP;
	+ zone->uz_ctlname = malloc(strlen(zone->uz_name) +
	+ sizeof(__XSTRING(UMA_MAX_DUP)) + 1 , M_UMA, M_WAITOK);
	+ sprintf(zone->uz_ctlname, "%s_%d", zone->uz_name,
	+ zone->uz_namecnt);
	+ } else
	+ zone->uz_ctlname = strdup(zone->uz_name, M_UMA);
	+ for (c = zone->uz_ctlname; *c != '\0'; c++)
	+ if (strchr("./\\ -", *c) != NULL)
	+ *c = '_';
	+
	+ /*
	+ * Basic parameters at the root.
	+ */
	+ zone->uz_oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_vm_uma),
	+ OID_AUTO, zone->uz_ctlname, CTLFLAG_RD, NULL, "");
	+ oid = zone->uz_oid;
	+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "size", CTLFLAG_RD, &zone->uz_size, 0, "Allocation size");
	+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "flags", CTLFLAG_RD, &zone->uz_flags, 0,
	+ "Allocator configuration flags");
	+ SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "bucket_size", CTLFLAG_RD, &zone->uz_bucket_size, 0,
	+ "Desired per-cpu cache size");
	+ SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0,
	+ "Maximum allowed per-cpu cache size");
	+
	+ /*
	+ * keg if present.
	+ */
	+ oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
	+ "keg", CTLFLAG_RD, NULL, "");
	+ keg = zone->uz_keg;
	+ if ((zone->uz_flags & UMA_ZFLAG_CACHEONLY) == 0) {
	+ SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "name", CTLFLAG_RD, keg->uk_name, "Keg name");
	+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rsize", CTLFLAG_RD, &keg->uk_rsize, 0,
	+ "Real object size with alignment");
	+ SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "ppera", CTLFLAG_RD, &keg->uk_ppera, 0,
	+ "pages per-slab allocation");
	+ SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "ipers", CTLFLAG_RD, &keg->uk_ipers, 0,
	+ "items available per-slab");
	+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "align", CTLFLAG_RD, &keg->uk_align, 0,
	+ "item alignment mask");
	+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "pages", CTLFLAG_RD, &keg->uk_pages, 0,
	+ "Total pages currently allocated from VM");
	+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "free", CTLFLAG_RD, &keg->uk_free, 0,
	+ "items free in the slab layer");
	+ } else
	+ SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "name", CTLFLAG_RD, nokeg, "Keg name");
	+
	+ /*
	+ * Information about zone limits.
	+ */
	+ oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
	+ "limit", CTLFLAG_RD, NULL, "");
	+ SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "items", CTLFLAG_RD, &zone->uz_items, 0,
	+ "current number of cached items");
	+ SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "max_items", CTLFLAG_RD, &zone->uz_max_items, 0,
	+ "Maximum number of cached items");
	+ SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0,
	+ "Number of threads sleeping at limit");
	+ SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0,
	+ "Total zone limit sleeps");
	+
	+ /*
	+ * Per-domain information.
	+ */
	+ if ((zone->uz_flags & UMA_ZONE_NUMA) != 0)
	+ domains = vm_ndomains;
	+ else
	+ domains = 1;
	+ domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid),
	+ OID_AUTO, "domain", CTLFLAG_RD, NULL, "");
	+ for (i = 0; i < domains; i++) {
	+ zdom = &zone->uz_domain[i];
	+ oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid),
	+ OID_AUTO, VM_DOMAIN(i)->vmd_name, CTLFLAG_RD, NULL, "");
	+ SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "nitems", CTLFLAG_RD, &zdom->uzd_nitems,
	+ "number of items in this domain");
	+ SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "imax", CTLFLAG_RD, &zdom->uzd_imax,
	+ "maximum item count in this period");
	+ SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "imin", CTLFLAG_RD, &zdom->uzd_imin,
	+ "minimum item count in this period");
	+ SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "wss", CTLFLAG_RD, &zdom->uzd_wss,
	+ "Working set size");
	+ }
	+
	+ /*
	+ * General statistics.
	+ */
	+ oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO,
	+ "stats", CTLFLAG_RD, NULL, "");
	+ SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "current", CTLFLAG_RD \| CTLTYPE_INT \| CTLFLAG_MPSAFE,
	+ zone, 1, sysctl_handle_uma_zone_cur, "I",
	+ "Current number of allocated items");
	+ SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "allocs", CTLFLAG_RD \| CTLTYPE_U64 \| CTLFLAG_MPSAFE,
	+ zone, 0, sysctl_handle_uma_zone_allocs, "QU",
	+ "Total allocation calls");
	+ SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "frees", CTLFLAG_RD \| CTLTYPE_U64 \| CTLFLAG_MPSAFE,
	+ zone, 0, sysctl_handle_uma_zone_frees, "QU",
	+ "Total free calls");
	+ SYSCTL_ADD_COUNTER_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "fails", CTLFLAG_RD, &zone->uz_fails,
	+ "Number of allocation failures");
	+ SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "xdomain", CTLFLAG_RD, &zone->uz_xdomain, 0,
	+ "Free calls from the wrong domain");
	+}
	+
	+struct uma_zone_count {
	+ const char *name;
	+ int count;
	+};
	+
	+static void
	+zone_count(uma_zone_t zone, void *arg)
	+{
	+ struct uma_zone_count *cnt;
	+
	+ cnt = arg;
	+ if (strcmp(zone->uz_name, cnt->name) == 0)
	+ cnt->count++;
	+}
	+
	/*
	* Zone header ctor. This initializes all fields, locks, etc.
	*
	@@ -1835,6 +2001,7 @@
	static int
	zone_ctor(void mem, int size, void udata, int flags)
	{
	+ struct uma_zone_count cnt;
	struct uma_zctor_args *arg = udata;
	uma_zone_t zone = mem;
	uma_zone_t z;
	@@ -1849,9 +2016,9 @@
	zone->uz_fini = NULL;
	zone->uz_sleeps = 0;
	zone->uz_xdomain = 0;
	- zone->uz_count = 0;
	- zone->uz_count_min = 0;
	- zone->uz_count_max = BUCKET_MAX;
	+ zone->uz_bucket_size = 0;
	+ zone->uz_bucket_size_min = 0;
	+ zone->uz_bucket_size_max = BUCKET_MAX;
	zone->uz_flags = 0;
	zone->uz_warning = NULL;
	/* The domain structures follow the cpu structures. */
	@@ -1859,13 +2026,11 @@
	zone->uz_bkt_max = ULONG_MAX;
	timevalclear(&zone->uz_ratecheck);

	- if (__predict_true(booted == BOOT_RUNNING))
	- zone_alloc_counters(zone);
	- else {
	- zone->uz_allocs = EARLY_COUNTER;
	- zone->uz_frees = EARLY_COUNTER;
	- zone->uz_fails = EARLY_COUNTER;
	- }
	+ /* Count the number of duplicate names. */
	+ cnt.name = arg->name;
	+ cnt.count = 0;
	+ zone_foreach(zone_count, &cnt);
	+ zone->uz_namecnt = cnt.count;

	for (i = 0; i < vm_ndomains; i++)
	TAILQ_INIT(&zone->uz_domain[i].uzd_buckets);
	@@ -1903,6 +2068,8 @@
	keg = arg->keg;

	if (arg->flags & UMA_ZONE_SECONDARY) {
	+ KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
	+ ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
	KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
	zone->uz_init = arg->uminit;
	zone->uz_fini = arg->fini;
	@@ -1939,35 +2106,36 @@
	return (error);
	}

	+ /* Inherit properties from the keg. */
	zone->uz_keg = keg;
	zone->uz_size = keg->uk_size;
	zone->uz_flags \|= (keg->uk_flags &
	(UMA_ZONE_INHERIT \| UMA_ZFLAG_INHERIT));

	- /*
	- * Some internal zones don't have room allocated for the per cpu
	- * caches. If we're internal, bail out here.
	- */
	- if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
	- KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
	- ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
	- return (0);
	+out:
	+ if (__predict_true(booted == BOOT_RUNNING)) {
	+ zone_alloc_counters(zone, NULL);
	+ zone_alloc_sysctl(zone, NULL);
	+ } else {
	+ zone->uz_allocs = EARLY_COUNTER;
	+ zone->uz_frees = EARLY_COUNTER;
	+ zone->uz_fails = EARLY_COUNTER;
	}

	-out:
	KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET \| UMA_ZONE_NOBUCKET)) !=
	(UMA_ZONE_MAXBUCKET \| UMA_ZONE_NOBUCKET),
	("Invalid zone flag combination"));
	- if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) {
	- zone->uz_count = BUCKET_MAX;
	- } else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0) {
	- zone->uz_count = BUCKET_MIN;
	- zone->uz_count_max = BUCKET_MIN;
	- } else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0)
	- zone->uz_count = 0;
	+ if (arg->flags & UMA_ZFLAG_INTERNAL)
	+ zone->uz_bucket_size_max = zone->uz_bucket_size = 0;
	+ if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0)
	+ zone->uz_bucket_size = BUCKET_MAX;
	+ else if ((arg->flags & UMA_ZONE_MINBUCKET) != 0)
	+ zone->uz_bucket_size_max = zone->uz_bucket_size = BUCKET_MIN;
	+ else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0)
	+ zone->uz_bucket_size = 0;
	else
	- zone->uz_count = bucket_select(zone->uz_size);
	- zone->uz_count_min = zone->uz_count;
	+ zone->uz_bucket_size = bucket_select(zone->uz_size);
	+ zone->uz_bucket_size_min = zone->uz_bucket_size;

	return (0);
	}
	@@ -2013,6 +2181,8 @@

	zone = (uma_zone_t)arg;

	+ sysctl_remove_oid(zone->uz_oid, 1, 1);
	+
	if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
	cache_drain(zone);

	@@ -2039,6 +2209,7 @@
	counter_u64_free(zone->uz_allocs);
	counter_u64_free(zone->uz_frees);
	counter_u64_free(zone->uz_fails);
	+ free(zone->uz_ctlname, M_UMA);
	if (zone->uz_lockptr == &zone->uz_lock)
	ZONE_LOCK_FINI(zone);
	}
	@@ -2054,7 +2225,7 @@
	* Nothing
	*/
	static void
	-zone_foreach(void (*zfunc)(uma_zone_t))
	+zone_foreach(void (zfunc)(uma_zone_t, void arg), void *arg)
	{
	uma_keg_t keg;
	uma_zone_t zone;
	@@ -2068,10 +2239,10 @@
	rw_rlock(&uma_rwlock);
	LIST_FOREACH(keg, &uma_kegs, uk_link) {
	LIST_FOREACH(zone, &keg->uk_zones, uz_link)
	- zfunc(zone);
	+ zfunc(zone, arg);
	}
	LIST_FOREACH(zone, &uma_cachezones, uz_link)
	- zfunc(zone);
	+ zfunc(zone, arg);
	if (__predict_true(booted == BOOT_RUNNING))
	rw_runlock(&uma_rwlock);
	}
	@@ -2249,7 +2420,8 @@
	uma_dbg_cnt = counter_u64_alloc(M_WAITOK);
	uma_skip_cnt = counter_u64_alloc(M_WAITOK);
	#endif
	- zone_foreach(zone_alloc_counters);
	+ zone_foreach(zone_alloc_counters, NULL);
	+ zone_foreach(zone_alloc_sysctl, NULL);
	callout_init(&uma_callout, 1);
	callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
	booted = BOOT_RUNNING;
	@@ -2665,7 +2837,7 @@

	critical_enter();
	/* Short-circuit for zones without buckets and low memory. */
	- if (zone->uz_count == 0 \|\| bucketdisable)
	+ if (zone->uz_bucket_size == 0 \|\| bucketdisable)
	return (false);

	cpu = curcpu;
	@@ -2702,8 +2874,8 @@
	* We bump the uz count when the cache size is insufficient to
	* handle the working set.
	*/
	- if (lockfail && zone->uz_count < zone->uz_count_max)
	- zone->uz_count++;
	+ if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max)
	+ zone->uz_bucket_size++;

	/*
	* Fill a bucket and attempt to use it as the alloc bucket.
	@@ -3005,11 +3177,11 @@
	if (zone->uz_max_items > 0) {
	if (zone->uz_items >= zone->uz_max_items)
	return (false);
	- maxbucket = MIN(zone->uz_count,
	+ maxbucket = MIN(zone->uz_bucket_size,
	zone->uz_max_items - zone->uz_items);
	zone->uz_items += maxbucket;
	} else
	- maxbucket = zone->uz_count;
	+ maxbucket = zone->uz_bucket_size;
	ZONE_UNLOCK(zone);

	/* Don't wait for buckets, preserve caller's NOVM setting. */
	@@ -3285,8 +3457,8 @@
	if (ZONE_TRYLOCK(zone) == 0) {
	/* Record contention to size the buckets. */
	ZONE_LOCK(zone);
	- if (zone->uz_count < zone->uz_count_max)
	- zone->uz_count++;
	+ if (zone->uz_bucket_size < zone->uz_bucket_size_max)
	+ zone->uz_bucket_size++;
	}

	CTR3(KTR_UMA,
	@@ -3323,7 +3495,7 @@

	CRITICAL_ASSERT(curthread);

	- if (zone->uz_count == 0 \|\| bucketdisable)
	+ if (zone->uz_bucket_size == 0 \|\| bucketdisable)
	return false;

	cpu = curcpu;
	@@ -3523,9 +3695,9 @@
	ZONE_LOCK(zone);
	ubz = bucket_zone_max(zone, nitems);
	count = ubz != NULL ? ubz->ubz_entries : 0;
	- zone->uz_count_max = zone->uz_count = count;
	- if (zone->uz_count_min > zone->uz_count_max)
	- zone->uz_count_min = zone->uz_count_max;
	+ zone->uz_bucket_size_max = zone->uz_bucket_size = count;
	+ if (zone->uz_bucket_size_min > zone->uz_bucket_size_max)
	+ zone->uz_bucket_size_min = zone->uz_bucket_size_max;
	zone->uz_max_items = nitems;
	ZONE_UNLOCK(zone);

	@@ -3549,12 +3721,12 @@
	bpcpu++;
	#endif
	nitems -= ubz->ubz_entries * bpcpu * mp_ncpus;
	- zone->uz_count_max = ubz->ubz_entries;
	+ zone->uz_bucket_size_max = ubz->ubz_entries;
	} else {
	- zone->uz_count_max = zone->uz_count = 0;
	+ zone->uz_bucket_size_max = zone->uz_bucket_size = 0;
	}
	- if (zone->uz_count_min > zone->uz_count_max)
	- zone->uz_count_min = zone->uz_count_max;
	+ if (zone->uz_bucket_size_min > zone->uz_bucket_size_max)
	+ zone->uz_bucket_size_min = zone->uz_bucket_size_max;
	zone->uz_bkt_max = nitems;
	ZONE_UNLOCK(zone);
	}
	@@ -3602,20 +3774,71 @@
	ZONE_LOCK(zone);
	nitems = counter_u64_fetch(zone->uz_allocs) -
	counter_u64_fetch(zone->uz_frees);
	- CPU_FOREACH(i) {
	- /*
	- * See the comment in uma_vm_zone_stats() regarding the
	- * safety of accessing the per-cpu caches. With the zone lock
	- * held, it is safe, but can potentially result in stale data.
	- */
	- nitems += zone->uz_cpu[i].uc_allocs -
	- zone->uz_cpu[i].uc_frees;
	+ if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
	+ CPU_FOREACH(i) {
	+ /*
	+ * See the comment in uma_vm_zone_stats() regarding
	+ * the safety of accessing the per-cpu caches. With
	+ * the zone lock held, it is safe, but can potentially
	+ * result in stale data.
	+ */
	+ nitems += zone->uz_cpu[i].uc_allocs -
	+ zone->uz_cpu[i].uc_frees;
	+ }
	}
	ZONE_UNLOCK(zone);

	return (nitems < 0 ? 0 : nitems);
	}

	+static uint64_t
	+uma_zone_get_allocs(uma_zone_t zone)
	+{
	+ uint64_t nitems;
	+ u_int i;
	+
	+ ZONE_LOCK(zone);
	+ nitems = counter_u64_fetch(zone->uz_allocs);
	+ if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
	+ CPU_FOREACH(i) {
	+ /*
	+ * See the comment in uma_vm_zone_stats() regarding
	+ * the safety of accessing the per-cpu caches. With
	+ * the zone lock held, it is safe, but can potentially
	+ * result in stale data.
	+ */
	+ nitems += zone->uz_cpu[i].uc_allocs;
	+ }
	+ }
	+ ZONE_UNLOCK(zone);
	+
	+ return (nitems);
	+}
	+
	+static uint64_t
	+uma_zone_get_frees(uma_zone_t zone)
	+{
	+ uint64_t nitems;
	+ u_int i;
	+
	+ ZONE_LOCK(zone);
	+ nitems = counter_u64_fetch(zone->uz_frees);
	+ if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
	+ CPU_FOREACH(i) {
	+ /*
	+ * See the comment in uma_vm_zone_stats() regarding
	+ * the safety of accessing the per-cpu caches. With
	+ * the zone lock held, it is safe, but can potentially
	+ * result in stale data.
	+ */
	+ nitems += zone->uz_cpu[i].uc_frees;
	+ }
	+ }
	+ ZONE_UNLOCK(zone);
	+
	+ return (nitems);
	+}
	+
	/* See uma.h */
	void
	uma_zone_set_init(uma_zone_t zone, uma_init uminit)
	@@ -3800,14 +4023,14 @@

	switch (req) {
	case UMA_RECLAIM_TRIM:
	- zone_foreach(zone_trim);
	+ zone_foreach(zone_trim, NULL);
	break;
	case UMA_RECLAIM_DRAIN:
	case UMA_RECLAIM_DRAIN_CPU:
	- zone_foreach(zone_drain);
	+ zone_foreach(zone_drain, NULL);
	if (req == UMA_RECLAIM_DRAIN_CPU) {
	pcpu_cache_drain_safe(NULL);
	- zone_foreach(zone_drain);
	+ zone_foreach(zone_drain, NULL);
	}
	break;
	default:
	@@ -3819,7 +4042,7 @@
	* we visit again so that we can free pages that are empty once other
	* zones are drained. We have to do the same for buckets.
	*/
	- zone_drain(slabzone);
	+ zone_drain(slabzone, NULL);
	bucket_zone_drain();
	sx_xunlock(&uma_reclaim_lock);
	}
	@@ -3859,14 +4082,14 @@

	switch (req) {
	case UMA_RECLAIM_TRIM:
	- zone_trim(zone);
	+ zone_trim(zone, NULL);
	break;
	case UMA_RECLAIM_DRAIN:
	- zone_drain(zone);
	+ zone_drain(zone, NULL);
	break;
	case UMA_RECLAIM_DRAIN_CPU:
	pcpu_cache_drain_safe(zone);
	- zone_drain(zone);
	+ zone_drain(zone, NULL);
	break;
	default:
	panic("unhandled reclamation request %d", req);
	@@ -3977,76 +4200,6 @@
	return (uma_kmem_limit - uma_size());
	}

	-void
	-uma_print_stats(void)
	-{
	- zone_foreach(uma_print_zone);
	-}
	-
	-static void
	-slab_print(uma_slab_t slab)
	-{
	- printf("slab: keg %p, data %p, freecount %d\n",
	- slab->us_keg, slab->us_data, slab->us_freecount);
	-}
	-
	-static void
	-cache_print(uma_cache_t cache)
	-{
	- printf("alloc: %p(%d), free: %p(%d), cross: %p(%d)j\n",
	- cache->uc_allocbucket,
	- cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0,
	- cache->uc_freebucket,
	- cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0,
	- cache->uc_crossbucket,
	- cache->uc_crossbucket?cache->uc_crossbucket->ub_cnt:0);
	-}
	-
	-static void
	-uma_print_keg(uma_keg_t keg)
	-{
	- uma_domain_t dom;
	- uma_slab_t slab;
	- int i;
	-
	- printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d "
	- "out %d free %d\n",
	- keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags,
	- keg->uk_ipers, keg->uk_ppera,
	- (keg->uk_pages / keg->uk_ppera) * keg->uk_ipers - keg->uk_free,
	- keg->uk_free);
	- for (i = 0; i < vm_ndomains; i++) {
	- dom = &keg->uk_domain[i];
	- printf("Part slabs:\n");
	- LIST_FOREACH(slab, &dom->ud_part_slab, us_link)
	- slab_print(slab);
	- printf("Free slabs:\n");
	- LIST_FOREACH(slab, &dom->ud_free_slab, us_link)
	- slab_print(slab);
	- printf("Full slabs:\n");
	- LIST_FOREACH(slab, &dom->ud_full_slab, us_link)
	- slab_print(slab);
	- }
	-}
	-
	-void
	-uma_print_zone(uma_zone_t zone)
	-{
	- uma_cache_t cache;
	- int i;
	-
	- printf("zone: %s(%p) size %d maxitems %ju flags %#x\n",
	- zone->uz_name, zone, zone->uz_size, (uintmax_t)zone->uz_max_items,
	- zone->uz_flags);
	- if (zone->uz_lockptr != &zone->uz_lock)
	- uma_print_keg(zone->uz_keg);
	- CPU_FOREACH(i) {
	- cache = &zone->uz_cpu[i];
	- printf("CPU %d Cache:\n", i);
	- cache_print(cache);
	- }
	-}
	-
	#ifdef DDB
	/*
	* Generate statistics across both the zone and its per-cpu cache's. Return
	@@ -4274,13 +4427,41 @@
	int
	sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS)
	{
	- uma_zone_t zone = (uma_zone_t )arg1;
	+ uma_zone_t zone;
	int cur;

	+ /*
	+ * Some callers want to add sysctls for global zones that
	+ * may not yet exist so they pass a pointer to a pointer.
	+ */
	+ if (arg2 == 0)
	+ zone = (uma_zone_t )arg1;
	+ else
	+ zone = arg1;
	cur = uma_zone_get_cur(zone);
	return (sysctl_handle_int(oidp, &cur, 0, req));
	}

	+static int
	+sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS)
	+{
	+ uma_zone_t zone = arg1;
	+ uint64_t cur;
	+
	+ cur = uma_zone_get_allocs(zone);
	+ return (sysctl_handle_64(oidp, &cur, 0, req));
	+}
	+
	+static int
	+sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS)
	+{
	+ uma_zone_t zone = arg1;
	+ uint64_t cur;
	+
	+ cur = uma_zone_get_frees(zone);
	+ return (sysctl_handle_64(oidp, &cur, 0, req));
	+}
	+
	#ifdef INVARIANTS
	static uma_slab_t
	uma_dbg_getslab(uma_zone_t zone, void *item)
	@@ -4500,7 +4681,8 @@
	db_printf(fmt_entry, cur_zone->uz_name,
	(uintmax_t)cur_keg->uk_size, (intmax_t)used, cachefree,
	(uintmax_t)allocs, (uintmax_t)sleeps,
	- (unsigned)cur_zone->uz_count, (intmax_t)size, xdomain);
	+ (unsigned)cur_zone->uz_bucket_size, (intmax_t)size,
	+ xdomain);

	if (db_pager_quit)
	return;
	@@ -4525,7 +4707,7 @@
	db_printf("%18s %8ju %8jd %8ld %12ju %8u\n",
	z->uz_name, (uintmax_t)z->uz_size,
	(intmax_t)(allocs - frees), cachefree,
	- (uintmax_t)allocs, z->uz_count);
	+ (uintmax_t)allocs, z->uz_bucket_size);
	if (db_pager_quit)
	return;
	}
	Index: head/sys/vm/uma_int.h
	===================================================================
	--- head/sys/vm/uma_int.h
	+++ head/sys/vm/uma_int.h
	@@ -338,8 +338,8 @@
	uint64_t uz_items; /* Total items count */
	uint64_t uz_max_items; /* Maximum number of items to alloc */
	uint32_t uz_sleepers; /* Number of sleepers on memory */
	- uint16_t uz_count; /* Amount of items in full bucket */
	- uint16_t uz_count_max; /* Maximum amount of items there */
	+ uint16_t uz_bucket_size; /* Number of items in full bucket */
	+ uint16_t uz_bucket_size_max; /* Maximum number of bucket items */

	/* Offset 64, used in bucket replenish. */
	uma_import uz_import; /* Import new memory to cache. */
	@@ -364,14 +364,17 @@
	const char uz_warning; / Warning to print on failure */
	struct timeval uz_ratecheck; /* Warnings rate-limiting */
	struct task uz_maxaction; /* Task to run when at limit */
	- uint16_t uz_count_min; /* Minimal amount of items in bucket */
	+ uint16_t uz_bucket_size_min; /* Min number of items in bucket */

	- /* Offset 256, stats. */
	+ /* Offset 256+, stats and misc. */
	counter_u64_t uz_allocs; /* Total number of allocations */
	counter_u64_t uz_frees; /* Total number of frees */
	counter_u64_t uz_fails; /* Total number of alloc failures */
	uint64_t uz_sleeps; /* Total number of alloc sleeps */
	uint64_t uz_xdomain; /* Total number of cross-domain frees */
	+ char uz_ctlname; / sysctl safe name string. */
	+ struct sysctl_oid uz_oid; / sysctl oid pointer. */
	+ int uz_namecnt; /* duplicate name count. */

	/*
	* This HAS to be the last item because we adjust the zone size

File Metadata

Mime Type: text/plain
Expires: Fri, Dec 27, 11:22 PM (7 h, 33 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 15618563
Default Alt Text: D22554.diff (25 KB)

D22554.diffNo OneTemporaryActions

D22554.diffView Options

File Metadata

Event Timeline

D22554.diff
No OneTemporary
Actions

D22554.diff
View Options