D22828.id.diff
No OneTemporary
Actions

Size

13 KB

Referenced Files

None

Subscribers

None

D22828.id.diff
View Options

	Index: head/sys/kern/kern_mbuf.c
	===================================================================
	--- head/sys/kern/kern_mbuf.c
	+++ head/sys/kern/kern_mbuf.c
	@@ -715,7 +715,7 @@
	* is deliberate. We don't want to acquire the zone lock for every
	* mbuf free.
	*/
	- if (uma_zone_exhausted_nolock(zone_clust))
	+ if (uma_zone_exhausted(zone_clust))
	uma_zone_reclaim(zone_pack, UMA_RECLAIM_DRAIN);
	}

	Index: head/sys/vm/uma.h
	===================================================================
	--- head/sys/vm/uma.h
	+++ head/sys/vm/uma.h
	@@ -641,7 +641,6 @@
	* Non-zero if zone is exhausted.
	*/
	int uma_zone_exhausted(uma_zone_t zone);
	-int uma_zone_exhausted_nolock(uma_zone_t zone);

	/*
	* Common UMA_ZONE_PCPU zones.
	Index: head/sys/vm/uma_core.c
	===================================================================
	--- head/sys/vm/uma_core.c
	+++ head/sys/vm/uma_core.c
	@@ -922,7 +922,7 @@
	/*
	* Drains the per cpu caches for a zone.
	*
	- * NOTE: This may only be called while the zone is being turn down, and not
	+ * NOTE: This may only be called while the zone is being torn down, and not
	* during normal operation. This is necessary in order that we do not have
	* to migrate CPUs to drain the per-CPU caches.
	*
	@@ -1041,7 +1041,7 @@
	int cpu;

	/*
	- * Polite bucket sizes shrinking was not enouth, shrink aggressively.
	+ * Polite bucket sizes shrinking was not enough, shrink aggressively.
	*/
	if (zone)
	cache_shrink(zone, NULL);
	@@ -1222,7 +1222,7 @@
	while (zone->uz_flags & UMA_ZFLAG_RECLAIMING) {
	if (waitok == M_NOWAIT)
	goto out;
	- msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1);
	+ msleep(zone, &zone->uz_lock, PVM, "zonedrain", 1);
	}
	zone->uz_flags \|= UMA_ZFLAG_RECLAIMING;
	bucket_cache_reclaim(zone, drain);
	@@ -1258,8 +1258,8 @@

	/*
	* Allocate a new slab for a keg. This does not insert the slab onto a list.
	- * If the allocation was successful, the keg lock will be held upon return,
	- * otherwise the keg will be left unlocked.
	+ * The keg should be locked on entry and will be dropped and reacquired on
	+ * return.
	*
	* Arguments:
	* flags Wait flags for the item initialization routine
	@@ -1283,8 +1283,6 @@
	KASSERT(domain >= 0 && domain < vm_ndomains,
	("keg_alloc_slab: domain %d out of range", domain));
	KEG_LOCK_ASSERT(keg);
	- MPASS(zone->uz_lockptr == &keg->uk_lock);
	-
	allocf = keg->uk_allocf;
	KEG_UNLOCK(keg);

	@@ -1293,7 +1291,7 @@
	if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
	slab = zone_alloc_item(keg->uk_slabzone, NULL, domain, aflags);
	if (slab == NULL)
	- goto out;
	+ goto fail;
	}

	/*
	@@ -1317,8 +1315,7 @@
	if (mem == NULL) {
	if (keg->uk_flags & UMA_ZONE_OFFPAGE)
	zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
	- slab = NULL;
	- goto out;
	+ goto fail;
	}
	uma_total_inc(size);

	@@ -1348,8 +1345,7 @@
	break;
	if (i != keg->uk_ipers) {
	keg_free_slab(keg, slab, i);
	- slab = NULL;
	- goto out;
	+ goto fail;
	}
	}
	KEG_LOCK(keg);
	@@ -1363,8 +1359,11 @@
	keg->uk_pages += keg->uk_ppera;
	keg->uk_free += keg->uk_ipers;

	-out:
	return (slab);
	+
	+fail:
	+ KEG_LOCK(keg);
	+ return (NULL);
	}

	/*
	@@ -2237,6 +2236,7 @@
	cnt.count = 0;
	zone_foreach(zone_count, &cnt);
	zone->uz_namecnt = cnt.count;
	+ ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS));

	for (i = 0; i < vm_ndomains; i++)
	TAILQ_INIT(&zone->uz_domain[i].uzd_buckets);
	@@ -2250,6 +2250,8 @@
	* This is a pure cache zone, no kegs.
	*/
	if (arg->import) {
	+ KASSERT((arg->flags & UMA_ZFLAG_CACHE) != 0,
	+ ("zone_ctor: Import specified for non-cache zone."));
	if (arg->flags & UMA_ZONE_VM)
	arg->flags \|= UMA_ZFLAG_CACHEONLY;
	zone->uz_flags = arg->flags;
	@@ -2257,8 +2259,6 @@
	zone->uz_import = arg->import;
	zone->uz_release = arg->release;
	zone->uz_arg = arg->arg;
	- zone->uz_lockptr = &zone->uz_lock;
	- ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS));
	rw_wlock(&uma_rwlock);
	LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link);
	rw_wunlock(&uma_rwlock);
	@@ -2279,7 +2279,6 @@
	KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
	zone->uz_init = arg->uminit;
	zone->uz_fini = arg->fini;
	- zone->uz_lockptr = &keg->uk_lock;
	zone->uz_flags \|= UMA_ZONE_SECONDARY;
	rw_wlock(&uma_rwlock);
	ZONE_LOCK(zone);
	@@ -2419,8 +2418,7 @@
	counter_u64_free(zone->uz_frees);
	counter_u64_free(zone->uz_fails);
	free(zone->uz_ctlname, M_UMA);
	- if (zone->uz_lockptr == &zone->uz_lock)
	- ZONE_LOCK_FINI(zone);
	+ ZONE_LOCK_FINI(zone);
	}

	/*
	@@ -3223,7 +3221,6 @@
	LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);
	return (slab);
	}
	- KEG_LOCK(keg);
	if (!rr && (flags & M_WAITOK) == 0)
	break;
	if (rr && vm_domainset_iter_policy(&di, &domain) != 0) {
	@@ -3869,7 +3866,6 @@
	uint8_t freei;

	keg = zone->uz_keg;
	- MPASS(zone->uz_lockptr == &keg->uk_lock);
	KEG_LOCK_ASSERT(keg);

	dom = &keg->uk_domain[slab->us_domain];
	@@ -4012,9 +4008,7 @@
	{
	int nitems;

	- ZONE_LOCK(zone);
	- nitems = zone->uz_max_items;
	- ZONE_UNLOCK(zone);
	+ nitems = atomic_load_64(&zone->uz_max_items);

	return (nitems);
	}
	@@ -4024,9 +4018,8 @@
	uma_zone_set_warning(uma_zone_t zone, const char *warning)
	{

	- ZONE_LOCK(zone);
	+ ZONE_ASSERT_COLD(zone);
	zone->uz_warning = warning;
	- ZONE_UNLOCK(zone);
	}

	/* See uma.h */
	@@ -4034,9 +4027,8 @@
	uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction)
	{

	- ZONE_LOCK(zone);
	+ ZONE_ASSERT_COLD(zone);
	TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone);
	- ZONE_UNLOCK(zone);
	}

	/* See uma.h */
	@@ -4046,22 +4038,11 @@
	int64_t nitems;
	u_int i;

	- ZONE_LOCK(zone);
	nitems = counter_u64_fetch(zone->uz_allocs) -
	counter_u64_fetch(zone->uz_frees);
	- if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
	- CPU_FOREACH(i) {
	- /*
	- * See the comment in uma_vm_zone_stats() regarding
	- * the safety of accessing the per-cpu caches. With
	- * the zone lock held, it is safe, but can potentially
	- * result in stale data.
	- */
	- nitems += zone->uz_cpu[i].uc_allocs -
	- zone->uz_cpu[i].uc_frees;
	- }
	- }
	- ZONE_UNLOCK(zone);
	+ CPU_FOREACH(i)
	+ nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs) -
	+ atomic_load_64(&zone->uz_cpu[i].uc_frees);

	return (nitems < 0 ? 0 : nitems);
	}
	@@ -4072,20 +4053,9 @@
	uint64_t nitems;
	u_int i;

	- ZONE_LOCK(zone);
	nitems = counter_u64_fetch(zone->uz_allocs);
	- if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
	- CPU_FOREACH(i) {
	- /*
	- * See the comment in uma_vm_zone_stats() regarding
	- * the safety of accessing the per-cpu caches. With
	- * the zone lock held, it is safe, but can potentially
	- * result in stale data.
	- */
	- nitems += zone->uz_cpu[i].uc_allocs;
	- }
	- }
	- ZONE_UNLOCK(zone);
	+ CPU_FOREACH(i)
	+ nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs);

	return (nitems);
	}
	@@ -4096,20 +4066,9 @@
	uint64_t nitems;
	u_int i;

	- ZONE_LOCK(zone);
	nitems = counter_u64_fetch(zone->uz_frees);
	- if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) {
	- CPU_FOREACH(i) {
	- /*
	- * See the comment in uma_vm_zone_stats() regarding
	- * the safety of accessing the per-cpu caches. With
	- * the zone lock held, it is safe, but can potentially
	- * result in stale data.
	- */
	- nitems += zone->uz_cpu[i].uc_frees;
	- }
	- }
	- ZONE_UNLOCK(zone);
	+ CPU_FOREACH(i)
	+ nitems += atomic_load_64(&zone->uz_cpu[i].uc_frees);

	return (nitems);
	}
	@@ -4121,11 +4080,8 @@
	uma_keg_t keg;

	KEG_GET(zone, keg);
	- KEG_LOCK(keg);
	- KASSERT(keg->uk_pages == 0,
	- ("uma_zone_set_init on non-empty keg"));
	+ KEG_ASSERT_COLD(keg);
	keg->uk_init = uminit;
	- KEG_UNLOCK(keg);
	}

	/* See uma.h */
	@@ -4135,11 +4091,8 @@
	uma_keg_t keg;

	KEG_GET(zone, keg);
	- KEG_LOCK(keg);
	- KASSERT(keg->uk_pages == 0,
	- ("uma_zone_set_fini on non-empty keg"));
	+ KEG_ASSERT_COLD(keg);
	keg->uk_fini = fini;
	- KEG_UNLOCK(keg);
	}

	/* See uma.h */
	@@ -4147,11 +4100,8 @@
	uma_zone_set_zinit(uma_zone_t zone, uma_init zinit)
	{

	- ZONE_LOCK(zone);
	- KASSERT(zone->uz_keg->uk_pages == 0,
	- ("uma_zone_set_zinit on non-empty keg"));
	+ ZONE_ASSERT_COLD(zone);
	zone->uz_init = zinit;
	- ZONE_UNLOCK(zone);
	}

	/* See uma.h */
	@@ -4159,38 +4109,30 @@
	uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini)
	{

	- ZONE_LOCK(zone);
	- KASSERT(zone->uz_keg->uk_pages == 0,
	- ("uma_zone_set_zfini on non-empty keg"));
	+ ZONE_ASSERT_COLD(zone);
	zone->uz_fini = zfini;
	- ZONE_UNLOCK(zone);
	}

	/* See uma.h */
	-/* XXX uk_freef is not actually used with the zone locked */
	void
	uma_zone_set_freef(uma_zone_t zone, uma_free freef)
	{
	uma_keg_t keg;

	KEG_GET(zone, keg);
	- KASSERT(keg != NULL, ("uma_zone_set_freef: Invalid zone type"));
	- KEG_LOCK(keg);
	+ KEG_ASSERT_COLD(keg);
	keg->uk_freef = freef;
	- KEG_UNLOCK(keg);
	}

	/* See uma.h */
	-/* XXX uk_allocf is not actually used with the zone locked */
	void
	uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
	{
	uma_keg_t keg;

	KEG_GET(zone, keg);
	- KEG_LOCK(keg);
	+ KEG_ASSERT_COLD(keg);
	keg->uk_allocf = allocf;
	- KEG_UNLOCK(keg);
	}

	/* See uma.h */
	@@ -4200,9 +4142,8 @@
	uma_keg_t keg;

	KEG_GET(zone, keg);
	- KEG_LOCK(keg);
	+ KEG_ASSERT_COLD(keg);
	keg->uk_reserve = items;
	- KEG_UNLOCK(keg);
	}

	/* See uma.h */
	@@ -4214,6 +4155,8 @@
	u_int pages;

	KEG_GET(zone, keg);
	+ KEG_ASSERT_COLD(keg);
	+ ZONE_ASSERT_COLD(zone);

	pages = count / keg->uk_ipers;
	if (pages * keg->uk_ipers < count)
	@@ -4277,7 +4220,6 @@
	us_link);
	break;
	}
	- KEG_LOCK(keg);
	if (vm_domainset_iter_policy(&di, &domain) != 0) {
	KEG_UNLOCK(keg);
	vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask);
	@@ -4376,20 +4318,10 @@
	int
	uma_zone_exhausted(uma_zone_t zone)
	{
	- int full;

	- ZONE_LOCK(zone);
	- full = zone->uz_sleepers > 0;
	- ZONE_UNLOCK(zone);
	- return (full);
	+ return (atomic_load_32(&zone->uz_sleepers) > 0);
	}

	-int
	-uma_zone_exhausted_nolock(uma_zone_t zone)
	-{
	- return (zone->uz_sleepers > 0);
	-}
	-
	unsigned long
	uma_limit(void)
	{
	@@ -4725,25 +4657,22 @@
	uma_keg_t keg;
	uint8_t *mem;

	+ /*
	+ * It is safe to return the slab here even though the
	+ * zone is unlocked because the item's allocation state
	+ * essentially holds a reference.
	+ */
	mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
	- if (zone->uz_flags & UMA_ZONE_VTOSLAB) {
	- slab = vtoslab((vm_offset_t)mem);
	- } else {
	- /*
	- * It is safe to return the slab here even though the
	- * zone is unlocked because the item's allocation state
	- * essentially holds a reference.
	- */
	- if (zone->uz_lockptr == &zone->uz_lock)
	- return (NULL);
	- ZONE_LOCK(zone);
	- keg = zone->uz_keg;
	- if (keg->uk_flags & UMA_ZONE_HASH)
	- slab = hash_sfind(&keg->uk_hash, mem);
	- else
	- slab = (uma_slab_t)(mem + keg->uk_pgoff);
	- ZONE_UNLOCK(zone);
	- }
	+ if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0)
	+ return (NULL);
	+ if (zone->uz_flags & UMA_ZONE_VTOSLAB)
	+ return (vtoslab((vm_offset_t)mem));
	+ keg = zone->uz_keg;
	+ if ((keg->uk_flags & UMA_ZONE_HASH) == 0)
	+ return ((uma_slab_t)(mem + keg->uk_pgoff));
	+ KEG_LOCK(keg);
	+ slab = hash_sfind(&keg->uk_hash, mem);
	+ KEG_UNLOCK(keg);

	return (slab);
	}
	@@ -4752,7 +4681,7 @@
	uma_dbg_zskip(uma_zone_t zone, void *mem)
	{

	- if (zone->uz_lockptr == &zone->uz_lock)
	+ if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0)
	return (true);

	return (uma_dbg_kskip(zone->uz_keg, mem));
	Index: head/sys/vm/uma_int.h
	===================================================================
	--- head/sys/vm/uma_int.h
	+++ head/sys/vm/uma_int.h
	@@ -257,7 +257,7 @@
	struct slabhead ud_part_slab; /* partially allocated slabs */
	struct slabhead ud_free_slab; /* completely unallocated slabs */
	struct slabhead ud_full_slab; /* fully allocated slabs */
	-};
	+} __aligned(CACHE_LINE_SIZE);

	typedef struct uma_domain * uma_domain_t;

	@@ -268,9 +268,7 @@
	*
	*/
	struct uma_keg {
	- struct mtx uk_lock; /* Lock for the keg must be first.
	- * See shared uz_keg/uz_lockptr
	- * member of struct uma_zone. */
	+ struct mtx uk_lock; /* Lock for the keg. */
	struct uma_hash uk_hash;
	LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */

	@@ -306,6 +304,10 @@
	typedef struct uma_keg * uma_keg_t;

	#ifdef _KERNEL
	+#define KEG_ASSERT_COLD(k) \
	+ KASSERT((k)->uk_pages == 0, ("keg %s initialization after use.",\
	+ (k)->uk_name))
	+
	/*
	* Free bits per-slab.
	*/
	@@ -401,7 +403,7 @@
	long uzd_imax; /* maximum item count this period */
	long uzd_imin; /* minimum item count this period */
	long uzd_wss; /* working set size estimate */
	-};
	+} __aligned(CACHE_LINE_SIZE);

	typedef struct uma_zone_domain * uma_zone_domain_t;

	@@ -410,10 +412,7 @@
	*/
	struct uma_zone {
	/* Offset 0, used in alloc/free fast/medium fast path and const. */
	- union {
	- uma_keg_t uz_keg; /* This zone's keg */
	- struct mtx uz_lockptr; / To keg or to self */
	- };
	+ uma_keg_t uz_keg; /* This zone's keg if !CACHE */
	struct uma_zone_domain uz_domain; / per-domain buckets */
	uint32_t uz_flags; /* Flags inherited from kegs */
	uint32_t uz_size; /* Size inherited from kegs */
	@@ -525,6 +524,10 @@
	#define UZ_ITEMS_SLEEPERS(x) ((x) >> UZ_ITEMS_SLEEPER_SHIFT)
	#define UZ_ITEMS_SLEEPER (1LL << UZ_ITEMS_SLEEPER_SHIFT)

	+#define ZONE_ASSERT_COLD(z) \
	+ KASSERT((z)->uz_bkt_count == 0, \
	+ ("zone %s initialization after use.", (z)->uz_name))
	+
	#undef UMA_ALIGN

	#ifdef _KERNEL
	@@ -564,11 +567,11 @@
	"UMA zone", MTX_DEF \| MTX_DUPOK); \
	} while (0)

	-#define ZONE_LOCK(z) mtx_lock((z)->uz_lockptr)
	-#define ZONE_TRYLOCK(z) mtx_trylock((z)->uz_lockptr)
	-#define ZONE_UNLOCK(z) mtx_unlock((z)->uz_lockptr)
	+#define ZONE_LOCK(z) mtx_lock(&(z)->uz_lock)
	+#define ZONE_TRYLOCK(z) mtx_trylock(&(z)->uz_lock)
	+#define ZONE_UNLOCK(z) mtx_unlock(&(z)->uz_lock)
	#define ZONE_LOCK_FINI(z) mtx_destroy(&(z)->uz_lock)
	-#define ZONE_LOCK_ASSERT(z) mtx_assert((z)->uz_lockptr, MA_OWNED)
	+#define ZONE_LOCK_ASSERT(z) mtx_assert(&(z)->uz_lock, MA_OWNED)

	/*
	* Find a slab within a hash table. This is used for OFFPAGE zones to lookup

File Metadata

Mime Type: text/plain
Expires: Tue, Mar 24, 1:35 AM (8 h, 10 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 30219233
Default Alt Text: D22828.id.diff (13 KB)

D22828.id.diffNo OneTemporaryActions

D22828.id.diffView Options

File Metadata

Event Timeline

D22828.id.diff
No OneTemporary
Actions

D22828.id.diff
View Options