Index: sys/kern/kern_mbuf.c =================================================================== --- sys/kern/kern_mbuf.c +++ sys/kern/kern_mbuf.c @@ -715,7 +715,7 @@ * is deliberate. We don't want to acquire the zone lock for every * mbuf free. */ - if (uma_zone_exhausted_nolock(zone_clust)) + if (uma_zone_exhausted(zone_clust)) uma_zone_reclaim(zone_pack, UMA_RECLAIM_DRAIN); } Index: sys/vm/uma.h =================================================================== --- sys/vm/uma.h +++ sys/vm/uma.h @@ -641,7 +641,6 @@ * Non-zero if zone is exhausted. */ int uma_zone_exhausted(uma_zone_t zone); -int uma_zone_exhausted_nolock(uma_zone_t zone); /* * Common UMA_ZONE_PCPU zones. Index: sys/vm/uma_core.c =================================================================== --- sys/vm/uma_core.c +++ sys/vm/uma_core.c @@ -914,7 +914,7 @@ /* * Drains the per cpu caches for a zone. * - * NOTE: This may only be called while the zone is being turn down, and not + * NOTE: This may only be called while the zone is being torn down, and not * during normal operation. This is necessary in order that we do not have * to migrate CPUs to drain the per-CPU caches. * @@ -1033,7 +1033,7 @@ int cpu; /* - * Polite bucket sizes shrinking was not enouth, shrink aggressively. + * Polite bucket sizes shrinking was not enough, shrink aggressively. */ if (zone) cache_shrink(zone, NULL); @@ -1214,7 +1214,7 @@ while (zone->uz_flags & UMA_ZFLAG_RECLAIMING) { if (waitok == M_NOWAIT) goto out; - msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1); + msleep(zone, &zone->uz_lock, PVM, "zonedrain", 1); } zone->uz_flags |= UMA_ZFLAG_RECLAIMING; bucket_cache_reclaim(zone, drain); @@ -1250,8 +1250,8 @@ /* * Allocate a new slab for a keg. This does not insert the slab onto a list. - * If the allocation was successful, the keg lock will be held upon return, - * otherwise the keg will be left unlocked. + * The keg should be locked on entry and will be dropped and reacquired on + * return. * * Arguments: * flags Wait flags for the item initialization routine @@ -1275,8 +1275,6 @@ KASSERT(domain >= 0 && domain < vm_ndomains, ("keg_alloc_slab: domain %d out of range", domain)); KEG_LOCK_ASSERT(keg); - MPASS(zone->uz_lockptr == &keg->uk_lock); - allocf = keg->uk_allocf; KEG_UNLOCK(keg); @@ -1285,7 +1283,7 @@ if (keg->uk_flags & UMA_ZONE_OFFPAGE) { slab = zone_alloc_item(keg->uk_slabzone, NULL, domain, aflags); if (slab == NULL) - goto out; + goto fail; } /* @@ -1309,8 +1307,7 @@ if (mem == NULL) { if (keg->uk_flags & UMA_ZONE_OFFPAGE) zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); - slab = NULL; - goto out; + goto fail; } uma_total_inc(size); @@ -1340,8 +1337,7 @@ break; if (i != keg->uk_ipers) { keg_free_slab(keg, slab, i); - slab = NULL; - goto out; + goto fail; } } KEG_LOCK(keg); @@ -1355,8 +1351,11 @@ keg->uk_pages += keg->uk_ppera; keg->uk_free += keg->uk_ipers; -out: return (slab); + +fail: + KEG_LOCK(keg); + return (NULL); } /* @@ -2229,6 +2228,7 @@ cnt.count = 0; zone_foreach(zone_count, &cnt); zone->uz_namecnt = cnt.count; + ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS)); for (i = 0; i < vm_ndomains; i++) TAILQ_INIT(&zone->uz_domain[i].uzd_buckets); @@ -2242,6 +2242,8 @@ * This is a pure cache zone, no kegs. */ if (arg->import) { + KASSERT((arg->flags & UMA_ZFLAG_CACHE) != 0, + ("zone_ctor: Import specified for non-cache zone.")); if (arg->flags & UMA_ZONE_VM) arg->flags |= UMA_ZFLAG_CACHEONLY; zone->uz_flags = arg->flags; @@ -2249,8 +2251,6 @@ zone->uz_import = arg->import; zone->uz_release = arg->release; zone->uz_arg = arg->arg; - zone->uz_lockptr = &zone->uz_lock; - ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS)); rw_wlock(&uma_rwlock); LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link); rw_wunlock(&uma_rwlock); @@ -2271,7 +2271,6 @@ KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); zone->uz_init = arg->uminit; zone->uz_fini = arg->fini; - zone->uz_lockptr = &keg->uk_lock; zone->uz_flags |= UMA_ZONE_SECONDARY; rw_wlock(&uma_rwlock); ZONE_LOCK(zone); @@ -2411,8 +2410,7 @@ counter_u64_free(zone->uz_frees); counter_u64_free(zone->uz_fails); free(zone->uz_ctlname, M_UMA); - if (zone->uz_lockptr == &zone->uz_lock) - ZONE_LOCK_FINI(zone); + ZONE_LOCK_FINI(zone); } /* @@ -3218,7 +3216,6 @@ LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); return (slab); } - KEG_LOCK(keg); if (rr && vm_domainset_iter_policy(&di, &domain) != 0) { if ((flags & M_WAITOK) != 0) { KEG_UNLOCK(keg); @@ -3851,7 +3848,6 @@ uint8_t freei; keg = zone->uz_keg; - MPASS(zone->uz_lockptr == &keg->uk_lock); KEG_LOCK_ASSERT(keg); dom = &keg->uk_domain[slab->us_domain]; @@ -3990,9 +3986,7 @@ { int nitems; - ZONE_LOCK(zone); - nitems = zone->uz_max_items; - ZONE_UNLOCK(zone); + nitems = atomic_load_64(&zone->uz_max_items); return (nitems); } @@ -4002,9 +3996,8 @@ uma_zone_set_warning(uma_zone_t zone, const char *warning) { - ZONE_LOCK(zone); + ZONE_ASSERT_COLD(zone); zone->uz_warning = warning; - ZONE_UNLOCK(zone); } /* See uma.h */ @@ -4012,9 +4005,8 @@ uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction) { - ZONE_LOCK(zone); + ZONE_ASSERT_COLD(zone); TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone); - ZONE_UNLOCK(zone); } /* See uma.h */ @@ -4024,22 +4016,11 @@ int64_t nitems; u_int i; - ZONE_LOCK(zone); nitems = counter_u64_fetch(zone->uz_allocs) - counter_u64_fetch(zone->uz_frees); - if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) { - CPU_FOREACH(i) { - /* - * See the comment in uma_vm_zone_stats() regarding - * the safety of accessing the per-cpu caches. With - * the zone lock held, it is safe, but can potentially - * result in stale data. - */ - nitems += zone->uz_cpu[i].uc_allocs - - zone->uz_cpu[i].uc_frees; - } - } - ZONE_UNLOCK(zone); + CPU_FOREACH(i) + nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs) - + atomic_load_64(&zone->uz_cpu[i].uc_frees); return (nitems < 0 ? 0 : nitems); } @@ -4050,20 +4031,9 @@ uint64_t nitems; u_int i; - ZONE_LOCK(zone); nitems = counter_u64_fetch(zone->uz_allocs); - if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) { - CPU_FOREACH(i) { - /* - * See the comment in uma_vm_zone_stats() regarding - * the safety of accessing the per-cpu caches. With - * the zone lock held, it is safe, but can potentially - * result in stale data. - */ - nitems += zone->uz_cpu[i].uc_allocs; - } - } - ZONE_UNLOCK(zone); + CPU_FOREACH(i) + nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs); return (nitems); } @@ -4074,20 +4044,9 @@ uint64_t nitems; u_int i; - ZONE_LOCK(zone); nitems = counter_u64_fetch(zone->uz_frees); - if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) == 0) { - CPU_FOREACH(i) { - /* - * See the comment in uma_vm_zone_stats() regarding - * the safety of accessing the per-cpu caches. With - * the zone lock held, it is safe, but can potentially - * result in stale data. - */ - nitems += zone->uz_cpu[i].uc_frees; - } - } - ZONE_UNLOCK(zone); + CPU_FOREACH(i) + nitems += atomic_load_64(&zone->uz_cpu[i].uc_frees); return (nitems); } @@ -4099,11 +4058,8 @@ uma_keg_t keg; KEG_GET(zone, keg); - KEG_LOCK(keg); - KASSERT(keg->uk_pages == 0, - ("uma_zone_set_init on non-empty keg")); + KEG_ASSERT_COLD(keg); keg->uk_init = uminit; - KEG_UNLOCK(keg); } /* See uma.h */ @@ -4113,11 +4069,8 @@ uma_keg_t keg; KEG_GET(zone, keg); - KEG_LOCK(keg); - KASSERT(keg->uk_pages == 0, - ("uma_zone_set_fini on non-empty keg")); + KEG_ASSERT_COLD(keg); keg->uk_fini = fini; - KEG_UNLOCK(keg); } /* See uma.h */ @@ -4125,11 +4078,8 @@ uma_zone_set_zinit(uma_zone_t zone, uma_init zinit) { - ZONE_LOCK(zone); - KASSERT(zone->uz_keg->uk_pages == 0, - ("uma_zone_set_zinit on non-empty keg")); + ZONE_ASSERT_COLD(zone); zone->uz_init = zinit; - ZONE_UNLOCK(zone); } /* See uma.h */ @@ -4137,38 +4087,30 @@ uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini) { - ZONE_LOCK(zone); - KASSERT(zone->uz_keg->uk_pages == 0, - ("uma_zone_set_zfini on non-empty keg")); + ZONE_ASSERT_COLD(zone); zone->uz_fini = zfini; - ZONE_UNLOCK(zone); } /* See uma.h */ -/* XXX uk_freef is not actually used with the zone locked */ void uma_zone_set_freef(uma_zone_t zone, uma_free freef) { uma_keg_t keg; KEG_GET(zone, keg); - KASSERT(keg != NULL, ("uma_zone_set_freef: Invalid zone type")); - KEG_LOCK(keg); + KEG_ASSERT_COLD(keg); keg->uk_freef = freef; - KEG_UNLOCK(keg); } /* See uma.h */ -/* XXX uk_allocf is not actually used with the zone locked */ void uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) { uma_keg_t keg; KEG_GET(zone, keg); - KEG_LOCK(keg); + KEG_ASSERT_COLD(keg); keg->uk_allocf = allocf; - KEG_UNLOCK(keg); } /* See uma.h */ @@ -4178,9 +4120,8 @@ uma_keg_t keg; KEG_GET(zone, keg); - KEG_LOCK(keg); + KEG_ASSERT_COLD(keg); keg->uk_reserve = items; - KEG_UNLOCK(keg); } /* See uma.h */ @@ -4192,6 +4133,8 @@ u_int pages; KEG_GET(zone, keg); + KEG_ASSERT_COLD(keg); + ZONE_ASSERT_COLD(zone); pages = count / keg->uk_ipers; if (pages * keg->uk_ipers < count) @@ -4255,7 +4198,6 @@ us_link); break; } - KEG_LOCK(keg); if (vm_domainset_iter_policy(&di, &domain) != 0) { KEG_UNLOCK(keg); vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask); @@ -4354,18 +4296,8 @@ int uma_zone_exhausted(uma_zone_t zone) { - int full; - - ZONE_LOCK(zone); - full = zone->uz_sleepers > 0; - ZONE_UNLOCK(zone); - return (full); -} -int -uma_zone_exhausted_nolock(uma_zone_t zone) -{ - return (zone->uz_sleepers > 0); + return (atomic_load_32(&zone->uz_sleepers) > 0); } unsigned long @@ -4701,25 +4633,22 @@ uma_keg_t keg; uint8_t *mem; + /* + * It is safe to return the slab here even though the + * zone is unlocked because the item's allocation state + * essentially holds a reference. + */ mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); - if (zone->uz_flags & UMA_ZONE_VTOSLAB) { - slab = vtoslab((vm_offset_t)mem); - } else { - /* - * It is safe to return the slab here even though the - * zone is unlocked because the item's allocation state - * essentially holds a reference. - */ - if (zone->uz_lockptr == &zone->uz_lock) - return (NULL); - ZONE_LOCK(zone); - keg = zone->uz_keg; - if (keg->uk_flags & UMA_ZONE_HASH) - slab = hash_sfind(&keg->uk_hash, mem); - else - slab = (uma_slab_t)(mem + keg->uk_pgoff); - ZONE_UNLOCK(zone); - } + if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) + return (NULL); + if (zone->uz_flags & UMA_ZONE_VTOSLAB) + return (vtoslab((vm_offset_t)mem)); + keg = zone->uz_keg; + if ((keg->uk_flags & UMA_ZONE_HASH) == 0) + return ((uma_slab_t)(mem + keg->uk_pgoff)); + KEG_LOCK(keg); + slab = hash_sfind(&keg->uk_hash, mem); + KEG_UNLOCK(keg); return (slab); } @@ -4728,7 +4657,7 @@ uma_dbg_zskip(uma_zone_t zone, void *mem) { - if (zone->uz_lockptr == &zone->uz_lock) + if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) return (true); return (uma_dbg_kskip(zone->uz_keg, mem)); Index: sys/vm/uma_int.h =================================================================== --- sys/vm/uma_int.h +++ sys/vm/uma_int.h @@ -258,7 +258,7 @@ struct slabhead ud_part_slab; /* partially allocated slabs */ struct slabhead ud_free_slab; /* completely unallocated slabs */ struct slabhead ud_full_slab; /* fully allocated slabs */ -}; +} __aligned(CACHE_LINE_SIZE); typedef struct uma_domain * uma_domain_t; @@ -269,9 +269,7 @@ * */ struct uma_keg { - struct mtx uk_lock; /* Lock for the keg must be first. - * See shared uz_keg/uz_lockptr - * member of struct uma_zone. */ + struct mtx uk_lock; /* Lock for the keg. */ struct uma_hash uk_hash; LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */ @@ -307,6 +305,10 @@ typedef struct uma_keg * uma_keg_t; #ifdef _KERNEL +#define KEG_ASSERT_COLD(k) \ + KASSERT((k)->uk_pages == 0, ("keg %s initialization after use.",\ + (k)->uk_name)) + /* * Free bits per-slab. */ @@ -402,7 +404,7 @@ long uzd_imax; /* maximum item count this period */ long uzd_imin; /* minimum item count this period */ long uzd_wss; /* working set size estimate */ -}; +} __aligned(CACHE_LINE_SIZE); typedef struct uma_zone_domain * uma_zone_domain_t; @@ -411,10 +413,7 @@ */ struct uma_zone { /* Offset 0, used in alloc/free fast/medium fast path and const. */ - union { - uma_keg_t uz_keg; /* This zone's keg */ - struct mtx *uz_lockptr; /* To keg or to self */ - }; + uma_keg_t uz_keg; /* This zone's keg if !CACHE */ struct uma_zone_domain *uz_domain; /* per-domain buckets */ uint32_t uz_flags; /* Flags inherited from kegs */ uint32_t uz_size; /* Size inherited from kegs */ @@ -523,6 +522,10 @@ #define UZ_ITEMS_SLEEPERS(x) ((x) >> UZ_ITEMS_SLEEPER_SHIFT) #define UZ_ITEMS_SLEEPER (1LL << UZ_ITEMS_SLEEPER_SHIFT) +#define ZONE_ASSERT_COLD(z) \ + KASSERT((z)->uz_bkt_count == 0, \ + ("zone %s initialization after use.", (z)->uz_name)) + #undef UMA_ALIGN #ifdef _KERNEL @@ -562,11 +565,11 @@ "UMA zone", MTX_DEF | MTX_DUPOK); \ } while (0) -#define ZONE_LOCK(z) mtx_lock((z)->uz_lockptr) -#define ZONE_TRYLOCK(z) mtx_trylock((z)->uz_lockptr) -#define ZONE_UNLOCK(z) mtx_unlock((z)->uz_lockptr) +#define ZONE_LOCK(z) mtx_lock(&(z)->uz_lock) +#define ZONE_TRYLOCK(z) mtx_trylock(&(z)->uz_lock) +#define ZONE_UNLOCK(z) mtx_unlock(&(z)->uz_lock) #define ZONE_LOCK_FINI(z) mtx_destroy(&(z)->uz_lock) -#define ZONE_LOCK_ASSERT(z) mtx_assert((z)->uz_lockptr, MA_OWNED) +#define ZONE_LOCK_ASSERT(z) mtx_assert(&(z)->uz_lock, MA_OWNED) /* * Find a slab within a hash table. This is used for OFFPAGE zones to lookup