Page MenuHomeFreeBSD
Files F150527425

D22470.id64651.diff
No OneTemporary
Actions

D22470.id64651.diff
View Options

Index: sys/vm/uma_core.c
===================================================================
--- sys/vm/uma_core.c
+++ sys/vm/uma_core.c
@@ -273,7 +273,7 @@
static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int);
static void bucket_free(uma_zone_t zone, uma_bucket_t, void *);
static void bucket_zone_drain(void);
-static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int, int);
+static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int);
static uma_slab_t zone_fetch_slab(uma_zone_t, uma_keg_t, int, int);
static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item);
@@ -282,6 +282,7 @@
static int zone_import(uma_zone_t, void **, int, int, int);
static void zone_release(uma_zone_t, void **, int);
static void uma_zero_item(void *, uma_zone_t);
+static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int);
void uma_print_zone(uma_zone_t);
void uma_print_stats(void);
@@ -2418,18 +2419,58 @@
uma_zfree_arg(zone, item, udata);
}
+static inline void *
+bucket_pop(uma_zone_t zone, uma_cache_t cache, uma_bucket_t bucket)
+{
+ void *item;
+
+ bucket->ub_cnt--;
+ item = bucket->ub_bucket[bucket->ub_cnt];
+#ifdef INVARIANTS
+ bucket->ub_bucket[bucket->ub_cnt] = NULL;
+ KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled."));
+#endif
+ cache->uc_allocs++;
+
+ return (item);
+}
+
+static void *
+item_ctor(uma_zone_t zone, void *udata, int flags, void *item)
+{
+#ifdef INVARIANTS
+ int skipdbg;
+
+ skipdbg = uma_dbg_zskip(zone, item);
+ if (zone->uz_ctor != NULL &&
+ (!skipdbg || zone->uz_ctor != trash_ctor ||
+ zone->uz_dtor != trash_dtor) &&
+#else
+ if (__predict_false(zone->uz_ctor != NULL) &&
+#endif
+ zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
+ counter_u64_add(zone->uz_fails, 1);
+ zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT);
+ return (NULL);
+ }
+#ifdef INVARIANTS
+ if (!skipdbg)
+ uma_dbg_alloc(zone, NULL, item);
+#endif
+ if (flags & M_ZERO)
+ uma_zero_item(item, zone);
+
+ return (item);
+}
+
/* See uma.h */
void *
uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
{
- uma_zone_domain_t zdom;
uma_bucket_t bucket;
uma_cache_t cache;
void *item;
- int cpu, domain, lockfail, maxbucket;
-#ifdef INVARIANTS
- bool skipdbg;
-#endif
+ int cpu, domain;
/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */
random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA);
@@ -2478,56 +2519,45 @@
* the current cache; when we re-acquire the critical section, we
* must detect and handle migration if it has occurred.
*/
-zalloc_restart:
critical_enter();
- cpu = curcpu;
- cache = &zone->uz_cpu[cpu];
-
-zalloc_start:
- bucket = cache->uc_allocbucket;
- if (bucket != NULL && bucket->ub_cnt > 0) {
- bucket->ub_cnt--;
- item = bucket->ub_bucket[bucket->ub_cnt];
-#ifdef INVARIANTS
- bucket->ub_bucket[bucket->ub_cnt] = NULL;
-#endif
- KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled."));
- cache->uc_allocs++;
- critical_exit();
-#ifdef INVARIANTS
- skipdbg = uma_dbg_zskip(zone, item);
-#endif
- if (zone->uz_ctor != NULL &&
-#ifdef INVARIANTS
- (!skipdbg || zone->uz_ctor != trash_ctor ||
- zone->uz_dtor != trash_dtor) &&
-#endif
- zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
- counter_u64_add(zone->uz_fails, 1);
- zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT);
- return (NULL);
+ do {
+ cpu = curcpu;
+ cache = &zone->uz_cpu[cpu];
+ bucket = cache->uc_allocbucket;
+ if (__predict_true(bucket != NULL && bucket->ub_cnt != 0)) {
+ item = bucket_pop(zone, cache, bucket);
+ critical_exit();
+ return (item_ctor(zone, udata, flags, item));
}
-#ifdef INVARIANTS
- if (!skipdbg)
- uma_dbg_alloc(zone, NULL, item);
-#endif
- if (flags & M_ZERO)
- uma_zero_item(item, zone);
- return (item);
- }
+ } while (cache_alloc(zone, cache, udata, flags));
+ critical_exit();
/*
- * We have run out of items in our alloc bucket.
- * See if we can switch with our free bucket.
+ * We can not get a bucket so try to return a single item.
+ */
+ if (zone->uz_flags & UMA_ZONE_NUMA)
+ domain = PCPU_GET(domain);
+ else
+ domain = UMA_ANYDOMAIN;
+ return zone_alloc_item_locked(zone, udata, domain, flags);
+}
+
+static __noinline bool
+cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags)
+{
+ uma_zone_domain_t zdom;
+ uma_bucket_t bucket;
+ int cpu, domain, lockfail;
+
+ /*
+ * If we have run out of items in our alloc bucket see
+ * if we can switch with the free bucket.
*/
bucket = cache->uc_freebucket;
- if (bucket != NULL && bucket->ub_cnt > 0) {
- CTR2(KTR_UMA,
- "uma_zalloc: zone %s(%p) swapping empty with alloc",
- zone->uz_name, zone);
+ if (bucket != NULL && bucket->ub_cnt != 0) {
cache->uc_freebucket = cache->uc_allocbucket;
cache->uc_allocbucket = bucket;
- goto zalloc_start;
+ return (true);
}
/*
@@ -2539,16 +2569,6 @@
if (bucket != NULL)
bucket_free(zone, bucket, udata);
- /* Short-circuit for zones without buckets and low memory. */
- if (zone->uz_count == 0 || bucketdisable) {
- ZONE_LOCK(zone);
- if (zone->uz_flags & UMA_ZONE_NUMA)
- domain = PCPU_GET(domain);
- else
- domain = UMA_ANYDOMAIN;
- goto zalloc_item;
- }
-
/*
* Attempt to retrieve the item from the per-CPU cache has failed, so
* we must go back to the zone. This requires the zone lock, so we
@@ -2564,14 +2584,19 @@
ZONE_LOCK(zone);
lockfail = 1;
}
+
critical_enter();
+ /* Short-circuit for zones without buckets and low memory. */
+ if (zone->uz_count == 0 || bucketdisable)
+ return (false);
+
cpu = curcpu;
cache = &zone->uz_cpu[cpu];
/* See if we lost the race to fill the cache. */
if (cache->uc_allocbucket != NULL) {
ZONE_UNLOCK(zone);
- goto zalloc_start;
+ return (true);
}
/*
@@ -2586,11 +2611,11 @@
}
if ((bucket = zone_fetch_bucket(zone, zdom)) != NULL) {
+ ZONE_UNLOCK(zone);
KASSERT(bucket->ub_cnt != 0,
("uma_zalloc_arg: Returning an empty bucket."));
cache->uc_allocbucket = bucket;
- ZONE_UNLOCK(zone);
- goto zalloc_start;
+ return (true);
}
/* We are no longer associated with this CPU. */
critical_exit();
@@ -2602,34 +2627,15 @@
if (lockfail && zone->uz_count < zone->uz_count_max)
zone->uz_count++;
- if (zone->uz_max_items > 0) {
- if (zone->uz_items >= zone->uz_max_items)
- goto zalloc_item;
- maxbucket = MIN(zone->uz_count,
- zone->uz_max_items - zone->uz_items);
- zone->uz_items += maxbucket;
- } else
- maxbucket = zone->uz_count;
- ZONE_UNLOCK(zone);
-
/*
- * Now lets just fill a bucket and put it on the free list. If that
- * works we'll restart the allocation from the beginning and it
- * will use the just filled bucket.
+ * Now lets just fill a bucket and attempt to use it as the alloc
+ * bucket.
*/
- bucket = zone_alloc_bucket(zone, udata, domain, flags, maxbucket);
+ bucket = zone_alloc_bucket(zone, udata, domain, flags);
CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p",
zone->uz_name, zone, bucket);
- ZONE_LOCK(zone);
+ critical_enter();
if (bucket != NULL) {
- if (zone->uz_max_items > 0 && bucket->ub_cnt < maxbucket) {
- MPASS(zone->uz_items >= maxbucket - bucket->ub_cnt);
- zone->uz_items -= maxbucket - bucket->ub_cnt;
- if (zone->uz_sleepers > 0 &&
- zone->uz_items < zone->uz_max_items)
- wakeup_one(zone);
- }
- critical_enter();
cpu = curcpu;
cache = &zone->uz_cpu[cpu];
@@ -2648,25 +2654,14 @@
ZONE_UNLOCK(zone);
bucket_drain(zone, bucket);
bucket_free(zone, bucket, udata);
- goto zalloc_restart;
+ critical_enter();
+ return (true);
} else
zone_put_bucket(zone, zdom, bucket, false);
ZONE_UNLOCK(zone);
- goto zalloc_start;
- } else if (zone->uz_max_items > 0) {
- zone->uz_items -= maxbucket;
- if (zone->uz_sleepers > 0 &&
- zone->uz_items + 1 < zone->uz_max_items)
- wakeup_one(zone);
+ return (true);
}
-
- /*
- * We may not be able to get a bucket so return an actual item.
- */
-zalloc_item:
- item = zone_alloc_item_locked(zone, udata, domain, flags);
-
- return (item);
+ return (false);
}
void *
@@ -2920,9 +2915,10 @@
}
static uma_bucket_t
-zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags, int max)
+zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags)
{
uma_bucket_t bucket;
+ int maxbucket, cnt;
CTR1(KTR_UMA, "zone_alloc:_bucket domain %d)", domain);
@@ -2930,13 +2926,25 @@
if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain))
domain = UMA_ANYDOMAIN;
+ if (zone->uz_max_items > 0) {
+ if (zone->uz_items >= zone->uz_max_items)
+ return (false);
+ maxbucket = MIN(zone->uz_count,
+ zone->uz_max_items - zone->uz_items);
+ zone->uz_items += maxbucket;
+ } else
+ maxbucket = zone->uz_count;
+ ZONE_UNLOCK(zone);
+
/* Don't wait for buckets, preserve caller's NOVM setting. */
bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM));
- if (bucket == NULL)
- return (NULL);
+ if (bucket == NULL) {
+ cnt = 0;
+ goto out;
+ }
bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
- MIN(max, bucket->ub_entries), domain, flags);
+ MIN(maxbucket, bucket->ub_entries), domain, flags);
/*
* Initialize the memory if necessary.
@@ -2963,10 +2971,21 @@
}
}
+ cnt = bucket->ub_cnt;
if (bucket->ub_cnt == 0) {
bucket_free(zone, bucket, udata);
counter_u64_add(zone->uz_fails, 1);
- return (NULL);
+ bucket = NULL;
+ }
+out:
+ ZONE_LOCK(zone);
+ if (zone->uz_max_items > 0 && cnt < maxbucket) {
+ MPASS(zone->uz_items >= maxbucket - cnt);
+ zone->uz_items -= maxbucket - cnt;
+ if (zone->uz_sleepers > 0 &&
+ (cnt == 0 ? zone->uz_items + 1 : zone->uz_items) <
+ zone->uz_max_items)
+ wakeup_one(zone);
}
return (bucket);
@@ -3001,9 +3020,6 @@
zone_alloc_item_locked(uma_zone_t zone, void *udata, int domain, int flags)
{
void *item;
-#ifdef INVARIANTS
- bool skipdbg;
-#endif
ZONE_LOCK_ASSERT(zone);
@@ -3036,9 +3052,6 @@
if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1)
goto fail;
-#ifdef INVARIANTS
- skipdbg = uma_dbg_zskip(zone, item);
-#endif
/*
* We have to call both the zone's init (not the keg's init)
* and the zone's ctor. This is because the item is going from
@@ -3051,21 +3064,9 @@
goto fail;
}
}
- if (zone->uz_ctor != NULL &&
-#ifdef INVARIANTS
- (!skipdbg || zone->uz_ctor != trash_ctor ||
- zone->uz_dtor != trash_dtor) &&
-#endif
- zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
- zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT);
+ item = item_ctor(zone, udata, flags, item);
+ if (item == NULL)
goto fail;
- }
-#ifdef INVARIANTS
- if (!skipdbg)
- uma_dbg_alloc(zone, NULL, item);
-#endif
- if (flags & M_ZERO)
- uma_zero_item(item, zone);
counter_u64_add(zone->uz_allocs, 1);
CTR3(KTR_UMA, "zone_alloc_item item %p from %s(%p)", item,
@@ -3076,6 +3077,7 @@
fail:
if (zone->uz_max_items > 0) {
ZONE_LOCK(zone);
+ /* XXX Decrement without wakeup */
zone->uz_items--;
ZONE_UNLOCK(zone);
}

File Metadata

Mime Type
text/plain
Expires
Fri, Apr 3, 1:26 AM (1 h, 33 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
30702549
Default Alt Text
D22470.id64651.diff (10 KB)

Event Timeline