Page MenuHomeFreeBSD
Files F153579034

D22831.id65699.diff
No OneTemporary
Actions

D22831.id65699.diff
View Options

Index: sys/conf/options
===================================================================
--- sys/conf/options
+++ sys/conf/options
@@ -621,8 +621,6 @@
MALLOC_MAKE_FAILURES opt_vm.h
MALLOC_PROFILE opt_vm.h
MALLOC_DEBUG_MAXZONES opt_vm.h
-UMA_XDOMAIN opt_vm.h
-UMA_FIRSTTOUCH opt_vm.h
# The MemGuard replacement allocator used for tamper-after-free detection
DEBUG_MEMGUARD opt_vm.h
Index: sys/vm/uma.h
===================================================================
--- sys/vm/uma.h
+++ sys/vm/uma.h
@@ -268,11 +268,9 @@
#define UMA_ZONE_PCPU 0x8000 /*
* Allocates mp_maxid + 1 slabs of PAGE_SIZE
*/
-#define UMA_ZONE_NUMA 0x10000 /*
- * NUMA aware Zone. Implements a best
- * effort first-touch policy.
- */
-#define UMA_ZONE_MINBUCKET 0x20000 /* Use smallest buckets. */
+#define UMA_ZONE_MINBUCKET 0x10000 /* Use smallest buckets. */
+#define UMA_ZONE_FIRSTTOUCH 0x20000 /* First touch NUMA policy */
+#define UMA_ZONE_ROUNDROBIN 0x40000 /* Round-robin NUMA policy. */
/*
* These flags are shared between the keg and zone. In zones wishing to add
@@ -281,7 +279,8 @@
*/
#define UMA_ZONE_INHERIT \
(UMA_ZONE_OFFPAGE | UMA_ZONE_MALLOC | UMA_ZONE_NOFREE | \
- UMA_ZONE_HASH | UMA_ZONE_VTOSLAB | UMA_ZONE_PCPU | UMA_ZONE_NUMA)
+ UMA_ZONE_HASH | UMA_ZONE_VTOSLAB | UMA_ZONE_PCPU | \
+ UMA_ZONE_FIRSTTOUCH | UMA_ZONE_ROUNDROBIN)
/* Definitions for align */
#define UMA_ALIGN_PTR (sizeof(void *) - 1) /* Alignment fit for ptr */
Index: sys/vm/uma_core.c
===================================================================
--- sys/vm/uma_core.c
+++ sys/vm/uma_core.c
@@ -360,7 +360,8 @@
size += sizeof(void *) * ubz->ubz_entries;
ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
- UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET | UMA_ZONE_NUMA);
+ UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET |
+ UMA_ZONE_FIRSTTOUCH);
}
}
@@ -387,11 +388,9 @@
int bpcpu;
bpcpu = 2;
-#ifdef UMA_XDOMAIN
- if ((zone->uz_flags & UMA_ZONE_NUMA) != 0)
+ if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0)
/* Count the cross-domain bucket. */
bpcpu++;
-#endif
for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
if (ubz->ubz_entries * bpcpu * mp_ncpus > nitems)
@@ -631,7 +630,7 @@
cache_bucket_store(&cache->uc_freebucket, b);
}
-#ifdef UMA_XDOMAIN
+#ifdef NUMA
static inline void
cache_bucket_store_cross(uma_cache_t cache, uma_bucket_t b)
{
@@ -997,7 +996,7 @@
b1 = b2 = b3 = NULL;
ZONE_LOCK(zone);
critical_enter();
- if (zone->uz_flags & UMA_ZONE_NUMA)
+ if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH)
domain = PCPU_GET(domain);
else
domain = 0;
@@ -1193,7 +1192,7 @@
dom->ud_pages -= n * keg->uk_ppera;
dom->ud_free -= n * keg->uk_ipers;
KEG_UNLOCK(keg, i);
- if ((keg->uk_flags & UMA_ZONE_NUMA) == 0)
+ if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0)
break;
}
@@ -1328,7 +1327,7 @@
slab->us_flags = sflags;
/* For non-numa zones all pages go to the same uma_domain. */
- if ((keg->uk_flags & UMA_ZONE_NUMA) != 0)
+ if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) != 0)
slab->us_domain = domain;
else
slab->us_domain = 0;
@@ -1893,8 +1892,8 @@
/*
* We use a global round-robin policy by default. Zones with
- * UMA_ZONE_NUMA set will use first-touch instead, in which case the
- * iterator is never run.
+ * UMA_ZONE_FIRSTTOUCH set will use first-touch instead, in which
+ * case the iterator is never run.
*/
keg->uk_dr.dr_policy = DOMAINSET_RR();
keg->uk_dr.dr_iter = 0;
@@ -1931,12 +1930,18 @@
}
/*
- * Sets all kegs with memory that comes from the page array to a
- * first-touch domain policy.
+ * Use a first-touch NUMA policy for all kegs that pmap_extract()
+ * will work on with the exception of critical VM structures
+ * necessary for paging.
+ *
+ * Zones may override the default by specifying either.
*/
-#ifdef UMA_FIRSTTOUCH
- if ((keg->uk_flags & UMA_ZONE_HASH) == 0)
- keg->uk_flags |= UMA_ZONE_NUMA;
+#ifdef NUMA
+ if ((keg->uk_flags &
+ (UMA_ZONE_HASH | UMA_ZONE_VM | UMA_ZONE_ROUNDROBIN)) == 0)
+ keg->uk_flags |= UMA_ZONE_FIRSTTOUCH;
+ else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0)
+ keg->uk_flags |= UMA_ZONE_ROUNDROBIN;
#endif
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
@@ -2068,7 +2073,7 @@
/*
* keg if present.
*/
- if ((zone->uz_flags & UMA_ZONE_NUMA) != 0)
+ if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0)
domains = vm_ndomains;
else
domains = 1;
@@ -2979,7 +2984,7 @@
/*
* We can not get a bucket so try to return a single item.
*/
- if (uz_flags & UMA_ZONE_NUMA)
+ if (uz_flags & UMA_ZONE_FIRSTTOUCH)
domain = PCPU_GET(domain);
else
domain = UMA_ANYDOMAIN;
@@ -3053,7 +3058,7 @@
/*
* Check the zone's cache of buckets.
*/
- if (zone->uz_flags & UMA_ZONE_NUMA) {
+ if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH) {
domain = PCPU_GET(domain);
zdom = &zone->uz_domain[domain];
} else {
@@ -3099,7 +3104,7 @@
critical_enter();
cache = &zone->uz_cpu[curcpu];
if (cache->uc_allocbucket.ucb_bucket == NULL &&
- ((zone->uz_flags & UMA_ZONE_NUMA) == 0 ||
+ ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) == 0 ||
domain == PCPU_GET(domain))) {
cache_bucket_store_alloc(cache, bucket);
zdom->uzd_imax += bucket->ub_cnt;
@@ -3186,7 +3191,7 @@
uint32_t reserve;
/* !NUMA has a single free list. */
- if ((keg->uk_flags & UMA_ZONE_NUMA) == 0)
+ if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0)
domain = 0;
KEG_LOCK(keg, domain);
@@ -3325,7 +3330,7 @@
* produces more fragmentation and requires more cpu
* time but yields better distribution.
*/
- if ((zone->uz_flags & UMA_ZONE_NUMA) == 0 &&
+ if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0 &&
vm_ndomains > 1 && --stripe == 0)
break;
#endif
@@ -3678,20 +3683,24 @@
do {
cache = &zone->uz_cpu[curcpu];
bucket = &cache->uc_allocbucket;
-#ifdef UMA_XDOMAIN
- if ((uz_flags & UMA_ZONE_NUMA) != 0) {
- itemdomain = _vm_phys_domain(pmap_kextract((vm_offset_t)item));
+#ifdef NUMA
+ if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) {
domain = PCPU_GET(domain);
- }
- if ((uz_flags & UMA_ZONE_NUMA) != 0 && domain != itemdomain) {
- bucket = &cache->uc_crossbucket;
+ itemdomain =
+ _vm_phys_domain(pmap_kextract((vm_offset_t)item));
+ if (domain != itemdomain)
+ bucket = &cache->uc_crossbucket;
+ else if (__predict_false(bucket->ucb_cnt >=
+ bucket->ucb_entries))
+ bucket = &cache->uc_freebucket;
} else
#endif
/*
- * Try to free into the allocbucket first to give LIFO ordering
- * for cache-hot datastructures. Spill over into the freebucket
- * if necessary. Alloc will swap them if one runs dry.
+ * Try to free into the allocbucket first to give LIFO
+ * ordering for cache-hot datastructures. Spill over
+ * into the freebucket if necessary. Alloc will swap
+ * them if one runs dry.
*/
if (__predict_false(bucket->ucb_cnt >= bucket->ucb_entries))
bucket = &cache->uc_freebucket;
@@ -3710,7 +3719,7 @@
zone_free_item(zone, item, udata, SKIP_DTOR);
}
-#ifdef UMA_XDOMAIN
+#ifdef NUMA
/*
* sort crossdomain free buckets to domain correct buckets and cache
* them.
@@ -3778,7 +3787,7 @@
{
uma_zone_domain_t zdom;
-#ifdef UMA_XDOMAIN
+#ifdef NUMA
/*
* Buckets coming from the wrong domain will be entirely for the
* only other domain on two domain systems. In this case we can
@@ -3844,27 +3853,26 @@
cache = &zone->uz_cpu[curcpu];
/*
- * NUMA domains need to free to the correct zdom. When XDOMAIN
- * is enabled this is the zdom of the item and the bucket may be
- * the cross bucket if they do not match.
+ * FIRSTTOUCH domains need to free to the correct zdom. When
+ * enabled this is the zdom of the item and the bucket
+ * is the cross bucket if they do not match.
*/
- if ((zone->uz_flags & UMA_ZONE_NUMA) != 0)
-#ifdef UMA_XDOMAIN
+#ifdef NUMA
+ if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) {
domain = PCPU_GET(domain);
-#else
- itemdomain = domain = PCPU_GET(domain);
-#endif
- else
- itemdomain = domain = 0;
-#ifdef UMA_XDOMAIN
- if (domain != itemdomain) {
- bucket = cache_bucket_load_cross(cache);
- if (bucket != NULL)
- atomic_add_64(&zone->uz_xdomain, bucket->ub_cnt);
+ if (domain != itemdomain) {
+ bucket = cache_bucket_load_cross(cache);
+ if (bucket != NULL)
+ atomic_add_64(&zone->uz_xdomain,
+ bucket->ub_cnt);
+ } else
+ bucket = cache_bucket_load_free(cache);
} else
#endif
+ {
+ itemdomain = domain = 0;
bucket = cache_bucket_load_free(cache);
-
+ }
/* We are no longer associated with this CPU. */
critical_exit();
@@ -3879,13 +3887,13 @@
if (bucket == NULL)
return (false);
cache = &zone->uz_cpu[curcpu];
-#ifdef UMA_XDOMAIN
+#ifdef NUMA
/*
* Check to see if we should be populating the cross bucket. If it
* is already populated we will fall through and attempt to populate
* the free bucket.
*/
- if ((zone->uz_flags & UMA_ZONE_NUMA) != 0) {
+ if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) {
domain = PCPU_GET(domain);
if (domain != itemdomain &&
cache->uc_crossbucket.ucb_bucket == NULL) {
@@ -4057,11 +4065,9 @@
ubz = bucket_zone_max(zone, nitems);
if (ubz != NULL) {
bpcpu = 2;
-#ifdef UMA_XDOMAIN
- if ((zone->uz_flags & UMA_ZONE_NUMA) != 0)
+ if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0)
/* Count the cross-domain bucket. */
bpcpu++;
-#endif
nitems -= ubz->ubz_entries * bpcpu * mp_ncpus;
zone->uz_bucket_size_max = ubz->ubz_entries;
} else {
Index: sys/vm/uma_int.h
===================================================================
--- sys/vm/uma_int.h
+++ sys/vm/uma_int.h
@@ -496,8 +496,9 @@
"\35BUCKET" \
"\34RECLAIMING" \
"\33CACHE" \
- "\22MINBUCKET" \
- "\21NUMA" \
+ "\23ROUNDROBIN" \
+ "\22FIRSTTOUCH" \
+ "\21MINBUCKET" \
"\20PCPU" \
"\17NODUMP" \
"\16VTOSLAB" \
Index: sys/vm/vm_glue.c
===================================================================
--- sys/vm/vm_glue.c
+++ sys/vm/vm_glue.c
@@ -474,7 +474,7 @@
kstack_cache = uma_zcache_create("kstack_cache",
kstack_pages * PAGE_SIZE, NULL, NULL, NULL, NULL,
kstack_import, kstack_release, NULL,
- UMA_ZONE_NUMA);
+ UMA_ZONE_FIRSTTOUCH);
kstack_cache_size = imax(128, mp_ncpus * 4);
uma_zone_set_maxcache(kstack_cache, kstack_cache_size);
}

File Metadata

Mime Type
text/plain
Expires
Thu, Apr 23, 2:10 AM (7 h, 23 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
32007334
Default Alt Text
D22831.id65699.diff (10 KB)

Event Timeline