Page MenuHomeFreeBSD
Files F106132780

D17420.diff
No OneTemporary
Actions

D17420.diff
View Options

Index: head/sys/vm/uma_core.c
===================================================================
--- head/sys/vm/uma_core.c
+++ head/sys/vm/uma_core.c
@@ -59,6 +59,7 @@
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bitset.h>
+#include <sys/domainset.h>
#include <sys/eventhandler.h>
#include <sys/kernel.h>
#include <sys/types.h>
@@ -79,6 +80,7 @@
#include <sys/vmmeter.h>
#include <vm/vm.h>
+#include <vm/vm_domainset.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
@@ -991,6 +993,8 @@
/*
* Allocate a new slab for a keg. This does not insert the slab onto a list.
+ * If the allocation was successful, the keg lock will be held upon return,
+ * otherwise the keg will be left unlocked.
*
* Arguments:
* wait Shall we wait?
@@ -1012,13 +1016,12 @@
KASSERT(domain >= 0 && domain < vm_ndomains,
("keg_alloc_slab: domain %d out of range", domain));
mtx_assert(&keg->uk_lock, MA_OWNED);
- slab = NULL;
- mem = NULL;
allocf = keg->uk_allocf;
KEG_UNLOCK(keg);
- size = keg->uk_ppera * PAGE_SIZE;
+ slab = NULL;
+ mem = NULL;
if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
slab = zone_alloc_item(keg->uk_slabzone, NULL, domain, wait);
if (slab == NULL)
@@ -1041,6 +1044,7 @@
wait |= M_NODUMP;
/* zone is passed for legacy reasons. */
+ size = keg->uk_ppera * PAGE_SIZE;
mem = allocf(zone, size, domain, &flags, wait);
if (mem == NULL) {
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
@@ -1079,20 +1083,18 @@
goto out;
}
}
-out:
KEG_LOCK(keg);
CTR3(KTR_UMA, "keg_alloc_slab: allocated slab %p for %s(%p)",
slab, keg->uk_name, keg);
- if (slab != NULL) {
- if (keg->uk_flags & UMA_ZONE_HASH)
- UMA_HASH_INSERT(&keg->uk_hash, slab, mem);
+ if (keg->uk_flags & UMA_ZONE_HASH)
+ UMA_HASH_INSERT(&keg->uk_hash, slab, mem);
- keg->uk_pages += keg->uk_ppera;
- keg->uk_free += keg->uk_ipers;
- }
+ keg->uk_pages += keg->uk_ppera;
+ keg->uk_free += keg->uk_ipers;
+out:
return (slab);
}
@@ -1559,7 +1561,6 @@
keg->uk_init = arg->uminit;
keg->uk_fini = arg->fini;
keg->uk_align = arg->align;
- keg->uk_cursor = 0;
keg->uk_free = 0;
keg->uk_reserve = 0;
keg->uk_pages = 0;
@@ -1567,6 +1568,14 @@
keg->uk_slabzone = NULL;
/*
+ * We use a global round-robin policy by default. Zones with
+ * UMA_ZONE_NUMA set will use first-touch instead, in which case the
+ * iterator is never run.
+ */
+ keg->uk_dr.dr_policy = DOMAINSET_RR();
+ keg->uk_dr.dr_iter = 0;
+
+ /*
* The master zone is passed to us at keg-creation time.
*/
zone = arg->zone;
@@ -2607,7 +2616,7 @@
* only 'domain'.
*/
static uma_slab_t
-keg_first_slab(uma_keg_t keg, int domain, int rr)
+keg_first_slab(uma_keg_t keg, int domain, bool rr)
{
uma_domain_t dom;
uma_slab_t slab;
@@ -2636,43 +2645,51 @@
}
static uma_slab_t
-keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, int flags)
+keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags)
{
+ uint32_t reserve;
+
+ mtx_assert(&keg->uk_lock, MA_OWNED);
+
+ reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve;
+ if (keg->uk_free <= reserve)
+ return (NULL);
+ return (keg_first_slab(keg, domain, rr));
+}
+
+static uma_slab_t
+keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags)
+{
+ struct vm_domainset_iter di;
uma_domain_t dom;
uma_slab_t slab;
- int allocflags, domain, reserve, rr, start;
+ int aflags, domain;
+ bool rr;
+restart:
mtx_assert(&keg->uk_lock, MA_OWNED);
- slab = NULL;
- reserve = 0;
- allocflags = flags;
- if ((flags & M_USE_RESERVE) == 0)
- reserve = keg->uk_reserve;
/*
- * Round-robin for non first-touch zones when there is more than one
- * domain.
+ * Use the keg's policy if upper layers haven't already specified a
+ * domain (as happens with first-touch zones).
+ *
+ * To avoid races we run the iterator with the keg lock held, but that
+ * means that we cannot allow the vm_domainset layer to sleep. Thus,
+ * clear M_WAITOK and handle low memory conditions locally.
*/
- if (vm_ndomains == 1)
- rdomain = 0;
rr = rdomain == UMA_ANYDOMAIN;
if (rr) {
- start = keg->uk_cursor;
- do {
- keg->uk_cursor = (keg->uk_cursor + 1) % vm_ndomains;
- domain = keg->uk_cursor;
- } while (VM_DOMAIN_EMPTY(domain) && domain != start);
- domain = start = keg->uk_cursor;
- /* Only block on the second pass. */
- if ((flags & (M_WAITOK | M_NOVM)) == M_WAITOK)
- allocflags = (allocflags & ~M_WAITOK) | M_NOWAIT;
- } else
- domain = start = rdomain;
+ aflags = (flags & ~M_WAITOK) | M_NOWAIT;
+ vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain,
+ &aflags);
+ } else {
+ aflags = flags;
+ domain = rdomain;
+ }
-again:
- do {
- if (keg->uk_free > reserve &&
- (slab = keg_first_slab(keg, domain, rr)) != NULL) {
+ for (;;) {
+ slab = keg_fetch_free_slab(keg, domain, rr, flags);
+ if (slab != NULL) {
MPASS(slab->us_keg == keg);
return (slab);
}
@@ -2700,7 +2717,7 @@
msleep(keg, &keg->uk_lock, PVM, "keglimit", 0);
continue;
}
- slab = keg_alloc_slab(keg, zone, domain, allocflags);
+ slab = keg_alloc_slab(keg, zone, domain, aflags);
/*
* If we got a slab here it's safe to mark it partially used
* and return. We assume that the caller is going to remove
@@ -2712,17 +2729,16 @@
LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link);
return (slab);
}
- if (rr) {
- do {
- domain = (domain + 1) % vm_ndomains;
- } while (VM_DOMAIN_EMPTY(domain) && domain != start);
+ KEG_LOCK(keg);
+ if (rr && vm_domainset_iter_policy(&di, &domain) != 0) {
+ if ((flags & M_WAITOK) != 0) {
+ KEG_UNLOCK(keg);
+ vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask);
+ KEG_LOCK(keg);
+ goto restart;
+ }
+ break;
}
- } while (domain != start);
-
- /* Retry domain scan with blocking. */
- if (allocflags != flags) {
- allocflags = flags;
- goto again;
}
/*
@@ -2730,8 +2746,7 @@
* could have while we were unlocked. Check again before we
* fail.
*/
- if (keg->uk_free > reserve &&
- (slab = keg_first_slab(keg, domain, rr)) != NULL) {
+ if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) {
MPASS(slab->us_keg == keg);
return (slab);
}
@@ -3606,14 +3621,13 @@
domain = 0;
if (slabs * keg->uk_ipers < items)
slabs++;
- while (slabs > 0) {
+ while (slabs-- > 0) {
slab = keg_alloc_slab(keg, zone, domain, M_WAITOK);
if (slab == NULL)
- break;
+ return;
MPASS(slab->us_keg == keg);
dom = &keg->uk_domain[slab->us_domain];
LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link);
- slabs--;
do {
domain = (domain + 1) % vm_ndomains;
} while (VM_DOMAIN_EMPTY(domain));
Index: head/sys/vm/uma_int.h
===================================================================
--- head/sys/vm/uma_int.h
+++ head/sys/vm/uma_int.h
@@ -226,7 +226,7 @@
struct uma_hash uk_hash;
LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */
- uint32_t uk_cursor; /* Domain alloc cursor. */
+ struct domainset_ref uk_dr; /* Domain selection policy. */
uint32_t uk_align; /* Alignment mask */
uint32_t uk_pages; /* Total page count */
uint32_t uk_free; /* Count of items free in slabs */

File Metadata

Mime Type
text/plain
Expires
Thu, Dec 26, 10:51 PM (11 h, 17 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
15608633
Default Alt Text
D17420.diff (7 KB)

Event Timeline