Index: head/sys/sys/_domainset.h
===================================================================
--- head/sys/sys/_domainset.h
+++ head/sys/sys/_domainset.h
@@ -54,7 +54,7 @@
 struct domainset;
 struct domainset_ref {
 	struct domainset * volatile	dr_policy;
-	unsigned int			dr_iterator;
+	unsigned int			dr_iter;
 };
 
 #endif /* !_SYS__DOMAINSET_H_ */
Index: head/sys/vm/vm_domainset.h
===================================================================
--- head/sys/vm/vm_domainset.h
+++ head/sys/vm/vm_domainset.h
@@ -40,12 +40,15 @@
 	bool			di_minskip;
 };
 
-int	vm_domainset_iter_page(struct vm_domainset_iter *, int *, int *);
+int	vm_domainset_iter_page(struct vm_domainset_iter *, struct vm_object *,
+	    int *);
 void	vm_domainset_iter_page_init(struct vm_domainset_iter *,
 	    struct vm_object *, vm_pindex_t, int *, int *);
-int	vm_domainset_iter_malloc(struct vm_domainset_iter *, int *, int *);
-void	vm_domainset_iter_malloc_init(struct vm_domainset_iter *,
-	    struct vm_object *, int *, int *);
+int	vm_domainset_iter_policy(struct vm_domainset_iter *, int *);
+void	vm_domainset_iter_policy_init(struct vm_domainset_iter *,
+	    struct domainset *, int *, int *);
+void	vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *,
+	    struct domainset_ref *, int *, int *);
 
 void	vm_wait_doms(const domainset_t *);
 
Index: head/sys/vm/vm_domainset.c
===================================================================
--- head/sys/vm/vm_domainset.c
+++ head/sys/vm/vm_domainset.c
@@ -40,6 +40,7 @@
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/malloc.h>
+#include <sys/rwlock.h>
 #include <sys/vmmeter.h>
 
 #include <vm/vm.h>
@@ -62,26 +63,13 @@
  * Determine which policy is to be used for this allocation.
  */
 static void
-vm_domainset_iter_init(struct vm_domainset_iter *di, struct vm_object *obj,
-    vm_pindex_t pindex)
+vm_domainset_iter_init(struct vm_domainset_iter *di, struct domainset *ds,
+    int *iter, struct vm_object *obj, vm_pindex_t pindex)
 {
-	struct domainset *domain;
-	struct thread *td;
 
-	/*
-	 * object policy takes precedence over thread policy.  The policies
-	 * are immutable and unsynchronized.  Updates can race but pointer
-	 * loads are assumed to be atomic.
-	 */
-	if (obj != NULL && (domain = obj->domain.dr_policy) != NULL) {
-		di->di_domain = domain;
-		di->di_iter = &obj->domain.dr_iterator;
-	} else {
-		td = curthread;
-		di->di_domain = td->td_domain.dr_policy;
-		di->di_iter = &td->td_domain.dr_iterator;
-	}
-	di->di_policy = di->di_domain->ds_policy;
+	di->di_domain = ds;
+	di->di_iter = iter;
+	di->di_policy = ds->ds_policy;
 	if (di->di_policy == DOMAINSET_POLICY_INTERLEAVE) {
 #if VM_NRESERVLEVEL > 0
 		if (vm_object_reserv(obj)) {
@@ -211,33 +199,39 @@
 vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
     vm_pindex_t pindex, int *domain, int *req)
 {
+	struct domainset_ref *dr;
 
-	vm_domainset_iter_init(di, obj, pindex);
+	/*
+	 * Object policy takes precedence over thread policy.  The policies
+	 * are immutable and unsynchronized.  Updates can race but pointer
+	 * loads are assumed to be atomic.
+	 */
+	if (obj != NULL && obj->domain.dr_policy != NULL)
+		dr = &obj->domain;
+	else
+		dr = &curthread->td_domain;
+	vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, obj, pindex);
 	di->di_flags = *req;
 	*req = (di->di_flags & ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) |
 	    VM_ALLOC_NOWAIT;
 	vm_domainset_iter_first(di, domain);
 	if (vm_page_count_min_domain(*domain))
-		vm_domainset_iter_page(di, domain, req);
+		vm_domainset_iter_page(di, obj, domain);
 }
 
 int
-vm_domainset_iter_page(struct vm_domainset_iter *di, int *domain, int *req)
+vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
+    int *domain)
 {
 
-	/*
-	 * If we exhausted all options with NOWAIT and did a WAITFAIL it
-	 * is time to return an error to the caller.
-	 */
-	if ((*req & VM_ALLOC_WAITFAIL) != 0)
-		return (ENOMEM);
-
 	/* If there are more domains to visit we run the iterator. */
 	while (--di->di_n != 0) {
 		vm_domainset_iter_next(di, domain);
 		if (!di->di_minskip || !vm_page_count_min_domain(*domain))
 			return (0);
 	}
+
+	/* If we skipped domains below min restart the search. */
 	if (di->di_minskip) {
 		di->di_minskip = false;
 		vm_domainset_iter_first(di, domain);
@@ -248,34 +242,53 @@
 	if ((di->di_flags & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) == 0)
 		return (ENOMEM);
 
-	/*
-	 * We have visited all domains with non-blocking allocations, try
-	 * from the beginning with a blocking allocation.
-	 */
+	/* Wait for one of the domains to accumulate some free pages. */
+	if (obj != NULL)
+		VM_OBJECT_WUNLOCK(obj);
+	vm_wait_doms(&di->di_domain->ds_mask);
+	if (obj != NULL)
+		VM_OBJECT_WLOCK(obj);
+	if ((di->di_flags & VM_ALLOC_WAITFAIL) != 0)
+		return (ENOMEM);
+
+	/* Restart the search. */
 	vm_domainset_iter_first(di, domain);
-	*req = di->di_flags;
 
 	return (0);
 }
 
-
-void
-vm_domainset_iter_malloc_init(struct vm_domainset_iter *di,
-    struct vm_object *obj, int *domain, int *flags)
+static void
+_vm_domainset_iter_policy_init(struct vm_domainset_iter *di, int *domain,
+    int *flags)
 {
 
-	vm_domainset_iter_init(di, obj, 0);
-	if (di->di_policy == DOMAINSET_POLICY_INTERLEAVE)
-		di->di_policy = DOMAINSET_POLICY_ROUNDROBIN;
 	di->di_flags = *flags;
 	*flags = (di->di_flags & ~M_WAITOK) | M_NOWAIT;
 	vm_domainset_iter_first(di, domain);
 	if (vm_page_count_min_domain(*domain))
-		vm_domainset_iter_malloc(di, domain, flags);
+		vm_domainset_iter_policy(di, domain);
 }
 
+void
+vm_domainset_iter_policy_init(struct vm_domainset_iter *di,
+    struct domainset *ds, int *domain, int *flags)
+{
+
+	vm_domainset_iter_init(di, ds, &curthread->td_domain.dr_iter, NULL, 0);
+	_vm_domainset_iter_policy_init(di, domain, flags);
+}
+
+void
+vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di,
+    struct domainset_ref *dr, int *domain, int *flags)
+{
+
+	vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, NULL, 0);
+	_vm_domainset_iter_policy_init(di, domain, flags);
+}
+
 int
-vm_domainset_iter_malloc(struct vm_domainset_iter *di, int *domain, int *flags)
+vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
 {
 
 	/* If there are more domains to visit we run the iterator. */
@@ -296,45 +309,46 @@
 	if ((di->di_flags & M_WAITOK) == 0)
 		return (ENOMEM);
 
-	/*
-	 * We have visited all domains with non-blocking allocations, try
-	 * from the beginning with a blocking allocation.
-	 */
+	/* Wait for one of the domains to accumulate some free pages. */
+	vm_wait_doms(&di->di_domain->ds_mask);
+
+	/* Restart the search. */
 	vm_domainset_iter_first(di, domain);
-	*flags = di->di_flags;
 
 	return (0);
 }
 
 #else /* !NUMA */
+
 int
-vm_domainset_iter_page(struct vm_domainset_iter *di, int *domain, int *flags)
+vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
+    int *domain)
 {
 
 	return (EJUSTRETURN);
 }
 
 void
-vm_domainset_iter_page_init(struct vm_domainset_iter *di,
-            struct vm_object *obj, vm_pindex_t pindex, int *domain, int *flags)
+vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
+    vm_pindex_t pindex, int *domain, int *flags)
 {
 
 	*domain = 0;
 }
 
 int
-vm_domainset_iter_malloc(struct vm_domainset_iter *di, int *domain, int *flags)
+vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
 {
 
 	return (EJUSTRETURN);
 }
 
 void
-vm_domainset_iter_malloc_init(struct vm_domainset_iter *di,
-            struct vm_object *obj, int *domain, int *flags)
+vm_domainset_iter_policy_init(struct vm_domainset_iter *di,
+    struct domainset *ds, int *domain, int *flags)
 {
 
 	*domain = 0;
 }
 
-#endif
+#endif /* NUMA */
Index: head/sys/vm/vm_glue.c
===================================================================
--- head/sys/vm/vm_glue.c
+++ head/sys/vm/vm_glue.c
@@ -377,7 +377,7 @@
 	 */
 	if (vm_ndomains > 1) {
 		ksobj->domain.dr_policy = DOMAINSET_RR();
-		ksobj->domain.dr_iterator =
+		ksobj->domain.dr_iter =
 		    atomic_fetchadd_int(&kstack_domain_iter, 1);
 	}
 
Index: head/sys/vm/vm_kern.c
===================================================================
--- head/sys/vm/vm_kern.c
+++ head/sys/vm/vm_kern.c
@@ -235,13 +235,13 @@
 	vm_offset_t addr;
 	int domain;
 
-	vm_domainset_iter_malloc_init(&di, kernel_object, &domain, &flags);
+	vm_domainset_iter_policy_init(&di, DOMAINSET_RR(), &domain, &flags);
 	do {
 		addr = kmem_alloc_attr_domain(domain, size, flags, low, high,
 		    memattr);
 		if (addr != 0)
 			break;
-	} while (vm_domainset_iter_malloc(&di, &domain, &flags) == 0);
+	} while (vm_domainset_iter_policy(&di, &domain) == 0);
 
 	return (addr);
 }
@@ -319,13 +319,13 @@
 	vm_offset_t addr;
 	int domain;
 
-	vm_domainset_iter_malloc_init(&di, kernel_object, &domain, &flags);
+	vm_domainset_iter_policy_init(&di, DOMAINSET_RR(), &domain, &flags);
 	do {
 		addr = kmem_alloc_contig_domain(domain, size, flags, low, high,
 		    alignment, boundary, memattr);
 		if (addr != 0)
 			break;
-	} while (vm_domainset_iter_malloc(&di, &domain, &flags) == 0);
+	} while (vm_domainset_iter_policy(&di, &domain) == 0);
 
 	return (addr);
 }
@@ -406,12 +406,12 @@
 	vm_offset_t addr;
 	int domain;
 
-	vm_domainset_iter_malloc_init(&di, kernel_object, &domain, &flags);
+	vm_domainset_iter_policy_init(&di, DOMAINSET_RR(), &domain, &flags);
 	do {
 		addr = kmem_malloc_domain(domain, size, flags);
 		if (addr != 0)
 			break;
-	} while (vm_domainset_iter_malloc(&di, &domain, &flags) == 0);
+	} while (vm_domainset_iter_policy(&di, &domain) == 0);
 
 	return (addr);
 }
Index: head/sys/vm/vm_object.c
===================================================================
--- head/sys/vm/vm_object.c
+++ head/sys/vm/vm_object.c
@@ -274,6 +274,7 @@
 		panic("_vm_object_allocate: type %d is undefined", type);
 	}
 	object->size = size;
+	object->domain.dr_policy = NULL;
 	object->generation = 1;
 	object->ref_count = 1;
 	object->memattr = VM_MEMATTR_DEFAULT;
Index: head/sys/vm/vm_page.c
===================================================================
--- head/sys/vm/vm_page.c
+++ head/sys/vm/vm_page.c
@@ -1753,7 +1753,7 @@
 		    mpred);
 		if (m != NULL)
 			break;
-	} while (vm_domainset_iter_page(&di, &domain, &req) == 0);
+	} while (vm_domainset_iter_page(&di, object, &domain) == 0);
 
 	return (m);
 }
@@ -1990,7 +1990,7 @@
 		    npages, low, high, alignment, boundary, memattr);
 		if (m != NULL)
 			break;
-	} while (vm_domainset_iter_page(&di, &domain, &req) == 0);
+	} while (vm_domainset_iter_page(&di, object, &domain) == 0);
 
 	return (m);
 }
@@ -2191,7 +2191,7 @@
 		m = vm_page_alloc_freelist_domain(domain, freelist, req);
 		if (m != NULL)
 			break;
-	} while (vm_domainset_iter_page(&di, &domain, &req) == 0);
+	} while (vm_domainset_iter_page(&di, NULL, &domain) == 0);
 
 	return (m);
 }
@@ -2830,7 +2830,7 @@
 		    high, alignment, boundary);
 		if (ret)
 			break;
-	} while (vm_domainset_iter_page(&di, &domain, &req) == 0);
+	} while (vm_domainset_iter_page(&di, NULL, &domain) == 0);
 
 	return (ret);
 }