diff --git a/sys/vm/vm_domainset.h b/sys/vm/vm_domainset.h --- a/sys/vm/vm_domainset.h +++ b/sys/vm/vm_domainset.h @@ -28,6 +28,8 @@ #ifndef __VM_DOMAINSET_H__ #define __VM_DOMAINSET_H__ +struct pctrie_iter; + struct vm_domainset_iter { struct domainset *di_domain; unsigned int *di_iter; @@ -40,9 +42,10 @@ }; int vm_domainset_iter_page(struct vm_domainset_iter *, struct vm_object *, - int *); + int *, struct pctrie_iter *); void vm_domainset_iter_page_init(struct vm_domainset_iter *, - struct vm_object *, vm_pindex_t, int *, int *); + struct vm_object *, vm_pindex_t, int *, int *, + struct pctrie_iter *); int vm_domainset_iter_policy(struct vm_domainset_iter *, int *); void vm_domainset_iter_policy_init(struct vm_domainset_iter *, struct domainset *, int *, int *); diff --git a/sys/vm/vm_domainset.c b/sys/vm/vm_domainset.c --- a/sys/vm/vm_domainset.c +++ b/sys/vm/vm_domainset.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -199,7 +200,7 @@ void vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj, - vm_pindex_t pindex, int *domain, int *req) + vm_pindex_t pindex, int *domain, int *req, struct pctrie_iter *pages) { struct domainset_ref *dr; @@ -218,12 +219,12 @@ VM_ALLOC_NOWAIT; vm_domainset_iter_first(di, domain); if (vm_page_count_min_domain(*domain)) - vm_domainset_iter_page(di, obj, domain); + vm_domainset_iter_page(di, obj, domain, pages); } int vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj, - int *domain) + int *domain, struct pctrie_iter *pages) { if (__predict_false(DOMAINSET_EMPTY(&di->di_valid_mask))) return (ENOMEM); @@ -248,8 +249,11 @@ return (ENOMEM); /* Wait for one of the domains to accumulate some free pages. */ - if (obj != NULL) + if (obj != NULL) { VM_OBJECT_WUNLOCK(obj); + if (pages != NULL) + pctrie_iter_reset(pages); + } vm_wait_doms(&di->di_valid_mask, 0); if (obj != NULL) VM_OBJECT_WLOCK(obj); @@ -339,7 +343,7 @@ int vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj, - int *domain) + int *domain, struct pctrie_iter *pages) { return (EJUSTRETURN); @@ -347,7 +351,7 @@ void vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj, - vm_pindex_t pindex, int *domain, int *flags) + vm_pindex_t pindex, int *domain, int *flags, struct pctrie_iter *pages) { *domain = 0; diff --git a/sys/vm/vm_glue.c b/sys/vm/vm_glue.c --- a/sys/vm/vm_glue.c +++ b/sys/vm/vm_glue.c @@ -453,7 +453,7 @@ obj = vm_thread_kstack_size_to_obj(pages); if (vm_ndomains > 1) obj->domain.dr_policy = ds; - vm_domainset_iter_page_init(&di, obj, 0, &domain, &req); + vm_domainset_iter_page_init(&di, obj, 0, &domain, &req, NULL); do { /* * Get a kernel virtual address for this thread's kstack. @@ -480,7 +480,7 @@ vm_page_valid(ma[i]); pmap_qenter(ks, ma, pages); return (ks); - } while (vm_domainset_iter_page(&di, obj, &domain) == 0); + } while (vm_domainset_iter_page(&di, obj, &domain, NULL) == 0); return (0); } diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -2007,13 +2007,14 @@ vm_page_t m; int domain; - vm_domainset_iter_page_init(&di, object, pindex, &domain, &req); + vm_domainset_iter_page_init(&di, object, pindex, &domain, &req, + pages); do { m = vm_page_alloc_domain_iter(object, pindex, domain, req, pages); if (m != NULL) break; - } while (vm_domainset_iter_page(&di, object, &domain) == 0); + } while (vm_domainset_iter_page(&di, object, &domain, pages) == 0); return (m); } @@ -2142,7 +2143,8 @@ * Not allocatable, give up. */ (void)vm_domain_alloc_fail(vmd, object, req); - pctrie_iter_reset(pages); + if ((req & VM_ALLOC_WAITFAIL) != 0) + pctrie_iter_reset(pages); return (NULL); } @@ -2254,7 +2256,7 @@ start_segind = -1; - vm_domainset_iter_page_init(&di, object, pindex, &domain, &req); + vm_domainset_iter_page_init(&di, object, pindex, &domain, &req, NULL); do { m = vm_page_alloc_contig_domain(object, pindex, domain, req, npages, low, high, alignment, boundary, memattr); @@ -2266,7 +2268,7 @@ npages, low, high) == -1) { vm_domainset_iter_ignore(&di, domain); } - } while (vm_domainset_iter_page(&di, object, &domain) == 0); + } while (vm_domainset_iter_page(&di, object, &domain, NULL) == 0); return (m); } @@ -2576,12 +2578,12 @@ vm_page_t m; int domain; - vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); + vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req, NULL); do { m = vm_page_alloc_noobj_domain(domain, req); if (m != NULL) break; - } while (vm_domainset_iter_page(&di, NULL, &domain) == 0); + } while (vm_domainset_iter_page(&di, NULL, &domain, NULL) == 0); return (m); } @@ -2595,13 +2597,13 @@ vm_page_t m; int domain; - vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); + vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req, NULL); do { m = vm_page_alloc_noobj_contig_domain(domain, req, npages, low, high, alignment, boundary, memattr); if (m != NULL) break; - } while (vm_domainset_iter_page(&di, NULL, &domain) == 0); + } while (vm_domainset_iter_page(&di, NULL, &domain, NULL) == 0); return (m); } @@ -3316,7 +3318,7 @@ ret = ERANGE; - vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req); + vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req, NULL); do { status = vm_page_reclaim_contig_domain(domain, req, npages, low, high, alignment, boundary); @@ -3329,7 +3331,7 @@ "from vm_page_reclaim_contig_domain()", status)); ret = ENOMEM; } - } while (vm_domainset_iter_page(&di, NULL, &domain) == 0); + } while (vm_domainset_iter_page(&di, NULL, &domain, NULL) == 0); return (ret); }