diff --git a/sys/vm/vm_domainset.h b/sys/vm/vm_domainset.h --- a/sys/vm/vm_domainset.h +++ b/sys/vm/vm_domainset.h @@ -39,6 +39,12 @@ bool di_minskip; }; +struct vm_domainset_batch_iter { + struct vm_domainset_iter di; + u_short dbi_obj_color; + int dbi_npages; +}; + int vm_domainset_iter_page(struct vm_domainset_iter *, struct vm_object *, int *); void vm_domainset_iter_page_init(struct vm_domainset_iter *, @@ -50,6 +56,11 @@ struct domainset_ref *, int *, int *); void vm_domainset_iter_ignore(struct vm_domainset_iter *, int); +int vm_domainset_batch_iter_page(struct vm_domainset_batch_iter *, + struct vm_object *, vm_pindex_t, int *, int *); +void vm_domainset_batch_iter_page_init(struct vm_domainset_batch_iter *, + struct vm_object *, vm_pindex_t, int, int *, int *, int *); + int vm_wait_doms(const domainset_t *, int mflags); #endif /* __VM_DOMAINSET_H__ */ diff --git a/sys/vm/vm_domainset.c b/sys/vm/vm_domainset.c --- a/sys/vm/vm_domainset.c +++ b/sys/vm/vm_domainset.c @@ -335,6 +335,142 @@ DOMAINSET_CLR(domain, &di->di_valid_mask); } +static void +vm_domainset_batch_iter_npages_interleave(struct vm_domainset_batch_iter *dbi, + struct vm_object *obj, vm_pindex_t pindex, int *npages) +{ + vm_pindex_t mask; + +#if VM_NRESERVLEVEL > 0 + mask = ((1 << VM_LEVEL_0_ORDER) - 1); + if (obj != NULL) + pindex += obj->pg_color; +#else + mask = vm_domainset_default_stride - 1; +#endif + + *npages = (mask + 1) - (pindex & mask); +} + +static void +vm_domainset_batch_iter_npages_next(struct vm_domainset_batch_iter *dbi, + int *npages) +{ + struct vm_domainset_iter *di = &dbi->di; + + switch (di->di_policy) { + case DOMAINSET_POLICY_INTERLEAVE: +#if VM_NRESERVLEVEL > 0 + *npages = 1 << VM_LEVEL_0_ORDER; +#else + *npages = vm_domainset_default_stride; +#endif + break; + case DOMAINSET_POLICY_FIRSTTOUCH: + /* FALLTHROUGH */ + case DOMAINSET_POLICY_ROUNDROBIN: + *npages = dbi->dbi_npages / vm_ndomains; + break; + case DOMAINSET_POLICY_PREFER: + *npages = dbi->dbi_npages; + break; + default: + panic("%s: Unknown policy %d", __func__, di->di_policy); + } +} + +static void +vm_domainset_batch_iter_npages_first(struct vm_domainset_batch_iter *dbi, + struct vm_object *obj, vm_pindex_t pindex, int *npages) +{ + struct vm_domainset_iter *di = &dbi->di; + + switch (di->di_policy) { + case DOMAINSET_POLICY_FIRSTTOUCH: + /* FALLTHROUGH */ + case DOMAINSET_POLICY_PREFER: + *npages = dbi->dbi_npages; + break; + case DOMAINSET_POLICY_ROUNDROBIN: + *npages = dbi->dbi_npages / vm_ndomains; + break; + case DOMAINSET_POLICY_INTERLEAVE: + vm_domainset_batch_iter_npages_interleave(dbi, obj, pindex, + npages); + break; + default: + panic("%s: Unknown policy %d", __func__, di->di_policy); + } +} + +static void +vm_domainset_batch_iter_next(struct vm_domainset_batch_iter *dbi, int *domain, + int *npages) +{ + + vm_domainset_iter_next(&dbi->di, domain); + vm_domainset_batch_iter_npages_next(dbi, npages); +} + +static void +vm_domainset_batch_iter_first(struct vm_domainset_batch_iter *dbi, + vm_object_t obj, vm_pindex_t pindex, int *domain, int *npages) +{ + + vm_domainset_iter_first(&dbi->di, domain); + vm_domainset_batch_iter_npages_first(dbi, obj, pindex, npages); +} + +void +vm_domainset_batch_iter_page_init(struct vm_domainset_batch_iter *dbi, + struct vm_object *obj, vm_pindex_t pindex, int req_npages, int *domain, + int *npages, int *req) +{ + + vm_domainset_iter_page_init(&dbi->di, obj, pindex, domain, req); + dbi->dbi_npages = req_npages; + vm_domainset_batch_iter_npages_first(dbi, obj, pindex, npages); +} + +int +vm_domainset_batch_iter_page(struct vm_domainset_batch_iter *dbi, + struct vm_object *obj, vm_pindex_t pindex, int *domain, int *npages) +{ + struct vm_domainset_iter *di = &dbi->di; + + /* If there are more domains to visit we run the iterator. */ + while (--di->di_n != 0) { + vm_domainset_batch_iter_next(dbi, domain, npages); + if (!di->di_minskip || !vm_page_count_min_domain(*domain)) + return (0); + } + + /* If we skipped domains below min restart the search. */ + if (di->di_minskip) { + di->di_minskip = false; + vm_domainset_batch_iter_first(dbi, obj, pindex, domain, npages); + return (0); + } + + /* If we visited all domains and this was a NOWAIT we return error. */ + if ((di->di_flags & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) == 0) + return (ENOMEM); + + /* Wait for one of the domains to accumulate some free pages. */ + if (obj != NULL) + VM_OBJECT_WUNLOCK(obj); + vm_wait_doms(&di->di_domain->ds_mask, 0); + if (obj != NULL) + VM_OBJECT_WLOCK(obj); + if ((di->di_flags & VM_ALLOC_WAITFAIL) != 0) + return (ENOMEM); + + /* Restart the search. */ + vm_domainset_batch_iter_first(dbi, obj, pindex, domain, npages); + + return (0); +} + #else /* !NUMA */ int