Index: sys/kern/kern_cpuset.c =================================================================== --- sys/kern/kern_cpuset.c +++ sys/kern/kern_cpuset.c @@ -119,6 +119,7 @@ */ LIST_HEAD(domainlist, domainset); +struct domainset __read_mostly domainset_firsttouch; struct domainset __read_mostly domainset_fixed[MAXMEMDOM]; struct domainset __read_mostly domainset_prefer[MAXMEMDOM]; struct domainset __read_mostly domainset_roundrobin; @@ -130,7 +131,7 @@ static struct domainlist cpuset_domains; static struct unrhdr *cpuset_unr; static struct cpuset *cpuset_zero, *cpuset_default, *cpuset_kernel; -static struct domainset domainset0, domainset2; +static struct domainset *domainset0, domainset2; /* Return the size of cpuset_t at the kernel level */ SYSCTL_INT(_kern_sched, OID_AUTO, cpusetsize, CTLFLAG_RD | CTLFLAG_CAPRD, @@ -568,7 +569,7 @@ if (domain->ds_policy == DOMAINSET_POLICY_PREFER && !DOMAINSET_ISSET(domain->ds_prefer, &domain->ds_mask)) return (NULL); - if (!DOMAINSET_SUBSET(&domainset0.ds_mask, &domain->ds_mask)) + if (!DOMAINSET_SUBSET(&domainset0->ds_mask, &domain->ds_mask)) return (NULL); ndomain = uma_zalloc(domainset_zone, M_WAITOK | M_ZERO); domainset_copy(domain, ndomain); @@ -1532,6 +1533,12 @@ struct domainset *dset; int i; + dset = &domainset_firsttouch; + DOMAINSET_COPY(&all_domains, &dset->ds_mask); + dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH; + dset->ds_prefer = -1; + _domainset_create(dset, NULL); + dset = &domainset_roundrobin; DOMAINSET_COPY(&all_domains, &dset->ds_mask); dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; @@ -1563,15 +1570,14 @@ mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE); - dset = &domainset0; + domainset0 = &domainset_firsttouch; + curthread->td_domain.dr_policy = domainset0; + + dset = &domainset2; DOMAINSET_COPY(&all_domains, &dset->ds_mask); - dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH; + dset->ds_policy = DOMAINSET_POLICY_INTERLEAVE; dset->ds_prefer = -1; - curthread->td_domain.dr_policy = _domainset_create(dset, NULL); - - domainset_copy(dset, &domainset2); - domainset2.ds_policy = DOMAINSET_POLICY_INTERLEAVE; - kernel_object->domain.dr_policy = _domainset_create(&domainset2, NULL); + kernel_object->domain.dr_policy = _domainset_create(dset, NULL); /* Remove empty domains from the global policies. */ LIST_FOREACH_SAFE(dset, &cpuset_domains, ds_link, tmp) @@ -1613,7 +1619,7 @@ LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); refcount_init(&set->cs_ref, 1); set->cs_flags = CPU_SET_ROOT | CPU_SET_RDONLY; - set->cs_domain = &domainset0; + set->cs_domain = DOMAINSET_FT(); cpuset_zero = set; cpuset_root = &set->cs_mask; Index: sys/sys/domainset.h =================================================================== --- sys/sys/domainset.h +++ sys/sys/domainset.h @@ -95,6 +95,8 @@ domainid_t ds_order[MAXMEMDOM]; /* nth domain table. */ }; +extern struct domainset domainset_firsttouch; +#define DOMAINSET_FT() (&domainset_firsttouch) extern struct domainset domainset_fixed[MAXMEMDOM], domainset_prefer[MAXMEMDOM]; #define DOMAINSET_FIXED(domain) (&domainset_fixed[(domain)]) #define DOMAINSET_PREF(domain) (&domainset_prefer[(domain)]) Index: sys/vm/uma_core.c =================================================================== --- sys/vm/uma_core.c +++ sys/vm/uma_core.c @@ -2217,14 +2217,6 @@ keg->uk_reserve = 0; keg->uk_flags = arg->flags; - /* - * We use a global round-robin policy by default. Zones with - * UMA_ZONE_FIRSTTOUCH set will use first-touch instead, in which - * case the iterator is never run. - */ - keg->uk_dr.dr_policy = DOMAINSET_RR(); - keg->uk_dr.dr_iter = 0; - /* * The primary zone is passed to us at keg-creation time. */ @@ -2244,19 +2236,32 @@ keg_layout(keg); /* - * Use a first-touch NUMA policy for kegs that pmap_extract() will + * Use a first-touch NUMA policy for kegs that pmap_kextract() will * work on. Use round-robin for everything else. * * Zones may override the default by specifying either. */ #ifdef NUMA + KASSERT((keg->uk_flags & (UMA_ZONE_ROUNDROBIN | UMA_ZONE_FIRSTTOUCH)) != + (UMA_ZONE_ROUNDROBIN | UMA_ZONE_FIRSTTOUCH), + ("%s: invalid keg NUMA flags %x", __func__, keg->uk_flags)); + if ((keg->uk_flags & - (UMA_ZONE_ROUNDROBIN | UMA_ZFLAG_CACHE | UMA_ZONE_NOTPAGE)) == 0) + (UMA_ZONE_ROUNDROBIN | UMA_ZFLAG_CACHE | UMA_ZONE_NOTPAGE)) == 0) { keg->uk_flags |= UMA_ZONE_FIRSTTOUCH; - else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0) + keg->uk_dr.dr_policy = DOMAINSET_FT(); + keg->uk_dr.dr_iter = 0; + } else { keg->uk_flags |= UMA_ZONE_ROUNDROBIN; + keg->uk_dr.dr_policy = DOMAINSET_RR(); + keg->uk_dr.dr_iter = 0; + } #endif + /* + * Zones use a first-touch policy by default. + */ + /* * If we haven't booted yet we need allocations to go through the * startup cache until the vm is ready. @@ -3643,32 +3648,26 @@ return (slab); } +/* + * Fetch a cached slab containing one or more free items, or attempt to allocate + * a new slab. Returns with the per-domain keg lock held if successful. + */ static uma_slab_t -keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags) +keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, int flags) { struct vm_domainset_iter di; uma_slab_t slab; - int aflags, domain; + int domain; bool rr; -restart: /* * Use the keg's policy if upper layers haven't already specified a * domain (as happens with first-touch zones). * - * To avoid races we run the iterator with the keg lock held, but that - * means that we cannot allow the vm_domainset layer to sleep. Thus, - * clear M_WAITOK and handle low memory conditions locally. + * XXXMJ keg iterator update is racy */ rr = rdomain == UMA_ANYDOMAIN; - if (rr) { - aflags = (flags & ~M_WAITOK) | M_NOWAIT; - vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, - &aflags); - } else { - aflags = flags; - domain = rdomain; - } + vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, &flags); for (;;) { slab = keg_fetch_free_slab(keg, domain, rr, flags); @@ -3681,29 +3680,18 @@ if (flags & M_NOVM) break; - slab = keg_alloc_slab(keg, zone, domain, flags, aflags); + slab = keg_alloc_slab(keg, zone, domain, flags, flags); if (slab != NULL) return (slab); - if (!rr && (flags & M_WAITOK) == 0) + if (vm_domainset_iter_policy(&di, &domain) != 0) break; - if (rr && vm_domainset_iter_policy(&di, &domain) != 0) { - if ((flags & M_WAITOK) != 0) { - vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask, 0); - goto restart; - } - break; - } } /* - * We might not have been able to get a slab but another cpu - * could have while we were unlocked. Check again before we - * fail. + * We might not have been able to get a slab but another cpu could have + * while we were unlocked. Check again before we fail. */ - if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) - return (slab); - - return (NULL); + return (keg_fetch_free_slab(keg, rr ? domain : rdomain, true, flags)); } static void *