Index: sys/vm/uma_core.c =================================================================== --- sys/vm/uma_core.c +++ sys/vm/uma_core.c @@ -2217,14 +2217,6 @@ keg->uk_reserve = 0; keg->uk_flags = arg->flags; - /* - * We use a global round-robin policy by default. Zones with - * UMA_ZONE_FIRSTTOUCH set will use first-touch instead, in which - * case the iterator is never run. - */ - keg->uk_dr.dr_policy = DOMAINSET_RR(); - keg->uk_dr.dr_iter = 0; - /* * The primary zone is passed to us at keg-creation time. */ @@ -2244,17 +2236,26 @@ keg_layout(keg); /* - * Use a first-touch NUMA policy for kegs that pmap_extract() will + * Use a first-touch NUMA policy for kegs that pmap_kextract() will * work on. Use round-robin for everything else. * * Zones may override the default by specifying either. */ + KASSERT((keg->uk_flags & (UMA_ZONE_ROUNDROBIN | UMA_ZONE_FIRSTTOUCH)) != + (UMA_ZONE_ROUNDROBIN | UMA_ZONE_FIRSTTOUCH), + ("%s: invalid keg NUMA flags %x", __func__, keg->uk_flags)); + #ifdef NUMA if ((keg->uk_flags & - (UMA_ZONE_ROUNDROBIN | UMA_ZFLAG_CACHE | UMA_ZONE_NOTPAGE)) == 0) + (UMA_ZONE_ROUNDROBIN | UMA_ZFLAG_CACHE | UMA_ZONE_NOTPAGE)) == 0) { keg->uk_flags |= UMA_ZONE_FIRSTTOUCH; - else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0) + keg->uk_dr.dr_policy = DOMAINSET_FT(); + keg->uk_dr.dr_iter = 0; + } else { keg->uk_flags |= UMA_ZONE_ROUNDROBIN; + keg->uk_dr.dr_policy = DOMAINSET_RR(); + keg->uk_dr.dr_iter = 0; + } #endif /* @@ -3648,34 +3649,28 @@ return (slab); } +/* + * Fetch a cached slab containing one or more free items, or attempt to allocate + * a new slab. Returns with the per-domain keg lock held if successful. + */ static uma_slab_t -keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags) +keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, int flags) { struct vm_domainset_iter di; uma_slab_t slab; - int aflags, domain; + int domain; bool rr; -restart: /* * Use the keg's policy if upper layers haven't already specified a * domain (as happens with first-touch zones). * - * To avoid races we run the iterator with the keg lock held, but that - * means that we cannot allow the vm_domainset layer to sleep. Thus, - * clear M_WAITOK and handle low memory conditions locally. + * XXXMJ keg iterator update is racy */ rr = rdomain == UMA_ANYDOMAIN; - if (rr) { - aflags = (flags & ~M_WAITOK) | M_NOWAIT; - vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, - &aflags); - } else { - aflags = flags; - domain = rdomain; - } + vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, &flags); - for (;;) { + do { slab = keg_fetch_free_slab(keg, domain, rr, flags); if (slab != NULL) return (slab); @@ -3684,31 +3679,18 @@ * M_NOVM means don't ask at all! */ if (flags & M_NOVM) - break; + continue; - slab = keg_alloc_slab(keg, zone, domain, flags, aflags); + slab = keg_alloc_slab(keg, zone, domain, flags, flags); if (slab != NULL) return (slab); - if (!rr && (flags & M_WAITOK) == 0) - break; - if (rr && vm_domainset_iter_policy(&di, &domain) != 0) { - if ((flags & M_WAITOK) != 0) { - vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask, 0); - goto restart; - } - break; - } - } + } while (vm_domainset_iter_policy(&di, &domain) == 0); /* - * We might not have been able to get a slab but another cpu - * could have while we were unlocked. Check again before we - * fail. + * We might not have been able to get a slab but another cpu could have + * while we were unlocked. Check again before we fail. */ - if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) - return (slab); - - return (NULL); + return (keg_fetch_free_slab(keg, rr ? domain : rdomain, true, flags)); } static void *