Index: head/sys/vm/uma_core.c =================================================================== --- head/sys/vm/uma_core.c +++ head/sys/vm/uma_core.c @@ -1830,6 +1830,39 @@ return (slab_ipers_hdr(size, rsize, UMA_SLAB_SIZE, true)); } +struct keg_layout_result { + u_int format; + u_int slabsize; + u_int ipers; + u_int eff; +}; + +static void +keg_layout_one(uma_keg_t keg, u_int rsize, u_int slabsize, u_int fmt, + struct keg_layout_result *kl) +{ + u_int total; + + kl->format = fmt; + kl->slabsize = slabsize; + + /* Handle INTERNAL as inline with an extra page. */ + if ((fmt & UMA_ZFLAG_INTERNAL) != 0) { + kl->format &= ~UMA_ZFLAG_INTERNAL; + kl->slabsize += PAGE_SIZE; + } + + kl->ipers = slab_ipers_hdr(keg->uk_size, rsize, kl->slabsize, + (fmt & UMA_ZFLAG_OFFPAGE) == 0); + + /* Account for memory used by an offpage slab header. */ + total = kl->slabsize; + if ((fmt & UMA_ZFLAG_OFFPAGE) != 0) + total += slabzone(kl->ipers)->uz_keg->uk_rsize; + + kl->eff = UMA_FRAC_FIXPT(kl->ipers * rsize, total); +} + /* * Determine the format of a uma keg. This determines where the slab header * will be placed (inline or offpage) and calculates ipers, rsize, and ppera. @@ -1843,15 +1876,14 @@ static void keg_layout(uma_keg_t keg) { + struct keg_layout_result kl = {}, kl_tmp; + u_int fmts[2]; u_int alignsize; - u_int eff; - u_int eff_offpage; - u_int format; - u_int ipers; - u_int ipers_offpage; + u_int nfmt; u_int pages; u_int rsize; u_int slabsize; + u_int i, j; KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || (keg->uk_size <= UMA_PCPU_ALLOC_SIZE && @@ -1866,8 +1898,6 @@ PRINT_UMA_ZFLAGS)); alignsize = keg->uk_align + 1; - format = 0; - ipers = 0; /* * Calculate the size of each allocation (rsize) according to @@ -1877,10 +1907,7 @@ rsize = MAX(keg->uk_size, UMA_SMALLEST_UNIT); rsize = roundup2(rsize, alignsize); - if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) { - slabsize = UMA_PCPU_ALLOC_SIZE; - pages = mp_maxid + 1; - } else if ((keg->uk_flags & UMA_ZONE_CACHESPREAD) != 0) { + if ((keg->uk_flags & UMA_ZONE_CACHESPREAD) != 0) { /* * We want one item to start on every align boundary in a page. * To do this we will span pages. We will also extend the item @@ -1892,23 +1919,22 @@ slabsize = rsize * (PAGE_SIZE / alignsize); slabsize = MIN(slabsize, rsize * SLAB_MAX_SETSIZE); slabsize = MIN(slabsize, UMA_CACHESPREAD_MAX_SIZE); - pages = howmany(slabsize, PAGE_SIZE); - slabsize = ptoa(pages); + slabsize = round_page(slabsize); } else { /* - * Choose a slab size of as many pages as it takes to represent - * a single item. We will then try to fit as many additional - * items into the slab as possible. At some point, we may want - * to increase the slab size for awkward item sizes in order to - * increase efficiency. + * Start with a slab size of as many pages as it takes to + * represent a single item. We will try to fit as many + * additional items into the slab as possible. */ - pages = howmany(keg->uk_size, PAGE_SIZE); - slabsize = ptoa(pages); + slabsize = round_page(keg->uk_size); } + /* Build a list of all of the available formats for this keg. */ + nfmt = 0; + /* Evaluate an inline slab layout. */ if ((keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0) - ipers = slab_ipers_hdr(keg->uk_size, rsize, slabsize, true); + fmts[nfmt++] = 0; /* TODO: vm_page-embedded slab. */ @@ -1917,65 +1943,91 @@ * asked to not go to the VM for buckets. If we do this we * may end up going to the VM for slabs which we do not * want to do if we're UMA_ZFLAG_CACHEONLY as a result - * of UMA_ZONE_VM, which clearly forbids it. + * of UMA_ZONE_VM, which clearly forbids it. In those cases, + * evaluate a pseudo-format called INTERNAL which has an inline + * slab header and one extra page to guarantee that it fits. + * + * Otherwise, see if using an OFFPAGE slab will improve our + * efficiency. */ - if ((keg->uk_flags & - (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY)) != 0) { - if (ipers == 0) { - /* We need an extra page for the slab header. */ - pages++; - slabsize = ptoa(pages); - ipers = slab_ipers_hdr(keg->uk_size, rsize, slabsize, - true); - } - goto out; - } + if ((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY)) != 0) + fmts[nfmt++] = UMA_ZFLAG_INTERNAL; + else + fmts[nfmt++] = UMA_ZFLAG_OFFPAGE; /* - * See if using an OFFPAGE slab will improve our efficiency. - * Only do this if we are below our efficiency threshold. + * Choose a slab size and format which satisfy the minimum efficiency. + * Prefer the smallest slab size that meets the constraints. * - * XXX We could try growing slabsize to limit max waste as well. - * Historically this was not done because the VM could not - * efficiently handle contiguous allocations. + * Start with a minimum slab size, to accommodate CACHESPREAD. Then, + * for small items (up to PAGE_SIZE), the iteration increment is one + * page; and for large items, the increment is one item. */ - eff = UMA_FRAC_FIXPT(ipers * rsize, slabsize); - ipers_offpage = slab_ipers_hdr(keg->uk_size, rsize, slabsize, false); - eff_offpage = UMA_FRAC_FIXPT(ipers_offpage * rsize, - slabsize + slabzone(ipers_offpage)->uz_keg->uk_rsize); - if (ipers == 0 || (eff < UMA_MIN_EFF && eff < eff_offpage)) { - CTR5(KTR_UMA, "UMA decided we need offpage slab headers for " - "keg: %s(%p), minimum efficiency allowed = %u%%, " - "old efficiency = %u%%, offpage efficiency = %u%%", - keg->uk_name, keg, UMA_FIXPT_PCT(UMA_MIN_EFF), - UMA_FIXPT_PCT(eff), UMA_FIXPT_PCT(eff_offpage)); - format = UMA_ZFLAG_OFFPAGE; - ipers = ipers_offpage; + i = (slabsize + rsize - keg->uk_size) / MAX(PAGE_SIZE, rsize); + KASSERT(i >= 1, ("keg %s(%p) flags=0x%b slabsize=%u, rsize=%u, i=%u", + keg->uk_name, keg, keg->uk_flags, PRINT_UMA_ZFLAGS, slabsize, + rsize, i)); + for ( ; ; i++) { + slabsize = (rsize <= PAGE_SIZE) ? ptoa(i) : + round_page(rsize * (i - 1) + keg->uk_size); + + for (j = 0; j < nfmt; j++) { + /* Only if we have no viable format yet. */ + if ((fmts[j] & UMA_ZFLAG_INTERNAL) != 0 && + kl.ipers > 0) + continue; + + keg_layout_one(keg, rsize, slabsize, fmts[j], &kl_tmp); + if (kl_tmp.eff <= kl.eff) + continue; + + kl = kl_tmp; + + CTR6(KTR_UMA, "keg %s layout: format %#x " + "(ipers %u * rsize %u) / slabsize %#x = %u%% eff", + keg->uk_name, kl.format, kl.ipers, rsize, + kl.slabsize, UMA_FIXPT_PCT(kl.eff)); + + /* Stop when we reach the minimum efficiency. */ + if (kl.eff >= UMA_MIN_EFF) + break; + } + + if (kl.eff >= UMA_MIN_EFF || + slabsize >= SLAB_MAX_SETSIZE * rsize || + (keg->uk_flags & (UMA_ZONE_PCPU | UMA_ZONE_CONTIG)) != 0) + break; } -out: + pages = atop(kl.slabsize); + if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) + pages *= mp_maxid + 1; + + keg->uk_rsize = rsize; + keg->uk_ipers = kl.ipers; + keg->uk_ppera = pages; + keg->uk_flags |= kl.format; + /* * How do we find the slab header if it is offpage or if not all item * start addresses are in the same page? We could solve the latter * case with vaddr alignment, but we don't. */ - if ((format & UMA_ZFLAG_OFFPAGE) != 0 || - (ipers - 1) * rsize >= PAGE_SIZE) { + if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0 || + (keg->uk_ipers - 1) * rsize >= PAGE_SIZE) { if ((keg->uk_flags & UMA_ZONE_NOTPAGE) != 0) - format |= UMA_ZFLAG_HASH; + keg->uk_flags |= UMA_ZFLAG_HASH; else - format |= UMA_ZFLAG_VTOSLAB; + keg->uk_flags |= UMA_ZFLAG_VTOSLAB; } - keg->uk_ipers = ipers; - keg->uk_rsize = rsize; - keg->uk_flags |= format; - keg->uk_ppera = pages; + CTR6(KTR_UMA, "%s: keg=%s, flags=%#x, rsize=%u, ipers=%u, ppera=%u", - __func__, keg->uk_name, keg->uk_flags, rsize, ipers, pages); + __func__, keg->uk_name, keg->uk_flags, rsize, keg->uk_ipers, + pages); KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_MAX_SETSIZE, ("%s: keg=%s, flags=0x%b, rsize=%u, ipers=%u, ppera=%u", __func__, - keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS, rsize, ipers, - pages)); + keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS, rsize, + keg->uk_ipers, pages)); } /*