Index: sys/vm/uma_core.c =================================================================== --- sys/vm/uma_core.c +++ sys/vm/uma_core.c @@ -1785,15 +1785,16 @@ static void keg_layout(uma_keg_t keg) { + u_int fmts[2]; u_int alignsize; u_int eff; - u_int eff_offpage; u_int format; u_int ipers; - u_int ipers_offpage; + u_int nfmt; u_int pages; u_int rsize; u_int slabsize; + u_int i, j; KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || (keg->uk_size <= UMA_PCPU_ALLOC_SIZE && @@ -1810,6 +1811,7 @@ alignsize = keg->uk_align + 1; format = 0; ipers = 0; + nfmt = 0; /* * Calculate the size of each allocation (rsize) according to @@ -1819,10 +1821,7 @@ rsize = MAX(keg->uk_size, UMA_SMALLEST_UNIT); rsize = roundup2(rsize, alignsize); - if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) { - slabsize = UMA_PCPU_ALLOC_SIZE; - pages = mp_maxid + 1; - } else if ((keg->uk_flags & UMA_ZONE_CACHESPREAD) != 0) { + if ((keg->uk_flags & UMA_ZONE_CACHESPREAD) != 0) { /* * We want one item to start on every align boundary in a page. * To do this we will span pages. We will also extend the item @@ -1834,23 +1833,19 @@ slabsize = rsize * (PAGE_SIZE / alignsize); slabsize = MIN(slabsize, rsize * SLAB_MAX_SETSIZE); slabsize = MIN(slabsize, UMA_CACHESPREAD_MAX_SIZE); - pages = howmany(slabsize, PAGE_SIZE); - slabsize = ptoa(pages); + slabsize = round_page(slabsize); } else { /* - * Choose a slab size of as many pages as it takes to represent - * a single item. We will then try to fit as many additional - * items into the slab as possible. At some point, we may want - * to increase the slab size for awkward item sizes in order to - * increase efficiency. + * Start with a slab size of as many pages as it takes to + * represent a single item. We will try to fit as many + * additional items into the slab as possible. */ - pages = howmany(keg->uk_size, PAGE_SIZE); - slabsize = ptoa(pages); + slabsize = round_page(keg->uk_size); } /* Evaluate an inline slab layout. */ if ((keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0) - ipers = slab_ipers_hdr(keg->uk_size, rsize, slabsize, true); + fmts[nfmt++] = 0; /* TODO: vm_page-embedded slab. */ @@ -1860,42 +1855,86 @@ * may end up going to the VM for slabs which we do not * want to do if we're UMA_ZFLAG_CACHEONLY as a result * of UMA_ZONE_VM, which clearly forbids it. + * + * Otherwise, see if using an OFFPAGE slab will improve our + * efficiency. */ - if ((keg->uk_flags & - (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY)) != 0) { - if (ipers == 0) { - /* We need an extra page for the slab header. */ - pages++; - slabsize = ptoa(pages); - ipers = slab_ipers_hdr(keg->uk_size, rsize, slabsize, - true); - } - goto out; - } + if ((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY)) != 0) + fmts[nfmt++] = UMA_ZFLAG_INTERNAL; + else + fmts[nfmt++] = UMA_ZFLAG_OFFPAGE; /* - * See if using an OFFPAGE slab will improve our efficiency. - * Only do this if we are below our efficiency threshold. + * Choose a slab size and format which satisfy the minimum efficiency. + * Prefer the smallest slab size that meets the constraints. * - * XXX We could try growing slabsize to limit max waste as well. - * Historically this was not done because the VM could not - * efficiently handle contiguous allocations. + * Start with a minimum slab size, to accommodate CACHESPREAD. Then, + * for small items (up to PAGE_SIZE), the iteration increment is one + * page; and for large items, the increment is one item. */ - eff = UMA_FRAC_FIXPT(ipers * rsize, slabsize); - ipers_offpage = slab_ipers_hdr(keg->uk_size, rsize, slabsize, false); - eff_offpage = UMA_FRAC_FIXPT(ipers_offpage * rsize, - slabsize + slabzone(ipers_offpage)->uz_keg->uk_rsize); - if (ipers == 0 || (eff < UMA_MIN_EFF && eff < eff_offpage)) { - CTR5(KTR_UMA, "UMA decided we need offpage slab headers for " - "keg: %s(%p), minimum efficiency allowed = %u%%, " - "old efficiency = %u%%, offpage efficiency = %u%%", - keg->uk_name, keg, UMA_FIXPT_PCT(UMA_MIN_EFF), - UMA_FIXPT_PCT(eff), UMA_FIXPT_PCT(eff_offpage)); - format = UMA_ZFLAG_OFFPAGE; - ipers = ipers_offpage; + i = (slabsize + rsize - keg->uk_size) / MAX(PAGE_SIZE, rsize); + KASSERT(i >= 1, ("keg %s(%p) flags=0x%b slabsize=%u, rsize=%u, i=%u", + keg->uk_name, keg, keg->uk_flags, PRINT_UMA_ZFLAGS, slabsize, + rsize, i)); + for ( ; ; i++) { + u_int slabsize_i; + + slabsize_i = (rsize <= PAGE_SIZE) ? ptoa(i) : + round_page(rsize * (i - 1) + keg->uk_size); + + for (j = 0; j < nfmt; j++) { + u_int eff_ij; + u_int fmt_j; + u_int ipers_ij; + u_int slabsize_ij; + u_int total; + + fmt_j = fmts[j]; + + slabsize_ij = slabsize_i; + if ((fmt_j & UMA_ZFLAG_INTERNAL) != 0) { + /* Only if we have no viable format yet. */ + if (ipers > 0) + continue; + slabsize_ij += PAGE_SIZE; + } + + ipers_ij = slab_ipers_hdr(keg->uk_size, rsize, + slabsize_ij, (fmt_j & UMA_ZFLAG_OFFPAGE) == 0); + + total = slabsize_ij; + if ((fmt_j & UMA_ZFLAG_OFFPAGE) != 0) + total += slabzone(ipers_ij)->uz_keg->uk_rsize; + + eff_ij = UMA_FRAC_FIXPT(ipers_ij * rsize, total); + if (eff_ij <= eff) + continue; + + format = fmt_j; + eff = eff_ij; + ipers = ipers_ij; + slabsize = slabsize_ij; + + CTR6(KTR_UMA, "keg %s layout: format %#x " + "(ipers %u * rsize %u) / slabsize %#x = %u%% eff", + keg->uk_name, format, ipers, rsize, slabsize, + UMA_FIXPT_PCT(eff)); + + /* Stop when we reach the minimum efficiency. */ + if (eff >= UMA_MIN_EFF) + break; + } + + if (eff >= UMA_MIN_EFF || + slabsize_i >= SLAB_MAX_SETSIZE * rsize || + (keg->uk_flags & (UMA_ZONE_PCPU | UMA_ZONE_CONTIG)) != 0) + break; } -out: + pages = atop(slabsize); + if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) + pages *= mp_maxid + 1; + /* * How do we find the slab header if it is offpage or if not all item * start addresses are in the same page? We could solve the latter