Index: vm_reserv.c =================================================================== --- vm_reserv.c +++ vm_reserv.c @@ -1269,6 +1269,73 @@ } /* + * Determine whether this reservation has free pages that satisfy the given + * request for contiguous physical memory. Start searching from the lower + * bound, defined by low_index. + * + * The free page queue lock must be held. + */ +static bool +vm_reserv_test_contig(vm_reserv_t rv, vm_paddr_t size, vm_paddr_t high, + int low_index, u_long alignment, vm_paddr_t boundary) +{ + vm_paddr_t pa; + u_long changes; + int bitpos, i, hi, lo; + + vm_reserv_assert_locked(rv); + i = low_index / NBPOPMAP; + changes = rv->popmap[i] | ((1UL << (low_index % NBPOPMAP)) - 1); + hi = lo = -1; + for (;;) { + /* + * "changes" is a bitmask that marks where a new sequence of + * 0s or 1s begins in popmap[i], with last bit in popmap[i-1] + * considered to be 1 if and only if lo == hi. The bits of + * popmap[-1] and popmap[NPOPMAP] are considered all 1s. + */ + changes ^= (changes << 1) | (lo == hi); + while (changes != 0) { + /* + * If the next change marked begins a run of 0s, set + * lo to mark that position. Otherwise set hi and + * look for a satisfactory first page from lo up to hi. + */ + bitpos = ffsl(changes) - 1; + changes ^= 1UL << bitpos; + if (lo == hi) { + lo = NBPOPMAP * i + bitpos; + continue; + } + hi = NBPOPMAP * i + bitpos; + pa = VM_PAGE_TO_PHYS(&rv->pages[lo]); + /* Skip to next aligned page. */ + lo += (((pa - 1) | (alignment - 1)) + 1) >> PAGE_SHIFT; + pa = VM_PAGE_TO_PHYS(&rv->pages[lo]); + if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) { + /* Skip to next boundary-matching page. */ + lo += (((pa - 1) | (boundary - 1)) + 1) >> + PAGE_SHIFT; + pa = VM_PAGE_TO_PHYS(&rv->pages[lo]); + } + if (pa + size > high) { + /* The rest of this reservation is too high. */ + return (FALSE); + } + if (lo * PAGE_SIZE + size <= hi * PAGE_SIZE) + return (TRUE); + lo = hi; + } + if (++i < NPOPMAP) + changes = rv->popmap[i]; + else if (i == NPOPMAP) + changes = ~0UL; + else + return (FALSE); + } +} + +/* * Searches the partially populated reservation queue for the least recently * changed reservation with free pages that satisfy the given request for * contiguous physical memory. If a satisfactory reservation is found, it is @@ -1282,7 +1349,7 @@ { vm_paddr_t pa, size; vm_reserv_t rv, rvn; - int hi, i, lo, low_index, next_free; + int low_index; if (npages > VM_LEVEL_0_NPAGES - 1) return (FALSE); @@ -1312,66 +1379,14 @@ } } else vm_reserv_domain_unlock(domain); - if (pa < low) { - /* Start the search for free pages at "low". */ - low_index = (low + PAGE_MASK - pa) >> PAGE_SHIFT; - i = low_index / NBPOPMAP; - hi = low_index % NBPOPMAP; - } else - i = hi = 0; - do { - /* Find the next free page. */ - lo = ffsl(~(((1UL << hi) - 1) | rv->popmap[i])); - while (lo == 0 && ++i < NPOPMAP) - lo = ffsl(~rv->popmap[i]); - if (i == NPOPMAP) - break; - /* Convert from ffsl() to ordinary bit numbering. */ - lo--; - next_free = NBPOPMAP * i + lo; - pa = VM_PAGE_TO_PHYS(&rv->pages[next_free]); - KASSERT(pa >= low, - ("vm_reserv_reclaim_contig: pa is too low")); - if (pa + size > high) { - /* The rest of this reservation is too high. */ - break; - } else if ((pa & (alignment - 1)) != 0 || - ((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) { - /* - * The current page doesn't meet the alignment - * and/or boundary requirements. Continue - * searching this reservation until the rest - * of its free pages are either excluded or - * exhausted. - */ - hi = lo + 1; - if (hi >= NBPOPMAP) { - hi = 0; - i++; - } - continue; - } - /* Find the next used page. */ - hi = ffsl(rv->popmap[i] & ~((1UL << lo) - 1)); - while (hi == 0 && ++i < NPOPMAP) { - if ((NBPOPMAP * i - next_free) * PAGE_SIZE >= - size) { - vm_reserv_reclaim(rv); - vm_reserv_unlock(rv); - return (TRUE); - } - hi = ffsl(rv->popmap[i]); - } - /* Convert from ffsl() to ordinary bit numbering. */ - if (i != NPOPMAP) - hi--; - if ((NBPOPMAP * i + hi - next_free) * PAGE_SIZE >= - size) { - vm_reserv_reclaim(rv); - vm_reserv_unlock(rv); - return (TRUE); - } - } while (i < NPOPMAP); + low_index = (pa < low) ? + ((low + PAGE_MASK - pa) >> PAGE_SHIFT) : 0; + if (vm_reserv_test_contig(rv, size, high, low_index, + alignment, boundary)) { + vm_reserv_reclaim(rv); + vm_reserv_unlock(rv); + return (TRUE); + } vm_reserv_unlock(rv); vm_reserv_domain_lock(domain); if (rvn != NULL && !rvn->inpartpopq)