Index: vm_reserv.c =================================================================== --- vm_reserv.c +++ vm_reserv.c @@ -1030,56 +1030,38 @@ static void vm_reserv_break(vm_reserv_t rv) { - int begin_zeroes, hi, i, lo; + u_long changes; + int bitpos, hi, i, lo; vm_reserv_assert_locked(rv); CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); vm_reserv_remove(rv); rv->pages->psind = 0; - i = hi = 0; - do { - /* Find the next 0 bit. Any previous 0 bits are < "hi". */ - lo = ffsl(~(((1UL << hi) - 1) | rv->popmap[i])); - if (lo == 0) { - /* Redundantly clears bits < "hi". */ + hi = lo = 0; + for (i = 0; i <= NPOPMAP; i++) { + if (i == NPOPMAP) + changes = 1; + else { + changes = rv->popmap[i]; + changes ^= (changes << 1) | (lo == hi); rv->popmap[i] = 0; - rv->popcnt -= NBPOPMAP - hi; - while (++i < NPOPMAP) { - lo = ffsl(~rv->popmap[i]); - if (lo == 0) { - rv->popmap[i] = 0; - rv->popcnt -= NBPOPMAP; - } else - break; + } + while (changes != 0) { + bitpos = ffsl(changes) - 1; + changes ^= 1UL << bitpos; + if (lo == hi) + lo = NBPOPMAP * i + bitpos; + else { + hi = NBPOPMAP * i + bitpos; + vm_domain_free_lock(VM_DOMAIN(rv->domain)); + vm_phys_free_contig(&rv->pages[lo], hi - lo); + vm_domain_free_unlock(VM_DOMAIN(rv->domain)); + lo = hi; } - if (i == NPOPMAP) - break; - hi = 0; } - KASSERT(lo > 0, ("vm_reserv_break: lo is %d", lo)); - /* Convert from ffsl() to ordinary bit numbering. */ - lo--; - if (lo > 0) { - /* Redundantly clears bits < "hi". */ - rv->popmap[i] &= ~((1UL << lo) - 1); - rv->popcnt -= lo - hi; - } - begin_zeroes = NBPOPMAP * i + lo; - /* Find the next 1 bit. */ - do - hi = ffsl(rv->popmap[i]); - while (hi == 0 && ++i < NPOPMAP); - if (i != NPOPMAP) - /* Convert from ffsl() to ordinary bit numbering. */ - hi--; - vm_domain_free_lock(VM_DOMAIN(rv->domain)); - vm_phys_free_contig(&rv->pages[begin_zeroes], NBPOPMAP * i + - hi - begin_zeroes); - vm_domain_free_unlock(VM_DOMAIN(rv->domain)); - } while (i < NPOPMAP); - KASSERT(rv->popcnt == 0, - ("vm_reserv_break: reserv %p's popcnt is corrupted", rv)); + } + rv->popcnt = 0; counter_u64_add(vm_reserv_broken, 1); } @@ -1289,7 +1271,8 @@ { vm_paddr_t pa, size; vm_reserv_t rv, rvn; - int hi, i, lo, low_index, next_free; + u_long changes; + int bitpos, hi, i, lo, low_index; if (npages > VM_LEVEL_0_NPAGES - 1) return (FALSE); @@ -1323,26 +1306,40 @@ /* Start the search for free pages at "low". */ low_index = (low + PAGE_MASK - pa) >> PAGE_SHIFT; i = low_index / NBPOPMAP; - hi = low_index % NBPOPMAP; - } else - i = hi = 0; + bitpos = low_index % NBPOPMAP; + changes = rv->popmap[i] | ((1UL << bitpos) - 1); + } else { + i = 0; + changes = rv->popmap[i]; + } + lo = hi = 0; + changes ^= (changes << 1) | 1; do { - /* Find the next free page. */ - lo = ffsl(~(((1UL << hi) - 1) | rv->popmap[i])); - while (lo == 0 && ++i < NPOPMAP) - lo = ffsl(~rv->popmap[i]); - if (i == NPOPMAP) - break; - /* Convert from ffsl() to ordinary bit numbering. */ - lo--; - next_free = NBPOPMAP * i + lo; - pa = VM_PAGE_TO_PHYS(&rv->pages[next_free]); + if (lo >= hi) { + while (changes == 0 && ++i < NPOPMAP) { + changes = rv->popmap[i]; + changes ^= (changes << 1) | (lo == hi); + } + if (i == NPOPMAP) + bitpos = 0; + else { + bitpos = ffsl(changes) - 1; + changes ^= 1UL << bitpos; + } + if (lo == hi) { + lo = NBPOPMAP * i + bitpos; + continue; + } + hi = NBPOPMAP * i + bitpos; + } + pa = VM_PAGE_TO_PHYS(&rv->pages[lo]); KASSERT(pa >= low, ("vm_reserv_reclaim_contig: pa is too low")); if (pa + size > high) { /* The rest of this reservation is too high. */ break; - } else if ((pa & (alignment - 1)) != 0 || + } + if ((pa & (alignment - 1)) != 0 || ((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) { /* * The current page doesn't meet the alignment @@ -1351,33 +1348,16 @@ * of its free pages are either excluded or * exhausted. */ - hi = lo + 1; - if (hi >= NBPOPMAP) { - hi = 0; - i++; - } + ++lo; continue; } - /* Find the next used page. */ - hi = ffsl(rv->popmap[i] & ~((1UL << lo) - 1)); - while (hi == 0 && ++i < NPOPMAP) { - if ((NBPOPMAP * i - next_free) * PAGE_SIZE >= - size) { - vm_reserv_reclaim(rv); - vm_reserv_unlock(rv); - return (TRUE); - } - hi = ffsl(rv->popmap[i]); + if ((hi - lo) * PAGE_SIZE < size) { + lo = hi; + continue; } - /* Convert from ffsl() to ordinary bit numbering. */ - if (i != NPOPMAP) - hi--; - if ((NBPOPMAP * i + hi - next_free) * PAGE_SIZE >= - size) { - vm_reserv_reclaim(rv); - vm_reserv_unlock(rv); - return (TRUE); - } + vm_reserv_reclaim(rv); + vm_reserv_unlock(rv); + return (TRUE); } while (i < NPOPMAP); vm_reserv_unlock(rv); vm_reserv_domain_lock(domain);