Index: vm_reserv.c =================================================================== --- vm_reserv.c +++ vm_reserv.c @@ -138,16 +138,46 @@ } /* - * Set a bit in the population map. + * Set all bits in the population map range. */ static __inline void -popmap_set(popmap_t popmap[], int i) +popmap_range_set(popmap_t popmap[], int start, int count) { + popmap_t mask; + int end, i; - popmap[i / NBPOPMAP] |= 1UL << (i % NBPOPMAP); + end = start + count; + for (i = start / NBPOPMAP, mask = -1UL << (start % NBPOPMAP); + i < end / NBPOPMAP; i++, mask = -1UL) + popmap[i] |= mask; + if (end % NBPOPMAP == 0) + return; + mask &= ~(-1UL << (end % NBPOPMAP)); + popmap[i] |= mask; } /* + * Are all bits in the population map range clear? + */ +static __inline boolean_t +popmap_range_is_clear(popmap_t popmap[], int start, int count) +{ + popmap_t mask; + int end, i; + + end = start + count; + for (i = start / NBPOPMAP, mask = -1UL << (start % NBPOPMAP); + i < end / NBPOPMAP && (mask & popmap[i]) == 0; + i++, mask = -1UL); + if (i < end / NBPOPMAP) + return (false); + if (end % NBPOPMAP == 0) + return (true); + mask &= ~(-1UL << (end % NBPOPMAP)); + return ((mask & popmap[i]) == 0); +} + +/* * Is a bit in the population map clear? */ static __inline boolean_t @@ -295,7 +325,7 @@ static vm_reserv_t vm_reserv_from_page(vm_page_t m); static boolean_t vm_reserv_has_pindex(vm_reserv_t rv, vm_pindex_t pindex); -static void vm_reserv_populate(vm_reserv_t rv, int index); +static void vm_reserv_populate(vm_reserv_t rv, int index, int num); static void vm_reserv_reclaim(vm_reserv_t rv); /* @@ -534,7 +564,7 @@ * The free page queue must be locked. */ static void -vm_reserv_populate(vm_reserv_t rv, int index) +vm_reserv_populate(vm_reserv_t rv, int start, int count) { vm_reserv_assert_locked(rv); @@ -542,9 +572,9 @@ __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); KASSERT(rv->object != NULL, ("vm_reserv_populate: reserv %p is free", rv)); - KASSERT(popmap_is_clear(rv->popmap, index), - ("vm_reserv_populate: reserv %p's popmap[%d] is set", rv, - index)); + KASSERT(popmap_range_is_clear(rv->popmap, start, count), + ("vm_reserv_populate: reserv %p's popmap[%d..%d) is not clear", rv, + start, start + count)); KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES, ("vm_reserv_populate: reserv %p is already full", rv)); KASSERT(rv->pages->psind == 0, @@ -552,8 +582,8 @@ KASSERT(rv->domain < vm_ndomains, ("vm_reserv_populate: reserv %p's domain is corrupted %d", rv, rv->domain)); - popmap_set(rv->popmap, index); - rv->popcnt++; + popmap_range_set(rv->popmap, start, count); + rv->popcnt += count; if ((unsigned)(ticks - rv->lasttick) < PARTPOPSLOP && rv->inpartpopq && rv->popcnt != VM_LEVEL_0_NPAGES) return; @@ -594,7 +624,7 @@ vm_paddr_t pa, size; vm_page_t m, msucc; vm_reserv_t rv; - int i, index; + int index; VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(npages != 0, ("vm_reserv_alloc_contig: npages is 0")); @@ -639,27 +669,21 @@ vmd = VM_DOMAIN(domain); vm_reserv_lock(rv); if (rv->object != object) - goto out; - m = &rv->pages[index]; - pa = VM_PAGE_TO_PHYS(m); - if (pa < low || pa + size > high || (pa & (alignment - 1)) != 0 || - ((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) - goto out; - /* Handle vm_page_rename(m, new_object, ...). */ - for (i = 0; i < npages; i++) { - if (popmap_is_set(rv->popmap, index + i)) - goto out; + m = NULL; + else { + m = &rv->pages[index]; + pa = VM_PAGE_TO_PHYS(m); + if (pa >= low && pa + size <= high && + (pa & (alignment - 1)) == 0 && + ((pa ^ (pa + size - 1)) & ~(boundary - 1)) == 0 && + popmap_range_is_clear(rv->popmap, index, npages) && + vm_domain_allocate(vmd, req, npages)) + vm_reserv_populate(rv, index, npages); + else + m = NULL; } - if (!vm_domain_allocate(vmd, req, npages)) - goto out; - for (i = 0; i < npages; i++) - vm_reserv_populate(rv, index + i); vm_reserv_unlock(rv); return (m); - -out: - vm_reserv_unlock(rv); - return (NULL); } /* @@ -691,7 +715,7 @@ vm_pindex_t first, leftcap, rightcap; vm_reserv_t rv; u_long allocpages, maxpages, minpages; - int i, index, n; + int index, n; VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(npages != 0, ("vm_reserv_alloc_contig: npages is 0")); @@ -833,8 +857,7 @@ vm_reserv_lock(rv); vm_reserv_insert(rv, object, first); n = ulmin(VM_LEVEL_0_NPAGES - index, npages); - for (i = 0; i < n; i++) - vm_reserv_populate(rv, index + i); + vm_reserv_populate(rv, index, n); npages -= n; if (m_ret == NULL) { m_ret = &rv->pages[index]; @@ -899,7 +922,7 @@ if (vm_domain_allocate(vmd, req, 1) == 0) m = NULL; else - vm_reserv_populate(rv, index); + vm_reserv_populate(rv, index, 1); out: vm_reserv_unlock(rv); @@ -1013,7 +1036,7 @@ ("vm_reserv_alloc_page: reserv %p's pages is corrupted", rv)); vm_reserv_insert(rv, object, first); index = VM_RESERV_INDEX(object, pindex); - vm_reserv_populate(rv, index); + vm_reserv_populate(rv, index, 1); vm_reserv_unlock(rv); return (&rv->pages[index]);