Index: sys/vm/vm_reserv.c =================================================================== --- sys/vm/vm_reserv.c +++ sys/vm/vm_reserv.c @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -105,22 +106,6 @@ #define VM_RESERV_INDEX(object, pindex) \ (((object)->pg_color + (pindex)) & (VM_LEVEL_0_NPAGES - 1)) -/* - * The size of a population map entry - */ -typedef u_long popmap_t; - -/* - * The number of bits in a population map entry - */ -#define NBPOPMAP (NBBY * sizeof(popmap_t)) - -/* - * The number of population map entries in a reservation - */ -#define NPOPMAP howmany(VM_LEVEL_0_NPAGES, NBPOPMAP) -#define NPOPMAP_MAX howmany(VM_LEVEL_0_NPAGES_MAX, NBPOPMAP) - /* * Number of elapsed ticks before we update the LRU queue position. Used * to reduce contention and churn on the list. @@ -128,43 +113,15 @@ #define PARTPOPSLOP 1 /* - * Clear a bit in the population map. - */ -static __inline void -popmap_clear(popmap_t popmap[], int i) -{ - - popmap[i / NBPOPMAP] &= ~(1UL << (i % NBPOPMAP)); -} - -/* - * Set a bit in the population map. + * Return the first position of a set bit, or -1. */ -static __inline void -popmap_set(popmap_t popmap[], int i) +static inline int +bit_ffs_val(bitstr_t *bitstr, int nbits) { + int result; - popmap[i / NBPOPMAP] |= 1UL << (i % NBPOPMAP); -} - -/* - * Is a bit in the population map clear? - */ -static __inline boolean_t -popmap_is_clear(popmap_t popmap[], int i) -{ - - return ((popmap[i / NBPOPMAP] & (1UL << (i % NBPOPMAP))) == 0); -} - -/* - * Is a bit in the population map set? - */ -static __inline boolean_t -popmap_is_set(popmap_t popmap[], int i) -{ - - return ((popmap[i / NBPOPMAP] & (1UL << (i % NBPOPMAP))) != 0); + bit_ffs(bitstr, nbits, &result); + return (result); } /* @@ -198,7 +155,8 @@ uint8_t domain; /* (c) NUMA domain. */ char inpartpopq; /* (d, r) */ int lasttick; /* (r) last pop update tick. */ - popmap_t popmap[NPOPMAP_MAX]; /* (r) bit vector, used pages */ + bitstr_t bit_decl(popmap, VM_LEVEL_0_NPAGES_MAX); + /* (r) bit vector, used pages */ }; TAILQ_HEAD(vm_reserv_queue, vm_reserv); @@ -414,7 +372,6 @@ static void vm_reserv_insert(vm_reserv_t rv, vm_object_t object, vm_pindex_t pindex) { - int i; vm_reserv_assert_locked(rv); CTR6(KTR_VM, @@ -427,9 +384,8 @@ ("vm_reserv_insert: reserv %p's popcnt is corrupted", rv)); KASSERT(!rv->inpartpopq, ("vm_reserv_insert: reserv %p's inpartpopq is TRUE", rv)); - for (i = 0; i < NPOPMAP; i++) - KASSERT(rv->popmap[i] == 0, - ("vm_reserv_insert: reserv %p's popmap is corrupted", rv)); + KASSERT(bit_ffs_val(rv->popmap, VM_LEVEL_0_NPAGES) == -1, + ("vm_reserv_insert: reserv %p's popmap is corrupted", rv)); vm_reserv_object_lock(object); rv->pindex = pindex; rv->object = object; @@ -454,7 +410,7 @@ __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); KASSERT(rv->object != NULL, ("vm_reserv_depopulate: reserv %p is free", rv)); - KASSERT(popmap_is_set(rv->popmap, index), + KASSERT(bit_test(rv->popmap, index), ("vm_reserv_depopulate: reserv %p's popmap[%d] is clear", rv, index)); KASSERT(rv->popcnt > 0, @@ -468,7 +424,7 @@ rv)); rv->pages->psind = 0; } - popmap_clear(rv->popmap, index); + bit_clear(rv->popmap, index); rv->popcnt--; if ((unsigned)(ticks - rv->lasttick) >= PARTPOPSLOP || rv->popcnt == 0) { @@ -574,7 +530,7 @@ __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); KASSERT(rv->object != NULL, ("vm_reserv_populate: reserv %p is free", rv)); - KASSERT(popmap_is_clear(rv->popmap, index), + KASSERT(!bit_test(rv->popmap, index), ("vm_reserv_populate: reserv %p's popmap[%d] is set", rv, index)); KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES, @@ -584,7 +540,7 @@ KASSERT(rv->domain < vm_ndomains, ("vm_reserv_populate: reserv %p's domain is corrupted %d", rv, rv->domain)); - popmap_set(rv->popmap, index); + bit_set(rv->popmap, index); rv->popcnt++; if ((unsigned)(ticks - rv->lasttick) < PARTPOPSLOP && rv->inpartpopq && rv->popcnt != VM_LEVEL_0_NPAGES) @@ -686,9 +642,9 @@ ((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) goto out; /* Handle vm_page_rename(m, new_object, ...). */ - for (i = 0; i < npages; i++) - if (popmap_is_set(rv->popmap, index + i)) - goto out; + bit_ffs_at(rv->popmap, index, index + npages, &i); + if (i != -1) + goto out; if (!vm_domain_allocate(vmd, req, npages)) goto out; for (i = 0; i < npages; i++) @@ -856,7 +812,7 @@ /* Handle reclaim race. */ if (rv->object != object || /* Handle vm_page_rename(m, new_object, ...). */ - popmap_is_set(rv->popmap, index)) { + bit_test(rv->popmap, index)) { m = NULL; goto out; } @@ -950,48 +906,23 @@ static void vm_reserv_break(vm_reserv_t rv) { - u_long changes; - int bitpos, hi, i, lo; + int hi, lo; vm_reserv_assert_locked(rv); CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); vm_reserv_remove(rv); rv->pages->psind = 0; - hi = lo = -1; - for (i = 0; i <= NPOPMAP; i++) { - /* - * "changes" is a bitmask that marks where a new sequence of - * 0s or 1s begins in popmap[i], with last bit in popmap[i-1] - * considered to be 1 if and only if lo == hi. The bits of - * popmap[-1] and popmap[NPOPMAP] are considered all 1s. - */ - if (i == NPOPMAP) - changes = lo != hi; - else { - changes = rv->popmap[i]; - changes ^= (changes << 1) | (lo == hi); - rv->popmap[i] = 0; - } - while (changes != 0) { - /* - * If the next change marked begins a run of 0s, set - * lo to mark that position. Otherwise set hi and - * free pages from lo up to hi. - */ - bitpos = ffsl(changes) - 1; - changes ^= 1UL << bitpos; - if (lo == hi) - lo = NBPOPMAP * i + bitpos; - else { - hi = NBPOPMAP * i + bitpos; - vm_domain_free_lock(VM_DOMAIN(rv->domain)); - vm_phys_enqueue_contig(&rv->pages[lo], hi - lo); - vm_domain_free_unlock(VM_DOMAIN(rv->domain)); - lo = hi; - } - } + hi = 0; + while (bit_ffc_at(rv->popmap, hi, VM_LEVEL_0_NPAGES, &lo), lo != -1) { + bit_ffs_at(rv->popmap, lo, VM_LEVEL_0_NPAGES, &hi); + if (hi == -1) + hi = VM_LEVEL_0_NPAGES; + vm_domain_free_lock(VM_DOMAIN(rv->domain)); + vm_phys_enqueue_contig(&rv->pages[lo], hi - lo); + vm_domain_free_unlock(VM_DOMAIN(rv->domain)); } + bit_nclear(rv->popmap, 0, VM_LEVEL_0_NPAGES - 1); rv->popcnt = 0; counter_u64_add(vm_reserv_broken, 1); } @@ -1070,7 +1001,7 @@ #ifdef VM_PHYSSEG_SPARSE vm_pindex_t used; #endif - int i, j, segind; + int i, segind; /* * Initialize the reservation array. Specifically, initialize the @@ -1112,8 +1043,7 @@ * partially populated reservation queues. */ rvd->marker.popcnt = VM_LEVEL_0_NPAGES; - for (j = 0; j < VM_LEVEL_0_NPAGES; j++) - popmap_set(rvd->marker.popmap, j); + bit_nset(rvd->marker.popmap, 0, VM_LEVEL_0_NPAGES - 1); } for (i = 0; i < VM_RESERV_OBJ_LOCK_COUNT; i++) @@ -1133,7 +1063,7 @@ rv = vm_reserv_from_page(m); if (rv->object == NULL) return (false); - return (popmap_is_clear(rv->popmap, m - rv->pages)); + return (!bit_test(rv->popmap, m - rv->pages)); } /* @@ -1233,78 +1163,43 @@ /* * Determine whether this reservation has free pages that satisfy the given * request for contiguous physical memory. Start searching from the lower - * bound, defined by low_index. + * bound, defined by 'low'. */ static bool vm_reserv_test_contig(vm_reserv_t rv, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) { vm_paddr_t pa, size; - u_long changes; - int bitpos, bits_left, i, hi, lo, n; + int hi, lo, page_ubound; vm_reserv_assert_locked(rv); size = npages << PAGE_SHIFT; pa = VM_PAGE_TO_PHYS(&rv->pages[0]); - lo = (pa < low) ? + hi = (pa < low) ? ((low + PAGE_MASK - pa) >> PAGE_SHIFT) : 0; - i = lo / NBPOPMAP; - changes = rv->popmap[i] | ((1UL << (lo % NBPOPMAP)) - 1); - hi = (pa + VM_LEVEL_0_SIZE > high) ? + page_ubound = (pa + VM_LEVEL_0_SIZE > high) ? ((high + PAGE_MASK - pa) >> PAGE_SHIFT) : VM_LEVEL_0_NPAGES; - n = hi / NBPOPMAP; - bits_left = hi % NBPOPMAP; - hi = lo = -1; - for (;;) { - /* - * "changes" is a bitmask that marks where a new sequence of - * 0s or 1s begins in popmap[i], with last bit in popmap[i-1] - * considered to be 1 if and only if lo == hi. The bits of - * popmap[-1] and popmap[NPOPMAP] are considered all 1s. - */ - changes ^= (changes << 1) | (lo == hi); - while (changes != 0) { - /* - * If the next change marked begins a run of 0s, set - * lo to mark that position. Otherwise set hi and - * look for a satisfactory first page from lo up to hi. - */ - bitpos = ffsl(changes) - 1; - changes ^= 1UL << bitpos; - if (lo == hi) { - lo = NBPOPMAP * i + bitpos; - continue; - } - hi = NBPOPMAP * i + bitpos; + while (bit_ffc_area_at(rv->popmap, hi, page_ubound, npages, &lo), + lo != -1) { + pa = VM_PAGE_TO_PHYS(&rv->pages[lo]); + if ((pa & (alignment - 1)) != 0) { + /* Skip to next aligned page. */ + lo += (((pa - 1) | (alignment - 1)) + 1) >> PAGE_SHIFT; + if (lo + npages >= page_ubound) + return (false); pa = VM_PAGE_TO_PHYS(&rv->pages[lo]); - if ((pa & (alignment - 1)) != 0) { - /* Skip to next aligned page. */ - lo += (((pa - 1) | (alignment - 1)) + 1) >> - PAGE_SHIFT; - if (lo >= VM_LEVEL_0_NPAGES) - return (false); - pa = VM_PAGE_TO_PHYS(&rv->pages[lo]); - } - if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) { - /* Skip to next boundary-matching page. */ - lo += (((pa - 1) | (boundary - 1)) + 1) >> - PAGE_SHIFT; - if (lo >= VM_LEVEL_0_NPAGES) - return (false); - pa = VM_PAGE_TO_PHYS(&rv->pages[lo]); - } - if (lo * PAGE_SIZE + size <= hi * PAGE_SIZE) - return (true); - lo = hi; } - if (++i < n) - changes = rv->popmap[i]; - else if (i == n) - changes = bits_left == 0 ? -1UL : - (rv->popmap[n] | (-1UL << bits_left)); - else - return (false); + if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) { + /* Skip to next boundary-matching page. */ + lo += (((pa - 1) | (boundary - 1)) + 1) >> PAGE_SHIFT; + if (lo + npages >= page_ubound) + return (false); + } + bit_ffs_at(rv->popmap, lo, npages, &hi); + if (hi == -1) + return (true); } + return (false); } /*