Page MenuHomeFreeBSD
Files F107239507

D33312.id99617.diff
No OneTemporary
Actions

D33312.id99617.diff
View Options

Index: sys/vm/vm_reserv.c
===================================================================
--- sys/vm/vm_reserv.c
+++ sys/vm/vm_reserv.c
@@ -53,6 +53,7 @@
#include <sys/sbuf.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
+#include <sys/bitstring.h>
#include <sys/counter.h>
#include <sys/ktr.h>
#include <sys/vmmeter.h>
@@ -105,22 +106,6 @@
#define VM_RESERV_INDEX(object, pindex) \
(((object)->pg_color + (pindex)) & (VM_LEVEL_0_NPAGES - 1))
-/*
- * The size of a population map entry
- */
-typedef u_long popmap_t;
-
-/*
- * The number of bits in a population map entry
- */
-#define NBPOPMAP (NBBY * sizeof(popmap_t))
-
-/*
- * The number of population map entries in a reservation
- */
-#define NPOPMAP howmany(VM_LEVEL_0_NPAGES, NBPOPMAP)
-#define NPOPMAP_MAX howmany(VM_LEVEL_0_NPAGES_MAX, NBPOPMAP)
-
/*
* Number of elapsed ticks before we update the LRU queue position. Used
* to reduce contention and churn on the list.
@@ -128,43 +113,15 @@
#define PARTPOPSLOP 1
/*
- * Clear a bit in the population map.
- */
-static __inline void
-popmap_clear(popmap_t popmap[], int i)
-{
-
- popmap[i / NBPOPMAP] &= ~(1UL << (i % NBPOPMAP));
-}
-
-/*
- * Set a bit in the population map.
+ * Return the first position of a set bit, or -1.
*/
-static __inline void
-popmap_set(popmap_t popmap[], int i)
+static inline int
+bit_ffs_val(bitstr_t *bitstr, int nbits)
{
+ int result;
- popmap[i / NBPOPMAP] |= 1UL << (i % NBPOPMAP);
-}
-
-/*
- * Is a bit in the population map clear?
- */
-static __inline boolean_t
-popmap_is_clear(popmap_t popmap[], int i)
-{
-
- return ((popmap[i / NBPOPMAP] & (1UL << (i % NBPOPMAP))) == 0);
-}
-
-/*
- * Is a bit in the population map set?
- */
-static __inline boolean_t
-popmap_is_set(popmap_t popmap[], int i)
-{
-
- return ((popmap[i / NBPOPMAP] & (1UL << (i % NBPOPMAP))) != 0);
+ bit_ffs(bitstr, nbits, &result);
+ return (result);
}
/*
@@ -198,7 +155,8 @@
uint8_t domain; /* (c) NUMA domain. */
char inpartpopq; /* (d, r) */
int lasttick; /* (r) last pop update tick. */
- popmap_t popmap[NPOPMAP_MAX]; /* (r) bit vector, used pages */
+ bitstr_t bit_decl(popmap, VM_LEVEL_0_NPAGES_MAX);
+ /* (r) bit vector, used pages */
};
TAILQ_HEAD(vm_reserv_queue, vm_reserv);
@@ -414,7 +372,6 @@
static void
vm_reserv_insert(vm_reserv_t rv, vm_object_t object, vm_pindex_t pindex)
{
- int i;
vm_reserv_assert_locked(rv);
CTR6(KTR_VM,
@@ -427,9 +384,8 @@
("vm_reserv_insert: reserv %p's popcnt is corrupted", rv));
KASSERT(!rv->inpartpopq,
("vm_reserv_insert: reserv %p's inpartpopq is TRUE", rv));
- for (i = 0; i < NPOPMAP; i++)
- KASSERT(rv->popmap[i] == 0,
- ("vm_reserv_insert: reserv %p's popmap is corrupted", rv));
+ KASSERT(bit_ffs_val(rv->popmap, VM_LEVEL_0_NPAGES) == -1,
+ ("vm_reserv_insert: reserv %p's popmap is corrupted", rv));
vm_reserv_object_lock(object);
rv->pindex = pindex;
rv->object = object;
@@ -454,7 +410,7 @@
__FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
KASSERT(rv->object != NULL,
("vm_reserv_depopulate: reserv %p is free", rv));
- KASSERT(popmap_is_set(rv->popmap, index),
+ KASSERT(bit_test(rv->popmap, index),
("vm_reserv_depopulate: reserv %p's popmap[%d] is clear", rv,
index));
KASSERT(rv->popcnt > 0,
@@ -468,7 +424,7 @@
rv));
rv->pages->psind = 0;
}
- popmap_clear(rv->popmap, index);
+ bit_clear(rv->popmap, index);
rv->popcnt--;
if ((unsigned)(ticks - rv->lasttick) >= PARTPOPSLOP ||
rv->popcnt == 0) {
@@ -574,7 +530,7 @@
__FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
KASSERT(rv->object != NULL,
("vm_reserv_populate: reserv %p is free", rv));
- KASSERT(popmap_is_clear(rv->popmap, index),
+ KASSERT(!bit_test(rv->popmap, index),
("vm_reserv_populate: reserv %p's popmap[%d] is set", rv,
index));
KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES,
@@ -584,7 +540,7 @@
KASSERT(rv->domain < vm_ndomains,
("vm_reserv_populate: reserv %p's domain is corrupted %d",
rv, rv->domain));
- popmap_set(rv->popmap, index);
+ bit_set(rv->popmap, index);
rv->popcnt++;
if ((unsigned)(ticks - rv->lasttick) < PARTPOPSLOP &&
rv->inpartpopq && rv->popcnt != VM_LEVEL_0_NPAGES)
@@ -686,9 +642,9 @@
((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0)
goto out;
/* Handle vm_page_rename(m, new_object, ...). */
- for (i = 0; i < npages; i++)
- if (popmap_is_set(rv->popmap, index + i))
- goto out;
+ bit_ffs_at(rv->popmap, index, index + npages, &i);
+ if (i != -1)
+ goto out;
if (!vm_domain_allocate(vmd, req, npages))
goto out;
for (i = 0; i < npages; i++)
@@ -856,7 +812,7 @@
/* Handle reclaim race. */
if (rv->object != object ||
/* Handle vm_page_rename(m, new_object, ...). */
- popmap_is_set(rv->popmap, index)) {
+ bit_test(rv->popmap, index)) {
m = NULL;
goto out;
}
@@ -950,48 +906,23 @@
static void
vm_reserv_break(vm_reserv_t rv)
{
- u_long changes;
- int bitpos, hi, i, lo;
+ int hi, lo;
vm_reserv_assert_locked(rv);
CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
__FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
vm_reserv_remove(rv);
rv->pages->psind = 0;
- hi = lo = -1;
- for (i = 0; i <= NPOPMAP; i++) {
- /*
- * "changes" is a bitmask that marks where a new sequence of
- * 0s or 1s begins in popmap[i], with last bit in popmap[i-1]
- * considered to be 1 if and only if lo == hi. The bits of
- * popmap[-1] and popmap[NPOPMAP] are considered all 1s.
- */
- if (i == NPOPMAP)
- changes = lo != hi;
- else {
- changes = rv->popmap[i];
- changes ^= (changes << 1) | (lo == hi);
- rv->popmap[i] = 0;
- }
- while (changes != 0) {
- /*
- * If the next change marked begins a run of 0s, set
- * lo to mark that position. Otherwise set hi and
- * free pages from lo up to hi.
- */
- bitpos = ffsl(changes) - 1;
- changes ^= 1UL << bitpos;
- if (lo == hi)
- lo = NBPOPMAP * i + bitpos;
- else {
- hi = NBPOPMAP * i + bitpos;
- vm_domain_free_lock(VM_DOMAIN(rv->domain));
- vm_phys_enqueue_contig(&rv->pages[lo], hi - lo);
- vm_domain_free_unlock(VM_DOMAIN(rv->domain));
- lo = hi;
- }
- }
+ hi = 0;
+ while (bit_ffc_at(rv->popmap, hi, VM_LEVEL_0_NPAGES, &lo), lo != -1) {
+ bit_ffs_at(rv->popmap, lo, VM_LEVEL_0_NPAGES, &hi);
+ if (hi == -1)
+ hi = VM_LEVEL_0_NPAGES;
+ vm_domain_free_lock(VM_DOMAIN(rv->domain));
+ vm_phys_enqueue_contig(&rv->pages[lo], hi - lo);
+ vm_domain_free_unlock(VM_DOMAIN(rv->domain));
}
+ bit_nclear(rv->popmap, 0, VM_LEVEL_0_NPAGES - 1);
rv->popcnt = 0;
counter_u64_add(vm_reserv_broken, 1);
}
@@ -1070,7 +1001,7 @@
#ifdef VM_PHYSSEG_SPARSE
vm_pindex_t used;
#endif
- int i, j, segind;
+ int i, segind;
/*
* Initialize the reservation array. Specifically, initialize the
@@ -1112,8 +1043,7 @@
* partially populated reservation queues.
*/
rvd->marker.popcnt = VM_LEVEL_0_NPAGES;
- for (j = 0; j < VM_LEVEL_0_NPAGES; j++)
- popmap_set(rvd->marker.popmap, j);
+ bit_nset(rvd->marker.popmap, 0, VM_LEVEL_0_NPAGES - 1);
}
for (i = 0; i < VM_RESERV_OBJ_LOCK_COUNT; i++)
@@ -1133,7 +1063,7 @@
rv = vm_reserv_from_page(m);
if (rv->object == NULL)
return (false);
- return (popmap_is_clear(rv->popmap, m - rv->pages));
+ return (!bit_test(rv->popmap, m - rv->pages));
}
/*
@@ -1233,78 +1163,43 @@
/*
* Determine whether this reservation has free pages that satisfy the given
* request for contiguous physical memory. Start searching from the lower
- * bound, defined by low_index.
+ * bound, defined by 'low'.
*/
static bool
vm_reserv_test_contig(vm_reserv_t rv, u_long npages, vm_paddr_t low,
vm_paddr_t high, u_long alignment, vm_paddr_t boundary)
{
vm_paddr_t pa, size;
- u_long changes;
- int bitpos, bits_left, i, hi, lo, n;
+ int hi, lo, page_ubound;
vm_reserv_assert_locked(rv);
size = npages << PAGE_SHIFT;
pa = VM_PAGE_TO_PHYS(&rv->pages[0]);
- lo = (pa < low) ?
+ hi = (pa < low) ?
((low + PAGE_MASK - pa) >> PAGE_SHIFT) : 0;
- i = lo / NBPOPMAP;
- changes = rv->popmap[i] | ((1UL << (lo % NBPOPMAP)) - 1);
- hi = (pa + VM_LEVEL_0_SIZE > high) ?
+ page_ubound = (pa + VM_LEVEL_0_SIZE > high) ?
((high + PAGE_MASK - pa) >> PAGE_SHIFT) : VM_LEVEL_0_NPAGES;
- n = hi / NBPOPMAP;
- bits_left = hi % NBPOPMAP;
- hi = lo = -1;
- for (;;) {
- /*
- * "changes" is a bitmask that marks where a new sequence of
- * 0s or 1s begins in popmap[i], with last bit in popmap[i-1]
- * considered to be 1 if and only if lo == hi. The bits of
- * popmap[-1] and popmap[NPOPMAP] are considered all 1s.
- */
- changes ^= (changes << 1) | (lo == hi);
- while (changes != 0) {
- /*
- * If the next change marked begins a run of 0s, set
- * lo to mark that position. Otherwise set hi and
- * look for a satisfactory first page from lo up to hi.
- */
- bitpos = ffsl(changes) - 1;
- changes ^= 1UL << bitpos;
- if (lo == hi) {
- lo = NBPOPMAP * i + bitpos;
- continue;
- }
- hi = NBPOPMAP * i + bitpos;
+ while (bit_ffc_area_at(rv->popmap, hi, page_ubound, npages, &lo),
+ lo != -1) {
+ pa = VM_PAGE_TO_PHYS(&rv->pages[lo]);
+ if ((pa & (alignment - 1)) != 0) {
+ /* Skip to next aligned page. */
+ lo += (((pa - 1) | (alignment - 1)) + 1) >> PAGE_SHIFT;
+ if (lo + npages >= page_ubound)
+ return (false);
pa = VM_PAGE_TO_PHYS(&rv->pages[lo]);
- if ((pa & (alignment - 1)) != 0) {
- /* Skip to next aligned page. */
- lo += (((pa - 1) | (alignment - 1)) + 1) >>
- PAGE_SHIFT;
- if (lo >= VM_LEVEL_0_NPAGES)
- return (false);
- pa = VM_PAGE_TO_PHYS(&rv->pages[lo]);
- }
- if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) {
- /* Skip to next boundary-matching page. */
- lo += (((pa - 1) | (boundary - 1)) + 1) >>
- PAGE_SHIFT;
- if (lo >= VM_LEVEL_0_NPAGES)
- return (false);
- pa = VM_PAGE_TO_PHYS(&rv->pages[lo]);
- }
- if (lo * PAGE_SIZE + size <= hi * PAGE_SIZE)
- return (true);
- lo = hi;
}
- if (++i < n)
- changes = rv->popmap[i];
- else if (i == n)
- changes = bits_left == 0 ? -1UL :
- (rv->popmap[n] | (-1UL << bits_left));
- else
- return (false);
+ if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) {
+ /* Skip to next boundary-matching page. */
+ lo += (((pa - 1) | (boundary - 1)) + 1) >> PAGE_SHIFT;
+ if (lo + npages >= page_ubound)
+ return (false);
+ }
+ bit_ffs_at(rv->popmap, lo, npages, &hi);
+ if (hi == -1)
+ return (true);
}
+ return (false);
}
/*

File Metadata

Mime Type
text/plain
Expires
Sun, Jan 12, 5:41 PM (17 h, 52 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
15771723
Default Alt Text
D33312.id99617.diff (10 KB)

Event Timeline