Page MenuHomeFreeBSD
Differential D33312 Diff 101306 sys/vm/vm_reserv.c

Changeset View

Standalone View

View Options

sys/vm/vm_reserv.c

Show First 20 Lines • Show All 47 Lines • ▼ Show 20 Lines
#include <sys/lock.h> #include <sys/lock.h>
#include <sys/malloc.h> #include <sys/malloc.h>
#include <sys/mutex.h> #include <sys/mutex.h>
#include <sys/queue.h> #include <sys/queue.h>
#include <sys/rwlock.h> #include <sys/rwlock.h>
#include <sys/sbuf.h> #include <sys/sbuf.h>
#include <sys/sysctl.h> #include <sys/sysctl.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/bitstring.h>
#include <sys/counter.h> #include <sys/counter.h>
#include <sys/ktr.h> #include <sys/ktr.h>
#include <sys/vmmeter.h> #include <sys/vmmeter.h>
#include <sys/smp.h> #include <sys/smp.h>
#include <vm/vm.h> #include <vm/vm.h>
#include <vm/vm_extern.h> #include <vm/vm_extern.h>
#include <vm/vm_param.h> #include <vm/vm_param.h>
Show All 38 Lines
/* /*
* Computes the index of the small page underlying the given (object, pindex) * Computes the index of the small page underlying the given (object, pindex)
* within the reservation's array of small pages. * within the reservation's array of small pages.
*/ */
#define VM_RESERV_INDEX(object, pindex) \ #define VM_RESERV_INDEX(object, pindex) \
(((object)->pg_color + (pindex)) & (VM_LEVEL_0_NPAGES - 1)) (((object)->pg_color + (pindex)) & (VM_LEVEL_0_NPAGES - 1))
/* /*
* The size of a population map entry
*/
typedef u_long popmap_t;
/*
* The number of bits in a population map entry
*/
#define NBPOPMAP (NBBY * sizeof(popmap_t))
/*
* The number of population map entries in a reservation
*/
#define NPOPMAP howmany(VM_LEVEL_0_NPAGES, NBPOPMAP)
#define NPOPMAP_MAX howmany(VM_LEVEL_0_NPAGES_MAX, NBPOPMAP)
/*
* Number of elapsed ticks before we update the LRU queue position. Used * Number of elapsed ticks before we update the LRU queue position. Used
* to reduce contention and churn on the list. * to reduce contention and churn on the list.
*/ */
#define PARTPOPSLOP 1 #define PARTPOPSLOP 1
/* /*
* Clear a bit in the population map.
*/
static __inline void
popmap_clear(popmap_t popmap[], int i)
{
popmap[i / NBPOPMAP] &= ~(1UL << (i % NBPOPMAP));
}
/*
* Set a bit in the population map.
*/
static __inline void
popmap_set(popmap_t popmap[], int i)
{
popmap[i / NBPOPMAP] |= 1UL << (i % NBPOPMAP);
}
/*
* Is a bit in the population map clear?
*/
static __inline boolean_t
popmap_is_clear(popmap_t popmap[], int i)
{
return ((popmap[i / NBPOPMAP] & (1UL << (i % NBPOPMAP))) == 0);
}
/*
* Is a bit in the population map set?
*/
static __inline boolean_t
popmap_is_set(popmap_t popmap[], int i)
{
return ((popmap[i / NBPOPMAP] & (1UL << (i % NBPOPMAP))) != 0);
}
/*
* The reservation structure * The reservation structure
* *
* A reservation structure is constructed whenever a large physical page is * A reservation structure is constructed whenever a large physical page is
* speculatively allocated to an object. The reservation provides the small * speculatively allocated to an object. The reservation provides the small
* physical pages for the range [pindex, pindex + VM_LEVEL_0_NPAGES) of offsets * physical pages for the range [pindex, pindex + VM_LEVEL_0_NPAGES) of offsets
* within that object. The reservation's "popcnt" tracks the number of these * within that object. The reservation's "popcnt" tracks the number of these
* small physical pages that are in use at any given time. When and if the * small physical pages that are in use at any given time. When and if the
* reservation is not fully utilized, it appears in the queue of partially * reservation is not fully utilized, it appears in the queue of partially
Show All 14 Linesstruct vm_reserv {
LIST_ENTRY(vm_reserv) objq; /* (o, r) object queue */ LIST_ENTRY(vm_reserv) objq; /* (o, r) object queue */
vm_object_t object; /* (o, r) containing object */ vm_object_t object; /* (o, r) containing object */
vm_pindex_t pindex; /* (o, r) offset in object */ vm_pindex_t pindex; /* (o, r) offset in object */
vm_page_t pages; /* (c) first page */ vm_page_t pages; /* (c) first page */
uint16_t popcnt; /* (r) # of pages in use */ uint16_t popcnt; /* (r) # of pages in use */
uint8_t domain; /* (c) NUMA domain. */ uint8_t domain; /* (c) NUMA domain. */
char inpartpopq; /* (d, r) */ char inpartpopq; /* (d, r) */
int lasttick; /* (r) last pop update tick. */ int lasttick; /* (r) last pop update tick. */
popmap_t popmap[NPOPMAP_MAX]; /* (r) bit vector, used pages */ bitstr_t bit_decl(popmap, VM_LEVEL_0_NPAGES_MAX);
/* (r) bit vector, used pages */
}; };
TAILQ_HEAD(vm_reserv_queue, vm_reserv); TAILQ_HEAD(vm_reserv_queue, vm_reserv);
#define vm_reserv_lockptr(rv) (&(rv)->lock) #define vm_reserv_lockptr(rv) (&(rv)->lock)
#define vm_reserv_assert_locked(rv) \ #define vm_reserv_assert_locked(rv) \
mtx_assert(vm_reserv_lockptr(rv), MA_OWNED) mtx_assert(vm_reserv_lockptr(rv), MA_OWNED)
#define vm_reserv_lock(rv) mtx_lock(vm_reserv_lockptr(rv)) #define vm_reserv_lock(rv) mtx_lock(vm_reserv_lockptr(rv))
▲ Show 20 Lines • Show All 199 Lines • ▼ Show 20 Lines
} }
/* /*
* Insert a new reservation into the object's objq. * Insert a new reservation into the object's objq.
*/ */
static void static void
vm_reserv_insert(vm_reserv_t rv, vm_object_t object, vm_pindex_t pindex) vm_reserv_insert(vm_reserv_t rv, vm_object_t object, vm_pindex_t pindex)
{ {
int i;
vm_reserv_assert_locked(rv); vm_reserv_assert_locked(rv);
CTR6(KTR_VM, CTR6(KTR_VM,
"%s: rv %p(%p) object %p new %p popcnt %d", "%s: rv %p(%p) object %p new %p popcnt %d",
__FUNCTION__, rv, rv->pages, rv->object, object, __FUNCTION__, rv, rv->pages, rv->object, object,
rv->popcnt); rv->popcnt);
KASSERT(rv->object == NULL, KASSERT(rv->object == NULL,
("vm_reserv_insert: reserv %p isn't free", rv)); ("vm_reserv_insert: reserv %p isn't free", rv));
KASSERT(rv->popcnt == 0, KASSERT(rv->popcnt == 0,
("vm_reserv_insert: reserv %p's popcnt is corrupted", rv)); ("vm_reserv_insert: reserv %p's popcnt is corrupted", rv));
KASSERT(!rv->inpartpopq, KASSERT(!rv->inpartpopq,
("vm_reserv_insert: reserv %p's inpartpopq is TRUE", rv)); ("vm_reserv_insert: reserv %p's inpartpopq is TRUE", rv));
for (i = 0; i < NPOPMAP; i++) KASSERT(bit_ntest(rv->popmap, 0, VM_LEVEL_0_NPAGES - 1, 0),
KASSERT(rv->popmap[i] == 0,
("vm_reserv_insert: reserv %p's popmap is corrupted", rv)); ("vm_reserv_insert: reserv %p's popmap is corrupted", rv));
vm_reserv_object_lock(object); vm_reserv_object_lock(object);
rv->pindex = pindex; rv->pindex = pindex;
rv->object = object; rv->object = object;
rv->lasttick = ticks; rv->lasttick = ticks;
LIST_INSERT_HEAD(&object->rvq, rv, objq); LIST_INSERT_HEAD(&object->rvq, rv, objq);
vm_reserv_object_unlock(object); vm_reserv_object_unlock(object);
} }
/* /*
* Reduces the given reservation's population count. If the population count * Reduces the given reservation's population count. If the population count
* becomes zero, the reservation is destroyed. Additionally, moves the * becomes zero, the reservation is destroyed. Additionally, moves the
* reservation to the tail of the partially populated reservation queue if the * reservation to the tail of the partially populated reservation queue if the
* population count is non-zero. * population count is non-zero.
*/ */
static void static void
vm_reserv_depopulate(vm_reserv_t rv, int index) vm_reserv_depopulate(vm_reserv_t rv, int index)
{ {
struct vm_domain *vmd; struct vm_domain *vmd;
vm_reserv_assert_locked(rv); vm_reserv_assert_locked(rv);
CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
__FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
KASSERT(rv->object != NULL, KASSERT(rv->object != NULL,
("vm_reserv_depopulate: reserv %p is free", rv)); ("vm_reserv_depopulate: reserv %p is free", rv));
KASSERT(popmap_is_set(rv->popmap, index), KASSERT(bit_test(rv->popmap, index),
("vm_reserv_depopulate: reserv %p's popmap[%d] is clear", rv, ("vm_reserv_depopulate: reserv %p's popmap[%d] is clear", rv,
index)); index));
KASSERT(rv->popcnt > 0, KASSERT(rv->popcnt > 0,
("vm_reserv_depopulate: reserv %p's popcnt is corrupted", rv)); ("vm_reserv_depopulate: reserv %p's popcnt is corrupted", rv));
KASSERT(rv->domain < vm_ndomains, KASSERT(rv->domain < vm_ndomains,
("vm_reserv_depopulate: reserv %p's domain is corrupted %d", ("vm_reserv_depopulate: reserv %p's domain is corrupted %d",
rv, rv->domain)); rv, rv->domain));
if (rv->popcnt == VM_LEVEL_0_NPAGES) { if (rv->popcnt == VM_LEVEL_0_NPAGES) {
KASSERT(rv->pages->psind == 1, KASSERT(rv->pages->psind == 1,
("vm_reserv_depopulate: reserv %p is already demoted", ("vm_reserv_depopulate: reserv %p is already demoted",
rv)); rv));
rv->pages->psind = 0; rv->pages->psind = 0;
} }
popmap_clear(rv->popmap, index); bit_clear(rv->popmap, index);
rv->popcnt--; rv->popcnt--;
if ((unsigned)(ticks - rv->lasttick) >= PARTPOPSLOP || if ((unsigned)(ticks - rv->lasttick) >= PARTPOPSLOP ||
rv->popcnt == 0) { rv->popcnt == 0) {
vm_reserv_domain_lock(rv->domain); vm_reserv_domain_lock(rv->domain);
if (rv->inpartpopq) { if (rv->inpartpopq) {
TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq); TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq);
rv->inpartpopq = FALSE; rv->inpartpopq = FALSE;
} }
▲ Show 20 Lines • Show All 89 Lines • ▼ Show 20 Lines
vm_reserv_populate(vm_reserv_t rv, int index) vm_reserv_populate(vm_reserv_t rv, int index)
{ {
vm_reserv_assert_locked(rv); vm_reserv_assert_locked(rv);
CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
__FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
KASSERT(rv->object != NULL, KASSERT(rv->object != NULL,
("vm_reserv_populate: reserv %p is free", rv)); ("vm_reserv_populate: reserv %p is free", rv));
KASSERT(popmap_is_clear(rv->popmap, index), KASSERT(!bit_test(rv->popmap, index),
("vm_reserv_populate: reserv %p's popmap[%d] is set", rv, ("vm_reserv_populate: reserv %p's popmap[%d] is set", rv,
index)); index));
KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES, KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES,
("vm_reserv_populate: reserv %p is already full", rv)); ("vm_reserv_populate: reserv %p is already full", rv));
KASSERT(rv->pages->psind == 0, KASSERT(rv->pages->psind == 0,
("vm_reserv_populate: reserv %p is already promoted", rv)); ("vm_reserv_populate: reserv %p is already promoted", rv));
KASSERT(rv->domain < vm_ndomains, KASSERT(rv->domain < vm_ndomains,
("vm_reserv_populate: reserv %p's domain is corrupted %d", ("vm_reserv_populate: reserv %p's domain is corrupted %d",
rv, rv->domain)); rv, rv->domain));
popmap_set(rv->popmap, index); bit_set(rv->popmap, index);
rv->popcnt++; rv->popcnt++;
if ((unsigned)(ticks - rv->lasttick) < PARTPOPSLOP && if ((unsigned)(ticks - rv->lasttick) < PARTPOPSLOP &&
rv->inpartpopq && rv->popcnt != VM_LEVEL_0_NPAGES) rv->inpartpopq && rv->popcnt != VM_LEVEL_0_NPAGES)
return; return;
rv->lasttick = ticks; rv->lasttick = ticks;
vm_reserv_domain_lock(rv->domain); vm_reserv_domain_lock(rv->domain);
if (rv->inpartpopq) { if (rv->inpartpopq) {
TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq); TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq);
▲ Show 20 Lines • Show All 82 Lines • ▼ Show 20 Lines if (rv != NULL) {
if (rv->object != object) if (rv->object != object)
goto out; goto out;
m = &rv->pages[index]; m = &rv->pages[index];
pa = VM_PAGE_TO_PHYS(m); pa = VM_PAGE_TO_PHYS(m);
if (pa < low || pa + size > high || if (pa < low || pa + size > high ||
!vm_addr_ok(pa, size, alignment, boundary)) !vm_addr_ok(pa, size, alignment, boundary))
goto out; goto out;
/* Handle vm_page_rename(m, new_object, ...). */ /* Handle vm_page_rename(m, new_object, ...). */
for (i = 0; i < npages; i++) if (!bit_ntest(rv->popmap, index, index + npages - 1, 0))
if (popmap_is_set(rv->popmap, index + i))
goto out; goto out;
markjUnsubmitted
Done
Inline Actions

Shouldn't the third parameter be npages?

markj: Shouldn't the third parameter be `npages`?
dougmAuthorUnsubmitted
Done
Inline Actions

Yes.

dougm: Yes.
dougmAuthorUnsubmitted
Done
Inline Actions

No, on second thought. This is supposed to test npages bits, from index to index + npages -1. The first page not to be checked is at index + npages.

I'm changing it back.

dougm: No, on second thought. This is supposed to test npages bits, from index to index + npages -1.
dougmAuthorUnsubmitted
Done
Inline Actions

bit_ffs_at(str, start, nbits, result) examines up to nbits-start bits in a string of total length nbits. We want to examine up to npages bits, so nbits-start must be npages. With start and nbits as index and index+npages, we get that. The total length of popmap is likely more than index+npages, but we don't need to look at all the bits to the end of the map.

I can just write an ntest function for bitstring.h that would have parameters like those of nset and nclear, if that would make this clearer.

dougm: bit_ffs_at(str, start, nbits, result) examines up to nbits-start bits in a string of total…
markjUnsubmitted
Not Done
Inline Actions

I see now. bit_ntest() seems like a sensible addition. The code is clear to me now for what it's worth, I was just unfamiliar with the bitstring.h functions.

markj: I see now. bit_ntest() seems like a sensible addition. The code is clear to me now for what…
if (!vm_domain_allocate(vmd, req, npages)) if (!vm_domain_allocate(vmd, req, npages))
goto out; goto out;
for (i = 0; i < npages; i++) for (i = 0; i < npages; i++)
vm_reserv_populate(rv, index + i); vm_reserv_populate(rv, index + i);
vm_reserv_unlock(rv); vm_reserv_unlock(rv);
return (m); return (m);
out: out:
vm_reserv_unlock(rv); vm_reserv_unlock(rv);
▲ Show 20 Lines • Show All 151 Lines • ▼ Show 20 Lines if (rv != NULL) {
domain = rv->domain; domain = rv->domain;
vmd = VM_DOMAIN(domain); vmd = VM_DOMAIN(domain);
index = VM_RESERV_INDEX(object, pindex); index = VM_RESERV_INDEX(object, pindex);
m = &rv->pages[index]; m = &rv->pages[index];
vm_reserv_lock(rv); vm_reserv_lock(rv);
/* Handle reclaim race. */ /* Handle reclaim race. */
if (rv->object != object || if (rv->object != object ||
/* Handle vm_page_rename(m, new_object, ...). */ /* Handle vm_page_rename(m, new_object, ...). */
popmap_is_set(rv->popmap, index)) { bit_test(rv->popmap, index)) {
m = NULL; m = NULL;
goto out; goto out;
} }
if (vm_domain_allocate(vmd, req, 1) == 0) if (vm_domain_allocate(vmd, req, 1) == 0)
m = NULL; m = NULL;
else else
vm_reserv_populate(rv, index); vm_reserv_populate(rv, index);
out: out:
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines
* population count and map are reset to their initial state. * population count and map are reset to their initial state.
* *
* The given reservation must not be in the partially populated reservation * The given reservation must not be in the partially populated reservation
* queue. * queue.
*/ */
static void static void
vm_reserv_break(vm_reserv_t rv) vm_reserv_break(vm_reserv_t rv)
{ {
u_long changes; int hi, lo, pos;
int bitpos, hi, i, lo;
vm_reserv_assert_locked(rv); vm_reserv_assert_locked(rv);
CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d",
__FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq);
vm_reserv_remove(rv); vm_reserv_remove(rv);
rv->pages->psind = 0; rv->pages->psind = 0;
hi = lo = -1; hi = lo = -1;
for (i = 0; i <= NPOPMAP; i++) { pos = 0;
/* for (;;) {
* "changes" is a bitmask that marks where a new sequence of bit_ff_at(rv->popmap, pos, VM_LEVEL_0_NPAGES, lo != hi, &pos);
* 0s or 1s begins in popmap[i], with last bit in popmap[i-1] if (lo == hi) {
* considered to be 1 if and only if lo == hi. The bits of if (pos == -1)
* popmap[-1] and popmap[NPOPMAP] are considered all 1s. break;
*/ lo = pos;
if (i == NPOPMAP) continue;
changes = lo != hi;
else {
changes = rv->popmap[i];
changes ^= (changes << 1) | (lo == hi);
rv->popmap[i] = 0;
} }
while (changes != 0) { if (pos == -1)
/* pos = VM_LEVEL_0_NPAGES;
* If the next change marked begins a run of 0s, set hi = pos;
* lo to mark that position. Otherwise set hi and
* free pages from lo up to hi.
*/
bitpos = ffsl(changes) - 1;
changes ^= 1UL << bitpos;
if (lo == hi)
lo = NBPOPMAP * i + bitpos;
else {
hi = NBPOPMAP * i + bitpos;
vm_domain_free_lock(VM_DOMAIN(rv->domain)); vm_domain_free_lock(VM_DOMAIN(rv->domain));
vm_phys_enqueue_contig(&rv->pages[lo], hi - lo); vm_phys_enqueue_contig(&rv->pages[lo], hi - lo);
vm_domain_free_unlock(VM_DOMAIN(rv->domain)); vm_domain_free_unlock(VM_DOMAIN(rv->domain));
lo = hi; lo = hi;
} }
} bit_nclear(rv->popmap, 0, VM_LEVEL_0_NPAGES - 1);
}
rv->popcnt = 0; rv->popcnt = 0;
counter_u64_add(vm_reserv_broken, 1); counter_u64_add(vm_reserv_broken, 1);
} }
/* /*
* Breaks all reservations belonging to the given object. * Breaks all reservations belonging to the given object.
markjUnsubmitted
Done
Inline Actions

As a micro-optimization, I wonder if it'd be preferable to have a bit_clear_all() or so which simply memset()s the map.

markj: As a micro-optimization, I wonder if it'd be preferable to have a bit_clear_all() or so which…
dougmAuthorUnsubmitted
Done
Inline Actions

I conjecture, without testing, that with VM_LEVEL_0_NPAGES being a multiple of 64, a decent compiler could optimize out checks for writing <64 bits at the beginning or end of the map.

dougm: I conjecture, without testing, that with VM_LEVEL_0_NPAGES being a multiple of 64, a decent…
alcUnsubmitted
Not Done
Inline Actions

The implementation of bit_nclear() is already hand-optimized for clearing ranges. That said, it handles the first and last long specially, and uses a loop for the longs in between:

ffffffff80f41b21: 49 c7 40 60 00 00 00 00       movq    $0, 96(%r8)
ffffffff80f41b29: 4d 39 e5              cmpq    %r12, %r13
ffffffff80f41b2c: 73 13                 jae     0xffffffff80f41b41 <vm_reserv_break+0x271>
ffffffff80f41b2e: 66 90                 nop
ffffffff80f41b30: 49 c7 45 00 00 00 00 00       movq    $0, (%r13)
ffffffff80f41b38: 49 83 c5 08           addq    $8, %r13
ffffffff80f41b3c: 4d 39 e5              cmpq    %r12, %r13
ffffffff80f41b3f: 72 ef                 jb      0xffffffff80f41b30 <vm_reserv_break+0x260>
ffffffff80f41b41: 49 c7 80 98 00 00 00 00 00 00 00      movq    $0, 152(%r8)

I omitted the extra instructions for initializing %r12 and %r13.

alc: The implementation of bit_nclear() is already hand-optimized for clearing ranges. That said…
*/ */
void void
vm_reserv_break_all(vm_object_t object) vm_reserv_break_all(vm_object_t object)
{ {
vm_reserv_t rv; vm_reserv_t rv;
/* /*
* This access of object->rvq is unsynchronized so that the * This access of object->rvq is unsynchronized so that the
▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines
{ {
vm_paddr_t paddr; vm_paddr_t paddr;
struct vm_phys_seg *seg; struct vm_phys_seg *seg;
struct vm_reserv *rv; struct vm_reserv *rv;
struct vm_reserv_domain *rvd; struct vm_reserv_domain *rvd;
#ifdef VM_PHYSSEG_SPARSE #ifdef VM_PHYSSEG_SPARSE
vm_pindex_t used; vm_pindex_t used;
#endif #endif
int i, j, segind; int i, segind;
/* /*
* Initialize the reservation array. Specifically, initialize the * Initialize the reservation array. Specifically, initialize the
* "pages" field for every element that has an underlying superpage. * "pages" field for every element that has an underlying superpage.
*/ */
#ifdef VM_PHYSSEG_SPARSE #ifdef VM_PHYSSEG_SPARSE
used = 0; used = 0;
#endif #endif
Show All 25 Lines for (i = 0; i < MAXMEMDOM; i++) {
TAILQ_INIT(&rvd->partpop); TAILQ_INIT(&rvd->partpop);
mtx_init(&rvd->marker.lock, "vm reserv marker", NULL, MTX_DEF); mtx_init(&rvd->marker.lock, "vm reserv marker", NULL, MTX_DEF);
/* /*
* Fully populated reservations should never be present in the * Fully populated reservations should never be present in the
* partially populated reservation queues. * partially populated reservation queues.
*/ */
rvd->marker.popcnt = VM_LEVEL_0_NPAGES; rvd->marker.popcnt = VM_LEVEL_0_NPAGES;
for (j = 0; j < VM_LEVEL_0_NPAGES; j++) bit_nset(rvd->marker.popmap, 0, VM_LEVEL_0_NPAGES - 1);
popmap_set(rvd->marker.popmap, j);
} }
for (i = 0; i < VM_RESERV_OBJ_LOCK_COUNT; i++) for (i = 0; i < VM_RESERV_OBJ_LOCK_COUNT; i++)
mtx_init(&vm_reserv_object_mtx[i], "resv obj lock", NULL, mtx_init(&vm_reserv_object_mtx[i], "resv obj lock", NULL,
MTX_DEF); MTX_DEF);
} }
/* /*
* Returns true if the given page belongs to a reservation and that page is * Returns true if the given page belongs to a reservation and that page is
* free. Otherwise, returns false. * free. Otherwise, returns false.
*/ */
bool bool
vm_reserv_is_page_free(vm_page_t m) vm_reserv_is_page_free(vm_page_t m)
{ {
vm_reserv_t rv; vm_reserv_t rv;
rv = vm_reserv_from_page(m); rv = vm_reserv_from_page(m);
if (rv->object == NULL) if (rv->object == NULL)
return (false); return (false);
return (popmap_is_clear(rv->popmap, m - rv->pages)); return (!bit_test(rv->popmap, m - rv->pages));
} }
/* /*
* If the given page belongs to a reservation, returns the level of that * If the given page belongs to a reservation, returns the level of that
* reservation. Otherwise, returns -1. * reservation. Otherwise, returns -1.
*/ */
int int
vm_reserv_level(vm_page_t m) vm_reserv_level(vm_page_t m)
▲ Show 20 Lines • Show All 90 Lines • ▼ Show 20 Lines
* request for contiguous physical memory. Start searching from the lower * request for contiguous physical memory. Start searching from the lower
* bound, defined by lo, and stop at the upper bound, hi. Return the index * bound, defined by lo, and stop at the upper bound, hi. Return the index
* of the first satisfactory free page, or -1 if none is found. * of the first satisfactory free page, or -1 if none is found.
*/ */
static int static int
vm_reserv_find_contig(vm_reserv_t rv, int npages, int lo, vm_reserv_find_contig(vm_reserv_t rv, int npages, int lo,
int hi, int ppn_align, int ppn_bound) int hi, int ppn_align, int ppn_bound)
{ {
u_long changes;
int bitpos, bits_left, i, n;
vm_reserv_assert_locked(rv); vm_reserv_assert_locked(rv);
KASSERT(npages <= VM_LEVEL_0_NPAGES - 1, KASSERT(npages <= VM_LEVEL_0_NPAGES - 1,
("%s: Too many pages", __func__)); ("%s: Too many pages", __func__));
KASSERT(ppn_bound <= VM_LEVEL_0_NPAGES, KASSERT(ppn_bound <= VM_LEVEL_0_NPAGES,
("%s: Too big a boundary for reservation size", __func__)); ("%s: Too big a boundary for reservation size", __func__));
KASSERT(npages <= ppn_bound, KASSERT(npages <= ppn_bound,
("%s: Too many pages for given boundary", __func__)); ("%s: Too many pages for given boundary", __func__));
KASSERT(ppn_align != 0 && powerof2(ppn_align), KASSERT(ppn_align != 0 && powerof2(ppn_align),
("ppn_align is not a positive power of 2")); ("ppn_align is not a positive power of 2"));
KASSERT(ppn_bound != 0 && powerof2(ppn_bound), KASSERT(ppn_bound != 0 && powerof2(ppn_bound),
("ppn_bound is not a positive power of 2")); ("ppn_bound is not a positive power of 2"));
i = lo / NBPOPMAP; while (bit_ffc_area_at(rv->popmap, lo, hi, npages, &lo), lo != -1) {
changes = rv->popmap[i] | ((1UL << (lo % NBPOPMAP)) - 1);
n = hi / NBPOPMAP;
bits_left = hi % NBPOPMAP;
hi = lo = -1;
for (;;) {
/*
* "changes" is a bitmask that marks where a new sequence of
* 0s or 1s begins in popmap[i], with last bit in popmap[i-1]
* considered to be 1 if and only if lo == hi. The bits of
* popmap[-1] and popmap[NPOPMAP] are considered all 1s.
*/
changes ^= (changes << 1) | (lo == hi);
while (changes != 0) {
/*
* If the next change marked begins a run of 0s, set
* lo to mark that position. Otherwise set hi and
* look for a satisfactory first page from lo up to hi.
*/
bitpos = ffsl(changes) - 1;
changes ^= 1UL << bitpos;
if (lo == hi) {
lo = NBPOPMAP * i + bitpos;
continue;
}
hi = NBPOPMAP * i + bitpos;
if (lo < roundup2(lo, ppn_align)) { if (lo < roundup2(lo, ppn_align)) {
/* Skip to next aligned page. */ /* Skip to next aligned page. */
lo = roundup2(lo, ppn_align); lo = roundup2(lo, ppn_align);
if (lo >= VM_LEVEL_0_NPAGES) } else if (roundup2(lo + 1, ppn_bound) >= lo + npages)
return (-1); return (lo);
} if (roundup2(lo + 1, ppn_bound) < lo + npages) {
if (lo + npages > roundup2(lo, ppn_bound)) {
/* Skip to next boundary-matching page. */ /* Skip to next boundary-matching page. */
lo = roundup2(lo, ppn_bound); lo = roundup2(lo + 1, ppn_bound);
if (lo >= VM_LEVEL_0_NPAGES)
return (-1);
} }
if (lo + npages <= hi)
return (lo);
lo = hi;
} }
if (++i < n)
changes = rv->popmap[i];
else if (i == n)
changes = bits_left == 0 ? -1UL :
(rv->popmap[n] | (-1UL << bits_left));
else
return (-1); return (-1);
markjUnsubmitted
Done
Inline Actions

How do we know that 0 <= lo < VM_LEVEL_0_NPAGES here?

markj: How do we know that `0 <= lo < VM_LEVEL_0_NPAGES` here?
dougmAuthorUnsubmitted
Done
Inline Actions

After changes made elsewhere, ppn_align and ppn_bound should be <= VM_LEVEL_0_NPAGES, so lo should be also.

dougm: After changes made elsewhere, ppn_align and ppn_bound should be <= VM_LEVEL_0_NPAGES, so lo…
} }
}
/* /*
* Searches the partially populated reservation queue for the least recently * Searches the partially populated reservation queue for the least recently
* changed reservation with free pages that satisfy the given request for * changed reservation with free pages that satisfy the given request for
* contiguous physical memory. If a satisfactory reservation is found, it is * contiguous physical memory. If a satisfactory reservation is found, it is
* broken. Returns true if a reservation is broken and false otherwise. * broken. Returns true if a reservation is broken and false otherwise.
*/ */
vm_page_t vm_page_t
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines TAILQ_FOREACH_SAFE(rv, queue, partpopq, rvn) {
hi = (pa + VM_LEVEL_0_SIZE <= high) ? VM_LEVEL_0_NPAGES : hi = (pa + VM_LEVEL_0_SIZE <= high) ? VM_LEVEL_0_NPAGES :
(int)((high - pa) >> PAGE_SHIFT); (int)((high - pa) >> PAGE_SHIFT);
posn = vm_reserv_find_contig(rv, (int)npages, lo, hi, posn = vm_reserv_find_contig(rv, (int)npages, lo, hi,
ppn_align, ppn_bound); ppn_align, ppn_bound);
if (posn >= 0) { if (posn >= 0) {
vm_reserv_domain_scan_unlock(domain); vm_reserv_domain_scan_unlock(domain);
/* Allocate requested space */ /* Allocate requested space */
rv->popcnt += npages; rv->popcnt += npages;
while (npages-- > 0) bit_nset(rv->popmap, posn, posn + npages - 1);
popmap_set(rv->popmap, posn + npages);
vm_reserv_reclaim(rv); vm_reserv_reclaim(rv);
vm_reserv_unlock(rv); vm_reserv_unlock(rv);
m_ret = &rv->pages[posn]; m_ret = &rv->pages[posn];
pa = VM_PAGE_TO_PHYS(m_ret); pa = VM_PAGE_TO_PHYS(m_ret);
KASSERT(vm_addr_ok(pa, size, alignment, boundary), KASSERT(vm_addr_ok(pa, size, alignment, boundary),
("%s: adjusted address not aligned/bounded to " ("%s: adjusted address not aligned/bounded to "
"%lx/%jx", "%lx/%jx",
__func__, alignment, (uintmax_t)boundary)); __func__, alignment, (uintmax_t)boundary));
▲ Show 20 Lines • Show All 138 Lines • Show Last 20 Lines