Changeset View
Standalone View
sys/vm/vm_reserv.c
Show First 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | |||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/queue.h> | #include <sys/queue.h> | ||||
#include <sys/rwlock.h> | #include <sys/rwlock.h> | ||||
#include <sys/sbuf.h> | #include <sys/sbuf.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <sys/bitstring.h> | |||||
#include <sys/counter.h> | #include <sys/counter.h> | ||||
#include <sys/ktr.h> | #include <sys/ktr.h> | ||||
#include <sys/vmmeter.h> | #include <sys/vmmeter.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/vm_param.h> | #include <vm/vm_param.h> | ||||
#include <vm/vm_object.h> | #include <vm/vm_object.h> | ||||
Show All 37 Lines | |||||
/* | /* | ||||
* Computes the index of the small page underlying the given (object, pindex) | * Computes the index of the small page underlying the given (object, pindex) | ||||
* within the reservation's array of small pages. | * within the reservation's array of small pages. | ||||
*/ | */ | ||||
#define VM_RESERV_INDEX(object, pindex) \ | #define VM_RESERV_INDEX(object, pindex) \ | ||||
(((object)->pg_color + (pindex)) & (VM_LEVEL_0_NPAGES - 1)) | (((object)->pg_color + (pindex)) & (VM_LEVEL_0_NPAGES - 1)) | ||||
/* | /* | ||||
* The size of a population map entry | |||||
*/ | |||||
typedef u_long popmap_t; | |||||
/* | |||||
* The number of bits in a population map entry | |||||
*/ | |||||
#define NBPOPMAP (NBBY * sizeof(popmap_t)) | |||||
/* | |||||
* The number of population map entries in a reservation | |||||
*/ | |||||
#define NPOPMAP howmany(VM_LEVEL_0_NPAGES, NBPOPMAP) | |||||
#define NPOPMAP_MAX howmany(VM_LEVEL_0_NPAGES_MAX, NBPOPMAP) | |||||
/* | |||||
* Number of elapsed ticks before we update the LRU queue position. Used | * Number of elapsed ticks before we update the LRU queue position. Used | ||||
* to reduce contention and churn on the list. | * to reduce contention and churn on the list. | ||||
*/ | */ | ||||
#define PARTPOPSLOP 1 | #define PARTPOPSLOP 1 | ||||
/* | /* | ||||
* Clear a bit in the population map. | |||||
*/ | |||||
static __inline void | |||||
popmap_clear(popmap_t popmap[], int i) | |||||
{ | |||||
popmap[i / NBPOPMAP] &= ~(1UL << (i % NBPOPMAP)); | |||||
} | |||||
/* | |||||
* Set a bit in the population map. | |||||
*/ | |||||
static __inline void | |||||
popmap_set(popmap_t popmap[], int i) | |||||
{ | |||||
popmap[i / NBPOPMAP] |= 1UL << (i % NBPOPMAP); | |||||
} | |||||
/* | |||||
* Is a bit in the population map clear? | |||||
*/ | |||||
static __inline boolean_t | |||||
popmap_is_clear(popmap_t popmap[], int i) | |||||
{ | |||||
return ((popmap[i / NBPOPMAP] & (1UL << (i % NBPOPMAP))) == 0); | |||||
} | |||||
/* | |||||
* Is a bit in the population map set? | |||||
*/ | |||||
static __inline boolean_t | |||||
popmap_is_set(popmap_t popmap[], int i) | |||||
{ | |||||
return ((popmap[i / NBPOPMAP] & (1UL << (i % NBPOPMAP))) != 0); | |||||
} | |||||
/* | |||||
* The reservation structure | * The reservation structure | ||||
* | * | ||||
* A reservation structure is constructed whenever a large physical page is | * A reservation structure is constructed whenever a large physical page is | ||||
* speculatively allocated to an object. The reservation provides the small | * speculatively allocated to an object. The reservation provides the small | ||||
* physical pages for the range [pindex, pindex + VM_LEVEL_0_NPAGES) of offsets | * physical pages for the range [pindex, pindex + VM_LEVEL_0_NPAGES) of offsets | ||||
* within that object. The reservation's "popcnt" tracks the number of these | * within that object. The reservation's "popcnt" tracks the number of these | ||||
* small physical pages that are in use at any given time. When and if the | * small physical pages that are in use at any given time. When and if the | ||||
* reservation is not fully utilized, it appears in the queue of partially | * reservation is not fully utilized, it appears in the queue of partially | ||||
Show All 14 Lines | struct vm_reserv { | ||||
LIST_ENTRY(vm_reserv) objq; /* (o, r) object queue */ | LIST_ENTRY(vm_reserv) objq; /* (o, r) object queue */ | ||||
vm_object_t object; /* (o, r) containing object */ | vm_object_t object; /* (o, r) containing object */ | ||||
vm_pindex_t pindex; /* (o, r) offset in object */ | vm_pindex_t pindex; /* (o, r) offset in object */ | ||||
vm_page_t pages; /* (c) first page */ | vm_page_t pages; /* (c) first page */ | ||||
uint16_t popcnt; /* (r) # of pages in use */ | uint16_t popcnt; /* (r) # of pages in use */ | ||||
uint8_t domain; /* (c) NUMA domain. */ | uint8_t domain; /* (c) NUMA domain. */ | ||||
char inpartpopq; /* (d, r) */ | char inpartpopq; /* (d, r) */ | ||||
int lasttick; /* (r) last pop update tick. */ | int lasttick; /* (r) last pop update tick. */ | ||||
popmap_t popmap[NPOPMAP_MAX]; /* (r) bit vector, used pages */ | bitstr_t bit_decl(popmap, VM_LEVEL_0_NPAGES_MAX); | ||||
/* (r) bit vector, used pages */ | |||||
}; | }; | ||||
TAILQ_HEAD(vm_reserv_queue, vm_reserv); | TAILQ_HEAD(vm_reserv_queue, vm_reserv); | ||||
#define vm_reserv_lockptr(rv) (&(rv)->lock) | #define vm_reserv_lockptr(rv) (&(rv)->lock) | ||||
#define vm_reserv_assert_locked(rv) \ | #define vm_reserv_assert_locked(rv) \ | ||||
mtx_assert(vm_reserv_lockptr(rv), MA_OWNED) | mtx_assert(vm_reserv_lockptr(rv), MA_OWNED) | ||||
#define vm_reserv_lock(rv) mtx_lock(vm_reserv_lockptr(rv)) | #define vm_reserv_lock(rv) mtx_lock(vm_reserv_lockptr(rv)) | ||||
▲ Show 20 Lines • Show All 199 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
/* | /* | ||||
* Insert a new reservation into the object's objq. | * Insert a new reservation into the object's objq. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_reserv_insert(vm_reserv_t rv, vm_object_t object, vm_pindex_t pindex) | vm_reserv_insert(vm_reserv_t rv, vm_object_t object, vm_pindex_t pindex) | ||||
{ | { | ||||
int i; | int i __diagused; | ||||
vm_reserv_assert_locked(rv); | vm_reserv_assert_locked(rv); | ||||
CTR6(KTR_VM, | CTR6(KTR_VM, | ||||
"%s: rv %p(%p) object %p new %p popcnt %d", | "%s: rv %p(%p) object %p new %p popcnt %d", | ||||
__FUNCTION__, rv, rv->pages, rv->object, object, | __FUNCTION__, rv, rv->pages, rv->object, object, | ||||
rv->popcnt); | rv->popcnt); | ||||
KASSERT(rv->object == NULL, | KASSERT(rv->object == NULL, | ||||
("vm_reserv_insert: reserv %p isn't free", rv)); | ("vm_reserv_insert: reserv %p isn't free", rv)); | ||||
KASSERT(rv->popcnt == 0, | KASSERT(rv->popcnt == 0, | ||||
("vm_reserv_insert: reserv %p's popcnt is corrupted", rv)); | ("vm_reserv_insert: reserv %p's popcnt is corrupted", rv)); | ||||
KASSERT(!rv->inpartpopq, | KASSERT(!rv->inpartpopq, | ||||
("vm_reserv_insert: reserv %p's inpartpopq is TRUE", rv)); | ("vm_reserv_insert: reserv %p's inpartpopq is TRUE", rv)); | ||||
for (i = 0; i < NPOPMAP; i++) | KASSERT((bit_ffs(rv->popmap, VM_LEVEL_0_NPAGES, &i), i == -1), | ||||
KASSERT(rv->popmap[i] == 0, | |||||
("vm_reserv_insert: reserv %p's popmap is corrupted", rv)); | ("vm_reserv_insert: reserv %p's popmap is corrupted", rv)); | ||||
vm_reserv_object_lock(object); | vm_reserv_object_lock(object); | ||||
rv->pindex = pindex; | rv->pindex = pindex; | ||||
rv->object = object; | rv->object = object; | ||||
rv->lasttick = ticks; | rv->lasttick = ticks; | ||||
LIST_INSERT_HEAD(&object->rvq, rv, objq); | LIST_INSERT_HEAD(&object->rvq, rv, objq); | ||||
vm_reserv_object_unlock(object); | vm_reserv_object_unlock(object); | ||||
} | } | ||||
/* | /* | ||||
* Reduces the given reservation's population count. If the population count | * Reduces the given reservation's population count. If the population count | ||||
* becomes zero, the reservation is destroyed. Additionally, moves the | * becomes zero, the reservation is destroyed. Additionally, moves the | ||||
* reservation to the tail of the partially populated reservation queue if the | * reservation to the tail of the partially populated reservation queue if the | ||||
* population count is non-zero. | * population count is non-zero. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_reserv_depopulate(vm_reserv_t rv, int index) | vm_reserv_depopulate(vm_reserv_t rv, int index) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
vm_reserv_assert_locked(rv); | vm_reserv_assert_locked(rv); | ||||
CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", | CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", | ||||
__FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); | __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); | ||||
KASSERT(rv->object != NULL, | KASSERT(rv->object != NULL, | ||||
("vm_reserv_depopulate: reserv %p is free", rv)); | ("vm_reserv_depopulate: reserv %p is free", rv)); | ||||
KASSERT(popmap_is_set(rv->popmap, index), | KASSERT(bit_test(rv->popmap, index), | ||||
("vm_reserv_depopulate: reserv %p's popmap[%d] is clear", rv, | ("vm_reserv_depopulate: reserv %p's popmap[%d] is clear", rv, | ||||
index)); | index)); | ||||
KASSERT(rv->popcnt > 0, | KASSERT(rv->popcnt > 0, | ||||
("vm_reserv_depopulate: reserv %p's popcnt is corrupted", rv)); | ("vm_reserv_depopulate: reserv %p's popcnt is corrupted", rv)); | ||||
KASSERT(rv->domain < vm_ndomains, | KASSERT(rv->domain < vm_ndomains, | ||||
("vm_reserv_depopulate: reserv %p's domain is corrupted %d", | ("vm_reserv_depopulate: reserv %p's domain is corrupted %d", | ||||
rv, rv->domain)); | rv, rv->domain)); | ||||
if (rv->popcnt == VM_LEVEL_0_NPAGES) { | if (rv->popcnt == VM_LEVEL_0_NPAGES) { | ||||
KASSERT(rv->pages->psind == 1, | KASSERT(rv->pages->psind == 1, | ||||
("vm_reserv_depopulate: reserv %p is already demoted", | ("vm_reserv_depopulate: reserv %p is already demoted", | ||||
rv)); | rv)); | ||||
rv->pages->psind = 0; | rv->pages->psind = 0; | ||||
} | } | ||||
popmap_clear(rv->popmap, index); | bit_clear(rv->popmap, index); | ||||
rv->popcnt--; | rv->popcnt--; | ||||
if ((unsigned)(ticks - rv->lasttick) >= PARTPOPSLOP || | if ((unsigned)(ticks - rv->lasttick) >= PARTPOPSLOP || | ||||
rv->popcnt == 0) { | rv->popcnt == 0) { | ||||
vm_reserv_domain_lock(rv->domain); | vm_reserv_domain_lock(rv->domain); | ||||
if (rv->inpartpopq) { | if (rv->inpartpopq) { | ||||
TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq); | TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq); | ||||
rv->inpartpopq = FALSE; | rv->inpartpopq = FALSE; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 89 Lines • ▼ Show 20 Lines | |||||
vm_reserv_populate(vm_reserv_t rv, int index) | vm_reserv_populate(vm_reserv_t rv, int index) | ||||
{ | { | ||||
vm_reserv_assert_locked(rv); | vm_reserv_assert_locked(rv); | ||||
CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", | CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", | ||||
__FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); | __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); | ||||
KASSERT(rv->object != NULL, | KASSERT(rv->object != NULL, | ||||
("vm_reserv_populate: reserv %p is free", rv)); | ("vm_reserv_populate: reserv %p is free", rv)); | ||||
KASSERT(popmap_is_clear(rv->popmap, index), | KASSERT(!bit_test(rv->popmap, index), | ||||
("vm_reserv_populate: reserv %p's popmap[%d] is set", rv, | ("vm_reserv_populate: reserv %p's popmap[%d] is set", rv, | ||||
index)); | index)); | ||||
KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES, | KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES, | ||||
("vm_reserv_populate: reserv %p is already full", rv)); | ("vm_reserv_populate: reserv %p is already full", rv)); | ||||
KASSERT(rv->pages->psind == 0, | KASSERT(rv->pages->psind == 0, | ||||
("vm_reserv_populate: reserv %p is already promoted", rv)); | ("vm_reserv_populate: reserv %p is already promoted", rv)); | ||||
KASSERT(rv->domain < vm_ndomains, | KASSERT(rv->domain < vm_ndomains, | ||||
("vm_reserv_populate: reserv %p's domain is corrupted %d", | ("vm_reserv_populate: reserv %p's domain is corrupted %d", | ||||
rv, rv->domain)); | rv, rv->domain)); | ||||
popmap_set(rv->popmap, index); | bit_set(rv->popmap, index); | ||||
rv->popcnt++; | rv->popcnt++; | ||||
if ((unsigned)(ticks - rv->lasttick) < PARTPOPSLOP && | if ((unsigned)(ticks - rv->lasttick) < PARTPOPSLOP && | ||||
rv->inpartpopq && rv->popcnt != VM_LEVEL_0_NPAGES) | rv->inpartpopq && rv->popcnt != VM_LEVEL_0_NPAGES) | ||||
return; | return; | ||||
rv->lasttick = ticks; | rv->lasttick = ticks; | ||||
vm_reserv_domain_lock(rv->domain); | vm_reserv_domain_lock(rv->domain); | ||||
if (rv->inpartpopq) { | if (rv->inpartpopq) { | ||||
TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq); | TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq); | ||||
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines | if (rv->object != object) | ||||
goto out; | goto out; | ||||
m = &rv->pages[index]; | m = &rv->pages[index]; | ||||
pa = VM_PAGE_TO_PHYS(m); | pa = VM_PAGE_TO_PHYS(m); | ||||
if (pa < low || pa + size > high || | if (pa < low || pa + size > high || | ||||
(pa & (alignment - 1)) != 0 || | (pa & (alignment - 1)) != 0 || | ||||
((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) | ((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) | ||||
goto out; | goto out; | ||||
/* Handle vm_page_rename(m, new_object, ...). */ | /* Handle vm_page_rename(m, new_object, ...). */ | ||||
for (i = 0; i < npages; i++) | bit_ffs_at(rv->popmap, index, index + npages, &i); | ||||
if (popmap_is_set(rv->popmap, index + i)) | if (i != -1) | ||||
goto out; | goto out; | ||||
markj: Shouldn't the third parameter be `npages`? | |||||
Done Inline ActionsYes. dougm: Yes. | |||||
Done Inline ActionsNo, on second thought. This is supposed to test npages bits, from index to index + npages -1. The first page not to be checked is at index + npages. I'm changing it back. dougm: No, on second thought. This is supposed to test npages bits, from index to index + npages -1. | |||||
Done Inline Actionsbit_ffs_at(str, start, nbits, result) examines up to nbits-start bits in a string of total length nbits. We want to examine up to npages bits, so nbits-start must be npages. With start and nbits as index and index+npages, we get that. The total length of popmap is likely more than index+npages, but we don't need to look at all the bits to the end of the map. I can just write an ntest function for bitstring.h that would have parameters like those of nset and nclear, if that would make this clearer. dougm: bit_ffs_at(str, start, nbits, result) examines up to nbits-start bits in a string of total… | |||||
Not Done Inline ActionsI see now. bit_ntest() seems like a sensible addition. The code is clear to me now for what it's worth, I was just unfamiliar with the bitstring.h functions. markj: I see now. bit_ntest() seems like a sensible addition. The code is clear to me now for what… | |||||
if (!vm_domain_allocate(vmd, req, npages)) | if (!vm_domain_allocate(vmd, req, npages)) | ||||
goto out; | goto out; | ||||
for (i = 0; i < npages; i++) | for (i = 0; i < npages; i++) | ||||
vm_reserv_populate(rv, index + i); | vm_reserv_populate(rv, index + i); | ||||
vm_reserv_unlock(rv); | vm_reserv_unlock(rv); | ||||
return (m); | return (m); | ||||
out: | out: | ||||
vm_reserv_unlock(rv); | vm_reserv_unlock(rv); | ||||
▲ Show 20 Lines • Show All 151 Lines • ▼ Show 20 Lines | if (rv != NULL) { | ||||
domain = rv->domain; | domain = rv->domain; | ||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
index = VM_RESERV_INDEX(object, pindex); | index = VM_RESERV_INDEX(object, pindex); | ||||
m = &rv->pages[index]; | m = &rv->pages[index]; | ||||
vm_reserv_lock(rv); | vm_reserv_lock(rv); | ||||
/* Handle reclaim race. */ | /* Handle reclaim race. */ | ||||
if (rv->object != object || | if (rv->object != object || | ||||
/* Handle vm_page_rename(m, new_object, ...). */ | /* Handle vm_page_rename(m, new_object, ...). */ | ||||
popmap_is_set(rv->popmap, index)) { | bit_test(rv->popmap, index)) { | ||||
m = NULL; | m = NULL; | ||||
goto out; | goto out; | ||||
} | } | ||||
if (vm_domain_allocate(vmd, req, 1) == 0) | if (vm_domain_allocate(vmd, req, 1) == 0) | ||||
m = NULL; | m = NULL; | ||||
else | else | ||||
vm_reserv_populate(rv, index); | vm_reserv_populate(rv, index); | ||||
out: | out: | ||||
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines | |||||
* population count and map are reset to their initial state. | * population count and map are reset to their initial state. | ||||
* | * | ||||
* The given reservation must not be in the partially populated reservation | * The given reservation must not be in the partially populated reservation | ||||
* queue. | * queue. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_reserv_break(vm_reserv_t rv) | vm_reserv_break(vm_reserv_t rv) | ||||
{ | { | ||||
u_long changes; | int hi, lo; | ||||
int bitpos, hi, i, lo; | |||||
vm_reserv_assert_locked(rv); | vm_reserv_assert_locked(rv); | ||||
CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", | CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", | ||||
__FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); | __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); | ||||
vm_reserv_remove(rv); | vm_reserv_remove(rv); | ||||
rv->pages->psind = 0; | rv->pages->psind = 0; | ||||
hi = lo = -1; | hi = 0; | ||||
for (i = 0; i <= NPOPMAP; i++) { | while (bit_ffc_at(rv->popmap, hi, VM_LEVEL_0_NPAGES, &lo), lo != -1) { | ||||
/* | bit_ffs_at(rv->popmap, lo, VM_LEVEL_0_NPAGES, &hi); | ||||
* "changes" is a bitmask that marks where a new sequence of | if (hi == -1) | ||||
* 0s or 1s begins in popmap[i], with last bit in popmap[i-1] | hi = VM_LEVEL_0_NPAGES; | ||||
* considered to be 1 if and only if lo == hi. The bits of | |||||
* popmap[-1] and popmap[NPOPMAP] are considered all 1s. | |||||
*/ | |||||
if (i == NPOPMAP) | |||||
changes = lo != hi; | |||||
else { | |||||
changes = rv->popmap[i]; | |||||
changes ^= (changes << 1) | (lo == hi); | |||||
rv->popmap[i] = 0; | |||||
} | |||||
while (changes != 0) { | |||||
/* | |||||
* If the next change marked begins a run of 0s, set | |||||
* lo to mark that position. Otherwise set hi and | |||||
* free pages from lo up to hi. | |||||
*/ | |||||
bitpos = ffsl(changes) - 1; | |||||
changes ^= 1UL << bitpos; | |||||
if (lo == hi) | |||||
lo = NBPOPMAP * i + bitpos; | |||||
else { | |||||
hi = NBPOPMAP * i + bitpos; | |||||
vm_domain_free_lock(VM_DOMAIN(rv->domain)); | vm_domain_free_lock(VM_DOMAIN(rv->domain)); | ||||
vm_phys_enqueue_contig(&rv->pages[lo], hi - lo); | vm_phys_enqueue_contig(&rv->pages[lo], hi - lo); | ||||
vm_domain_free_unlock(VM_DOMAIN(rv->domain)); | vm_domain_free_unlock(VM_DOMAIN(rv->domain)); | ||||
lo = hi; | |||||
} | } | ||||
} | bit_nclear(rv->popmap, 0, VM_LEVEL_0_NPAGES - 1); | ||||
} | |||||
rv->popcnt = 0; | rv->popcnt = 0; | ||||
counter_u64_add(vm_reserv_broken, 1); | counter_u64_add(vm_reserv_broken, 1); | ||||
} | } | ||||
/* | /* | ||||
* Breaks all reservations belonging to the given object. | * Breaks all reservations belonging to the given object. | ||||
*/ | */ | ||||
void | void | ||||
vm_reserv_break_all(vm_object_t object) | vm_reserv_break_all(vm_object_t object) | ||||
{ | { | ||||
vm_reserv_t rv; | vm_reserv_t rv; | ||||
Done Inline ActionsAs a micro-optimization, I wonder if it'd be preferable to have a bit_clear_all() or so which simply memset()s the map. markj: As a micro-optimization, I wonder if it'd be preferable to have a bit_clear_all() or so which… | |||||
Done Inline ActionsI conjecture, without testing, that with VM_LEVEL_0_NPAGES being a multiple of 64, a decent compiler could optimize out checks for writing <64 bits at the beginning or end of the map. dougm: I conjecture, without testing, that with VM_LEVEL_0_NPAGES being a multiple of 64, a decent… | |||||
Not Done Inline ActionsThe implementation of bit_nclear() is already hand-optimized for clearing ranges. That said, it handles the first and last long specially, and uses a loop for the longs in between: ffffffff80f41b21: 49 c7 40 60 00 00 00 00 movq $0, 96(%r8) ffffffff80f41b29: 4d 39 e5 cmpq %r12, %r13 ffffffff80f41b2c: 73 13 jae 0xffffffff80f41b41 <vm_reserv_break+0x271> ffffffff80f41b2e: 66 90 nop ffffffff80f41b30: 49 c7 45 00 00 00 00 00 movq $0, (%r13) ffffffff80f41b38: 49 83 c5 08 addq $8, %r13 ffffffff80f41b3c: 4d 39 e5 cmpq %r12, %r13 ffffffff80f41b3f: 72 ef jb 0xffffffff80f41b30 <vm_reserv_break+0x260> ffffffff80f41b41: 49 c7 80 98 00 00 00 00 00 00 00 movq $0, 152(%r8) I omitted the extra instructions for initializing %r12 and %r13. alc: The implementation of bit_nclear() is already hand-optimized for clearing ranges. That said… | |||||
/* | /* | ||||
* This access of object->rvq is unsynchronized so that the | * This access of object->rvq is unsynchronized so that the | ||||
* object rvq lock can nest after the domain_free lock. We | * object rvq lock can nest after the domain_free lock. We | ||||
* must check for races in the results. However, the object | * must check for races in the results. However, the object | ||||
* lock prevents new additions, so we are guaranteed that when | * lock prevents new additions, so we are guaranteed that when | ||||
* it returns NULL the object is properly empty. | * it returns NULL the object is properly empty. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
vm_paddr_t paddr; | vm_paddr_t paddr; | ||||
struct vm_phys_seg *seg; | struct vm_phys_seg *seg; | ||||
struct vm_reserv *rv; | struct vm_reserv *rv; | ||||
struct vm_reserv_domain *rvd; | struct vm_reserv_domain *rvd; | ||||
#ifdef VM_PHYSSEG_SPARSE | #ifdef VM_PHYSSEG_SPARSE | ||||
vm_pindex_t used; | vm_pindex_t used; | ||||
#endif | #endif | ||||
int i, j, segind; | int i, segind; | ||||
/* | /* | ||||
* Initialize the reservation array. Specifically, initialize the | * Initialize the reservation array. Specifically, initialize the | ||||
* "pages" field for every element that has an underlying superpage. | * "pages" field for every element that has an underlying superpage. | ||||
*/ | */ | ||||
#ifdef VM_PHYSSEG_SPARSE | #ifdef VM_PHYSSEG_SPARSE | ||||
used = 0; | used = 0; | ||||
#endif | #endif | ||||
Show All 25 Lines | for (i = 0; i < MAXMEMDOM; i++) { | ||||
TAILQ_INIT(&rvd->partpop); | TAILQ_INIT(&rvd->partpop); | ||||
mtx_init(&rvd->marker.lock, "vm reserv marker", NULL, MTX_DEF); | mtx_init(&rvd->marker.lock, "vm reserv marker", NULL, MTX_DEF); | ||||
/* | /* | ||||
* Fully populated reservations should never be present in the | * Fully populated reservations should never be present in the | ||||
* partially populated reservation queues. | * partially populated reservation queues. | ||||
*/ | */ | ||||
rvd->marker.popcnt = VM_LEVEL_0_NPAGES; | rvd->marker.popcnt = VM_LEVEL_0_NPAGES; | ||||
for (j = 0; j < VM_LEVEL_0_NPAGES; j++) | bit_nset(rvd->marker.popmap, 0, VM_LEVEL_0_NPAGES - 1); | ||||
popmap_set(rvd->marker.popmap, j); | |||||
} | } | ||||
for (i = 0; i < VM_RESERV_OBJ_LOCK_COUNT; i++) | for (i = 0; i < VM_RESERV_OBJ_LOCK_COUNT; i++) | ||||
mtx_init(&vm_reserv_object_mtx[i], "resv obj lock", NULL, | mtx_init(&vm_reserv_object_mtx[i], "resv obj lock", NULL, | ||||
MTX_DEF); | MTX_DEF); | ||||
} | } | ||||
/* | /* | ||||
* Returns true if the given page belongs to a reservation and that page is | * Returns true if the given page belongs to a reservation and that page is | ||||
* free. Otherwise, returns false. | * free. Otherwise, returns false. | ||||
*/ | */ | ||||
bool | bool | ||||
vm_reserv_is_page_free(vm_page_t m) | vm_reserv_is_page_free(vm_page_t m) | ||||
{ | { | ||||
vm_reserv_t rv; | vm_reserv_t rv; | ||||
rv = vm_reserv_from_page(m); | rv = vm_reserv_from_page(m); | ||||
if (rv->object == NULL) | if (rv->object == NULL) | ||||
return (false); | return (false); | ||||
return (popmap_is_clear(rv->popmap, m - rv->pages)); | return (!bit_test(rv->popmap, m - rv->pages)); | ||||
} | } | ||||
/* | /* | ||||
* If the given page belongs to a reservation, returns the level of that | * If the given page belongs to a reservation, returns the level of that | ||||
* reservation. Otherwise, returns -1. | * reservation. Otherwise, returns -1. | ||||
*/ | */ | ||||
int | int | ||||
vm_reserv_level(vm_page_t m) | vm_reserv_level(vm_page_t m) | ||||
▲ Show 20 Lines • Show All 90 Lines • ▼ Show 20 Lines | |||||
* request for contiguous physical memory. Start searching from the lower | * request for contiguous physical memory. Start searching from the lower | ||||
* bound, defined by lo, and stop at the upper bound, hi. Return the index | * bound, defined by lo, and stop at the upper bound, hi. Return the index | ||||
* of the first satisfactory free page, or -1 if none is found. | * of the first satisfactory free page, or -1 if none is found. | ||||
*/ | */ | ||||
static int | static int | ||||
vm_reserv_find_contig(vm_reserv_t rv, int npages, int lo, | vm_reserv_find_contig(vm_reserv_t rv, int npages, int lo, | ||||
int hi, int ppn_align, int ppn_bound) | int hi, int ppn_align, int ppn_bound) | ||||
{ | { | ||||
u_long changes; | |||||
int bitpos, bits_left, i, n; | |||||
vm_reserv_assert_locked(rv); | vm_reserv_assert_locked(rv); | ||||
KASSERT(npages <= VM_LEVEL_0_NPAGES - 1, | KASSERT(npages <= VM_LEVEL_0_NPAGES - 1, | ||||
("%s: Too many pages", __func__)); | ("%s: Too many pages", __func__)); | ||||
KASSERT(ppn_bound <= VM_LEVEL_0_NPAGES, | KASSERT(ppn_bound <= VM_LEVEL_0_NPAGES, | ||||
("%s: Too big a boundary for reservation size", __func__)); | ("%s: Too big a boundary for reservation size", __func__)); | ||||
KASSERT(npages <= ppn_bound, | KASSERT(npages <= ppn_bound, | ||||
("%s: Too many pages for given boundary", __func__)); | ("%s: Too many pages for given boundary", __func__)); | ||||
KASSERT(ppn_align != 0 && powerof2(ppn_align), | KASSERT(ppn_align != 0 && powerof2(ppn_align), | ||||
("ppn_align is not a positive power of 2")); | ("ppn_align is not a positive power of 2")); | ||||
KASSERT(ppn_bound != 0 && powerof2(ppn_bound), | KASSERT(ppn_bound != 0 && powerof2(ppn_bound), | ||||
("ppn_bound is not a positive power of 2")); | ("ppn_bound is not a positive power of 2")); | ||||
i = lo / NBPOPMAP; | while (bit_ffc_area_at(rv->popmap, lo, hi, npages, &lo), lo != -1) { | ||||
changes = rv->popmap[i] | ((1UL << (lo % NBPOPMAP)) - 1); | if (lo == roundup2(lo, ppn_align)) { | ||||
n = hi / NBPOPMAP; | if (lo + npages <= roundup2(lo, ppn_bound)) | ||||
bits_left = hi % NBPOPMAP; | return (lo); | ||||
hi = lo = -1; | } else { | ||||
for (;;) { | |||||
/* | |||||
* "changes" is a bitmask that marks where a new sequence of | |||||
* 0s or 1s begins in popmap[i], with last bit in popmap[i-1] | |||||
* considered to be 1 if and only if lo == hi. The bits of | |||||
* popmap[-1] and popmap[NPOPMAP] are considered all 1s. | |||||
*/ | |||||
changes ^= (changes << 1) | (lo == hi); | |||||
while (changes != 0) { | |||||
/* | |||||
* If the next change marked begins a run of 0s, set | |||||
* lo to mark that position. Otherwise set hi and | |||||
* look for a satisfactory first page from lo up to hi. | |||||
*/ | |||||
bitpos = ffsl(changes) - 1; | |||||
changes ^= 1UL << bitpos; | |||||
if (lo == hi) { | |||||
lo = NBPOPMAP * i + bitpos; | |||||
continue; | |||||
} | |||||
hi = NBPOPMAP * i + bitpos; | |||||
if (lo < roundup2(lo, ppn_align)) { | |||||
/* Skip to next aligned page. */ | /* Skip to next aligned page. */ | ||||
lo = roundup2(lo, ppn_align); | lo = roundup2(lo, ppn_align); | ||||
if (lo >= VM_LEVEL_0_NPAGES) | |||||
return (-1); | |||||
} | } | ||||
if (lo + npages > roundup2(lo, ppn_bound)) { | if (lo + npages > roundup2(lo, ppn_bound)) { | ||||
/* Skip to next boundary-matching page. */ | /* Skip to next boundary-matching page. */ | ||||
lo = roundup2(lo, ppn_bound); | lo = roundup2(lo, ppn_bound); | ||||
if (lo >= VM_LEVEL_0_NPAGES) | |||||
return (-1); | |||||
} | } | ||||
if (lo + npages <= hi) | |||||
return (lo); | |||||
lo = hi; | |||||
} | } | ||||
if (++i < n) | |||||
changes = rv->popmap[i]; | |||||
else if (i == n) | |||||
changes = bits_left == 0 ? -1UL : | |||||
(rv->popmap[n] | (-1UL << bits_left)); | |||||
else | |||||
return (-1); | return (-1); | ||||
} | |||||
} | } | ||||
/* | /* | ||||
Done Inline ActionsHow do we know that 0 <= lo < VM_LEVEL_0_NPAGES here? markj: How do we know that `0 <= lo < VM_LEVEL_0_NPAGES` here? | |||||
Done Inline ActionsAfter changes made elsewhere, ppn_align and ppn_bound should be <= VM_LEVEL_0_NPAGES, so lo should be also. dougm: After changes made elsewhere, ppn_align and ppn_bound should be <= VM_LEVEL_0_NPAGES, so lo… | |||||
* Searches the partially populated reservation queue for the least recently | * Searches the partially populated reservation queue for the least recently | ||||
* changed reservation with free pages that satisfy the given request for | * changed reservation with free pages that satisfy the given request for | ||||
* contiguous physical memory. If a satisfactory reservation is found, it is | * contiguous physical memory. If a satisfactory reservation is found, it is | ||||
* broken. Returns true if a reservation is broken and false otherwise. | * broken. Returns true if a reservation is broken and false otherwise. | ||||
*/ | */ | ||||
bool | bool | ||||
vm_reserv_reclaim_contig(int domain, u_long npages, vm_paddr_t low, | vm_reserv_reclaim_contig(int domain, u_long npages, vm_paddr_t low, | ||||
vm_paddr_t high, u_long alignment, vm_paddr_t boundary) | vm_paddr_t high, u_long alignment, vm_paddr_t boundary) | ||||
▲ Show 20 Lines • Show All 220 Lines • Show Last 20 Lines |
Shouldn't the third parameter be npages?