Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_reserv.c
Show First 20 Lines • Show All 253 Lines • ▼ Show 20 Lines | #define vm_reserv_object_lock_idx(object) \ | ||||
(((uintptr_t)object / sizeof(*object)) % VM_RESERV_OBJ_LOCK_COUNT) | (((uintptr_t)object / sizeof(*object)) % VM_RESERV_OBJ_LOCK_COUNT) | ||||
#define vm_reserv_object_lock_ptr(object) \ | #define vm_reserv_object_lock_ptr(object) \ | ||||
&vm_reserv_object_mtx[vm_reserv_object_lock_idx((object))] | &vm_reserv_object_mtx[vm_reserv_object_lock_idx((object))] | ||||
#define vm_reserv_object_lock(object) \ | #define vm_reserv_object_lock(object) \ | ||||
mtx_lock(vm_reserv_object_lock_ptr((object))) | mtx_lock(vm_reserv_object_lock_ptr((object))) | ||||
#define vm_reserv_object_unlock(object) \ | #define vm_reserv_object_unlock(object) \ | ||||
mtx_unlock(vm_reserv_object_lock_ptr((object))) | mtx_unlock(vm_reserv_object_lock_ptr((object))) | ||||
static void vm_reserv_break(vm_reserv_t rv, vm_page_t m); | static void vm_reserv_break(vm_reserv_t rv); | ||||
static void vm_reserv_depopulate(vm_reserv_t rv, int index); | static void vm_reserv_depopulate(vm_reserv_t rv, int index); | ||||
static vm_reserv_t vm_reserv_from_page(vm_page_t m); | static vm_reserv_t vm_reserv_from_page(vm_page_t m); | ||||
static boolean_t vm_reserv_has_pindex(vm_reserv_t rv, | static boolean_t vm_reserv_has_pindex(vm_reserv_t rv, | ||||
vm_pindex_t pindex); | vm_pindex_t pindex); | ||||
static void vm_reserv_populate(vm_reserv_t rv, int index); | static void vm_reserv_populate(vm_reserv_t rv, int index); | ||||
static void vm_reserv_reclaim(vm_reserv_t rv); | static void vm_reserv_reclaim(vm_reserv_t rv); | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 658 Lines • ▼ Show 20 Lines | KASSERT(rv->pages == m, | ||||
("vm_reserv_alloc_page: reserv %p's pages is corrupted", rv)); | ("vm_reserv_alloc_page: reserv %p's pages is corrupted", rv)); | ||||
vm_reserv_insert(rv, object, first); | vm_reserv_insert(rv, object, first); | ||||
index = VM_RESERV_INDEX(object, pindex); | index = VM_RESERV_INDEX(object, pindex); | ||||
vm_reserv_populate(rv, index); | vm_reserv_populate(rv, index); | ||||
return (&rv->pages[index]); | return (&rv->pages[index]); | ||||
} | } | ||||
/* | /* | ||||
* Breaks the given reservation. Except for the specified free page, all free | * Breaks the given reservation. All free pages in the reservation | ||||
* pages in the reservation are returned to the physical memory allocator. | * are returned to the physical memory allocator. The reservation's | ||||
* The reservation's population count and map are reset to their initial | * population count and map are reset to their initial state. | ||||
* state. | |||||
* | * | ||||
* The given reservation must not be in the partially populated reservation | * The given reservation must not be in the partially populated reservation | ||||
* queue. The free page queue lock must be held. | * queue. The free page queue lock must be held. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_reserv_break(vm_reserv_t rv, vm_page_t m) | vm_reserv_break(vm_reserv_t rv) | ||||
{ | { | ||||
int begin_zeroes, hi, i, lo; | int begin_zeroes, hi, i, lo; | ||||
vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); | vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); | ||||
vm_reserv_remove(rv); | vm_reserv_remove(rv); | ||||
rv->pages->psind = 0; | rv->pages->psind = 0; | ||||
if (m != NULL) { | |||||
/* | |||||
* Since the reservation is being broken, there is no harm in | |||||
* abusing the population map to stop "m" from being returned | |||||
* to the physical memory allocator. | |||||
*/ | |||||
i = m - rv->pages; | |||||
KASSERT(popmap_is_clear(rv->popmap, i), | |||||
("vm_reserv_break: reserv %p's popmap is corrupted", rv)); | |||||
popmap_set(rv->popmap, i); | |||||
rv->popcnt++; | |||||
} | |||||
i = hi = 0; | i = hi = 0; | ||||
do { | do { | ||||
/* Find the next 0 bit. Any previous 0 bits are < "hi". */ | /* Find the next 0 bit. Any previous 0 bits are < "hi". */ | ||||
lo = ffsl(~(((1UL << hi) - 1) | rv->popmap[i])); | lo = ffsl(~(((1UL << hi) - 1) | rv->popmap[i])); | ||||
if (lo == 0) { | if (lo == 0) { | ||||
/* Redundantly clears bits < "hi". */ | /* Redundantly clears bits < "hi". */ | ||||
rv->popmap[i] = 0; | rv->popmap[i] = 0; | ||||
rv->popcnt -= NBPOPMAP - hi; | rv->popcnt -= NBPOPMAP - hi; | ||||
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines | while ((rv = LIST_FIRST(&object->rvq)) != NULL) { | ||||
if (rv->object != object) | if (rv->object != object) | ||||
continue; | continue; | ||||
KASSERT(rv->object == object, | KASSERT(rv->object == object, | ||||
("vm_reserv_break_all: reserv %p is corrupted", rv)); | ("vm_reserv_break_all: reserv %p is corrupted", rv)); | ||||
if (rv->inpartpopq) { | if (rv->inpartpopq) { | ||||
TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); | TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); | ||||
rv->inpartpopq = FALSE; | rv->inpartpopq = FALSE; | ||||
} | } | ||||
vm_reserv_break(rv, NULL); | vm_reserv_break(rv); | ||||
} | } | ||||
if (vmd != NULL) | if (vmd != NULL) | ||||
vm_domain_free_unlock(vmd); | vm_domain_free_unlock(vmd); | ||||
} | } | ||||
/* | /* | ||||
* Frees the given page if it belongs to a reservation. Returns TRUE if the | * Frees the given page if it belongs to a reservation. Returns TRUE if the | ||||
* page is freed and FALSE otherwise. | * page is freed and FALSE otherwise. | ||||
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Lines | vm_reserv_reclaim(vm_reserv_t rv) | ||||
vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); | vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); | ||||
KASSERT(rv->inpartpopq, | KASSERT(rv->inpartpopq, | ||||
("vm_reserv_reclaim: reserv %p's inpartpopq is FALSE", rv)); | ("vm_reserv_reclaim: reserv %p's inpartpopq is FALSE", rv)); | ||||
KASSERT(rv->domain >= 0 && rv->domain < vm_ndomains, | KASSERT(rv->domain >= 0 && rv->domain < vm_ndomains, | ||||
("vm_reserv_reclaim: reserv %p's domain is corrupted %d", | ("vm_reserv_reclaim: reserv %p's domain is corrupted %d", | ||||
rv, rv->domain)); | rv, rv->domain)); | ||||
TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); | TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); | ||||
rv->inpartpopq = FALSE; | rv->inpartpopq = FALSE; | ||||
vm_reserv_break(rv, NULL); | vm_reserv_break(rv); | ||||
vm_reserv_reclaimed++; | vm_reserv_reclaimed++; | ||||
} | } | ||||
/* | /* | ||||
* Breaks the reservation at the head of the partially populated reservation | * Breaks the reservation at the head of the partially populated reservation | ||||
* queue, releasing its free pages to the physical memory allocator. Returns | * queue, releasing its free pages to the physical memory allocator. Returns | ||||
* TRUE if a reservation is broken and FALSE otherwise. | * TRUE if a reservation is broken and FALSE otherwise. | ||||
* | * | ||||
▲ Show 20 Lines • Show All 209 Lines • Show Last 20 Lines |