Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_reserv.c
Show First 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | |||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <sys/vmmeter.h> | #include <sys/vmmeter.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/vm_param.h> | #include <vm/vm_param.h> | ||||
#include <vm/vm_object.h> | #include <vm/vm_object.h> | ||||
#include <vm/vm_page.h> | #include <vm/vm_page.h> | ||||
#include <vm/vm_pageout.h> | |||||
#include <vm/vm_phys.h> | #include <vm/vm_phys.h> | ||||
#include <vm/vm_pagequeue.h> | |||||
#include <vm/vm_radix.h> | #include <vm/vm_radix.h> | ||||
#include <vm/vm_reserv.h> | #include <vm/vm_reserv.h> | ||||
/* | /* | ||||
* The reservation system supports the speculative allocation of large physical | * The reservation system supports the speculative allocation of large physical | ||||
* pages ("superpages"). Speculative allocation enables the fully automatic | * pages ("superpages"). Speculative allocation enables the fully automatic | ||||
* utilization of superpages by the virtual memory system. In other words, no | * utilization of superpages by the virtual memory system. In other words, no | ||||
* programmatic directives are required to use superpages. | * programmatic directives are required to use superpages. | ||||
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Lines | |||||
* physical pages for the range [pindex, pindex + VM_LEVEL_0_NPAGES) of offsets | * physical pages for the range [pindex, pindex + VM_LEVEL_0_NPAGES) of offsets | ||||
* within that object. The reservation's "popcnt" tracks the number of these | * within that object. The reservation's "popcnt" tracks the number of these | ||||
* small physical pages that are in use at any given time. When and if the | * small physical pages that are in use at any given time. When and if the | ||||
* reservation is not fully utilized, it appears in the queue of partially | * reservation is not fully utilized, it appears in the queue of partially | ||||
* populated reservations. The reservation always appears on the containing | * populated reservations. The reservation always appears on the containing | ||||
* object's list of reservations. | * object's list of reservations. | ||||
* | * | ||||
* A partially populated reservation can be broken and reclaimed at any time. | * A partially populated reservation can be broken and reclaimed at any time. | ||||
* | |||||
* f - vm_domain_free_lock | |||||
* o - vm_reserv_object_lock | |||||
* c - constant after boot | |||||
*/ | */ | ||||
struct vm_reserv { | struct vm_reserv { | ||||
TAILQ_ENTRY(vm_reserv) partpopq; | TAILQ_ENTRY(vm_reserv) partpopq; /* (f) per-domain queue. */ | ||||
LIST_ENTRY(vm_reserv) objq; | LIST_ENTRY(vm_reserv) objq; /* (o, f) object queue */ | ||||
vm_object_t object; /* containing object */ | vm_object_t object; /* (o, f) containing object */ | ||||
vm_pindex_t pindex; /* offset within object */ | vm_pindex_t pindex; /* (o, f) offset in object */ | ||||
vm_page_t pages; /* first page of a superpage */ | vm_page_t pages; /* (c) first page */ | ||||
int domain; /* NUMA domain */ | int domain; /* (c) NUMA domain. */ | ||||
int popcnt; /* # of pages in use */ | int popcnt; /* (f) # of pages in use */ | ||||
char inpartpopq; | char inpartpopq; /* (f) */ | ||||
popmap_t popmap[NPOPMAP]; /* bit vector of used pages */ | popmap_t popmap[NPOPMAP]; /* (f) bit vector, used pages */ | ||||
}; | }; | ||||
/* | /* | ||||
* The reservation array | * The reservation array | ||||
* | * | ||||
* This array is analoguous in function to vm_page_array. It differs in the | * This array is analoguous in function to vm_page_array. It differs in the | ||||
* respect that it may contain a greater number of useful reservation | * respect that it may contain a greater number of useful reservation | ||||
* structures than there are (physical) superpages. These "invalid" | * structures than there are (physical) superpages. These "invalid" | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | |||||
SYSCTL_OID(_vm_reserv, OID_AUTO, partpopq, CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, | SYSCTL_OID(_vm_reserv, OID_AUTO, partpopq, CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, | ||||
sysctl_vm_reserv_partpopq, "A", "Partially populated reservation queues"); | sysctl_vm_reserv_partpopq, "A", "Partially populated reservation queues"); | ||||
static long vm_reserv_reclaimed; | static long vm_reserv_reclaimed; | ||||
SYSCTL_LONG(_vm_reserv, OID_AUTO, reclaimed, CTLFLAG_RD, | SYSCTL_LONG(_vm_reserv, OID_AUTO, reclaimed, CTLFLAG_RD, | ||||
&vm_reserv_reclaimed, 0, "Cumulative number of reclaimed reservations"); | &vm_reserv_reclaimed, 0, "Cumulative number of reclaimed reservations"); | ||||
/* | |||||
* The object lock pool is used to synchronize the rvq. We can not use a | |||||
* pool mutex because it is required before malloc works. | |||||
* | |||||
* The "hash" function could be made faster without divide and modulo. | |||||
*/ | |||||
#define VM_RESERV_OBJ_LOCK_COUNT MAXCPU | |||||
struct mtx_padalign vm_reserv_object_mtx[VM_RESERV_OBJ_LOCK_COUNT]; | |||||
#define vm_reserv_object_lock_idx(object) \ | |||||
(((uintptr_t)object / sizeof(*object)) % VM_RESERV_OBJ_LOCK_COUNT) | |||||
jeff: divide and modulo may be expensive | |||||
#define vm_reserv_object_lock_ptr(object) \ | |||||
&vm_reserv_object_mtx[vm_reserv_object_lock_idx((object))] | |||||
#define vm_reserv_object_lock(object) \ | |||||
mtx_lock(vm_reserv_object_lock_ptr((object))) | |||||
#define vm_reserv_object_unlock(object) \ | |||||
mtx_unlock(vm_reserv_object_lock_ptr((object))) | |||||
static void vm_reserv_break(vm_reserv_t rv, vm_page_t m); | static void vm_reserv_break(vm_reserv_t rv, vm_page_t m); | ||||
static void vm_reserv_depopulate(vm_reserv_t rv, int index); | static void vm_reserv_depopulate(vm_reserv_t rv, int index); | ||||
static vm_reserv_t vm_reserv_from_page(vm_page_t m); | static vm_reserv_t vm_reserv_from_page(vm_page_t m); | ||||
static boolean_t vm_reserv_has_pindex(vm_reserv_t rv, | static boolean_t vm_reserv_has_pindex(vm_reserv_t rv, | ||||
vm_pindex_t pindex); | vm_pindex_t pindex); | ||||
static void vm_reserv_populate(vm_reserv_t rv, int index); | static void vm_reserv_populate(vm_reserv_t rv, int index); | ||||
static void vm_reserv_reclaim(vm_reserv_t rv); | static void vm_reserv_reclaim(vm_reserv_t rv); | ||||
Show All 38 Lines | sysctl_vm_reserv_partpopq(SYSCTL_HANDLER_ARGS) | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
sbuf_new_for_sysctl(&sbuf, NULL, 128, req); | sbuf_new_for_sysctl(&sbuf, NULL, 128, req); | ||||
sbuf_printf(&sbuf, "\nDOMAIN LEVEL SIZE NUMBER\n\n"); | sbuf_printf(&sbuf, "\nDOMAIN LEVEL SIZE NUMBER\n\n"); | ||||
for (domain = 0; domain < vm_ndomains; domain++) { | for (domain = 0; domain < vm_ndomains; domain++) { | ||||
for (level = -1; level <= VM_NRESERVLEVEL - 2; level++) { | for (level = -1; level <= VM_NRESERVLEVEL - 2; level++) { | ||||
counter = 0; | counter = 0; | ||||
unused_pages = 0; | unused_pages = 0; | ||||
mtx_lock(&vm_page_queue_free_mtx); | vm_domain_free_lock(VM_DOMAIN(domain)); | ||||
TAILQ_FOREACH(rv, &vm_rvq_partpop[domain], partpopq) { | TAILQ_FOREACH(rv, &vm_rvq_partpop[domain], partpopq) { | ||||
counter++; | counter++; | ||||
unused_pages += VM_LEVEL_0_NPAGES - rv->popcnt; | unused_pages += VM_LEVEL_0_NPAGES - rv->popcnt; | ||||
} | } | ||||
mtx_unlock(&vm_page_queue_free_mtx); | vm_domain_free_unlock(VM_DOMAIN(domain)); | ||||
sbuf_printf(&sbuf, "%6d, %7d, %6dK, %6d\n", | sbuf_printf(&sbuf, "%6d, %7d, %6dK, %6d\n", | ||||
domain, level, | domain, level, | ||||
unused_pages * ((int)PAGE_SIZE / 1024), counter); | unused_pages * ((int)PAGE_SIZE / 1024), counter); | ||||
} | } | ||||
} | } | ||||
error = sbuf_finish(&sbuf); | error = sbuf_finish(&sbuf); | ||||
sbuf_delete(&sbuf); | sbuf_delete(&sbuf); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Remove a reservation from the object's objq. | |||||
*/ | |||||
static void | |||||
vm_reserv_remove(vm_reserv_t rv) | |||||
{ | |||||
vm_object_t object; | |||||
KASSERT(rv->object != NULL, | |||||
("vm_reserv_remove: reserv %p is free", rv)); | |||||
KASSERT(!rv->inpartpopq, | |||||
("vm_reserv_remove: reserv %p's inpartpopq is TRUE", rv)); | |||||
object = rv->object; | |||||
vm_reserv_object_lock(object); | |||||
LIST_REMOVE(rv, objq); | |||||
rv->object = NULL; | |||||
vm_reserv_object_unlock(object); | |||||
} | |||||
/* | |||||
* Insert a new reservation into the object's objq. | |||||
*/ | |||||
static void | |||||
vm_reserv_insert(vm_reserv_t rv, vm_object_t object, vm_pindex_t pindex) | |||||
{ | |||||
int i; | |||||
KASSERT(rv->object == NULL, | |||||
("vm_reserv_insert: reserv %p isn't free", rv)); | |||||
KASSERT(rv->popcnt == 0, | |||||
("vm_reserv_insert: reserv %p's popcnt is corrupted", rv)); | |||||
KASSERT(!rv->inpartpopq, | |||||
("vm_reserv_insert: reserv %p's inpartpopq is TRUE", rv)); | |||||
for (i = 0; i < NPOPMAP; i++) | |||||
KASSERT(rv->popmap[i] == 0, | |||||
("vm_reserv_insert: reserv %p's popmap is corrupted", rv)); | |||||
vm_reserv_object_lock(object); | |||||
rv->pindex = pindex; | |||||
rv->object = object; | |||||
LIST_INSERT_HEAD(&object->rvq, rv, objq); | |||||
vm_reserv_object_unlock(object); | |||||
} | |||||
/* | |||||
* Reduces the given reservation's population count. If the population count | * Reduces the given reservation's population count. If the population count | ||||
* becomes zero, the reservation is destroyed. Additionally, moves the | * becomes zero, the reservation is destroyed. Additionally, moves the | ||||
* reservation to the tail of the partially populated reservation queue if the | * reservation to the tail of the partially populated reservation queue if the | ||||
* population count is non-zero. | * population count is non-zero. | ||||
* | * | ||||
* The free page queue lock must be held. | * The free page queue lock must be held. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_reserv_depopulate(vm_reserv_t rv, int index) | vm_reserv_depopulate(vm_reserv_t rv, int index) | ||||
{ | { | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); | ||||
KASSERT(rv->object != NULL, | KASSERT(rv->object != NULL, | ||||
("vm_reserv_depopulate: reserv %p is free", rv)); | ("vm_reserv_depopulate: reserv %p is free", rv)); | ||||
KASSERT(popmap_is_set(rv->popmap, index), | KASSERT(popmap_is_set(rv->popmap, index), | ||||
("vm_reserv_depopulate: reserv %p's popmap[%d] is clear", rv, | ("vm_reserv_depopulate: reserv %p's popmap[%d] is clear", rv, | ||||
index)); | index)); | ||||
KASSERT(rv->popcnt > 0, | KASSERT(rv->popcnt > 0, | ||||
("vm_reserv_depopulate: reserv %p's popcnt is corrupted", rv)); | ("vm_reserv_depopulate: reserv %p's popcnt is corrupted", rv)); | ||||
KASSERT(rv->domain >= 0 && rv->domain < vm_ndomains, | KASSERT(rv->domain >= 0 && rv->domain < vm_ndomains, | ||||
("vm_reserv_depopulate: reserv %p's domain is corrupted %d", | ("vm_reserv_depopulate: reserv %p's domain is corrupted %d", | ||||
rv, rv->domain)); | rv, rv->domain)); | ||||
if (rv->inpartpopq) { | if (rv->inpartpopq) { | ||||
TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); | TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); | ||||
rv->inpartpopq = FALSE; | rv->inpartpopq = FALSE; | ||||
} else { | } else { | ||||
KASSERT(rv->pages->psind == 1, | KASSERT(rv->pages->psind == 1, | ||||
("vm_reserv_depopulate: reserv %p is already demoted", | ("vm_reserv_depopulate: reserv %p is already demoted", | ||||
rv)); | rv)); | ||||
rv->pages->psind = 0; | rv->pages->psind = 0; | ||||
} | } | ||||
popmap_clear(rv->popmap, index); | popmap_clear(rv->popmap, index); | ||||
rv->popcnt--; | rv->popcnt--; | ||||
if (rv->popcnt == 0) { | if (rv->popcnt == 0) { | ||||
LIST_REMOVE(rv, objq); | vm_reserv_remove(rv); | ||||
rv->object = NULL; | |||||
rv->domain = -1; | |||||
vm_phys_free_pages(rv->pages, VM_LEVEL_0_ORDER); | vm_phys_free_pages(rv->pages, VM_LEVEL_0_ORDER); | ||||
vm_reserv_freed++; | vm_reserv_freed++; | ||||
} else { | } else { | ||||
rv->inpartpopq = TRUE; | rv->inpartpopq = TRUE; | ||||
TAILQ_INSERT_TAIL(&vm_rvq_partpop[rv->domain], rv, partpopq); | TAILQ_INSERT_TAIL(&vm_rvq_partpop[rv->domain], rv, partpopq); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Returns the reservation to which the given page might belong. | * Returns the reservation to which the given page might belong. | ||||
*/ | */ | ||||
static __inline vm_reserv_t | static __inline vm_reserv_t | ||||
vm_reserv_from_page(vm_page_t m) | vm_reserv_from_page(vm_page_t m) | ||||
{ | { | ||||
return (&vm_reserv_array[VM_PAGE_TO_PHYS(m) >> VM_LEVEL_0_SHIFT]); | return (&vm_reserv_array[VM_PAGE_TO_PHYS(m) >> VM_LEVEL_0_SHIFT]); | ||||
} | } | ||||
/* | /* | ||||
* Returns an existing reservation or NULL and initialized successor pointer. | |||||
*/ | |||||
static vm_reserv_t | |||||
vm_reserv_from_object(vm_object_t object, vm_pindex_t pindex, | |||||
vm_page_t mpred, vm_page_t *msuccp) | |||||
{ | |||||
vm_reserv_t rv; | |||||
vm_page_t msucc; | |||||
msucc = NULL; | |||||
if (mpred != NULL) { | |||||
KASSERT(mpred->object == object, | |||||
("vm_reserv_from_object: object doesn't contain mpred")); | |||||
KASSERT(mpred->pindex < pindex, | |||||
("vm_reserv_from_object: mpred doesn't precede pindex")); | |||||
rv = vm_reserv_from_page(mpred); | |||||
if (rv->object == object && vm_reserv_has_pindex(rv, pindex)) | |||||
goto found; | |||||
msucc = TAILQ_NEXT(mpred, listq); | |||||
} else | |||||
msucc = TAILQ_FIRST(&object->memq); | |||||
if (msucc != NULL) { | |||||
KASSERT(msucc->pindex > pindex, | |||||
("vm_reserv_from_object: msucc doesn't succeed pindex")); | |||||
rv = vm_reserv_from_page(msucc); | |||||
if (rv->object == object && vm_reserv_has_pindex(rv, pindex)) | |||||
goto found; | |||||
} | |||||
rv = NULL; | |||||
found: | |||||
*msuccp = msucc; | |||||
return (rv); | |||||
} | |||||
/* | |||||
* Returns TRUE if the given reservation contains the given page index and | * Returns TRUE if the given reservation contains the given page index and | ||||
* FALSE otherwise. | * FALSE otherwise. | ||||
*/ | */ | ||||
static __inline boolean_t | static __inline boolean_t | ||||
vm_reserv_has_pindex(vm_reserv_t rv, vm_pindex_t pindex) | vm_reserv_has_pindex(vm_reserv_t rv, vm_pindex_t pindex) | ||||
{ | { | ||||
return (((pindex - rv->pindex) & ~(VM_LEVEL_0_NPAGES - 1)) == 0); | return (((pindex - rv->pindex) & ~(VM_LEVEL_0_NPAGES - 1)) == 0); | ||||
} | } | ||||
/* | /* | ||||
* Increases the given reservation's population count. Moves the reservation | * Increases the given reservation's population count. Moves the reservation | ||||
* to the tail of the partially populated reservation queue. | * to the tail of the partially populated reservation queue. | ||||
* | * | ||||
* The free page queue must be locked. | * The free page queue must be locked. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_reserv_populate(vm_reserv_t rv, int index) | vm_reserv_populate(vm_reserv_t rv, int index) | ||||
{ | { | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); | ||||
KASSERT(rv->object != NULL, | KASSERT(rv->object != NULL, | ||||
("vm_reserv_populate: reserv %p is free", rv)); | ("vm_reserv_populate: reserv %p is free", rv)); | ||||
KASSERT(popmap_is_clear(rv->popmap, index), | KASSERT(popmap_is_clear(rv->popmap, index), | ||||
("vm_reserv_populate: reserv %p's popmap[%d] is set", rv, | ("vm_reserv_populate: reserv %p's popmap[%d] is set", rv, | ||||
index)); | index)); | ||||
KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES, | KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES, | ||||
("vm_reserv_populate: reserv %p is already full", rv)); | ("vm_reserv_populate: reserv %p is already full", rv)); | ||||
KASSERT(rv->pages->psind == 0, | KASSERT(rv->pages->psind == 0, | ||||
Show All 25 Lines | |||||
* "alignment" and "boundary" must be a power of two. | * "alignment" and "boundary" must be a power of two. | ||||
* | * | ||||
* The page "mpred" must immediately precede the offset "pindex" within the | * The page "mpred" must immediately precede the offset "pindex" within the | ||||
* specified object. | * specified object. | ||||
* | * | ||||
* The object and free page queue must be locked. | * The object and free page queue must be locked. | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_reserv_extend_contig(int req, vm_object_t object, vm_pindex_t pindex, | |||||
int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, | |||||
u_long alignment, vm_paddr_t boundary, vm_page_t mpred) | |||||
{ | |||||
struct vm_domain *vmd; | |||||
vm_paddr_t pa, size; | |||||
vm_page_t m, msucc; | |||||
vm_reserv_t rv; | |||||
int i, index; | |||||
VM_OBJECT_ASSERT_WLOCKED(object); | |||||
KASSERT(npages != 0, ("vm_reserv_alloc_contig: npages is 0")); | |||||
/* | |||||
* Is a reservation fundamentally impossible? | |||||
*/ | |||||
if (pindex < VM_RESERV_INDEX(object, pindex) || | |||||
pindex + npages > object->size || object->resident_page_count == 0) | |||||
return (NULL); | |||||
/* | |||||
* All reservations of a particular size have the same alignment. | |||||
* Assuming that the first page is allocated from a reservation, the | |||||
* least significant bits of its physical address can be determined | |||||
* from its offset from the beginning of the reservation and the size | |||||
* of the reservation. | |||||
* | |||||
* Could the specified index within a reservation of the smallest | |||||
* possible size satisfy the alignment and boundary requirements? | |||||
*/ | |||||
pa = VM_RESERV_INDEX(object, pindex) << PAGE_SHIFT; | |||||
if ((pa & (alignment - 1)) != 0) | |||||
return (NULL); | |||||
size = npages << PAGE_SHIFT; | |||||
if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) | |||||
return (NULL); | |||||
/* | |||||
* Look for an existing reservation. | |||||
*/ | |||||
rv = vm_reserv_from_object(object, pindex, mpred, &msucc); | |||||
if (rv == NULL) | |||||
return (NULL); | |||||
KASSERT(object != kernel_object || rv->domain == domain, | |||||
("vm_reserv_extend_contig: Domain mismatch from reservation.")); | |||||
index = VM_RESERV_INDEX(object, pindex); | |||||
/* Does the allocation fit within the reservation? */ | |||||
if (index + npages > VM_LEVEL_0_NPAGES) | |||||
return (NULL); | |||||
domain = rv->domain; | |||||
vmd = VM_DOMAIN(domain); | |||||
vm_domain_free_lock(vmd); | |||||
if (rv->object != object || !vm_domain_available(vmd, req, npages)) { | |||||
m = NULL; | |||||
goto out; | |||||
} | |||||
m = &rv->pages[index]; | |||||
pa = VM_PAGE_TO_PHYS(m); | |||||
if (pa < low || pa + size > high || (pa & (alignment - 1)) != 0 || | |||||
((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) { | |||||
m = NULL; | |||||
goto out; | |||||
} | |||||
/* Handle vm_page_rename(m, new_object, ...). */ | |||||
for (i = 0; i < npages; i++) { | |||||
if (popmap_is_set(rv->popmap, index + i)) { | |||||
m = NULL; | |||||
goto out; | |||||
} | |||||
} | |||||
for (i = 0; i < npages; i++) | |||||
vm_reserv_populate(rv, index + i); | |||||
vm_domain_freecnt_adj(vmd, -npages); | |||||
out: | |||||
vm_domain_free_unlock(vmd); | |||||
return (m); | |||||
} | |||||
/* | |||||
* Allocates a contiguous set of physical pages of the given size "npages" | |||||
* from existing or newly created reservations. All of the physical pages | |||||
* must be at or above the given physical address "low" and below the given | |||||
* physical address "high". The given value "alignment" determines the | |||||
* alignment of the first physical page in the set. If the given value | |||||
* "boundary" is non-zero, then the set of physical pages cannot cross any | |||||
* physical address boundary that is a multiple of that value. Both | |||||
* "alignment" and "boundary" must be a power of two. | |||||
* | |||||
* The page "mpred" must immediately precede the offset "pindex" within the | |||||
* specified object. | |||||
* | |||||
* The object and free page queue must be locked. | |||||
*/ | |||||
vm_page_t | |||||
vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t pindex, int domain, | vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | ||||
vm_paddr_t boundary, vm_page_t mpred) | vm_paddr_t boundary, vm_page_t mpred) | ||||
{ | { | ||||
vm_paddr_t pa, size; | vm_paddr_t pa, size; | ||||
vm_page_t m, m_ret, msucc; | vm_page_t m, m_ret, msucc; | ||||
vm_pindex_t first, leftcap, rightcap; | vm_pindex_t first, leftcap, rightcap; | ||||
vm_reserv_t rv; | vm_reserv_t rv; | ||||
u_long allocpages, maxpages, minpages; | u_long allocpages, maxpages, minpages; | ||||
int i, index, n; | int i, index, n; | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | vm_domain_free_assert_locked(VM_DOMAIN(domain)); | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
KASSERT(npages != 0, ("vm_reserv_alloc_contig: npages is 0")); | KASSERT(npages != 0, ("vm_reserv_alloc_contig: npages is 0")); | ||||
/* | /* | ||||
* Is a reservation fundamentally impossible? | * Is a reservation fundamentally impossible? | ||||
*/ | */ | ||||
if (pindex < VM_RESERV_INDEX(object, pindex) || | if (pindex < VM_RESERV_INDEX(object, pindex) || | ||||
pindex + npages > object->size) | pindex + npages > object->size) | ||||
Show All 12 Lines | vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
pa = VM_RESERV_INDEX(object, pindex) << PAGE_SHIFT; | pa = VM_RESERV_INDEX(object, pindex) << PAGE_SHIFT; | ||||
if ((pa & (alignment - 1)) != 0) | if ((pa & (alignment - 1)) != 0) | ||||
return (NULL); | return (NULL); | ||||
size = npages << PAGE_SHIFT; | size = npages << PAGE_SHIFT; | ||||
if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) | if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) | ||||
return (NULL); | return (NULL); | ||||
/* | /* | ||||
* Look for an existing reservation. | * Callers should've extended an existing reservation prior to | ||||
* calling this function. If a reservation exists it is | |||||
* incompatible with the allocation. | |||||
*/ | */ | ||||
if (mpred != NULL) { | rv = vm_reserv_from_object(object, pindex, mpred, &msucc); | ||||
KASSERT(mpred->object == object, | if (rv != NULL) | ||||
("vm_reserv_alloc_contig: object doesn't contain mpred")); | return (NULL); | ||||
KASSERT(mpred->pindex < pindex, | |||||
("vm_reserv_alloc_contig: mpred doesn't precede pindex")); | |||||
rv = vm_reserv_from_page(mpred); | |||||
if (rv->object == object && vm_reserv_has_pindex(rv, pindex)) | |||||
goto found; | |||||
msucc = TAILQ_NEXT(mpred, listq); | |||||
} else | |||||
msucc = TAILQ_FIRST(&object->memq); | |||||
if (msucc != NULL) { | |||||
KASSERT(msucc->pindex > pindex, | |||||
("vm_reserv_alloc_contig: msucc doesn't succeed pindex")); | |||||
rv = vm_reserv_from_page(msucc); | |||||
if (rv->object == object && vm_reserv_has_pindex(rv, pindex)) | |||||
goto found; | |||||
} | |||||
/* | /* | ||||
* Could at least one reservation fit between the first index to the | * Could at least one reservation fit between the first index to the | ||||
* left that can be used ("leftcap") and the first index to the right | * left that can be used ("leftcap") and the first index to the right | ||||
* that cannot be used ("rightcap")? | * that cannot be used ("rightcap")? | ||||
* | |||||
* We must synchronize with the reserv object lock to protect the | |||||
* pindex/object of the resulting reservations against rename while | |||||
* we are inspecting. | |||||
*/ | */ | ||||
first = pindex - VM_RESERV_INDEX(object, pindex); | first = pindex - VM_RESERV_INDEX(object, pindex); | ||||
minpages = VM_RESERV_INDEX(object, pindex) + npages; | |||||
maxpages = roundup2(minpages, VM_LEVEL_0_NPAGES); | |||||
allocpages = maxpages; | |||||
vm_reserv_object_lock(object); | |||||
if (mpred != NULL) { | if (mpred != NULL) { | ||||
Not Done Inline ActionsThis section of code is similar to the one in vm_reserv_alloc_page() can could likely be consolidated. jeff: This section of code is similar to the one in vm_reserv_alloc_page() can could likely be… | |||||
if ((rv = vm_reserv_from_page(mpred))->object != object) | if ((rv = vm_reserv_from_page(mpred))->object != object) | ||||
leftcap = mpred->pindex + 1; | leftcap = mpred->pindex + 1; | ||||
else | else | ||||
leftcap = rv->pindex + VM_LEVEL_0_NPAGES; | leftcap = rv->pindex + VM_LEVEL_0_NPAGES; | ||||
if (leftcap > first) | if (leftcap > first) { | ||||
vm_reserv_object_unlock(object); | |||||
return (NULL); | return (NULL); | ||||
} | } | ||||
minpages = VM_RESERV_INDEX(object, pindex) + npages; | } | ||||
maxpages = roundup2(minpages, VM_LEVEL_0_NPAGES); | |||||
allocpages = maxpages; | |||||
if (msucc != NULL) { | if (msucc != NULL) { | ||||
if ((rv = vm_reserv_from_page(msucc))->object != object) | if ((rv = vm_reserv_from_page(msucc))->object != object) | ||||
rightcap = msucc->pindex; | rightcap = msucc->pindex; | ||||
else | else | ||||
rightcap = rv->pindex; | rightcap = rv->pindex; | ||||
if (first + maxpages > rightcap) { | if (first + maxpages > rightcap) { | ||||
if (maxpages == VM_LEVEL_0_NPAGES) | if (maxpages == VM_LEVEL_0_NPAGES) { | ||||
vm_reserv_object_unlock(object); | |||||
return (NULL); | return (NULL); | ||||
} | |||||
/* | /* | ||||
* At least one reservation will fit between "leftcap" | * At least one reservation will fit between "leftcap" | ||||
* and "rightcap". However, a reservation for the | * and "rightcap". However, a reservation for the | ||||
* last of the requested pages will not fit. Reduce | * last of the requested pages will not fit. Reduce | ||||
* the size of the upcoming allocation accordingly. | * the size of the upcoming allocation accordingly. | ||||
*/ | */ | ||||
allocpages = minpages; | allocpages = minpages; | ||||
} | } | ||||
} | } | ||||
vm_reserv_object_unlock(object); | |||||
/* | /* | ||||
* Would the last new reservation extend past the end of the object? | * Would the last new reservation extend past the end of the object? | ||||
*/ | */ | ||||
if (first + maxpages > object->size) { | if (first + maxpages > object->size) { | ||||
/* | /* | ||||
* Don't allocate the last new reservation if the object is a | * Don't allocate the last new reservation if the object is a | ||||
* vnode or backed by another object that is a vnode. | * vnode or backed by another object that is a vnode. | ||||
Show All 14 Lines | vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
* boundary specified for the requested pages. For instance, the | * boundary specified for the requested pages. For instance, the | ||||
* specified index may not be the first page within the first new | * specified index may not be the first page within the first new | ||||
* reservation. | * reservation. | ||||
*/ | */ | ||||
m = vm_phys_alloc_contig(domain, allocpages, low, high, ulmax(alignment, | m = vm_phys_alloc_contig(domain, allocpages, low, high, ulmax(alignment, | ||||
VM_LEVEL_0_SIZE), boundary > VM_LEVEL_0_SIZE ? boundary : 0); | VM_LEVEL_0_SIZE), boundary > VM_LEVEL_0_SIZE ? boundary : 0); | ||||
if (m == NULL) | if (m == NULL) | ||||
return (NULL); | return (NULL); | ||||
KASSERT(vm_phys_domidx(m) == domain, | KASSERT(vm_phys_domain(m) == domain, | ||||
("vm_reserv_alloc_contig: Page domain does not match requested.")); | ("vm_reserv_alloc_contig: Page domain does not match requested.")); | ||||
/* | /* | ||||
* The allocated physical pages always begin at a reservation | * The allocated physical pages always begin at a reservation | ||||
* boundary, but they do not always end at a reservation boundary. | * boundary, but they do not always end at a reservation boundary. | ||||
* Initialize every reservation that is completely covered by the | * Initialize every reservation that is completely covered by the | ||||
* allocated physical pages. | * allocated physical pages. | ||||
*/ | */ | ||||
m_ret = NULL; | m_ret = NULL; | ||||
index = VM_RESERV_INDEX(object, pindex); | index = VM_RESERV_INDEX(object, pindex); | ||||
do { | do { | ||||
rv = vm_reserv_from_page(m); | rv = vm_reserv_from_page(m); | ||||
KASSERT(rv->pages == m, | KASSERT(rv->pages == m, | ||||
("vm_reserv_alloc_contig: reserv %p's pages is corrupted", | ("vm_reserv_alloc_contig: reserv %p's pages is corrupted", | ||||
rv)); | rv)); | ||||
KASSERT(rv->object == NULL, | vm_reserv_insert(rv, object, first); | ||||
("vm_reserv_alloc_contig: reserv %p isn't free", rv)); | |||||
LIST_INSERT_HEAD(&object->rvq, rv, objq); | |||||
rv->object = object; | |||||
rv->pindex = first; | |||||
rv->domain = domain; | |||||
KASSERT(rv->popcnt == 0, | |||||
("vm_reserv_alloc_contig: reserv %p's popcnt is corrupted", | |||||
rv)); | |||||
KASSERT(!rv->inpartpopq, | |||||
("vm_reserv_alloc_contig: reserv %p's inpartpopq is TRUE", | |||||
rv)); | |||||
for (i = 0; i < NPOPMAP; i++) | |||||
KASSERT(rv->popmap[i] == 0, | |||||
("vm_reserv_alloc_contig: reserv %p's popmap is corrupted", | |||||
rv)); | |||||
n = ulmin(VM_LEVEL_0_NPAGES - index, npages); | n = ulmin(VM_LEVEL_0_NPAGES - index, npages); | ||||
for (i = 0; i < n; i++) | for (i = 0; i < n; i++) | ||||
vm_reserv_populate(rv, index + i); | vm_reserv_populate(rv, index + i); | ||||
npages -= n; | npages -= n; | ||||
if (m_ret == NULL) { | if (m_ret == NULL) { | ||||
m_ret = &rv->pages[index]; | m_ret = &rv->pages[index]; | ||||
index = 0; | index = 0; | ||||
} | } | ||||
m += VM_LEVEL_0_NPAGES; | m += VM_LEVEL_0_NPAGES; | ||||
first += VM_LEVEL_0_NPAGES; | first += VM_LEVEL_0_NPAGES; | ||||
allocpages -= VM_LEVEL_0_NPAGES; | allocpages -= VM_LEVEL_0_NPAGES; | ||||
} while (allocpages >= VM_LEVEL_0_NPAGES); | } while (allocpages >= VM_LEVEL_0_NPAGES); | ||||
return (m_ret); | return (m_ret); | ||||
} | |||||
/* | /* | ||||
* Found a matching reservation. | * Attempts to extend an existing reservation and allocate the page to the | ||||
* object. | |||||
* | |||||
* The page "mpred" must immediately precede the offset "pindex" within the | |||||
* specified object. | |||||
* | |||||
* The object must be locked. | |||||
*/ | */ | ||||
found: | vm_page_t | ||||
index = VM_RESERV_INDEX(object, pindex); | vm_reserv_extend(int req, vm_object_t object, vm_pindex_t pindex, int domain, | ||||
/* Does the allocation fit within the reservation? */ | vm_page_t mpred) | ||||
if (index + npages > VM_LEVEL_0_NPAGES) | { | ||||
struct vm_domain *vmd; | |||||
vm_page_t m, msucc; | |||||
vm_reserv_t rv; | |||||
int index, free_count; | |||||
VM_OBJECT_ASSERT_WLOCKED(object); | |||||
/* | |||||
* Could a reservation currently exist? | |||||
*/ | |||||
if (pindex < VM_RESERV_INDEX(object, pindex) || | |||||
pindex >= object->size || object->resident_page_count == 0) | |||||
return (NULL); | return (NULL); | ||||
m = &rv->pages[index]; | |||||
pa = VM_PAGE_TO_PHYS(m); | /* | ||||
if (pa < low || pa + size > high || (pa & (alignment - 1)) != 0 || | * Look for an existing reservation. | ||||
((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) | */ | ||||
rv = vm_reserv_from_object(object, pindex, mpred, &msucc); | |||||
if (rv == NULL) | |||||
return (NULL); | return (NULL); | ||||
KASSERT(object != kernel_object || rv->domain == domain, | |||||
("vm_reserv_extend: Domain mismatch from reservation.")); | |||||
domain = rv->domain; | |||||
vmd = VM_DOMAIN(domain); | |||||
index = VM_RESERV_INDEX(object, pindex); | |||||
m = &rv->pages[index]; | |||||
vm_domain_free_lock(vmd); | |||||
if (vm_domain_available(vmd, req, 1) == 0 || | |||||
/* Handle reclaim race. */ | |||||
rv->object != object || | |||||
/* Handle vm_page_rename(m, new_object, ...). */ | /* Handle vm_page_rename(m, new_object, ...). */ | ||||
for (i = 0; i < npages; i++) | popmap_is_set(rv->popmap, index)) | ||||
if (popmap_is_set(rv->popmap, index + i)) | m = NULL; | ||||
return (NULL); | if (m != NULL) | ||||
for (i = 0; i < npages; i++) | vm_reserv_populate(rv, index); | ||||
vm_reserv_populate(rv, index + i); | free_count = vm_domain_freecnt_adj(vmd, -1); | ||||
vm_domain_free_unlock(vmd); | |||||
if (vm_paging_needed(vmd, free_count)) | |||||
pagedaemon_wakeup(domain); | |||||
return (m); | return (m); | ||||
} | } | ||||
/* | /* | ||||
* Allocates a page from an existing or newly created reservation. | * Allocates a page from an existing reservation. | ||||
* | * | ||||
* The page "mpred" must immediately precede the offset "pindex" within the | * The page "mpred" must immediately precede the offset "pindex" within the | ||||
* specified object. | * specified object. | ||||
* | * | ||||
* The object and free page queue must be locked. | * The object and free page queue must be locked. | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex, int domain, | vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
vm_page_t mpred) | vm_page_t mpred) | ||||
{ | { | ||||
vm_page_t m, msucc; | vm_page_t m, msucc; | ||||
vm_pindex_t first, leftcap, rightcap; | vm_pindex_t first, leftcap, rightcap; | ||||
vm_reserv_t rv; | vm_reserv_t rv; | ||||
int i, index; | int index; | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | vm_domain_free_assert_locked(VM_DOMAIN(domain)); | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
/* | /* | ||||
* Is a reservation fundamentally impossible? | * Is a reservation fundamentally impossible? | ||||
*/ | */ | ||||
if (pindex < VM_RESERV_INDEX(object, pindex) || | if (pindex < VM_RESERV_INDEX(object, pindex) || | ||||
pindex >= object->size) | pindex >= object->size) | ||||
return (NULL); | return (NULL); | ||||
/* | /* | ||||
* Look for an existing reservation. | * Callers should've extended an existing reservation prior to | ||||
* calling this function. If a reservation exists it is | |||||
* incompatible with the allocation. | |||||
*/ | */ | ||||
if (mpred != NULL) { | rv = vm_reserv_from_object(object, pindex, mpred, &msucc); | ||||
KASSERT(mpred->object == object, | if (rv != NULL) | ||||
("vm_reserv_alloc_page: object doesn't contain mpred")); | return (NULL); | ||||
KASSERT(mpred->pindex < pindex, | |||||
("vm_reserv_alloc_page: mpred doesn't precede pindex")); | |||||
rv = vm_reserv_from_page(mpred); | |||||
if (rv->object == object && vm_reserv_has_pindex(rv, pindex)) | |||||
goto found; | |||||
msucc = TAILQ_NEXT(mpred, listq); | |||||
} else | |||||
msucc = TAILQ_FIRST(&object->memq); | |||||
if (msucc != NULL) { | |||||
KASSERT(msucc->pindex > pindex, | |||||
("vm_reserv_alloc_page: msucc doesn't succeed pindex")); | |||||
rv = vm_reserv_from_page(msucc); | |||||
if (rv->object == object && vm_reserv_has_pindex(rv, pindex)) | |||||
goto found; | |||||
} | |||||
/* | /* | ||||
* Could a reservation fit between the first index to the left that | * Could a reservation fit between the first index to the left that | ||||
* can be used and the first index to the right that cannot be used? | * can be used and the first index to the right that cannot be used? | ||||
* | |||||
* We must synchronize with the reserv object lock to protect the | |||||
* pindex/object of the resulting reservations against rename while | |||||
* we are inspecting. | |||||
*/ | */ | ||||
first = pindex - VM_RESERV_INDEX(object, pindex); | first = pindex - VM_RESERV_INDEX(object, pindex); | ||||
vm_reserv_object_lock(object); | |||||
if (mpred != NULL) { | if (mpred != NULL) { | ||||
if ((rv = vm_reserv_from_page(mpred))->object != object) | if ((rv = vm_reserv_from_page(mpred))->object != object) | ||||
leftcap = mpred->pindex + 1; | leftcap = mpred->pindex + 1; | ||||
else | else | ||||
leftcap = rv->pindex + VM_LEVEL_0_NPAGES; | leftcap = rv->pindex + VM_LEVEL_0_NPAGES; | ||||
if (leftcap > first) | if (leftcap > first) { | ||||
vm_reserv_object_unlock(object); | |||||
return (NULL); | return (NULL); | ||||
} | } | ||||
} | |||||
if (msucc != NULL) { | if (msucc != NULL) { | ||||
if ((rv = vm_reserv_from_page(msucc))->object != object) | if ((rv = vm_reserv_from_page(msucc))->object != object) | ||||
rightcap = msucc->pindex; | rightcap = msucc->pindex; | ||||
else | else | ||||
rightcap = rv->pindex; | rightcap = rv->pindex; | ||||
if (first + VM_LEVEL_0_NPAGES > rightcap) | if (first + VM_LEVEL_0_NPAGES > rightcap) { | ||||
vm_reserv_object_unlock(object); | |||||
return (NULL); | return (NULL); | ||||
} | } | ||||
} | |||||
vm_reserv_object_unlock(object); | |||||
/* | /* | ||||
* Would a new reservation extend past the end of the object? | * Would a new reservation extend past the end of the object? | ||||
*/ | */ | ||||
if (first + VM_LEVEL_0_NPAGES > object->size) { | if (first + VM_LEVEL_0_NPAGES > object->size) { | ||||
/* | /* | ||||
* Don't allocate a new reservation if the object is a vnode or | * Don't allocate a new reservation if the object is a vnode or | ||||
* backed by another object that is a vnode. | * backed by another object that is a vnode. | ||||
Show All 9 Lines | vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
* Allocate and populate the new reservation. | * Allocate and populate the new reservation. | ||||
*/ | */ | ||||
m = vm_phys_alloc_pages(domain, VM_FREEPOOL_DEFAULT, VM_LEVEL_0_ORDER); | m = vm_phys_alloc_pages(domain, VM_FREEPOOL_DEFAULT, VM_LEVEL_0_ORDER); | ||||
if (m == NULL) | if (m == NULL) | ||||
return (NULL); | return (NULL); | ||||
rv = vm_reserv_from_page(m); | rv = vm_reserv_from_page(m); | ||||
KASSERT(rv->pages == m, | KASSERT(rv->pages == m, | ||||
("vm_reserv_alloc_page: reserv %p's pages is corrupted", rv)); | ("vm_reserv_alloc_page: reserv %p's pages is corrupted", rv)); | ||||
KASSERT(rv->object == NULL, | vm_reserv_insert(rv, object, first); | ||||
("vm_reserv_alloc_page: reserv %p isn't free", rv)); | |||||
LIST_INSERT_HEAD(&object->rvq, rv, objq); | |||||
rv->object = object; | |||||
rv->pindex = first; | |||||
rv->domain = domain; | |||||
KASSERT(rv->popcnt == 0, | |||||
("vm_reserv_alloc_page: reserv %p's popcnt is corrupted", rv)); | |||||
KASSERT(!rv->inpartpopq, | |||||
("vm_reserv_alloc_page: reserv %p's inpartpopq is TRUE", rv)); | |||||
for (i = 0; i < NPOPMAP; i++) | |||||
KASSERT(rv->popmap[i] == 0, | |||||
("vm_reserv_alloc_page: reserv %p's popmap is corrupted", | |||||
rv)); | |||||
index = VM_RESERV_INDEX(object, pindex); | index = VM_RESERV_INDEX(object, pindex); | ||||
vm_reserv_populate(rv, index); | vm_reserv_populate(rv, index); | ||||
return (&rv->pages[index]); | return (&rv->pages[index]); | ||||
/* | |||||
* Found a matching reservation. | |||||
*/ | |||||
found: | |||||
index = VM_RESERV_INDEX(object, pindex); | |||||
m = &rv->pages[index]; | |||||
KASSERT(object != kernel_object || vm_phys_domidx(m) == domain, | |||||
("vm_reserv_alloc_page: Domain mismatch from reservation.")); | |||||
/* Handle vm_page_rename(m, new_object, ...). */ | |||||
if (popmap_is_set(rv->popmap, index)) | |||||
return (NULL); | |||||
vm_reserv_populate(rv, index); | |||||
return (m); | |||||
} | } | ||||
/* | /* | ||||
* Breaks the given reservation. Except for the specified free page, all free | * Breaks the given reservation. Except for the specified free page, all free | ||||
* pages in the reservation are returned to the physical memory allocator. | * pages in the reservation are returned to the physical memory allocator. | ||||
* The reservation's population count and map are reset to their initial | * The reservation's population count and map are reset to their initial | ||||
* state. | * state. | ||||
* | * | ||||
* The given reservation must not be in the partially populated reservation | * The given reservation must not be in the partially populated reservation | ||||
* queue. The free page queue lock must be held. | * queue. The free page queue lock must be held. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_reserv_break(vm_reserv_t rv, vm_page_t m) | vm_reserv_break(vm_reserv_t rv, vm_page_t m) | ||||
{ | { | ||||
int begin_zeroes, hi, i, lo; | int begin_zeroes, hi, i, lo; | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); | ||||
KASSERT(rv->object != NULL, | vm_reserv_remove(rv); | ||||
("vm_reserv_break: reserv %p is free", rv)); | |||||
KASSERT(!rv->inpartpopq, | |||||
("vm_reserv_break: reserv %p's inpartpopq is TRUE", rv)); | |||||
LIST_REMOVE(rv, objq); | |||||
rv->object = NULL; | |||||
rv->domain = -1; | |||||
if (m != NULL) { | if (m != NULL) { | ||||
/* | /* | ||||
* Since the reservation is being broken, there is no harm in | * Since the reservation is being broken, there is no harm in | ||||
* abusing the population map to stop "m" from being returned | * abusing the population map to stop "m" from being returned | ||||
* to the physical memory allocator. | * to the physical memory allocator. | ||||
*/ | */ | ||||
i = m - rv->pages; | i = m - rv->pages; | ||||
KASSERT(popmap_is_clear(rv->popmap, i), | KASSERT(popmap_is_clear(rv->popmap, i), | ||||
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Breaks all reservations belonging to the given object. | * Breaks all reservations belonging to the given object. | ||||
*/ | */ | ||||
void | void | ||||
vm_reserv_break_all(vm_object_t object) | vm_reserv_break_all(vm_object_t object) | ||||
{ | { | ||||
vm_reserv_t rv; | vm_reserv_t rv; | ||||
struct vm_domain *vmd; | |||||
markjUnsubmitted Not Done Inline ActionsWe should assert that the object is locked. markj: We should assert that the object is locked. | |||||
mtx_lock(&vm_page_queue_free_mtx); | /* | ||||
* This access of object->rvq is unsynchronized so that the | |||||
* object rvq lock can nest after the domain_free lock. We | |||||
* must check for races in the results. However, the object | |||||
* lock prevents new additions, so we are guaranteed that when | |||||
* it returns NULL the object is properly empty. | |||||
*/ | |||||
vmd = NULL; | |||||
while ((rv = LIST_FIRST(&object->rvq)) != NULL) { | while ((rv = LIST_FIRST(&object->rvq)) != NULL) { | ||||
if (vmd != VM_DOMAIN(rv->domain)) { | |||||
if (vmd != NULL) | |||||
vm_domain_free_unlock(vmd); | |||||
vmd = VM_DOMAIN(rv->domain); | |||||
vm_domain_free_lock(vmd); | |||||
} | |||||
/* Reclaim race. */ | |||||
if (rv->object != object) | |||||
continue; | |||||
KASSERT(rv->object == object, | KASSERT(rv->object == object, | ||||
("vm_reserv_break_all: reserv %p is corrupted", rv)); | ("vm_reserv_break_all: reserv %p is corrupted", rv)); | ||||
if (rv->inpartpopq) { | if (rv->inpartpopq) { | ||||
TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); | TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); | ||||
rv->inpartpopq = FALSE; | rv->inpartpopq = FALSE; | ||||
} | } | ||||
vm_reserv_break(rv, NULL); | vm_reserv_break(rv, NULL); | ||||
} | } | ||||
mtx_unlock(&vm_page_queue_free_mtx); | if (vmd != NULL) | ||||
vm_domain_free_unlock(vmd); | |||||
} | } | ||||
/* | /* | ||||
* Frees the given page if it belongs to a reservation. Returns TRUE if the | * Frees the given page if it belongs to a reservation. Returns TRUE if the | ||||
* page is freed and FALSE otherwise. | * page is freed and FALSE otherwise. | ||||
* | * | ||||
* The free page queue lock must be held. | * The free page queue lock must be held. | ||||
*/ | */ | ||||
boolean_t | boolean_t | ||||
vm_reserv_free_page(vm_page_t m) | vm_reserv_free_page(vm_page_t m) | ||||
{ | { | ||||
vm_reserv_t rv; | vm_reserv_t rv; | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | |||||
rv = vm_reserv_from_page(m); | rv = vm_reserv_from_page(m); | ||||
vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); | |||||
if (rv->object == NULL) | if (rv->object == NULL) | ||||
return (FALSE); | return (FALSE); | ||||
vm_reserv_depopulate(rv, m - rv->pages); | vm_reserv_depopulate(rv, m - rv->pages); | ||||
return (TRUE); | return (TRUE); | ||||
} | } | ||||
/* | /* | ||||
* Initializes the reservation management system. Specifically, initializes | * Initializes the reservation management system. Specifically, initializes | ||||
Show All 13 Lines | vm_reserv_init(void) | ||||
* "pages" field for every element that has an underlying superpage. | * "pages" field for every element that has an underlying superpage. | ||||
*/ | */ | ||||
for (segind = 0; segind < vm_phys_nsegs; segind++) { | for (segind = 0; segind < vm_phys_nsegs; segind++) { | ||||
seg = &vm_phys_segs[segind]; | seg = &vm_phys_segs[segind]; | ||||
paddr = roundup2(seg->start, VM_LEVEL_0_SIZE); | paddr = roundup2(seg->start, VM_LEVEL_0_SIZE); | ||||
while (paddr + VM_LEVEL_0_SIZE <= seg->end) { | while (paddr + VM_LEVEL_0_SIZE <= seg->end) { | ||||
vm_reserv_array[paddr >> VM_LEVEL_0_SHIFT].pages = | vm_reserv_array[paddr >> VM_LEVEL_0_SHIFT].pages = | ||||
PHYS_TO_VM_PAGE(paddr); | PHYS_TO_VM_PAGE(paddr); | ||||
vm_reserv_array[paddr >> VM_LEVEL_0_SHIFT].domain = | |||||
seg->domain; | |||||
paddr += VM_LEVEL_0_SIZE; | paddr += VM_LEVEL_0_SIZE; | ||||
} | } | ||||
} | } | ||||
for (i = 0; i < MAXMEMDOM; i++) | for (i = 0; i < MAXMEMDOM; i++) | ||||
TAILQ_INIT(&vm_rvq_partpop[i]); | TAILQ_INIT(&vm_rvq_partpop[i]); | ||||
} | } | ||||
/* | /* | ||||
* Returns true if the given page belongs to a reservation and that page is | * Returns true if the given page belongs to a reservation and that page is | ||||
* free. Otherwise, returns false. | * free. Otherwise, returns false. | ||||
*/ | */ | ||||
bool | bool | ||||
vm_reserv_is_page_free(vm_page_t m) | vm_reserv_is_page_free(vm_page_t m) | ||||
{ | { | ||||
vm_reserv_t rv; | vm_reserv_t rv; | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | |||||
rv = vm_reserv_from_page(m); | rv = vm_reserv_from_page(m); | ||||
vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); | |||||
if (rv->object == NULL) | if (rv->object == NULL) | ||||
return (false); | return (false); | ||||
return (popmap_is_clear(rv->popmap, m - rv->pages)); | return (popmap_is_clear(rv->popmap, m - rv->pages)); | ||||
} | } | ||||
/* | /* | ||||
* If the given page belongs to a reservation, returns the level of that | * If the given page belongs to a reservation, returns the level of that | ||||
* reservation. Otherwise, returns -1. | * reservation. Otherwise, returns -1. | ||||
Show All 25 Lines | |||||
* to the physical memory allocator. | * to the physical memory allocator. | ||||
* | * | ||||
* The free page queue lock must be held. | * The free page queue lock must be held. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_reserv_reclaim(vm_reserv_t rv) | vm_reserv_reclaim(vm_reserv_t rv) | ||||
{ | { | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | vm_domain_free_assert_locked(VM_DOMAIN(rv->domain)); | ||||
KASSERT(rv->inpartpopq, | KASSERT(rv->inpartpopq, | ||||
("vm_reserv_reclaim: reserv %p's inpartpopq is FALSE", rv)); | ("vm_reserv_reclaim: reserv %p's inpartpopq is FALSE", rv)); | ||||
KASSERT(rv->domain >= 0 && rv->domain < vm_ndomains, | KASSERT(rv->domain >= 0 && rv->domain < vm_ndomains, | ||||
("vm_reserv_reclaim: reserv %p's domain is corrupted %d", | ("vm_reserv_reclaim: reserv %p's domain is corrupted %d", | ||||
rv, rv->domain)); | rv, rv->domain)); | ||||
TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); | TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq); | ||||
rv->inpartpopq = FALSE; | rv->inpartpopq = FALSE; | ||||
vm_reserv_break(rv, NULL); | vm_reserv_break(rv, NULL); | ||||
vm_reserv_reclaimed++; | vm_reserv_reclaimed++; | ||||
} | } | ||||
/* | /* | ||||
* Breaks the reservation at the head of the partially populated reservation | * Breaks the reservation at the head of the partially populated reservation | ||||
* queue, releasing its free pages to the physical memory allocator. Returns | * queue, releasing its free pages to the physical memory allocator. Returns | ||||
* TRUE if a reservation is broken and FALSE otherwise. | * TRUE if a reservation is broken and FALSE otherwise. | ||||
* | * | ||||
* The free page queue lock must be held. | * The free page queue lock must be held. | ||||
*/ | */ | ||||
boolean_t | boolean_t | ||||
vm_reserv_reclaim_inactive(int domain) | vm_reserv_reclaim_inactive(int domain) | ||||
{ | { | ||||
vm_reserv_t rv; | vm_reserv_t rv; | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | vm_domain_free_assert_locked(VM_DOMAIN(domain)); | ||||
if ((rv = TAILQ_FIRST(&vm_rvq_partpop[domain])) != NULL) { | if ((rv = TAILQ_FIRST(&vm_rvq_partpop[domain])) != NULL) { | ||||
vm_reserv_reclaim(rv); | vm_reserv_reclaim(rv); | ||||
return (TRUE); | return (TRUE); | ||||
} | } | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
/* | /* | ||||
* Searches the partially populated reservation queue for the least recently | * Searches the partially populated reservation queue for the least recently | ||||
* changed reservation with free pages that satisfy the given request for | * changed reservation with free pages that satisfy the given request for | ||||
* contiguous physical memory. If a satisfactory reservation is found, it is | * contiguous physical memory. If a satisfactory reservation is found, it is | ||||
* broken. Returns TRUE if a reservation is broken and FALSE otherwise. | * broken. Returns TRUE if a reservation is broken and FALSE otherwise. | ||||
* | * | ||||
* The free page queue lock must be held. | * The free page queue lock must be held. | ||||
*/ | */ | ||||
boolean_t | boolean_t | ||||
vm_reserv_reclaim_contig(int domain, u_long npages, vm_paddr_t low, | vm_reserv_reclaim_contig(int domain, u_long npages, vm_paddr_t low, | ||||
vm_paddr_t high, u_long alignment, vm_paddr_t boundary) | vm_paddr_t high, u_long alignment, vm_paddr_t boundary) | ||||
{ | { | ||||
vm_paddr_t pa, size; | vm_paddr_t pa, size; | ||||
vm_reserv_t rv; | vm_reserv_t rv; | ||||
int hi, i, lo, low_index, next_free; | int hi, i, lo, low_index, next_free; | ||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | vm_domain_free_assert_locked(VM_DOMAIN(domain)); | ||||
if (npages > VM_LEVEL_0_NPAGES - 1) | if (npages > VM_LEVEL_0_NPAGES - 1) | ||||
return (FALSE); | return (FALSE); | ||||
size = npages << PAGE_SHIFT; | size = npages << PAGE_SHIFT; | ||||
TAILQ_FOREACH(rv, &vm_rvq_partpop[domain], partpopq) { | TAILQ_FOREACH(rv, &vm_rvq_partpop[domain], partpopq) { | ||||
pa = VM_PAGE_TO_PHYS(&rv->pages[VM_LEVEL_0_NPAGES - 1]); | pa = VM_PAGE_TO_PHYS(&rv->pages[VM_LEVEL_0_NPAGES - 1]); | ||||
if (pa + PAGE_SIZE - size < low) { | if (pa + PAGE_SIZE - size < low) { | ||||
/* This entire reservation is too low; go to next. */ | /* This entire reservation is too low; go to next. */ | ||||
continue; | continue; | ||||
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines | |||||
vm_reserv_rename(vm_page_t m, vm_object_t new_object, vm_object_t old_object, | vm_reserv_rename(vm_page_t m, vm_object_t new_object, vm_object_t old_object, | ||||
vm_pindex_t old_object_offset) | vm_pindex_t old_object_offset) | ||||
{ | { | ||||
vm_reserv_t rv; | vm_reserv_t rv; | ||||
VM_OBJECT_ASSERT_WLOCKED(new_object); | VM_OBJECT_ASSERT_WLOCKED(new_object); | ||||
rv = vm_reserv_from_page(m); | rv = vm_reserv_from_page(m); | ||||
if (rv->object == old_object) { | if (rv->object == old_object) { | ||||
mtx_lock(&vm_page_queue_free_mtx); | vm_domain_free_lock(VM_DOMAIN(rv->domain)); | ||||
if (rv->object == old_object) { | if (rv->object == old_object) { | ||||
vm_reserv_object_lock(old_object); | |||||
rv->object = NULL; | |||||
LIST_REMOVE(rv, objq); | LIST_REMOVE(rv, objq); | ||||
LIST_INSERT_HEAD(&new_object->rvq, rv, objq); | vm_reserv_object_unlock(old_object); | ||||
vm_reserv_object_lock(new_object); | |||||
rv->object = new_object; | rv->object = new_object; | ||||
rv->pindex -= old_object_offset; | rv->pindex -= old_object_offset; | ||||
LIST_INSERT_HEAD(&new_object->rvq, rv, objq); | |||||
vm_reserv_object_unlock(new_object); | |||||
} | } | ||||
mtx_unlock(&vm_page_queue_free_mtx); | vm_domain_free_unlock(VM_DOMAIN(rv->domain)); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Returns the size (in bytes) of a reservation of the specified level. | * Returns the size (in bytes) of a reservation of the specified level. | ||||
*/ | */ | ||||
int | int | ||||
vm_reserv_size(int level) | vm_reserv_size(int level) | ||||
Show All 13 Lines | |||||
* Allocates the virtual and physical memory required by the reservation | * Allocates the virtual and physical memory required by the reservation | ||||
* management system's data structures, in particular, the reservation array. | * management system's data structures, in particular, the reservation array. | ||||
*/ | */ | ||||
vm_paddr_t | vm_paddr_t | ||||
vm_reserv_startup(vm_offset_t *vaddr, vm_paddr_t end, vm_paddr_t high_water) | vm_reserv_startup(vm_offset_t *vaddr, vm_paddr_t end, vm_paddr_t high_water) | ||||
{ | { | ||||
vm_paddr_t new_end; | vm_paddr_t new_end; | ||||
size_t size; | size_t size; | ||||
int i; | |||||
/* | /* | ||||
* Calculate the size (in bytes) of the reservation array. Round up | * Calculate the size (in bytes) of the reservation array. Round up | ||||
* from "high_water" because every small page is mapped to an element | * from "high_water" because every small page is mapped to an element | ||||
* in the reservation array based on its physical address. Thus, the | * in the reservation array based on its physical address. Thus, the | ||||
* number of elements in the reservation array can be greater than the | * number of elements in the reservation array can be greater than the | ||||
* number of superpages. | * number of superpages. | ||||
*/ | */ | ||||
size = howmany(high_water, VM_LEVEL_0_SIZE) * sizeof(struct vm_reserv); | size = howmany(high_water, VM_LEVEL_0_SIZE) * sizeof(struct vm_reserv); | ||||
/* | /* | ||||
* Allocate and map the physical memory for the reservation array. The | * Allocate and map the physical memory for the reservation array. The | ||||
* next available virtual address is returned by reference. | * next available virtual address is returned by reference. | ||||
*/ | */ | ||||
new_end = end - round_page(size); | new_end = end - round_page(size); | ||||
vm_reserv_array = (void *)(uintptr_t)pmap_map(vaddr, new_end, end, | vm_reserv_array = (void *)(uintptr_t)pmap_map(vaddr, new_end, end, | ||||
VM_PROT_READ | VM_PROT_WRITE); | VM_PROT_READ | VM_PROT_WRITE); | ||||
bzero(vm_reserv_array, size); | bzero(vm_reserv_array, size); | ||||
for (i = 0; i < VM_RESERV_OBJ_LOCK_COUNT; i++) | |||||
mtx_init(&vm_reserv_object_mtx[i], "resv obj lock", NULL, | |||||
MTX_DEF); | |||||
/* | /* | ||||
* Return the next available physical address. | * Return the next available physical address. | ||||
*/ | */ | ||||
return (new_end); | return (new_end); | ||||
} | } | ||||
/* | /* | ||||
Show All 14 Lines |
divide and modulo may be expensive