Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_reserv.c
Show First 20 Lines • Show All 570 Lines • ▼ Show 20 Lines | KASSERT(rv->pages->psind == 0, | ||||
("vm_reserv_populate: reserv %p is already promoted", | ("vm_reserv_populate: reserv %p is already promoted", | ||||
rv)); | rv)); | ||||
rv->pages->psind = 1; | rv->pages->psind = 1; | ||||
} | } | ||||
vm_reserv_domain_unlock(rv->domain); | vm_reserv_domain_unlock(rv->domain); | ||||
} | } | ||||
/* | /* | ||||
* Attempts to allocate a contiguous set of physical pages from existing | * Allocates a contiguous set of physical pages of the given size "npages" | ||||
* reservations. See vm_reserv_alloc_contig() for a description of the | * from existing or newly created reservations. All of the physical pages | ||||
* function's parameters. | * must be at or above the given physical address "low" and below the given | ||||
* physical address "high". The given value "alignment" determines the | |||||
* alignment of the first physical page in the set. If the given value | |||||
* "boundary" is non-zero, then the set of physical pages cannot cross any | |||||
* physical address boundary that is a multiple of that value. Both | |||||
* "alignment" and "boundary" must be a power of two. | |||||
* | * | ||||
* The page "mpred" must immediately precede the offset "pindex" within the | * The page "mpred" must immediately precede the offset "pindex" within the | ||||
* specified object. | * specified object. | ||||
* | * | ||||
* The object must be locked. | * The object must be locked. | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_reserv_extend_contig(int req, vm_object_t object, vm_pindex_t pindex, | vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, | int req, vm_page_t mpred, u_long npages, vm_paddr_t low, vm_paddr_t high, | ||||
u_long alignment, vm_paddr_t boundary, vm_page_t mpred) | u_long alignment, vm_paddr_t boundary) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
vm_paddr_t pa, size; | vm_paddr_t pa, size; | ||||
vm_page_t m, msucc; | vm_page_t m, m_ret, msucc; | ||||
vm_pindex_t first, leftcap, rightcap; | |||||
vm_reserv_t rv; | vm_reserv_t rv; | ||||
int i, index; | u_long allocpages, maxpages, minpages; | ||||
int i, index, n; | |||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
KASSERT(npages != 0, ("vm_reserv_alloc_contig: npages is 0")); | KASSERT(npages != 0, ("vm_reserv_alloc_contig: npages is 0")); | ||||
/* | /* | ||||
* Is a reservation fundamentally impossible? | * Is a reservation fundamentally impossible? | ||||
*/ | */ | ||||
if (pindex < VM_RESERV_INDEX(object, pindex) || | if (pindex < VM_RESERV_INDEX(object, pindex) || | ||||
pindex + npages > object->size || object->resident_page_count == 0) | pindex + npages > object->size) | ||||
return (NULL); | return (NULL); | ||||
/* | /* | ||||
* All reservations of a particular size have the same alignment. | * All reservations of a particular size have the same alignment. | ||||
* Assuming that the first page is allocated from a reservation, the | * Assuming that the first page is allocated from a reservation, the | ||||
* least significant bits of its physical address can be determined | * least significant bits of its physical address can be determined | ||||
* from its offset from the beginning of the reservation and the size | * from its offset from the beginning of the reservation and the size | ||||
* of the reservation. | * of the reservation. | ||||
* | * | ||||
* Could the specified index within a reservation of the smallest | * Could the specified index within a reservation of the smallest | ||||
* possible size satisfy the alignment and boundary requirements? | * possible size satisfy the alignment and boundary requirements? | ||||
*/ | */ | ||||
pa = VM_RESERV_INDEX(object, pindex) << PAGE_SHIFT; | pa = VM_RESERV_INDEX(object, pindex) << PAGE_SHIFT; | ||||
if ((pa & (alignment - 1)) != 0) | if ((pa & (alignment - 1)) != 0) | ||||
return (NULL); | return (NULL); | ||||
size = npages << PAGE_SHIFT; | size = npages << PAGE_SHIFT; | ||||
if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) | if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) | ||||
return (NULL); | return (NULL); | ||||
/* | /* | ||||
* Look for an existing reservation. | * Look for an existing reservation. | ||||
*/ | */ | ||||
rv = vm_reserv_from_object(object, pindex, mpred, &msucc); | rv = vm_reserv_from_object(object, pindex, mpred, &msucc); | ||||
if (rv == NULL) | if (rv != NULL) { | ||||
return (NULL); | |||||
KASSERT(object != kernel_object || rv->domain == domain, | KASSERT(object != kernel_object || rv->domain == domain, | ||||
("vm_reserv_extend_contig: Domain mismatch from reservation.")); | ("vm_reserv_alloc_contig: domain mismatch")); | ||||
index = VM_RESERV_INDEX(object, pindex); | index = VM_RESERV_INDEX(object, pindex); | ||||
/* Does the allocation fit within the reservation? */ | /* Does the allocation fit within the reservation? */ | ||||
if (index + npages > VM_LEVEL_0_NPAGES) | if (index + npages > VM_LEVEL_0_NPAGES) | ||||
return (NULL); | return (NULL); | ||||
domain = rv->domain; | domain = rv->domain; | ||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
vm_reserv_lock(rv); | vm_reserv_lock(rv); | ||||
/* Handle reclaim race. */ | |||||
if (rv->object != object) | if (rv->object != object) | ||||
goto out; | goto out; | ||||
m = &rv->pages[index]; | m = &rv->pages[index]; | ||||
pa = VM_PAGE_TO_PHYS(m); | pa = VM_PAGE_TO_PHYS(m); | ||||
if (pa < low || pa + size > high || (pa & (alignment - 1)) != 0 || | if (pa < low || pa + size > high || | ||||
(pa & (alignment - 1)) != 0 || | |||||
((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) | ((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) | ||||
goto out; | goto out; | ||||
/* Handle vm_page_rename(m, new_object, ...). */ | /* Handle vm_page_rename(m, new_object, ...). */ | ||||
for (i = 0; i < npages; i++) { | for (i = 0; i < npages; i++) | ||||
if (popmap_is_set(rv->popmap, index + i)) | if (popmap_is_set(rv->popmap, index + i)) | ||||
goto out; | goto out; | ||||
} | |||||
if (!vm_domain_allocate(vmd, req, npages)) | if (!vm_domain_allocate(vmd, req, npages)) | ||||
goto out; | goto out; | ||||
for (i = 0; i < npages; i++) | for (i = 0; i < npages; i++) | ||||
vm_reserv_populate(rv, index + i); | vm_reserv_populate(rv, index + i); | ||||
vm_reserv_unlock(rv); | vm_reserv_unlock(rv); | ||||
return (m); | return (m); | ||||
out: | out: | ||||
vm_reserv_unlock(rv); | vm_reserv_unlock(rv); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
/* | /* | ||||
* Allocates a contiguous set of physical pages of the given size "npages" | |||||
* from newly created reservations. All of the physical pages | |||||
* must be at or above the given physical address "low" and below the given | |||||
* physical address "high". The given value "alignment" determines the | |||||
* alignment of the first physical page in the set. If the given value | |||||
* "boundary" is non-zero, then the set of physical pages cannot cross any | |||||
* physical address boundary that is a multiple of that value. Both | |||||
* "alignment" and "boundary" must be a power of two. | |||||
* | |||||
* Callers should first invoke vm_reserv_extend_contig() to attempt an | |||||
* allocation from existing reservations. | |||||
* | |||||
* The page "mpred" must immediately precede the offset "pindex" within the | |||||
* specified object. | |||||
* | |||||
* The object and free page queue must be locked. | |||||
*/ | |||||
vm_page_t | |||||
vm_reserv_alloc_contig(int req, vm_object_t object, vm_pindex_t pindex, int domain, | |||||
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | |||||
vm_paddr_t boundary, vm_page_t mpred) | |||||
{ | |||||
struct vm_domain *vmd; | |||||
vm_paddr_t pa, size; | |||||
vm_page_t m, m_ret, msucc; | |||||
vm_pindex_t first, leftcap, rightcap; | |||||
vm_reserv_t rv; | |||||
u_long allocpages, maxpages, minpages; | |||||
int i, index, n; | |||||
VM_OBJECT_ASSERT_WLOCKED(object); | |||||
KASSERT(npages != 0, ("vm_reserv_alloc_contig: npages is 0")); | |||||
/* | |||||
* Is a reservation fundamentally impossible? | |||||
*/ | |||||
if (pindex < VM_RESERV_INDEX(object, pindex) || | |||||
pindex + npages > object->size) | |||||
return (NULL); | |||||
/* | |||||
* All reservations of a particular size have the same alignment. | |||||
* Assuming that the first page is allocated from a reservation, the | |||||
* least significant bits of its physical address can be determined | |||||
* from its offset from the beginning of the reservation and the size | |||||
* of the reservation. | |||||
* | |||||
* Could the specified index within a reservation of the smallest | |||||
* possible size satisfy the alignment and boundary requirements? | |||||
*/ | |||||
pa = VM_RESERV_INDEX(object, pindex) << PAGE_SHIFT; | |||||
if ((pa & (alignment - 1)) != 0) | |||||
return (NULL); | |||||
size = npages << PAGE_SHIFT; | |||||
if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) | |||||
return (NULL); | |||||
/* | |||||
* Callers should've extended an existing reservation prior to | |||||
* calling this function. If a reservation exists it is | |||||
* incompatible with the allocation. | |||||
*/ | |||||
rv = vm_reserv_from_object(object, pindex, mpred, &msucc); | |||||
if (rv != NULL) | |||||
return (NULL); | |||||
/* | |||||
* Could at least one reservation fit between the first index to the | * Could at least one reservation fit between the first index to the | ||||
* left that can be used ("leftcap") and the first index to the right | * left that can be used ("leftcap") and the first index to the right | ||||
* that cannot be used ("rightcap")? | * that cannot be used ("rightcap")? | ||||
* | * | ||||
* We must synchronize with the reserv object lock to protect the | * We must synchronize with the reserv object lock to protect the | ||||
* pindex/object of the resulting reservations against rename while | * pindex/object of the resulting reservations against rename while | ||||
* we are inspecting. | * we are inspecting. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines | do { | ||||
m += VM_LEVEL_0_NPAGES; | m += VM_LEVEL_0_NPAGES; | ||||
first += VM_LEVEL_0_NPAGES; | first += VM_LEVEL_0_NPAGES; | ||||
allocpages -= VM_LEVEL_0_NPAGES; | allocpages -= VM_LEVEL_0_NPAGES; | ||||
} while (allocpages >= VM_LEVEL_0_NPAGES); | } while (allocpages >= VM_LEVEL_0_NPAGES); | ||||
return (m_ret); | return (m_ret); | ||||
} | } | ||||
/* | /* | ||||
* Attempts to extend an existing reservation and allocate the page to the | * Allocate a physical page from an existing or newly created reservation. | ||||
* object. | |||||
* | * | ||||
* The page "mpred" must immediately precede the offset "pindex" within the | * The page "mpred" must immediately precede the offset "pindex" within the | ||||
* specified object. | * specified object. | ||||
* | * | ||||
* The object must be locked. | * The object must be locked. | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_reserv_extend(int req, vm_object_t object, vm_pindex_t pindex, int domain, | vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex, int domain, | ||||
vm_page_t mpred) | int req, vm_page_t mpred) | ||||
{ | { | ||||
struct vm_domain *vmd; | struct vm_domain *vmd; | ||||
vm_page_t m, msucc; | vm_page_t m, msucc; | ||||
vm_pindex_t first, leftcap, rightcap; | |||||
vm_reserv_t rv; | vm_reserv_t rv; | ||||
int index; | int index; | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
/* | /* | ||||
* Could a reservation currently exist? | * Is a reservation fundamentally impossible? | ||||
*/ | */ | ||||
if (pindex < VM_RESERV_INDEX(object, pindex) || | if (pindex < VM_RESERV_INDEX(object, pindex) || | ||||
pindex >= object->size || object->resident_page_count == 0) | pindex >= object->size) | ||||
return (NULL); | return (NULL); | ||||
/* | /* | ||||
* Look for an existing reservation. | * Look for an existing reservation. | ||||
*/ | */ | ||||
rv = vm_reserv_from_object(object, pindex, mpred, &msucc); | rv = vm_reserv_from_object(object, pindex, mpred, &msucc); | ||||
if (rv == NULL) | if (rv != NULL) { | ||||
return (NULL); | |||||
KASSERT(object != kernel_object || rv->domain == domain, | KASSERT(object != kernel_object || rv->domain == domain, | ||||
("vm_reserv_extend: Domain mismatch from reservation.")); | ("vm_reserv_alloc_page: domain mismatch")); | ||||
domain = rv->domain; | domain = rv->domain; | ||||
vmd = VM_DOMAIN(domain); | vmd = VM_DOMAIN(domain); | ||||
index = VM_RESERV_INDEX(object, pindex); | index = VM_RESERV_INDEX(object, pindex); | ||||
m = &rv->pages[index]; | m = &rv->pages[index]; | ||||
vm_reserv_lock(rv); | vm_reserv_lock(rv); | ||||
/* Handle reclaim race. */ | /* Handle reclaim race. */ | ||||
if (rv->object != object || | if (rv->object != object || | ||||
/* Handle vm_page_rename(m, new_object, ...). */ | /* Handle vm_page_rename(m, new_object, ...). */ | ||||
popmap_is_set(rv->popmap, index)) { | popmap_is_set(rv->popmap, index)) { | ||||
m = NULL; | m = NULL; | ||||
goto out; | goto out; | ||||
} | } | ||||
if (vm_domain_allocate(vmd, req, 1) == 0) | if (vm_domain_allocate(vmd, req, 1) == 0) | ||||
m = NULL; | m = NULL; | ||||
else | else | ||||
vm_reserv_populate(rv, index); | vm_reserv_populate(rv, index); | ||||
out: | out: | ||||
vm_reserv_unlock(rv); | vm_reserv_unlock(rv); | ||||
return (m); | return (m); | ||||
} | } | ||||
/* | |||||
* Attempts to allocate a new reservation for the object, and allocates a | |||||
* page from that reservation. Callers should first invoke vm_reserv_extend() | |||||
* to attempt an allocation from an existing reservation. | |||||
* | |||||
* The page "mpred" must immediately precede the offset "pindex" within the | |||||
* specified object. | |||||
* | |||||
* The object and free page queue must be locked. | |||||
*/ | |||||
vm_page_t | |||||
vm_reserv_alloc_page(int req, vm_object_t object, vm_pindex_t pindex, int domain, | |||||
vm_page_t mpred) | |||||
{ | |||||
struct vm_domain *vmd; | |||||
vm_page_t m, msucc; | |||||
vm_pindex_t first, leftcap, rightcap; | |||||
vm_reserv_t rv; | |||||
int index; | |||||
VM_OBJECT_ASSERT_WLOCKED(object); | |||||
/* | |||||
* Is a reservation fundamentally impossible? | |||||
*/ | |||||
if (pindex < VM_RESERV_INDEX(object, pindex) || | |||||
pindex >= object->size) | |||||
return (NULL); | |||||
/* | |||||
* Callers should've extended an existing reservation prior to | |||||
* calling this function. If a reservation exists it is | |||||
* incompatible with the allocation. | |||||
*/ | |||||
rv = vm_reserv_from_object(object, pindex, mpred, &msucc); | |||||
if (rv != NULL) | |||||
return (NULL); | |||||
/* | /* | ||||
* Could a reservation fit between the first index to the left that | * Could a reservation fit between the first index to the left that | ||||
* can be used and the first index to the right that cannot be used? | * can be used and the first index to the right that cannot be used? | ||||
* | * | ||||
* We must synchronize with the reserv object lock to protect the | * We must synchronize with the reserv object lock to protect the | ||||
* pindex/object of the resulting reservations against rename while | * pindex/object of the resulting reservations against rename while | ||||
* we are inspecting. | * we are inspecting. | ||||
▲ Show 20 Lines • Show All 549 Lines • Show Last 20 Lines |