Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_phys.c
Show First 20 Lines • Show All 167 Lines • ▼ Show 20 Lines | SYSCTL_OID(_vm, OID_AUTO, phys_locality, | ||||
CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, | ||||
sysctl_vm_phys_locality, "A", | sysctl_vm_phys_locality, "A", | ||||
"Phys Locality Info"); | "Phys Locality Info"); | ||||
#endif | #endif | ||||
SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD, | SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD, | ||||
&vm_ndomains, 0, "Number of physical memory domains available."); | &vm_ndomains, 0, "Number of physical memory domains available."); | ||||
static vm_page_t vm_phys_alloc_seg_contig(struct vm_phys_seg *seg, | |||||
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | |||||
vm_paddr_t boundary); | |||||
static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain); | static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain); | ||||
static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end); | static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end); | ||||
static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, | static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, | ||||
int order, int tail); | int order, int tail); | ||||
/* | /* | ||||
* Red-black tree helpers for vm fictitious range management. | * Red-black tree helpers for vm fictitious range management. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 1,159 Lines • ▼ Show 20 Lines | while (order > 0) { | ||||
} | } | ||||
vm_freelist_add(fl, m_tmp, order, 0); | vm_freelist_add(fl, m_tmp, order, 0); | ||||
} | } | ||||
KASSERT(m_set == m, ("vm_phys_unfree_page: fatal inconsistency")); | KASSERT(m_set == m, ("vm_phys_unfree_page: fatal inconsistency")); | ||||
return (TRUE); | return (TRUE); | ||||
} | } | ||||
/* | /* | ||||
* Allocate a contiguous set of physical pages of the given size | * Allocate a run of contiguous physical pages from the specified free list | ||||
* "npages" from the free lists. All of the physical pages must be at | * table. | ||||
* or above the given physical address "low" and below the given | |||||
* physical address "high". The given value "alignment" determines the | |||||
* alignment of the first physical page in the set. If the given value | |||||
* "boundary" is non-zero, then the set of physical pages cannot cross | |||||
* any physical address boundary that is a multiple of that value. Both | |||||
* "alignment" and "boundary" must be a power of two. | |||||
*/ | */ | ||||
vm_page_t | static vm_page_t | ||||
vm_phys_alloc_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, | vm_phys_alloc_queues_contig( | ||||
struct vm_freelist (*queues)[VM_NFREEPOOL][VM_NFREEORDER_MAX], | |||||
u_long npages, vm_paddr_t low, vm_paddr_t high, | |||||
u_long alignment, vm_paddr_t boundary) | u_long alignment, vm_paddr_t boundary) | ||||
{ | { | ||||
vm_paddr_t pa_end, pa_start; | |||||
vm_page_t m_run; | |||||
struct vm_phys_seg *seg; | struct vm_phys_seg *seg; | ||||
int segind; | |||||
KASSERT(npages > 0, ("npages is 0")); | |||||
KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | |||||
KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | |||||
vm_domain_free_assert_locked(VM_DOMAIN(domain)); | |||||
if (low >= high) | |||||
return (NULL); | |||||
m_run = NULL; | |||||
for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) { | |||||
seg = &vm_phys_segs[segind]; | |||||
if (seg->start >= high || seg->domain != domain) | |||||
continue; | |||||
if (low >= seg->end) | |||||
break; | |||||
if (low <= seg->start) | |||||
pa_start = seg->start; | |||||
else | |||||
pa_start = low; | |||||
if (high < seg->end) | |||||
pa_end = high; | |||||
else | |||||
pa_end = seg->end; | |||||
if (pa_end - pa_start < ptoa(npages)) | |||||
continue; | |||||
m_run = vm_phys_alloc_seg_contig(seg, npages, low, high, | |||||
alignment, boundary); | |||||
if (m_run != NULL) | |||||
break; | |||||
} | |||||
return (m_run); | |||||
} | |||||
/* | |||||
* Allocate a run of contiguous physical pages from the free list for the | |||||
* specified segment. | |||||
*/ | |||||
static vm_page_t | |||||
vm_phys_alloc_seg_contig(struct vm_phys_seg *seg, u_long npages, | |||||
vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) | |||||
{ | |||||
struct vm_freelist *fl; | struct vm_freelist *fl; | ||||
vm_paddr_t pa, pa_end, size; | vm_paddr_t pa, pa_end, size; | ||||
vm_page_t m, m_ret; | vm_page_t m, m_ret; | ||||
u_long npages_end; | u_long npages_end; | ||||
int oind, order, pind; | int oind, order, pind; | ||||
KASSERT(npages > 0, ("npages is 0")); | KASSERT(npages > 0, ("npages is 0")); | ||||
KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | ||||
KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | ||||
vm_domain_free_assert_locked(VM_DOMAIN(seg->domain)); | |||||
/* Compute the queue that is the best fit for npages. */ | /* Compute the queue that is the best fit for npages. */ | ||||
order = flsl(npages - 1); | order = flsl(npages - 1); | ||||
/* Search for a run satisfying the specified conditions. */ | /* Search for a run satisfying the specified conditions. */ | ||||
size = npages << PAGE_SHIFT; | size = npages << PAGE_SHIFT; | ||||
for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; | for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; | ||||
oind++) { | oind++) { | ||||
for (pind = 0; pind < VM_NFREEPOOL; pind++) { | for (pind = 0; pind < VM_NFREEPOOL; pind++) { | ||||
fl = (*seg->free_queues)[pind]; | fl = (*queues)[pind]; | ||||
TAILQ_FOREACH(m_ret, &fl[oind].pl, listq) { | TAILQ_FOREACH(m_ret, &fl[oind].pl, listq) { | ||||
/* | /* | ||||
* Determine if the address range starting at pa | * Determine if the address range starting at pa | ||||
* is within the given range, satisfies the | * is within the given range, satisfies the | ||||
* given alignment, and does not cross the given | * given alignment, and does not cross the given | ||||
* boundary. | * boundary. | ||||
*/ | */ | ||||
pa = VM_PAGE_TO_PHYS(m_ret); | pa = VM_PAGE_TO_PHYS(m_ret); | ||||
Show All 9 Lines | for (pind = 0; pind < VM_NFREEPOOL; pind++) { | ||||
if (order < VM_NFREEORDER) | if (order < VM_NFREEORDER) | ||||
goto done; | goto done; | ||||
/* | /* | ||||
* Determine if the address range is valid | * Determine if the address range is valid | ||||
* (without overflow in pa_end calculation) | * (without overflow in pa_end calculation) | ||||
* and fits within the segment. | * and fits within the segment. | ||||
*/ | */ | ||||
if (pa_end < pa || | seg = &vm_phys_segs[m_ret->segind]; | ||||
pa < seg->start || seg->end < pa_end) | if (pa_end < pa || seg->end < pa_end) | ||||
continue; | continue; | ||||
/* | /* | ||||
* Determine if a sufficient number of | * Determine if a sufficient number of | ||||
* subsequent blocks to satisfy the | * subsequent blocks to satisfy the | ||||
* allocation request are free. | * allocation request are free. | ||||
*/ | */ | ||||
do { | do { | ||||
pa += 1 << | pa += 1 << | ||||
(PAGE_SHIFT + VM_NFREEORDER - 1); | (PAGE_SHIFT + VM_NFREEORDER - 1); | ||||
if (pa >= pa_end) | if (pa >= pa_end) | ||||
goto done; | goto done; | ||||
} while (VM_NFREEORDER - 1 == seg->first_page[ | } while (VM_NFREEORDER - 1 == seg->first_page[ | ||||
atop(pa - seg->start)].order); | atop(pa - seg->start)].order); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
return (NULL); | return (NULL); | ||||
done: | done: | ||||
for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) { | for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) { | ||||
fl = (*seg->free_queues)[m->pool]; | fl = (*queues)[m->pool]; | ||||
vm_freelist_rem(fl, m, oind); | vm_freelist_rem(fl, m, oind); | ||||
if (m->pool != VM_FREEPOOL_DEFAULT) | if (m->pool != VM_FREEPOOL_DEFAULT) | ||||
vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m, oind); | vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m, oind); | ||||
} | } | ||||
/* Return excess pages to the free lists. */ | /* Return excess pages to the free lists. */ | ||||
npages_end = roundup2(npages, 1 << oind); | npages_end = roundup2(npages, 1 << oind); | ||||
if (npages < npages_end) { | if (npages < npages_end) { | ||||
fl = (*seg->free_queues)[VM_FREEPOOL_DEFAULT]; | fl = (*queues)[VM_FREEPOOL_DEFAULT]; | ||||
vm_phys_enq_range(&m_ret[npages], npages_end - npages, fl, 0); | vm_phys_enq_range(&m_ret[npages], npages_end - npages, fl, 0); | ||||
} | } | ||||
return (m_ret); | return (m_ret); | ||||
} | |||||
/* | |||||
* Allocate a contiguous set of physical pages of the given size | |||||
* "npages" from the free lists. All of the physical pages must be at | |||||
* or above the given physical address "low" and below the given | |||||
* physical address "high". The given value "alignment" determines the | |||||
* alignment of the first physical page in the set. If the given value | |||||
* "boundary" is non-zero, then the set of physical pages cannot cross | |||||
* any physical address boundary that is a multiple of that value. Both | |||||
* "alignment" and "boundary" must be a power of two. | |||||
*/ | |||||
vm_page_t | |||||
vm_phys_alloc_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, | |||||
u_long alignment, vm_paddr_t boundary) | |||||
{ | |||||
vm_paddr_t pa_end, pa_start; | |||||
vm_page_t m_run; | |||||
struct vm_phys_seg *seg; | |||||
struct vm_freelist (*queues)[VM_NFREEPOOL][VM_NFREEORDER_MAX]; | |||||
int segind; | |||||
KASSERT(npages > 0, ("npages is 0")); | |||||
KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | |||||
KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | |||||
vm_domain_free_assert_locked(VM_DOMAIN(domain)); | |||||
if (low >= high) | |||||
return (NULL); | |||||
queues = NULL; | |||||
m_run = NULL; | |||||
for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) { | |||||
seg = &vm_phys_segs[segind]; | |||||
if (seg->start >= high || seg->domain != domain) | |||||
continue; | |||||
if (low >= seg->end) | |||||
break; | |||||
if (low <= seg->start) | |||||
pa_start = seg->start; | |||||
else | |||||
pa_start = low; | |||||
if (high < seg->end) | |||||
pa_end = high; | |||||
else | |||||
pa_end = seg->end; | |||||
if (pa_end - pa_start < ptoa(npages)) | |||||
continue; | |||||
/* | |||||
* If a previous segment led to a search using | |||||
* the same free lists as would this segment, then | |||||
* we've actually already searched within this | |||||
* too. So skip it. | |||||
*/ | |||||
if (seg->free_queues == queues) | |||||
continue; | |||||
queues = seg->free_queues; | |||||
m_run = vm_phys_alloc_queues_contig(queues, npages, | |||||
low, high, alignment, boundary); | |||||
if (m_run != NULL) | |||||
break; | |||||
} | |||||
return (m_run); | |||||
} | } | ||||
/* | /* | ||||
* Return the index of the first unused slot which may be the terminating | * Return the index of the first unused slot which may be the terminating | ||||
* entry. | * entry. | ||||
*/ | */ | ||||
static int | static int | ||||
vm_phys_avail_count(void) | vm_phys_avail_count(void) | ||||
▲ Show 20 Lines • Show All 291 Lines • Show Last 20 Lines |