Changeset View
Changeset View
Standalone View
Standalone View
user/alc/PQ_LAUNDRY/sys/vm/vm_phys.c
Show First 20 Lines • Show All 164 Lines • ▼ Show 20 Lines | |||||
#else | #else | ||||
/* Use round-robin so the domain policy code will only try once per allocation */ | /* Use round-robin so the domain policy code will only try once per allocation */ | ||||
static struct vm_domain_policy vm_default_policy = | static struct vm_domain_policy vm_default_policy = | ||||
VM_DOMAIN_POLICY_STATIC_INITIALISER(VM_POLICY_ROUND_ROBIN, 0); | VM_DOMAIN_POLICY_STATIC_INITIALISER(VM_POLICY_ROUND_ROBIN, 0); | ||||
#endif | #endif | ||||
static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool, | static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool, | ||||
int order); | int order); | ||||
static vm_page_t vm_phys_alloc_seg_contig(struct vm_phys_seg *seg, | |||||
u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, | |||||
vm_paddr_t boundary); | |||||
static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain); | static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain); | ||||
static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end); | static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end); | ||||
static int vm_phys_paddr_to_segind(vm_paddr_t pa); | static int vm_phys_paddr_to_segind(vm_paddr_t pa); | ||||
static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, | static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, | ||||
int order); | int order); | ||||
static int | static int | ||||
sysctl_vm_default_policy(SYSCTL_HANDLER_ARGS) | sysctl_vm_default_policy(SYSCTL_HANDLER_ARGS) | ||||
▲ Show 20 Lines • Show All 977 Lines • ▼ Show 20 Lines | for (; npages > 0; npages -= n) { | ||||
order = flsl(npages) - 1; | order = flsl(npages) - 1; | ||||
n = 1 << order; | n = 1 << order; | ||||
vm_phys_free_pages(m, order); | vm_phys_free_pages(m, order); | ||||
m += n; | m += n; | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Scan physical memory between the specified addresses "low" and "high" for a | |||||
* run of contiguous physical pages that satisfy the specified conditions, and | |||||
* return the lowest page in the run. The specified "alignment" determines | |||||
* the alignment of the lowest physical page in the run. If the specified | |||||
* "boundary" is non-zero, then the run of physical pages cannot span a | |||||
* physical address that is a multiple of "boundary". | |||||
* | |||||
* "npages" must be greater than zero. Both "alignment" and "boundary" must | |||||
* be a power of two. | |||||
*/ | |||||
vm_page_t | |||||
vm_phys_scan_contig(u_long npages, vm_paddr_t low, vm_paddr_t high, | |||||
u_long alignment, vm_paddr_t boundary, int options) | |||||
{ | |||||
vm_paddr_t pa_end; | |||||
vm_page_t m_end, m_run, m_start; | |||||
struct vm_phys_seg *seg; | |||||
int segind; | |||||
KASSERT(npages > 0, ("npages is 0")); | |||||
KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | |||||
KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | |||||
if (low >= high) | |||||
return (NULL); | |||||
for (segind = 0; segind < vm_phys_nsegs; segind++) { | |||||
seg = &vm_phys_segs[segind]; | |||||
if (seg->start >= high) | |||||
break; | |||||
if (low >= seg->end) | |||||
continue; | |||||
if (low <= seg->start) | |||||
m_start = seg->first_page; | |||||
else | |||||
m_start = &seg->first_page[atop(low - seg->start)]; | |||||
if (high < seg->end) | |||||
pa_end = high; | |||||
else | |||||
pa_end = seg->end; | |||||
if (pa_end - VM_PAGE_TO_PHYS(m_start) < ptoa(npages)) | |||||
continue; | |||||
m_end = &seg->first_page[atop(pa_end - seg->start)]; | |||||
m_run = vm_page_scan_contig(npages, m_start, m_end, | |||||
alignment, boundary, options); | |||||
if (m_run != NULL) | |||||
return (m_run); | |||||
} | |||||
return (NULL); | |||||
} | |||||
/* | |||||
* Set the pool for a contiguous, power of two-sized set of physical pages. | * Set the pool for a contiguous, power of two-sized set of physical pages. | ||||
*/ | */ | ||||
void | void | ||||
vm_phys_set_pool(int pool, vm_page_t m, int order) | vm_phys_set_pool(int pool, vm_page_t m, int order) | ||||
{ | { | ||||
vm_page_t m_tmp; | vm_page_t m_tmp; | ||||
for (m_tmp = m; m_tmp < &m[1 << order]; m_tmp++) | for (m_tmp = m; m_tmp < &m[1 << order]; m_tmp++) | ||||
▲ Show 20 Lines • Show All 121 Lines • ▼ Show 20 Lines | |||||
* "boundary" is non-zero, then the set of physical pages cannot cross | * "boundary" is non-zero, then the set of physical pages cannot cross | ||||
* any physical address boundary that is a multiple of that value. Both | * any physical address boundary that is a multiple of that value. Both | ||||
* "alignment" and "boundary" must be a power of two. | * "alignment" and "boundary" must be a power of two. | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_phys_alloc_contig(u_long npages, vm_paddr_t low, vm_paddr_t high, | vm_phys_alloc_contig(u_long npages, vm_paddr_t low, vm_paddr_t high, | ||||
u_long alignment, vm_paddr_t boundary) | u_long alignment, vm_paddr_t boundary) | ||||
{ | { | ||||
struct vm_freelist *fl; | vm_paddr_t pa_end, pa_start; | ||||
struct vm_phys_seg *seg; | vm_page_t m_run; | ||||
vm_paddr_t pa, pa_last, size; | |||||
vm_page_t m, m_ret; | |||||
u_long npages_end; | |||||
int domain, flind, oind, order, pind; | |||||
struct vm_domain_iterator vi; | struct vm_domain_iterator vi; | ||||
struct vm_phys_seg *seg; | |||||
int domain, segind; | |||||
KASSERT(npages > 0, ("npages is 0")); | |||||
KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | |||||
KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | |||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | ||||
size = npages << PAGE_SHIFT; | if (low >= high) | ||||
KASSERT(size != 0, | return (NULL); | ||||
("vm_phys_alloc_contig: size must not be 0")); | |||||
KASSERT((alignment & (alignment - 1)) == 0, | |||||
("vm_phys_alloc_contig: alignment must be a power of 2")); | |||||
KASSERT((boundary & (boundary - 1)) == 0, | |||||
("vm_phys_alloc_contig: boundary must be a power of 2")); | |||||
/* Compute the queue that is the best fit for npages. */ | |||||
for (order = 0; (1 << order) < npages; order++); | |||||
vm_policy_iterator_init(&vi); | vm_policy_iterator_init(&vi); | ||||
restartdom: | restartdom: | ||||
if (vm_domain_iterator_run(&vi, &domain) != 0) { | if (vm_domain_iterator_run(&vi, &domain) != 0) { | ||||
vm_policy_iterator_finish(&vi); | vm_policy_iterator_finish(&vi); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
m_run = NULL; | |||||
for (segind = 0; segind < vm_phys_nsegs; segind++) { | |||||
seg = &vm_phys_segs[segind]; | |||||
if (seg->start >= high) | |||||
break; | |||||
if (low >= seg->end || seg->domain != domain) | |||||
continue; | |||||
if (low <= seg->start) | |||||
pa_start = seg->start; | |||||
else | |||||
pa_start = low; | |||||
if (high < seg->end) | |||||
pa_end = high; | |||||
else | |||||
pa_end = seg->end; | |||||
if (pa_end - pa_start < ptoa(npages)) | |||||
continue; | |||||
m_run = vm_phys_alloc_seg_contig(seg, npages, low, high, | |||||
alignment, boundary); | |||||
if (m_run != NULL) | |||||
break; | |||||
} | |||||
if (m_run == NULL && !vm_domain_iterator_isdone(&vi)) | |||||
goto restartdom; | |||||
vm_policy_iterator_finish(&vi); | |||||
return (m_run); | |||||
} | |||||
for (flind = 0; flind < vm_nfreelists; flind++) { | |||||
for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; oind++) { | |||||
for (pind = 0; pind < VM_NFREEPOOL; pind++) { | |||||
fl = &vm_phys_free_queues[domain][flind][pind][0]; | |||||
TAILQ_FOREACH(m_ret, &fl[oind].pl, plinks.q) { | |||||
/* | /* | ||||
* A free list may contain physical pages | * Allocate a run of contiguous physical pages from the free list for the | ||||
* from one or more segments. | * specified segment. | ||||
*/ | */ | ||||
seg = &vm_phys_segs[m_ret->segind]; | static vm_page_t | ||||
if (seg->start > high || | vm_phys_alloc_seg_contig(struct vm_phys_seg *seg, u_long npages, | ||||
low >= seg->end) | vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) | ||||
continue; | { | ||||
struct vm_freelist *fl; | |||||
vm_paddr_t pa, pa_end, size; | |||||
vm_page_t m, m_ret; | |||||
u_long npages_end; | |||||
int oind, order, pind; | |||||
KASSERT(npages > 0, ("npages is 0")); | |||||
KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | |||||
KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | |||||
mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); | |||||
/* Compute the queue that is the best fit for npages. */ | |||||
for (order = 0; (1 << order) < npages; order++); | |||||
/* Search for a run satisfying the specified conditions. */ | |||||
size = npages << PAGE_SHIFT; | |||||
for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; | |||||
oind++) { | |||||
for (pind = 0; pind < VM_NFREEPOOL; pind++) { | |||||
fl = (*seg->free_queues)[pind]; | |||||
TAILQ_FOREACH(m_ret, &fl[oind].pl, plinks.q) { | |||||
/* | /* | ||||
* Is the size of this allocation request | * Is the size of this allocation request | ||||
* larger than the largest block size? | * larger than the largest block size? | ||||
*/ | */ | ||||
if (order >= VM_NFREEORDER) { | if (order >= VM_NFREEORDER) { | ||||
/* | /* | ||||
* Determine if a sufficient number | * Determine if a sufficient number of | ||||
* of subsequent blocks to satisfy | * subsequent blocks to satisfy the | ||||
* the allocation request are free. | * allocation request are free. | ||||
*/ | */ | ||||
pa = VM_PAGE_TO_PHYS(m_ret); | pa = VM_PAGE_TO_PHYS(m_ret); | ||||
pa_last = pa + size; | pa_end = pa + size; | ||||
for (;;) { | for (;;) { | ||||
pa += 1 << (PAGE_SHIFT + VM_NFREEORDER - 1); | pa += 1 << (PAGE_SHIFT + | ||||
if (pa >= pa_last) | VM_NFREEORDER - 1); | ||||
break; | if (pa >= pa_end || | ||||
if (pa < seg->start || | pa < seg->start || | ||||
pa >= seg->end) | pa >= seg->end) | ||||
break; | break; | ||||
m = &seg->first_page[atop(pa - seg->start)]; | m = &seg->first_page[atop(pa - | ||||
if (m->order != VM_NFREEORDER - 1) | seg->start)]; | ||||
if (m->order != VM_NFREEORDER - | |||||
1) | |||||
break; | break; | ||||
} | } | ||||
/* If not, continue to the next block. */ | /* If not, go to the next block. */ | ||||
if (pa < pa_last) | if (pa < pa_end) | ||||
continue; | continue; | ||||
} | } | ||||
/* | /* | ||||
* Determine if the blocks are within the given range, | * Determine if the blocks are within the | ||||
* satisfy the given alignment, and do not cross the | * given range, satisfy the given alignment, | ||||
* given boundary. | * and do not cross the given boundary. | ||||
*/ | */ | ||||
pa = VM_PAGE_TO_PHYS(m_ret); | pa = VM_PAGE_TO_PHYS(m_ret); | ||||
if (pa >= low && | pa_end = pa + size; | ||||
pa + size <= high && | if (pa >= low && pa_end <= high && (pa & | ||||
(pa & (alignment - 1)) == 0 && | (alignment - 1)) == 0 && ((pa ^ (pa_end - | ||||
((pa ^ (pa + size - 1)) & ~(boundary - 1)) == 0) | 1)) & ~(boundary - 1)) == 0) | ||||
goto done; | goto done; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
} | |||||
if (!vm_domain_iterator_isdone(&vi)) | |||||
goto restartdom; | |||||
vm_policy_iterator_finish(&vi); | |||||
return (NULL); | return (NULL); | ||||
done: | done: | ||||
for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) { | for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) { | ||||
fl = (*seg->free_queues)[m->pool]; | fl = (*seg->free_queues)[m->pool]; | ||||
vm_freelist_rem(fl, m, m->order); | vm_freelist_rem(fl, m, m->order); | ||||
} | } | ||||
if (m_ret->pool != VM_FREEPOOL_DEFAULT) | if (m_ret->pool != VM_FREEPOOL_DEFAULT) | ||||
vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m_ret, oind); | vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m_ret, oind); | ||||
▲ Show 20 Lines • Show All 45 Lines • Show Last 20 Lines |