Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_phys.c
Show First 20 Lines • Show All 76 Lines • ▼ Show 20 Lines | |||||
int __read_mostly *mem_locality; | int __read_mostly *mem_locality; | ||||
#endif | #endif | ||||
int __read_mostly vm_ndomains = 1; | int __read_mostly vm_ndomains = 1; | ||||
domainset_t __read_mostly all_domains = DOMAINSET_T_INITIALIZER(0x1); | domainset_t __read_mostly all_domains = DOMAINSET_T_INITIALIZER(0x1); | ||||
struct vm_phys_seg __read_mostly vm_phys_segs[VM_PHYSSEG_MAX]; | struct vm_phys_seg __read_mostly vm_phys_segs[VM_PHYSSEG_MAX]; | ||||
int __read_mostly vm_phys_nsegs; | int __read_mostly vm_phys_nsegs; | ||||
static short __read_mostly vm_phys_segs_root; | |||||
static struct vm_phys_seg vm_phys_early_segs[8]; | static struct vm_phys_seg vm_phys_early_segs[8]; | ||||
static int vm_phys_early_nsegs; | static int vm_phys_early_nsegs; | ||||
struct vm_phys_fictitious_seg; | struct vm_phys_fictitious_seg; | ||||
static int vm_phys_fictitious_cmp(struct vm_phys_fictitious_seg *, | static int vm_phys_fictitious_cmp(struct vm_phys_fictitious_seg *, | ||||
struct vm_phys_fictitious_seg *); | struct vm_phys_fictitious_seg *); | ||||
RB_HEAD(fict_tree, vm_phys_fictitious_seg) vm_phys_fictitious_tree = | RB_HEAD(fict_tree, vm_phys_fictitious_seg) vm_phys_fictitious_tree = | ||||
▲ Show 20 Lines • Show All 479 Lines • ▼ Show 20 Lines | #endif | ||||
("vm_phys_init: DEFAULT flind < 0")); | ("vm_phys_init: DEFAULT flind < 0")); | ||||
} | } | ||||
seg->free_queues = &vm_phys_free_queues[seg->domain][flind]; | seg->free_queues = &vm_phys_free_queues[seg->domain][flind]; | ||||
} | } | ||||
/* | /* | ||||
* Coalesce physical memory segments that are contiguous and share the | * Coalesce physical memory segments that are contiguous and share the | ||||
* same per-domain free queues. | * same per-domain free queues. | ||||
* | |||||
* Maintain a heap of coalesced memory segments, with biggest segments | |||||
* at the top of the heap. 'vm_phys_segs_root' identifies the top of | |||||
* the heap. Offsets 'left' and 'right' locate children of heap | |||||
* elements. Offset zero indicates no child. During construction, | |||||
* the links along the right fringe of the heap point backwards, | |||||
* and are reversed to complete the heap. | |||||
*/ | */ | ||||
prev_seg = vm_phys_segs; | prev_seg = vm_phys_segs; | ||||
prev_seg->left = 0; | |||||
prev_seg->right = 0; | |||||
seg = &vm_phys_segs[1]; | seg = &vm_phys_segs[1]; | ||||
end_seg = &vm_phys_segs[vm_phys_nsegs]; | end_seg = &vm_phys_segs[vm_phys_nsegs]; | ||||
while (seg < end_seg) { | while (seg < end_seg) { | ||||
if (prev_seg->end == seg->start && | if (prev_seg->end == seg->start && | ||||
prev_seg->free_queues == seg->free_queues) { | prev_seg->free_queues == seg->free_queues) { | ||||
prev_seg->end = seg->end; | prev_seg->end = seg->end; | ||||
KASSERT(prev_seg->domain == seg->domain, | KASSERT(prev_seg->domain == seg->domain, | ||||
("vm_phys_init: free queues cannot span domains")); | ("vm_phys_init: free queues cannot span domains")); | ||||
vm_phys_nsegs--; | |||||
end_seg--; | end_seg--; | ||||
for (tmp_seg = seg; tmp_seg < end_seg; tmp_seg++) | for (tmp_seg = seg; tmp_seg < end_seg; tmp_seg++) | ||||
*tmp_seg = *(tmp_seg + 1); | *tmp_seg = *(tmp_seg + 1); | ||||
} else { | } else { | ||||
segind = 0; | |||||
tmp_seg = seg; | |||||
do { | |||||
if (seg->end - seg->start <= | |||||
prev_seg->end - prev_seg->start) | |||||
break; | |||||
tmp_seg = prev_seg; | |||||
prev_seg -= prev_seg->right; | |||||
tmp_seg->right = segind; | |||||
segind = tmp_seg - prev_seg; | |||||
} while (tmp_seg != prev_seg); | |||||
if (tmp_seg == prev_seg) | |||||
prev_seg = seg; | prev_seg = seg; | ||||
seg->left = seg - tmp_seg; | |||||
seg->right = seg - prev_seg; | |||||
prev_seg = seg; | |||||
seg++; | seg++; | ||||
} | } | ||||
} | } | ||||
vm_phys_nsegs = end_seg - vm_phys_segs; | |||||
segind = 0; | |||||
do { | |||||
tmp_seg = prev_seg; | |||||
prev_seg -= prev_seg->right; | |||||
tmp_seg->right = segind; | |||||
segind = tmp_seg - prev_seg; | |||||
} while (tmp_seg != prev_seg); | |||||
vm_phys_segs_root = prev_seg - vm_phys_segs; | |||||
/* | /* | ||||
* Initialize the free queues. | * Initialize the free queues. | ||||
*/ | */ | ||||
for (dom = 0; dom < vm_ndomains; dom++) { | for (dom = 0; dom < vm_ndomains; dom++) { | ||||
for (flind = 0; flind < vm_nfreelists; flind++) { | for (flind = 0; flind < vm_nfreelists; flind++) { | ||||
for (pind = 0; pind < VM_NFREEPOOL; pind++) { | for (pind = 0; pind < VM_NFREEPOOL; pind++) { | ||||
fl = vm_phys_free_queues[dom][flind][pind]; | fl = vm_phys_free_queues[dom][flind][pind]; | ||||
for (oind = 0; oind < VM_NFREEORDER; oind++) | for (oind = 0; oind < VM_NFREEORDER; oind++) | ||||
▲ Show 20 Lines • Show All 263 Lines • ▼ Show 20 Lines | for (pind = 0; pind < VM_NFREEPOOL; pind++) { | ||||
return (m); | return (m); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
/* | /* | ||||
* Find the last segment before the given physical address. | |||||
*/ | |||||
static int | |||||
vm_phys_addr_to_seg(vm_paddr_t pa) | |||||
{ | |||||
int segind; | |||||
#if 0 | |||||
for (segind = vm_phys_nsegs - 1; | |||||
segind >= 0 && vm_phys_segs[segind].start >= pa; segind--); | |||||
#else | |||||
segind = vm_phys_segs_root; | |||||
for (;;) { | |||||
if (pa < vm_phys_segs[segind].start) { | |||||
if (vm_phys_segs[segind].left == 0) { | |||||
--segind; | |||||
break; | |||||
} | |||||
segind -= vm_phys_segs[segind].left; | |||||
} else if (vm_phys_segs[segind].end <= pa) { | |||||
if (vm_phys_segs[segind].right == 0) | |||||
break; | |||||
segind += vm_phys_segs[segind].right; | |||||
} else | |||||
break; | |||||
} | |||||
#endif | |||||
return (segind); | |||||
} | |||||
/* | |||||
* Find the vm_page corresponding to the given physical address. | * Find the vm_page corresponding to the given physical address. | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_phys_paddr_to_vm_page(vm_paddr_t pa) | vm_phys_paddr_to_vm_page(vm_paddr_t pa) | ||||
{ | { | ||||
struct vm_phys_seg *seg; | struct vm_phys_seg *seg; | ||||
int segind; | int segind; | ||||
for (segind = 0; segind < vm_phys_nsegs; segind++) { | segind = vm_phys_addr_to_seg(pa); | ||||
if (segind >= 0) { | |||||
seg = &vm_phys_segs[segind]; | seg = &vm_phys_segs[segind]; | ||||
if (pa >= seg->start && pa < seg->end) | if (pa < seg->end) | ||||
return (&seg->first_page[atop(pa - seg->start)]); | return (&seg->first_page[atop(pa - seg->start)]); | ||||
} | } | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
vm_page_t | vm_page_t | ||||
vm_phys_fictitious_to_vm_page(vm_paddr_t pa) | vm_phys_fictitious_to_vm_page(vm_paddr_t pa) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 333 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* "npages" must be greater than zero. Both "alignment" and "boundary" must | * "npages" must be greater than zero. Both "alignment" and "boundary" must | ||||
* be a power of two. | * be a power of two. | ||||
*/ | */ | ||||
vm_page_t | vm_page_t | ||||
vm_phys_scan_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, | vm_phys_scan_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, | ||||
u_long alignment, vm_paddr_t boundary, int options) | u_long alignment, vm_paddr_t boundary, int options) | ||||
{ | { | ||||
vm_paddr_t pa_end; | vm_paddr_t pa_end, pa_start; | ||||
vm_page_t m_end, m_run, m_start; | vm_page_t m_end, m_run, m_start; | ||||
struct vm_phys_seg *seg; | struct vm_phys_seg *seg; | ||||
int segind; | int segind; | ||||
KASSERT(npages > 0, ("npages is 0")); | KASSERT(npages > 0, ("npages is 0")); | ||||
KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | ||||
KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | ||||
if (low >= high) | if (low >= high) | ||||
return (NULL); | return (NULL); | ||||
for (segind = 0; segind < vm_phys_nsegs; segind++) { | for (segind = vm_phys_addr_to_seg(high); segind >= 0; segind--) { | ||||
seg = &vm_phys_segs[segind]; | seg = &vm_phys_segs[segind]; | ||||
if (seg->domain != domain) | if (seg->domain != domain) | ||||
continue; | continue; | ||||
if (seg->start >= high) | |||||
break; | |||||
if (low >= seg->end) | if (low >= seg->end) | ||||
continue; | break; | ||||
if (low <= seg->start) | if (low <= seg->start) | ||||
m_start = seg->first_page; | pa_start = seg->start; | ||||
else | else | ||||
m_start = &seg->first_page[atop(low - seg->start)]; | pa_start = low; | ||||
if (high < seg->end) | if (high < seg->end) | ||||
pa_end = high; | pa_end = high; | ||||
else | else | ||||
pa_end = seg->end; | pa_end = seg->end; | ||||
m_start = &seg->first_page[atop(pa_start - seg->start)]; | |||||
if (pa_end - VM_PAGE_TO_PHYS(m_start) < ptoa(npages)) | if (pa_end - VM_PAGE_TO_PHYS(m_start) < ptoa(npages)) | ||||
continue; | continue; | ||||
m_end = &seg->first_page[atop(pa_end - seg->start)]; | m_end = &seg->first_page[atop(pa_end - seg->start)]; | ||||
m_run = vm_page_scan_contig(npages, m_start, m_end, | m_run = vm_page_scan_contig(npages, m_start, m_end, | ||||
alignment, boundary, options); | alignment, boundary, options); | ||||
if (m_run != NULL) | if (m_run != NULL) | ||||
return (m_run); | return (m_run); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 96 Lines • ▼ Show 20 Lines | vm_phys_alloc_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high, | ||||
int segind; | int segind; | ||||
KASSERT(npages > 0, ("npages is 0")); | KASSERT(npages > 0, ("npages is 0")); | ||||
KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | KASSERT(powerof2(alignment), ("alignment is not a power of 2")); | ||||
KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | KASSERT(powerof2(boundary), ("boundary is not a power of 2")); | ||||
vm_domain_free_assert_locked(VM_DOMAIN(domain)); | vm_domain_free_assert_locked(VM_DOMAIN(domain)); | ||||
if (low >= high) | if (low >= high) | ||||
return (NULL); | return (NULL); | ||||
m_run = NULL; | for (segind = vm_phys_addr_to_seg(high); segind >= 0; segind--) { | ||||
for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) { | |||||
seg = &vm_phys_segs[segind]; | seg = &vm_phys_segs[segind]; | ||||
if (seg->start >= high || seg->domain != domain) | if (seg->domain != domain) | ||||
continue; | continue; | ||||
if (low >= seg->end) | if (low >= seg->end) | ||||
break; | break; | ||||
if (low <= seg->start) | if (low <= seg->start) | ||||
pa_start = seg->start; | pa_start = seg->start; | ||||
else | else | ||||
pa_start = low; | pa_start = low; | ||||
if (high < seg->end) | if (high < seg->end) | ||||
pa_end = high; | pa_end = high; | ||||
else | else | ||||
pa_end = seg->end; | pa_end = seg->end; | ||||
if (pa_end - pa_start < ptoa(npages)) | if (pa_end - pa_start < ptoa(npages)) | ||||
continue; | continue; | ||||
m_run = vm_phys_alloc_seg_contig(seg, npages, low, high, | m_run = vm_phys_alloc_seg_contig(seg, npages, low, high, | ||||
alignment, boundary); | alignment, boundary); | ||||
if (m_run != NULL) | if (m_run != NULL) | ||||
break; | |||||
} | |||||
return (m_run); | return (m_run); | ||||
} | |||||
return (NULL); | |||||
} | } | ||||
/* | /* | ||||
* Allocate a run of contiguous physical pages from the free list for the | * Allocate a run of contiguous physical pages from the free list for the | ||||
* specified segment. | * specified segment. | ||||
*/ | */ | ||||
static vm_page_t | static vm_page_t | ||||
vm_phys_alloc_seg_contig(struct vm_phys_seg *seg, u_long npages, | vm_phys_alloc_seg_contig(struct vm_phys_seg *seg, u_long npages, | ||||
▲ Show 20 Lines • Show All 362 Lines • Show Last 20 Lines |