Index: sys/vm/vm_page.h
===================================================================
--- sys/vm/vm_page.h
+++ sys/vm/vm_page.h
@@ -683,8 +683,6 @@
void vm_page_replace(vm_page_t mnew, vm_object_t object,
vm_pindex_t pindex, vm_page_t mold);
int vm_page_sbusied(vm_page_t m);
-vm_page_t vm_page_scan_contig(u_long npages, vm_page_t m_start,
- vm_page_t m_end, u_long alignment, vm_paddr_t boundary, int options);
vm_page_bits_t vm_page_set_dirty(vm_page_t m);
void vm_page_set_valid_range(vm_page_t m, int base, int size);
vm_offset_t vm_page_startup(vm_offset_t vaddr);
Index: sys/vm/vm_page.c
===================================================================
--- sys/vm/vm_page.c
+++ sys/vm/vm_page.c
@@ -2627,7 +2627,7 @@
* span a hole (or discontiguity) in the physical address space. Both
* "alignment" and "boundary" must be a power of two.
*/
-vm_page_t
+static vm_page_t
vm_page_scan_contig(u_long npages, vm_page_t m_start, vm_page_t m_end,
u_long alignment, vm_paddr_t boundary, int options)
{
@@ -3028,10 +3028,9 @@
int desired_runs)
{
struct vm_domain *vmd;
- vm_paddr_t curr_low;
- vm_page_t m_run, _m_runs[NRUNS], *m_runs;
+ vm_page_t bounds[2], m_run, _m_runs[NRUNS], *m_runs;
u_long count, minalign, reclaimed;
- int error, i, min_reclaim, nruns, options, req_class;
+ int error, i, min_reclaim, nruns, options, req_class, segind;
bool ret;
KASSERT(npages > 0, ("npages is 0"));
@@ -3098,16 +3097,16 @@
* Find the highest runs that satisfy the given constraints
* and restrictions, and record them in "m_runs".
*/
- curr_low = low;
count = 0;
- for (;;) {
- m_run = vm_phys_scan_contig(domain, npages, curr_low,
- high, alignment, boundary, options);
- if (m_run == NULL)
- break;
- curr_low = VM_PAGE_TO_PHYS(m_run) + ptoa(npages);
- m_runs[RUN_INDEX(count, nruns)] = m_run;
- count++;
+ segind = vm_phys_paddr_to_segind(low);
+ while ((segind = vm_phys_found_range(bounds, segind, domain,
+ npages, low, high)) != -1) {
+ while ((m_run = vm_page_scan_contig(npages, bounds[0],
+ bounds[1], alignment, boundary, options))) {
+ bounds[0] = m_run + npages;
+ m_runs[RUN_INDEX(count, nruns)] = m_run;
+ count++;
+ }
}
/*
Index: sys/vm/vm_phys.h
===================================================================
--- sys/vm/vm_phys.h
+++ sys/vm/vm_phys.h
@@ -42,6 +42,8 @@
#ifdef _KERNEL
+#include <vm/_vm_phys.h>
+
extern vm_paddr_t phys_avail[];
/* Domains must be dense (non-sparse) and zero-based. */
@@ -77,8 +79,8 @@
vm_page_t vm_phys_paddr_to_vm_page(vm_paddr_t pa);
void vm_phys_register_domains(int ndomains, struct mem_affinity *affinity,
int *locality);
-vm_page_t vm_phys_scan_contig(int domain, u_long npages, vm_paddr_t low,
- vm_paddr_t high, u_long alignment, vm_paddr_t boundary, int options);
+int vm_phys_found_range(vm_page_t bounds[], int segind, int domain,
+ u_long npages, vm_paddr_t low, vm_paddr_t high);
bool vm_phys_unfree_page(vm_page_t m);
int vm_phys_mem_affinity(int f, int t);
void vm_phys_early_add_seg(vm_paddr_t start, vm_paddr_t end);
@@ -106,5 +108,25 @@
#endif
}
+/*
+ * Find the segind corresponding to the given physical address.
+ */
+static inline int
+vm_phys_paddr_to_segind(vm_paddr_t pa)
+{
+ int hi, lo, mid;
+
+ lo = 0;
+ hi = vm_phys_nsegs;
+ while (lo != hi) {
+ mid = lo + (hi - lo) / 2;
+ if (pa >= vm_phys_segs[mid].end)
+ lo = mid + 1;
+ else
+ hi = mid;
+ }
+ return (lo);
+}
+
#endif /* _KERNEL */
#endif /* !_VM_PHYS_H_ */
Index: sys/vm/vm_phys.c
===================================================================
--- sys/vm/vm_phys.c
+++ sys/vm/vm_phys.c
@@ -900,12 +900,13 @@
struct vm_phys_seg *seg;
int segind;
- for (segind = 0; segind < vm_phys_nsegs; segind++) {
- seg = &vm_phys_segs[segind];
- if (pa >= seg->start && pa < seg->end)
- return (&seg->first_page[atop(pa - seg->start)]);
- }
- return (NULL);
+ segind = vm_phys_paddr_to_segind(pa);
+ if (segind == vm_phys_nsegs)
+ return (NULL);
+ seg = &vm_phys_segs[segind];
+ if (pa < seg->start)
+ return (NULL);
+ return (&seg->first_page[atop(pa - seg->start)]);
}
vm_page_t
@@ -1238,55 +1239,34 @@
}
/*
- * Scan physical memory between the specified addresses "low" and "high" for a
- * run of contiguous physical pages that satisfy the specified conditions, and
- * return the lowest page in the run. The specified "alignment" determines
- * the alignment of the lowest physical page in the run. If the specified
- * "boundary" is non-zero, then the run of physical pages cannot span a
- * physical address that is a multiple of "boundary".
- *
- * "npages" must be greater than zero. Both "alignment" and "boundary" must
- * be a power of two.
+ * Identify the first address range within segment segind or greater
+ * that matches the domain, lies within the low/high range, and has
+ * enough pages. Return -1 if there is none.
*/
-vm_page_t
-vm_phys_scan_contig(int domain, u_long npages, vm_paddr_t low, vm_paddr_t high,
- u_long alignment, vm_paddr_t boundary, int options)
+int
+vm_phys_found_range(vm_page_t bounds[], int segind, int domain,
+ u_long npages, vm_paddr_t low, vm_paddr_t high)
{
- vm_paddr_t pa_end;
- vm_page_t m_end, m_run, m_start;
- struct vm_phys_seg *seg;
- int segind;
+ vm_paddr_t pa_end, pa_start;
+ struct vm_phys_seg *end_seg, *seg;
- KASSERT(npages > 0, ("npages is 0"));
- KASSERT(powerof2(alignment), ("alignment is not a power of 2"));
- KASSERT(powerof2(boundary), ("boundary is not a power of 2"));
- if (low >= high)
- return (NULL);
- for (segind = 0; segind < vm_phys_nsegs; segind++) {
- seg = &vm_phys_segs[segind];
- if (seg->domain != domain)
- continue;
- if (seg->start >= high)
- break;
- if (low >= seg->end)
- continue;
- if (low <= seg->start)
- m_start = seg->first_page;
- else
- m_start = &seg->first_page[atop(low - seg->start)];
- if (high < seg->end)
- pa_end = high;
- else
- pa_end = seg->end;
- if (pa_end - VM_PAGE_TO_PHYS(m_start) < ptoa(npages))
+ KASSERT(npages > 0, ("npages is zero"));
+ end_seg = &vm_phys_segs[vm_phys_nsegs];
+ for (seg = &vm_phys_segs[segind];
+ seg < end_seg && seg->start < high; seg++) {
+ pa_start = MAX(low, seg->start);
+ pa_end = MIN(high, seg->end);
+ if (seg->domain != domain || pa_end - pa_start < ptoa(npages))
continue;
- m_end = &seg->first_page[atop(pa_end - seg->start)];
- m_run = vm_page_scan_contig(npages, m_start, m_end,
- alignment, boundary, options);
- if (m_run != NULL)
- return (m_run);
+ bounds[0] = &seg->first_page[atop(pa_start - seg->start)];
+ bounds[1] = &seg->first_page[atop(pa_end - seg->start)];
+ break;
}
- return (NULL);
+ if (seg < end_seg && seg->start >= high)
+ seg = end_seg;
+ if (seg == end_seg)
+ return (-1);
+ return (seg - vm_phys_segs);
}
/*