Page MenuHomeFreeBSD
Files F137312323

D33344.id99889.diff
No OneTemporary
Actions

D33344.id99889.diff
View Options

Index: sys/vm/vm_reserv.c
===================================================================
--- sys/vm/vm_reserv.c
+++ sys/vm/vm_reserv.c
@@ -1233,25 +1233,29 @@
/*
* Determine whether this reservation has free pages that satisfy the given
* request for contiguous physical memory. Start searching from the lower
- * bound, defined by low_index.
+ * bound, defined by lo, and stop at the upper bound, hi. Return the index
+ * of the first satisfactory free page, or -1 if none is found.
*/
-static bool
-vm_reserv_test_contig(vm_reserv_t rv, u_long npages, vm_paddr_t low,
- vm_paddr_t high, u_long alignment, vm_paddr_t boundary)
+static int
+vm_reserv_find_contig(vm_reserv_t rv, int npages, int lo,
+ int hi, int page_align, int page_bound)
{
- vm_paddr_t pa, size;
u_long changes;
- int bitpos, bits_left, i, hi, lo, n;
+ int bitpos, bits_left, i, n;
vm_reserv_assert_locked(rv);
- size = npages << PAGE_SHIFT;
- pa = VM_PAGE_TO_PHYS(&rv->pages[0]);
- lo = (pa < low) ?
- ((low + PAGE_MASK - pa) >> PAGE_SHIFT) : 0;
+ KASSERT(npages <= VM_LEVEL_0_NPAGES - 1,
+ ("%s: Too many pages", __func__));
+ KASSERT(page_bound <= VM_LEVEL_0_NPAGES,
+ ("%s: Too big a boundary for reservation size", __func__));
+ KASSERT(npages <= page_bound,
+ ("%s: Too many pages for given boundary", __func__));
+ KASSERT(page_align != 0 && powerof2(page_align),
+ ("page_align is not a power of 2"));
+ KASSERT(page_bound != 0 && powerof2(page_bound),
+ ("page_bound is not a power of 2"));
i = lo / NBPOPMAP;
changes = rv->popmap[i] | ((1UL << (lo % NBPOPMAP)) - 1);
- hi = (pa + VM_LEVEL_0_SIZE > high) ?
- ((high + PAGE_MASK - pa) >> PAGE_SHIFT) : VM_LEVEL_0_NPAGES;
n = hi / NBPOPMAP;
bits_left = hi % NBPOPMAP;
hi = lo = -1;
@@ -1276,25 +1280,20 @@
continue;
}
hi = NBPOPMAP * i + bitpos;
- pa = VM_PAGE_TO_PHYS(&rv->pages[lo]);
- if ((pa & (alignment - 1)) != 0) {
+ if (lo < roundup2(lo, page_align)) {
/* Skip to next aligned page. */
- lo += (((pa - 1) | (alignment - 1)) + 1) >>
- PAGE_SHIFT;
+ lo = roundup2(lo, page_align);
if (lo >= VM_LEVEL_0_NPAGES)
- return (false);
- pa = VM_PAGE_TO_PHYS(&rv->pages[lo]);
+ return (-1);
}
- if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) {
+ if (lo + npages > roundup2(lo, page_bound)) {
/* Skip to next boundary-matching page. */
- lo += (((pa - 1) | (boundary - 1)) + 1) >>
- PAGE_SHIFT;
+ lo = roundup2(lo, page_bound);
if (lo >= VM_LEVEL_0_NPAGES)
- return (false);
- pa = VM_PAGE_TO_PHYS(&rv->pages[lo]);
+ return (-1);
}
- if (lo * PAGE_SIZE + size <= hi * PAGE_SIZE)
- return (true);
+ if (lo + npages <= hi)
+ return (lo);
lo = hi;
}
if (++i < n)
@@ -1303,7 +1302,7 @@
changes = bits_left == 0 ? -1UL :
(rv->popmap[n] | (-1UL << bits_left));
else
- return (false);
+ return (-1);
}
}
@@ -1320,12 +1319,23 @@
struct vm_reserv_queue *queue;
vm_paddr_t pa, size;
vm_reserv_t marker, rv, rvn;
+ int hi, lo, posn;
+ KASSERT(npages > 0, ("npages is 0"));
+ KASSERT(powerof2(alignment), ("alignment is not a power of 2"));
+ KASSERT(powerof2(boundary), ("boundary is not a power of 2"));
if (npages > VM_LEVEL_0_NPAGES - 1)
return (false);
+ size = npages << PAGE_SHIFT;
+ /*
+ * Ensure that a free range starting at a boundary-multiple
+ * doesn't include a boundary-multiple within it. Otherwise,
+ * no boundary-constrained allocation is possible.
+ */
+ if (size > boundary)
+ return (false);
marker = &vm_rvd[domain].marker;
queue = &vm_rvd[domain].partpop;
- size = npages << PAGE_SHIFT;
vm_reserv_domain_scan_lock(domain);
vm_reserv_domain_lock(domain);
@@ -1339,6 +1349,10 @@
/* This entire reservation is too high; go to next. */
continue;
}
+ if ((pa & (alignment - 1)) != 0) {
+ /* This entire reservation is unaligned; go to next. */
+ continue;
+ }
if (vm_reserv_trylock(rv) == 0) {
TAILQ_INSERT_AFTER(queue, rv, marker, partpopq);
@@ -1356,8 +1370,22 @@
TAILQ_REMOVE(queue, marker, partpopq);
}
vm_reserv_domain_unlock(domain);
- if (vm_reserv_test_contig(rv, npages, low, high,
- alignment, boundary)) {
+ lo = (pa >= low) ? 0 :
+ (int)((low + PAGE_MASK - pa) >> PAGE_SHIFT);
+ hi = (pa + VM_LEVEL_0_SIZE <= high) ? VM_LEVEL_0_NPAGES :
+ (int)((high - pa) >> PAGE_SHIFT);
+ posn = vm_reserv_find_contig(rv, (int)npages, lo, hi,
+ (int)(alignment >> PAGE_SHIFT),
+ (int)(boundary >> PAGE_SHIFT));
+ if (posn >= 0) {
+ pa = VM_PAGE_TO_PHYS(&rv->pages[posn]);
+ KASSERT((pa & (alignment - 1)) == 0,
+ ("%s: adjusted address does not align to %lx",
+ __func__, alignment));
+ KASSERT(((pa ^ (pa + size - 1)) & -boundary) == 0,
+ ("%s: adjusted address spans boundary to %lx",
+ __func__, boundary));
+
vm_reserv_domain_scan_unlock(domain);
vm_reserv_reclaim(rv);
vm_reserv_unlock(rv);
@@ -1480,6 +1508,90 @@
return (new_end);
}
+/*
+ * XXX
+ */
+static vm_object_t
+debug_vm_reserv_reclaim_contig_pre(void)
+{
+ vm_object_t object;
+ int i;
+
+ object = vm_object_allocate(OBJT_DEFAULT, 512);
+ object->flags |= OBJ_COLORED;
+ object->pg_color = 0;
+
+ /*
+ * Change the set of vm_page_alloc() calls to vary the allocated versus
+ * free pattern that vm_reserv_reclaim_contig() has to handle.
+ */
+ VM_OBJECT_WLOCK(object);
+ for (i = 0; i < 512; i += 17)
+ (void)vm_page_alloc(object, i, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY);
+ VM_OBJECT_WUNLOCK(object);
+
+ return (object);
+}
+
+/*
+ * XXX
+ */
+static uint64_t
+debug_vm_reserv_reclaim_contig_test(vm_object_t object)
+{
+ vm_reserv_t rv;
+
+ rv = LIST_FIRST(&object->rvq);
+ vm_reserv_lock(rv);
+ if (rv->object != object) {
+ vm_reserv_unlock(rv);
+ return (~(uint64_t)0);
+ }
+ vm_reserv_domain_lock(rv->domain);
+ if (rv->inpartpopq) {
+ TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq);
+ rv->inpartpopq = FALSE;
+ }
+ vm_reserv_domain_unlock(rv->domain);
+ uint64_t tsc = rdtscp();
+ vm_reserv_reclaim_contig(rv->domain, 16, 0, 0x80000000, 0x10000, 1);
+ tsc = rdtscp() - tsc;
+ vm_reserv_unlock(rv);
+ return (tsc);
+}
+
+#define NOBJECTS 997
+
+/*
+ * To run a test, use the command: sysctl debug.vm_reserv_reclaim_contig=1
+ */
+static int
+debug_vm_reserv_reclaim_contig(SYSCTL_HANDLER_ARGS)
+{
+ vm_object_t object[NOBJECTS];
+ uint64_t total_tsc;
+ int error, i, j;
+
+ i = 0;
+ error = sysctl_handle_int(oidp, &i, 0, req);
+ if (error != 0)
+ return (error);
+ if (i != 0) {
+ for (j = 0; j < NOBJECTS; j++)
+ object[j] = debug_vm_reserv_reclaim_contig_pre();
+ total_tsc = 0;
+ for (j = 0; j < NOBJECTS; j++)
+ total_tsc += debug_vm_reserv_reclaim_contig_test(object[j]);
+ uprintf("\ncycles/break: %lu\n", total_tsc / NOBJECTS);
+ for (j = 0; j < NOBJECTS; j++)
+ vm_object_deallocate(object[j]);
+ }
+ return (0);
+}
+
+SYSCTL_PROC(_debug, OID_AUTO, vm_reserv_reclaim_contig, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
+ debug_vm_reserv_reclaim_contig, "I", "set to trigger vm_reserv_reclaim_contig event");
+
/*
* Returns the superpage containing the given page.
*/

File Metadata

Mime Type
text/plain
Expires
Sun, Nov 23, 8:31 AM (18 h, 31 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
26009344
Default Alt Text
D33344.id99889.diff (6 KB)

Event Timeline