Index: sys/vm/vm_reserv.c =================================================================== --- sys/vm/vm_reserv.c +++ sys/vm/vm_reserv.c @@ -1244,8 +1244,16 @@ int bitpos, bits_left, i, hi, lo, n; vm_reserv_assert_locked(rv); + KASSERT(npages <= VM_LEVEL_0_NPAGES - 1, + ("%s: Too many pages", __FUNCTION__)); size = npages << PAGE_SHIFT; + KASSERT(size <= boundary, + ("%s: Size overflows boundary", __FUNCTION__)); pa = VM_PAGE_TO_PHYS(&rv->pages[0]); + KASSERT(pa + VM_LEVEL_0_SIZE - size >= low, + ("%s: This entire reservation is too low", __FUNCTION__)); + KASSERT(pa + size <= high, + ("%s: This entire reservation is too high", __FUNCTION__)); lo = (pa < low) ? ((low + PAGE_MASK - pa) >> PAGE_SHIFT) : 0; i = lo / NBPOPMAP; @@ -1277,23 +1285,29 @@ } hi = NBPOPMAP * i + bitpos; pa = VM_PAGE_TO_PHYS(&rv->pages[lo]); - if ((pa & (alignment - 1)) != 0) { + + if (0 < (((alignment - 1) & -pa) >> PAGE_SHIFT)) { /* Skip to next aligned page. */ - lo += (((pa - 1) | (alignment - 1)) + 1) >> - PAGE_SHIFT; + lo += ((alignment - 1) & -pa) >> PAGE_SHIFT; if (lo >= VM_LEVEL_0_NPAGES) return (false); pa = VM_PAGE_TO_PHYS(&rv->pages[lo]); } - if (((pa ^ (pa + size - 1)) & ~(boundary - 1)) != 0) { + KASSERT((pa & (alignment - 1)) == 0, + ("%s: adjusted address does not align to %lx", + __FUNCTION__, alignment)); + + if (npages > (((boundary - 1) & -pa) >> PAGE_SHIFT)) { /* Skip to next boundary-matching page. */ - lo += (((pa - 1) | (boundary - 1)) + 1) >> - PAGE_SHIFT; + lo += ((boundary - 1) & -pa) >> PAGE_SHIFT; if (lo >= VM_LEVEL_0_NPAGES) return (false); pa = VM_PAGE_TO_PHYS(&rv->pages[lo]); } - if (lo * PAGE_SIZE + size <= hi * PAGE_SIZE) + KASSERT((((pa ^ (pa + size - 1)) & -boundary) == 0), + ("%s: adjusted address spans boundary to %lx", + __FUNCTION__, boundary)); + if (lo + npages <= hi) return (true); lo = hi; } @@ -1326,6 +1340,8 @@ marker = &vm_rvd[domain].marker; queue = &vm_rvd[domain].partpop; size = npages << PAGE_SHIFT; + if (size > boundary) + return (false); vm_reserv_domain_scan_lock(domain); vm_reserv_domain_lock(domain); @@ -1480,6 +1496,90 @@ return (new_end); } +/* + * XXX + */ +static vm_object_t +debug_vm_reserv_reclaim_contig_pre(void) +{ + vm_object_t object; + int i; + + object = vm_object_allocate(OBJT_DEFAULT, 512); + object->flags |= OBJ_COLORED; + object->pg_color = 0; + + /* + * Change the set of vm_page_alloc() calls to vary the allocated versus + * free pattern that vm_reserv_reclaim_contig() has to handle. + */ + VM_OBJECT_WLOCK(object); + for (i = 0; i < 512; i += 17) + (void)vm_page_alloc(object, i, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY); + VM_OBJECT_WUNLOCK(object); + + return (object); +} + +/* + * XXX + */ +static uint64_t +debug_vm_reserv_reclaim_contig_test(vm_object_t object) +{ + vm_reserv_t rv; + + rv = LIST_FIRST(&object->rvq); + vm_reserv_lock(rv); + if (rv->object != object) { + vm_reserv_unlock(rv); + return (~(uint64_t)0); + } + vm_reserv_domain_lock(rv->domain); + if (rv->inpartpopq) { + TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq); + rv->inpartpopq = FALSE; + } + vm_reserv_domain_unlock(rv->domain); + uint64_t tsc = rdtscp(); + vm_reserv_reclaim_contig(rv->domain, 16, 0, 0x80000000, 0x10000, 1); + tsc = rdtscp() - tsc; + vm_reserv_unlock(rv); + return (tsc); +} + +#define NOBJECTS 997 + +/* + * To run a test, use the command: sysctl debug.vm_reserv_reclaim_contig=1 + */ +static int +debug_vm_reserv_reclaim_contig(SYSCTL_HANDLER_ARGS) +{ + vm_object_t object[NOBJECTS]; + uint64_t total_tsc; + int error, i, j; + + i = 0; + error = sysctl_handle_int(oidp, &i, 0, req); + if (error != 0) + return (error); + if (i != 0) { + for (j = 0; j < NOBJECTS; j++) + object[j] = debug_vm_reserv_reclaim_contig_pre(); + total_tsc = 0; + for (j = 0; j < NOBJECTS; j++) + total_tsc += debug_vm_reserv_reclaim_contig_test(object[j]); + uprintf("\ncycles/break: %lu\n", total_tsc / NOBJECTS); + for (j = 0; j < NOBJECTS; j++) + vm_object_deallocate(object[j]); + } + return (0); +} + +SYSCTL_PROC(_debug, OID_AUTO, vm_reserv_reclaim_contig, CTLTYPE_INT | CTLFLAG_RW, 0, 0, + debug_vm_reserv_reclaim_contig, "I", "set to trigger vm_reserv_reclaim_contig event"); + /* * Returns the superpage containing the given page. */