Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -1520,7 +1520,7 @@ void vm_object_split(vm_map_entry_t entry) { - struct pctrie_iter pages; + struct pctrie_iter new_pages, orig_pages; vm_page_t m; vm_object_t orig_object, new_object, backing_object; vm_pindex_t offidxstart; @@ -1574,11 +1574,13 @@ * that the object is in transition. */ vm_object_set_flag(orig_object, OBJ_SPLIT); - vm_page_iter_limit_init(&pages, orig_object, offidxstart + size); + vm_page_iter_limit_init(&orig_pages, orig_object, offidxstart + size); + vm_page_iter_init(&new_pages, new_object); retry: - pctrie_iter_reset(&pages); - for (m = vm_page_iter_lookup_ge(&pages, offidxstart); m != NULL; - m = vm_radix_iter_step(&pages)) { + pctrie_iter_reset(&orig_pages); + pctrie_iter_reset(&new_pages); + for (m = vm_page_iter_lookup_ge(&orig_pages, offidxstart); m != NULL; + m = vm_radix_iter_step(&orig_pages)) { /* * We must wait for pending I/O to complete before we can * rename the page. @@ -1599,13 +1601,14 @@ * an incomplete fault. Just remove and ignore. */ if (vm_page_none_valid(m)) { - if (vm_page_iter_remove(&pages)) + if (vm_page_iter_remove(&orig_pages)) vm_page_free(m); continue; } /* vm_page_rename() will dirty the page. */ - if (vm_page_rename(&pages, new_object, m->pindex - offidxstart)) { + if (vm_page_rename(&orig_pages, &new_pages, new_object, + m->pindex - offidxstart)) { vm_page_xunbusy(m); VM_OBJECT_WUNLOCK(new_object); VM_OBJECT_WUNLOCK(orig_object); @@ -1651,8 +1654,8 @@ } static vm_page_t -vm_object_collapse_scan_wait(struct pctrie_iter *pages, vm_object_t object, - vm_page_t p) +vm_object_collapse_scan_wait(struct pctrie_iter *backups, + struct pctrie_iter *pages, vm_object_t object, vm_page_t p) { vm_object_t backing_object; @@ -1668,25 +1671,29 @@ VM_OBJECT_WUNLOCK(backing_object); vm_radix_wait(); VM_OBJECT_WLOCK(object); + vm_page_iter_init(pages, object); } else if (p->object == object) { VM_OBJECT_WUNLOCK(backing_object); - if (vm_page_busy_sleep(p, "vmocol", 0)) + if (vm_page_busy_sleep(p, "vmocol", 0)) { VM_OBJECT_WLOCK(object); + vm_page_iter_init(pages, object); + } } else { VM_OBJECT_WUNLOCK(object); if (!vm_page_busy_sleep(p, "vmocol", 0)) VM_OBJECT_WUNLOCK(backing_object); VM_OBJECT_WLOCK(object); + vm_page_iter_init(pages, object); } VM_OBJECT_WLOCK(backing_object); - vm_page_iter_init(pages, backing_object); - return (vm_page_iter_lookup_ge(pages, 0)); + vm_page_iter_init(backups, backing_object); + return (vm_page_iter_lookup_ge(backups, 0)); } static void vm_object_collapse_scan(vm_object_t object) { - struct pctrie_iter pages; + struct pctrie_iter backups, pages; vm_object_t backing_object; vm_page_t next, p, pp; vm_pindex_t backing_offset_index, new_pindex; @@ -1700,16 +1707,17 @@ /* * Our scan */ - vm_page_iter_init(&pages, backing_object); - for (p = vm_page_iter_lookup_ge(&pages, 0); p != NULL; p = next) { - next = TAILQ_NEXT(p, listq); + vm_page_iter_init(&backups, backing_object); + vm_page_iter_init(&pages, object); + for (p = vm_page_iter_lookup_ge(&backups, 0); p != NULL; p = next) { new_pindex = p->pindex - backing_offset_index; /* * Check for busy page */ if (vm_page_tryxbusy(p) == 0) { - next = vm_object_collapse_scan_wait(&pages, object, p); + next = vm_object_collapse_scan_wait( + &backups, &pages, object, p); continue; } @@ -1726,22 +1734,22 @@ KASSERT(!pmap_page_is_mapped(p), ("freeing mapped page %p", p)); - if (vm_page_iter_remove(&pages)) + if (vm_page_iter_remove(&backups)) vm_page_free(p); - next = vm_radix_iter_step(&pages); + next = vm_radix_iter_step(&backups); continue; } if (!vm_page_all_valid(p)) { KASSERT(!pmap_page_is_mapped(p), ("freeing mapped page %p", p)); - if (vm_page_iter_remove(&pages)) + if (vm_page_iter_remove(&backups)) vm_page_free(p); - next = vm_radix_iter_step(&pages); + next = vm_radix_iter_step(&backups); continue; } - pp = vm_page_lookup(object, new_pindex); + pp = vm_page_iter_lookup(&pages, new_pindex); if (pp != NULL && vm_page_tryxbusy(pp) == 0) { vm_page_xunbusy(p); /* @@ -1750,7 +1758,8 @@ * busy bit owner, we can't tell whether it shadows the * original page. */ - next = vm_object_collapse_scan_wait(&pages, object, pp); + next = vm_object_collapse_scan_wait( + &backups, &pages, object, pp); continue; } @@ -1760,7 +1769,7 @@ * there by an incomplete fault. Just remove and * ignore. p can replace it. */ - if (vm_page_remove(pp)) + if (vm_page_iter_remove(&pages)) vm_page_free(pp); pp = NULL; } @@ -1778,9 +1787,9 @@ ("freeing mapped page %p", p)); if (pp != NULL) vm_page_xunbusy(pp); - if (vm_page_iter_remove(&pages)) + if (vm_page_iter_remove(&backups)) vm_page_free(p); - next = vm_radix_iter_step(&pages); + next = vm_radix_iter_step(&backups); continue; } @@ -1791,10 +1800,10 @@ * If the page was mapped to a process, it can remain mapped * through the rename. vm_page_rename() will dirty the page. */ - if (vm_page_rename(&pages, object, new_pindex)) { + if (vm_page_rename(&backups, &pages, object, new_pindex)) { vm_page_xunbusy(p); - next = vm_object_collapse_scan_wait(&pages, object, - NULL); + next = vm_object_collapse_scan_wait( + &backups, &pages, object, NULL); continue; } @@ -1810,7 +1819,7 @@ backing_offset_index); #endif vm_page_xunbusy(p); - next = vm_radix_iter_step(&pages); + next = vm_radix_iter_step(&backups); } return; } Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h +++ sys/vm/vm_page.h @@ -682,7 +682,8 @@ bool vm_page_remove(vm_page_t); bool vm_page_iter_remove(struct pctrie_iter *); bool vm_page_remove_xbusy(vm_page_t); -int vm_page_rename(struct pctrie_iter *, vm_object_t, vm_pindex_t); +int vm_page_rename(struct pctrie_iter *from, struct pctrie_iter *to, + vm_object_t, vm_pindex_t); void vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex, vm_page_t mold); int vm_page_sbusied(vm_page_t m); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -172,7 +172,6 @@ static void vm_page_enqueue(vm_page_t m, uint8_t queue); static bool vm_page_free_prep(vm_page_t m); static void vm_page_free_toq(vm_page_t m); -static void vm_page_free_toq_impl(vm_page_t m, bool do_remove); static void vm_page_init(void *dummy); static int vm_page_insert_after(vm_page_t m, vm_object_t object, vm_pindex_t pindex, vm_page_t mpred); @@ -2086,7 +2085,7 @@ * The objects must be locked. */ int -vm_page_rename(struct pctrie_iter *pages, +vm_page_rename(struct pctrie_iter *pages, struct pctrie_iter *new_pages, vm_object_t new_object, vm_pindex_t new_pindex) { vm_page_t m, mpred; @@ -2102,9 +2101,12 @@ * by m_prev and can cheat on the implementation aspects of the * function. */ + mpred = vm_radix_iter_lookup_le(new_pages, new_pindex); + if (mpred != NULL && mpred->pindex == new_pindex) + return (1); opidx = m->pindex; m->pindex = new_pindex; - if (vm_radix_insert_lookup_lt(&new_object->rtree, m, &mpred) != 0) { + if (vm_radix_iter_insert(new_pages, m) != 0) { m->pindex = opidx; return (1); }