Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -1520,7 +1520,7 @@ void vm_object_split(vm_map_entry_t entry) { - struct pctrie_iter pages; + struct pctrie_iter new_pages, orig_pages; vm_page_t m; vm_object_t orig_object, new_object, backing_object; vm_pindex_t offidxstart; @@ -1574,11 +1574,11 @@ * that the object is in transition. */ vm_object_set_flag(orig_object, OBJ_SPLIT); - vm_page_iter_limit_init(&pages, orig_object, offidxstart + size); + vm_page_iter_limit_init(&orig_pages, orig_object, offidxstart + size); + vm_page_iter_init(&new_pages, new_object); retry: - pctrie_iter_reset(&pages); - for (m = vm_page_iter_lookup_ge(&pages, offidxstart); m != NULL; - m = vm_radix_iter_step(&pages)) { + for (m = vm_page_iter_lookup_ge(&orig_pages, offidxstart); m != NULL; + m = vm_radix_iter_step(&orig_pages)) { /* * We must wait for pending I/O to complete before we can * rename the page. @@ -1590,6 +1590,8 @@ VM_OBJECT_WUNLOCK(new_object); if (vm_page_busy_sleep(m, "spltwt", 0)) VM_OBJECT_WLOCK(orig_object); + pctrie_iter_reset(&orig_pages); + pctrie_iter_reset(&new_pages); VM_OBJECT_WLOCK(new_object); goto retry; } @@ -1599,18 +1601,20 @@ * an incomplete fault. Just remove and ignore. */ if (vm_page_none_valid(m)) { - if (vm_page_iter_remove(&pages, m)) + if (vm_page_iter_remove(&orig_pages, m)) vm_page_free(m); continue; } /* vm_page_iter_rename() will dirty the page. */ - if (!vm_page_iter_rename(&pages, m, new_object, m->pindex - - offidxstart)) { + if (!vm_page_iter_rename(&orig_pages, m, &new_pages, new_object, + m->pindex - offidxstart)) { vm_page_xunbusy(m); VM_OBJECT_WUNLOCK(new_object); VM_OBJECT_WUNLOCK(orig_object); vm_radix_wait(); + pctrie_iter_reset(&orig_pages); + pctrie_iter_reset(&new_pages); VM_OBJECT_WLOCK(orig_object); VM_OBJECT_WLOCK(new_object); goto retry; @@ -1652,8 +1656,8 @@ } static vm_page_t -vm_object_collapse_scan_wait(struct pctrie_iter *pages, vm_object_t object, - vm_page_t p) +vm_object_collapse_scan_wait(struct pctrie_iter *backups, + struct pctrie_iter *pages, vm_object_t object, vm_page_t p) { vm_object_t backing_object; @@ -1668,26 +1672,30 @@ VM_OBJECT_WUNLOCK(object); VM_OBJECT_WUNLOCK(backing_object); vm_radix_wait(); + vm_page_iter_init(pages, object); VM_OBJECT_WLOCK(object); } else if (p->object == object) { VM_OBJECT_WUNLOCK(backing_object); - if (vm_page_busy_sleep(p, "vmocol", 0)) + if (vm_page_busy_sleep(p, "vmocol", 0)) { + vm_page_iter_init(pages, object); VM_OBJECT_WLOCK(object); + } } else { VM_OBJECT_WUNLOCK(object); if (!vm_page_busy_sleep(p, "vmocol", 0)) VM_OBJECT_WUNLOCK(backing_object); + vm_page_iter_init(pages, object); VM_OBJECT_WLOCK(object); } + vm_page_iter_init(backups, backing_object); VM_OBJECT_WLOCK(backing_object); - vm_page_iter_init(pages, backing_object); - return (vm_page_iter_lookup_ge(pages, 0)); + return (vm_page_iter_lookup_ge(backups, 0)); } static void vm_object_collapse_scan(vm_object_t object) { - struct pctrie_iter pages; + struct pctrie_iter backups, pages; vm_object_t backing_object; vm_page_t next, p, pp; vm_pindex_t backing_offset_index, new_pindex; @@ -1701,13 +1709,15 @@ /* * Our scan */ - vm_page_iter_init(&pages, backing_object); - for (p = vm_page_iter_lookup_ge(&pages, 0); p != NULL; p = next) { + vm_page_iter_init(&backups, backing_object); + vm_page_iter_init(&pages, object); + for (p = vm_page_iter_lookup_ge(&backups, 0); p != NULL; p = next) { /* * Check for busy page */ if (vm_page_tryxbusy(p) == 0) { - next = vm_object_collapse_scan_wait(&pages, object, p); + next = vm_object_collapse_scan_wait( + &backups, &pages, object, p); continue; } @@ -1724,22 +1734,22 @@ KASSERT(!pmap_page_is_mapped(p), ("freeing mapped page %p", p)); - if (vm_page_iter_remove(&pages, p)) + if (vm_page_iter_remove(&backups, p)) vm_page_free(p); - next = vm_radix_iter_step(&pages); + next = vm_radix_iter_step(&backups); continue; } if (!vm_page_all_valid(p)) { KASSERT(!pmap_page_is_mapped(p), ("freeing mapped page %p", p)); - if (vm_page_iter_remove(&pages, p)) + if (vm_page_iter_remove(&backups, p)) vm_page_free(p); - next = vm_radix_iter_step(&pages); + next = vm_radix_iter_step(&backups); continue; } - pp = vm_page_lookup(object, new_pindex); + pp = vm_page_iter_lookup(&pages, new_pindex); if (pp != NULL && vm_page_tryxbusy(pp) == 0) { vm_page_xunbusy(p); /* @@ -1748,7 +1758,8 @@ * busy bit owner, we can't tell whether it shadows the * original page. */ - next = vm_object_collapse_scan_wait(&pages, object, pp); + next = vm_object_collapse_scan_wait( + &backups, &pages, object, pp); continue; } @@ -1758,7 +1769,7 @@ * there by an incomplete fault. Just remove and * ignore. p can replace it. */ - if (vm_page_remove(pp)) + if (vm_page_iter_remove(&pages, pp)) vm_page_free(pp); pp = NULL; } @@ -1776,9 +1787,9 @@ ("freeing mapped page %p", p)); if (pp != NULL) vm_page_xunbusy(pp); - if (vm_page_iter_remove(&pages, p)) + if (vm_page_iter_remove(&backups, p)) vm_page_free(p); - next = vm_radix_iter_step(&pages); + next = vm_radix_iter_step(&backups); continue; } @@ -1790,10 +1801,11 @@ * through the rename. vm_page_iter_rename() will dirty the * page. */ - if (!vm_page_iter_rename(&pages, p, object, new_pindex)) { + if (!vm_page_iter_rename(&backups, p, &pages, object, + new_pindex)) { vm_page_xunbusy(p); - next = vm_object_collapse_scan_wait(&pages, object, - NULL); + next = vm_object_collapse_scan_wait( + &backups, &pages, object, NULL); continue; } @@ -1809,7 +1821,7 @@ backing_offset_index); #endif vm_page_xunbusy(p); - next = vm_radix_iter_step(&pages); + next = vm_radix_iter_step(&backups); } return; } Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h +++ sys/vm/vm_page.h @@ -656,7 +656,8 @@ vm_page_t vm_page_iter_lookup(struct pctrie_iter *, vm_pindex_t); bool vm_page_iter_remove(struct pctrie_iter *pages, vm_page_t m); bool vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m, - vm_object_t new_object, vm_pindex_t new_pindex); + struct pctrie_iter *new_pages, vm_object_t new_object, + vm_pindex_t new_pindex); void vm_page_launder(vm_page_t m); vm_page_t vm_page_lookup(vm_object_t, vm_pindex_t); vm_page_t vm_page_lookup_unlocked(vm_object_t, vm_pindex_t); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -2087,7 +2087,8 @@ */ bool vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m, - vm_object_t new_object, vm_pindex_t new_pindex) + struct pctrie_iter *new_pages, vm_object_t new_object, + vm_pindex_t new_pindex) { vm_page_t mpred; vm_pindex_t opidx; @@ -2102,9 +2103,13 @@ * by m_prev and can cheat on the implementation aspects of the * function. */ + mpred = vm_radix_iter_lookup_le(new_pages, new_pindex); + if (mpred != NULL && mpred->pindex == new_pindex) + panic("%s: renaming page %p to duplicate page %p", + __func__, m, mpred); opidx = m->pindex; m->pindex = new_pindex; - if (vm_radix_insert_lookup_lt(&new_object->rtree, m, &mpred) != 0) { + if (vm_radix_iter_insert(new_pages, m) != 0) { m->pindex = opidx; return (false); }