Index: sys/kern/subr_pctrie.c =================================================================== --- sys/kern/subr_pctrie.c +++ sys/kern/subr_pctrie.c @@ -1074,7 +1074,6 @@ pctrie_iter_remove(struct pctrie_iter *it, struct pctrie_node **freenode) { struct pctrie_node *child, *node, *parent; - uint64_t *m; int slot; DEBUG_POISON_POINTER(parent); @@ -1089,12 +1088,11 @@ node = NULL; child = pctrie_root_load(it->ptree, NULL, PCTRIE_LOCKED); } - m = pctrie_match_value(child, it->index); - if (m != NULL) + if (pctrie_isleaf(child) && pctrie_toval(child) != NULL) pctrie_remove(it->ptree, it->index, parent, node, freenode); if (*freenode != NULL) --it->top; - return (m); + return (pctrie_toval(child)); } /* Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -1514,9 +1514,10 @@ void vm_object_split(vm_map_entry_t entry) { - vm_page_t m, m_next; + struct pctrie_iter pages; + vm_page_t m; vm_object_t orig_object, new_object, backing_object; - vm_pindex_t idx, offidxstart; + vm_pindex_t offidxstart; vm_size_t size; orig_object = entry->object.vm_object; @@ -1567,17 +1568,11 @@ * that the object is in transition. */ vm_object_set_flag(orig_object, OBJ_SPLIT); -#ifdef INVARIANTS - idx = 0; -#endif + vm_page_iter_limit_init(&pages, orig_object, offidxstart + size); retry: - m = vm_page_find_least(orig_object, offidxstart); - KASSERT(m == NULL || idx <= m->pindex - offidxstart, - ("%s: object %p was repopulated", __func__, orig_object)); - for (; m != NULL && (idx = m->pindex - offidxstart) < size; - m = m_next) { - m_next = TAILQ_NEXT(m, listq); - + pctrie_iter_reset(&pages); + for (m = vm_page_iter_lookup_ge(&pages, offidxstart); m != NULL; + m = vm_radix_iter_step(&pages)) { /* * We must wait for pending I/O to complete before we can * rename the page. @@ -1598,13 +1593,13 @@ * an incomplete fault. Just remove and ignore. */ if (vm_page_none_valid(m)) { - if (vm_page_remove(m)) - vm_page_free(m); + vm_page_remove_free(m); + vm_radix_iter_remove(&pages); continue; } /* vm_page_rename() will dirty the page. */ - if (vm_page_rename(m, new_object, idx)) { + if (vm_page_rename(m, new_object, m->pindex - offidxstart)) { vm_page_xunbusy(m); VM_OBJECT_WUNLOCK(new_object); VM_OBJECT_WUNLOCK(orig_object); @@ -1629,6 +1624,7 @@ */ vm_reserv_rename(m, new_object, orig_object, offidxstart); #endif + vm_radix_iter_remove(&pages); } /* @@ -1650,7 +1646,8 @@ } static vm_page_t -vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p) +vm_object_collapse_scan_wait(struct pctrie_iter *pages, vm_object_t object, + vm_page_t p) { vm_object_t backing_object; @@ -1677,7 +1674,8 @@ VM_OBJECT_WLOCK(object); } VM_OBJECT_WLOCK(backing_object); - return (TAILQ_FIRST(&backing_object->memq)); + vm_page_iter_init(pages, backing_object); + return (vm_page_iter_lookup_ge(pages, 0)); } static bool @@ -1770,6 +1768,7 @@ static void vm_object_collapse_scan(vm_object_t object) { + struct pctrie_iter pages; vm_object_t backing_object; vm_page_t next, p, pp; vm_pindex_t backing_offset_index, new_pindex; @@ -1783,7 +1782,8 @@ /* * Our scan */ - for (p = TAILQ_FIRST(&backing_object->memq); p != NULL; p = next) { + vm_page_iter_init(&pages, backing_object); + for (p = vm_page_iter_lookup_ge(&pages, 0); p != NULL; p = next) { next = TAILQ_NEXT(p, listq); new_pindex = p->pindex - backing_offset_index; @@ -1791,7 +1791,7 @@ * Check for busy page */ if (vm_page_tryxbusy(p) == 0) { - next = vm_object_collapse_scan_wait(object, p); + next = vm_object_collapse_scan_wait(&pages, object, p); continue; } @@ -1808,16 +1808,18 @@ KASSERT(!pmap_page_is_mapped(p), ("freeing mapped page %p", p)); - if (vm_page_remove(p)) - vm_page_free(p); + vm_page_remove_free(p); + vm_radix_iter_remove(&pages); + next = vm_radix_iter_step(&pages); continue; } if (!vm_page_all_valid(p)) { KASSERT(!pmap_page_is_mapped(p), ("freeing mapped page %p", p)); - if (vm_page_remove(p)) - vm_page_free(p); + vm_page_remove_free(p); + vm_radix_iter_remove(&pages); + next = vm_radix_iter_step(&pages); continue; } @@ -1830,7 +1832,7 @@ * busy bit owner, we can't tell whether it shadows the * original page. */ - next = vm_object_collapse_scan_wait(object, pp); + next = vm_object_collapse_scan_wait(&pages, object, pp); continue; } @@ -1856,10 +1858,11 @@ vm_pager_freespace(backing_object, p->pindex, 1); KASSERT(!pmap_page_is_mapped(p), ("freeing mapped page %p", p)); - if (vm_page_remove(p)) - vm_page_free(p); if (pp != NULL) vm_page_xunbusy(pp); + vm_page_remove_free(p); + vm_radix_iter_remove(&pages); + next = vm_radix_iter_step(&pages); continue; } @@ -1872,7 +1875,8 @@ */ if (vm_page_rename(p, object, new_pindex)) { vm_page_xunbusy(p); - next = vm_object_collapse_scan_wait(object, NULL); + next = vm_object_collapse_scan_wait(&pages, object, + NULL); continue; } @@ -1888,6 +1892,8 @@ backing_offset_index); #endif vm_page_xunbusy(p); + vm_radix_iter_remove(&pages); + next = vm_radix_iter_step(&pages); } return; } Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h +++ sys/vm/vm_page.h @@ -681,6 +681,7 @@ vm_page_t vm_page_relookup(vm_object_t, vm_pindex_t); bool vm_page_remove(vm_page_t); bool vm_page_remove_xbusy(vm_page_t); +void vm_page_remove_free(vm_page_t); int vm_page_rename(vm_page_t, vm_object_t, vm_pindex_t); void vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex, vm_page_t mold); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -1618,7 +1618,6 @@ vm_page_object_remove(vm_page_t m) { vm_object_t object; - vm_page_t mrem __diagused; vm_page_assert_xbusied(m); object = m->object; @@ -1631,10 +1630,7 @@ vm_pager_page_unswapped(m); vm_pager_page_removed(object, m); - m->object = NULL; - mrem = vm_radix_remove(&object->rtree, m->pindex); - KASSERT(mrem == m, ("removed page %p, expected page %p", mrem, m)); /* * Now remove from the object's list of backed pages. @@ -1685,11 +1681,32 @@ bool vm_page_remove_xbusy(vm_page_t m) { + vm_page_t mrem __diagused; + mrem = vm_radix_remove(&m->object->rtree, m->pindex); + KASSERT(mrem == m, ("removed page %p, expected page %p", mrem, m)); vm_page_object_remove(m); return (vm_page_drop(m, VPRC_OBJREF) == VPRC_OBJREF); } +/* + * vm_page_remove_free + * + * Like if "(vm_page_remove(m)) vm_page_free(m)" without removing m from + * its radix tree. + */ +void +vm_page_remove_free(vm_page_t m) +{ + bool dropped; + + vm_page_object_remove(m); + dropped = vm_page_drop(m, VPRC_OBJREF) == VPRC_OBJREF; + vm_page_xunbusy(m); + if (dropped) + vm_page_free(m); +} + /* * vm_page_lookup: * @@ -2000,11 +2017,9 @@ * The operation cannot fail anymore. The removal must happen before * the listq iterator is tainted. */ - m->pindex = opidx; vm_page_object_remove(m); /* Return back to the new pindex to complete vm_page_insert(). */ - m->pindex = new_pindex; m->object = new_object; vm_page_insert_radixdone(m, new_object, mpred); @@ -4074,6 +4089,8 @@ VM_CNT_INC(v_tfree); if (m->object != NULL) { + vm_page_t mrem __diagused; + KASSERT(((m->oflags & VPO_UNMANAGED) != 0) == ((m->object->flags & OBJ_UNMANAGED) != 0), ("vm_page_free_prep: managed flag mismatch for page %p", @@ -4088,6 +4105,9 @@ m->ref_count == VPRC_OBJREF, ("vm_page_free_prep: page %p has unexpected ref_count %u", m, m->ref_count)); + mrem = vm_radix_remove(&m->object->rtree, m->pindex); + KASSERT(mrem == m, ("removed page %p, expected page %p", + mrem, m)); vm_page_object_remove(m); m->ref_count -= VPRC_OBJREF; } else Index: sys/vm/vm_radix.h =================================================================== --- sys/vm/vm_radix.h +++ sys/vm/vm_radix.h @@ -187,6 +187,15 @@ { return (VM_RADIX_PCTRIE_REMOVE_LOOKUP(&rtree->rt_trie, index)); } + +/* + * Remove the current page from the trie. + */ +static __inline void +vm_radix_iter_remove(struct pctrie_iter *pages) +{ + VM_RADIX_PCTRIE_ITER_REMOVE(pages); +} /* * Reclaim all the interior nodes of the trie, and invoke the callback