Index: sys/kern/subr_pctrie.c =================================================================== --- sys/kern/subr_pctrie.c +++ sys/kern/subr_pctrie.c @@ -1074,7 +1074,6 @@ pctrie_iter_remove(struct pctrie_iter *it, struct pctrie_node **freenode) { struct pctrie_node *child, *node, *parent; - uint64_t *m; int slot; DEBUG_POISON_POINTER(parent); @@ -1089,12 +1088,11 @@ node = NULL; child = pctrie_root_load(it->ptree, NULL, PCTRIE_LOCKED); } - m = pctrie_match_value(child, it->index); - if (m != NULL) + if (pctrie_isleaf(child) && pctrie_toval(child) != NULL) pctrie_remove(it->ptree, it->index, parent, node, freenode); if (*freenode != NULL) --it->top; - return (m); + return (pctrie_toval(child)); } /* Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -1514,9 +1514,10 @@ void vm_object_split(vm_map_entry_t entry) { - vm_page_t m, m_next; + struct pctrie_iter pages; + vm_page_t m; vm_object_t orig_object, new_object, backing_object; - vm_pindex_t idx, offidxstart; + vm_pindex_t offidxstart; vm_size_t size; orig_object = entry->object.vm_object; @@ -1567,17 +1568,11 @@ * that the object is in transition. */ vm_object_set_flag(orig_object, OBJ_SPLIT); -#ifdef INVARIANTS - idx = 0; -#endif + vm_page_iter_limit_init(&pages, orig_object, offidxstart + size); retry: - m = vm_page_find_least(orig_object, offidxstart); - KASSERT(m == NULL || idx <= m->pindex - offidxstart, - ("%s: object %p was repopulated", __func__, orig_object)); - for (; m != NULL && (idx = m->pindex - offidxstart) < size; - m = m_next) { - m_next = TAILQ_NEXT(m, listq); - + pctrie_iter_reset(&pages); + for (m = vm_page_iter_lookup_ge(&pages, offidxstart); m != NULL; + m = vm_radix_iter_step(&pages)) { /* * We must wait for pending I/O to complete before we can * rename the page. @@ -1598,13 +1593,13 @@ * an incomplete fault. Just remove and ignore. */ if (vm_page_none_valid(m)) { - if (vm_page_remove(m)) - vm_page_free(m); + vm_radix_iter_remove(&pages); + vm_page_remove_free(m); continue; } /* vm_page_rename() will dirty the page. */ - if (vm_page_rename(m, new_object, idx)) { + if (vm_page_rename(m, new_object, m->pindex - offidxstart)) { vm_page_xunbusy(m); VM_OBJECT_WUNLOCK(new_object); VM_OBJECT_WUNLOCK(orig_object); @@ -1613,6 +1608,7 @@ VM_OBJECT_WLOCK(new_object); goto retry; } + vm_radix_iter_remove(&pages); #if VM_NRESERVLEVEL > 0 /* @@ -1771,7 +1767,7 @@ vm_object_collapse_scan(vm_object_t object) { vm_object_t backing_object; - vm_page_t next, p, pp; + vm_page_t next, p, pp, prem __diagused; vm_pindex_t backing_offset_index, new_pindex; VM_OBJECT_ASSERT_WLOCKED(object); @@ -1875,6 +1871,11 @@ next = vm_object_collapse_scan_wait(object, NULL); continue; } + p->pindex = new_pindex + backing_offset_index; + prem = vm_radix_remove(&backing_object->rtree, p->pindex); + KASSERT(prem == p, ("removed page %p, expected page %p", + prem, p)); + p->pindex = new_pindex; /* Use the old pindex to free the right page. */ vm_pager_freespace(backing_object, new_pindex + Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h +++ sys/vm/vm_page.h @@ -681,6 +681,7 @@ vm_page_t vm_page_relookup(vm_object_t, vm_pindex_t); bool vm_page_remove(vm_page_t); bool vm_page_remove_xbusy(vm_page_t); +void vm_page_remove_free(vm_page_t); int vm_page_rename(vm_page_t, vm_object_t, vm_pindex_t); void vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex, vm_page_t mold); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -1618,7 +1618,6 @@ vm_page_object_remove(vm_page_t m) { vm_object_t object; - vm_page_t mrem __diagused; vm_page_assert_xbusied(m); object = m->object; @@ -1631,10 +1630,7 @@ vm_pager_page_unswapped(m); vm_pager_page_removed(object, m); - m->object = NULL; - mrem = vm_radix_remove(&object->rtree, m->pindex); - KASSERT(mrem == m, ("removed page %p, expected page %p", mrem, m)); /* * Now remove from the object's list of backed pages. @@ -1685,11 +1681,32 @@ bool vm_page_remove_xbusy(vm_page_t m) { + vm_page_t mrem __diagused; + mrem = vm_radix_remove(&m->object->rtree, m->pindex); + KASSERT(mrem == m, ("removed page %p, expected page %p", mrem, m)); vm_page_object_remove(m); return (vm_page_drop(m, VPRC_OBJREF) == VPRC_OBJREF); } +/* + * vm_page_remove_free + * + * Like if "(vm_page_remove(m)) vm_page_free(m)" without removing m from + * its radix tree. + */ +void +vm_page_remove_free(vm_page_t m) +{ + bool dropped; + + vm_page_object_remove(m); + dropped = vm_page_drop(m, VPRC_OBJREF) == VPRC_OBJREF; + vm_page_xunbusy(m); + if (dropped) + vm_page_free(m); +} + /* * vm_page_lookup: * @@ -2000,11 +2017,9 @@ * The operation cannot fail anymore. The removal must happen before * the listq iterator is tainted. */ - m->pindex = opidx; vm_page_object_remove(m); /* Return back to the new pindex to complete vm_page_insert(). */ - m->pindex = new_pindex; m->object = new_object; vm_page_insert_radixdone(m, new_object, mpred); @@ -4074,6 +4089,8 @@ VM_CNT_INC(v_tfree); if (m->object != NULL) { + vm_page_t mrem __diagused; + KASSERT(((m->oflags & VPO_UNMANAGED) != 0) == ((m->object->flags & OBJ_UNMANAGED) != 0), ("vm_page_free_prep: managed flag mismatch for page %p", @@ -4088,6 +4105,9 @@ m->ref_count == VPRC_OBJREF, ("vm_page_free_prep: page %p has unexpected ref_count %u", m, m->ref_count)); + mrem = vm_radix_remove(&m->object->rtree, m->pindex); + KASSERT(mrem == m, ("removed page %p, expected page %p", + mrem, m)); vm_page_object_remove(m); m->ref_count -= VPRC_OBJREF; } else Index: sys/vm/vm_radix.h =================================================================== --- sys/vm/vm_radix.h +++ sys/vm/vm_radix.h @@ -187,6 +187,15 @@ { return (VM_RADIX_PCTRIE_REMOVE_LOOKUP(&rtree->rt_trie, index)); } + +/* + * Remove the current page from the trie. + */ +static __inline void +vm_radix_iter_remove(struct pctrie_iter *pages) +{ + VM_RADIX_PCTRIE_ITER_REMOVE(pages); +} /* * Reclaim all the interior nodes of the trie, and invoke the callback