Index: sys/sys/pctrie.h =================================================================== --- sys/sys/pctrie.h +++ sys/sys/pctrie.h @@ -473,6 +473,42 @@ /* Synchronize to make changes visible. */ \ pctrie_node_store(parentp, child, access); \ return (res); \ +} \ + \ +static __inline __unused int \ +name##_PCTRIE_TRANSFER(struct pctrie_iter *src, struct type *ptr, \ + struct pctrie_iter *dst, uint64_t dindex) \ +{ \ + struct pctrie_node *d_child, *freed, *node, *s_child; \ + smr_pctnode_t *dst_p, *src_p; \ + uint64_t *val = name##_PCTRIE_PTR2VAL(ptr); \ + \ + if (val == NULL) \ + panic("%s: key not found", __func__); \ + dst_p = pctrie_iter_insert_lookup(dst, dindex); \ + node = pctrie_node_load(dst_p, NULL, PCTRIE_UNSERIALIZED); \ + if (node != PCTRIE_NULL) { \ + d_child = allocfn(dst->ptree); \ + if (__predict_false(d_child == NULL)) \ + return (ENOMEM); \ + } \ + src_p = pctrie_iter_remove(src, &s_child); \ + if (s_child != PCTRIE_NULL) { \ + freed = pctrie_node_load(src_p, NULL, \ + PCTRIE_UNSERIALIZED); \ + } else \ + freed = NULL; \ + pctrie_node_store(src_p, s_child, PCTRIE_UNSERIALIZED); \ + *val = dindex; \ + if (node != PCTRIE_NULL) \ + pctrie_insert_node(val, dst->node, node, d_child); \ + else \ + d_child = pctrie_toleaf(val); \ + /* Synchronize to make changes visible. */ \ + pctrie_node_store(dst_p, d_child, access); \ + if (freed != NULL) \ + freefn(src->ptree, freed); \ + return (0); \ } smr_pctnode_t *pctrie_insert_lookup(struct pctrie *ptree, uint64_t index, Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -1937,38 +1937,19 @@ vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex) { - vm_page_t mpred; - vm_pindex_t opidx; + struct pctrie_iter new_pages; KASSERT((m->ref_count & VPRC_OBJREF) != 0, ("%s: page %p is missing object ref", __func__, m)); VM_OBJECT_ASSERT_WLOCKED(m->object); VM_OBJECT_ASSERT_WLOCKED(new_object); - - /* - * Create a custom version of vm_page_insert() which does not depend - * by m_prev and can cheat on the implementation aspects of the - * function. - */ - opidx = m->pindex; - m->pindex = new_pindex; - if (vm_radix_insert_lookup_lt(&new_object->rtree, m, &mpred) != 0) { - m->pindex = opidx; + vm_page_iter_init(&new_pages, new_object); + if (vm_radix_transfer(old_pages, m, &new_pages, new_pindex) != 0) return (false); - } - /* - * The operation cannot fail anymore. The removal must happen before - * the listq iterator is tainted. - */ - m->pindex = opidx; - vm_radix_iter_remove(old_pages); + /* The operation cannot fail anymore. */ vm_page_remove_radixdone(m); - - /* Return back to the new pindex to complete vm_page_insert(). */ - m->pindex = new_pindex; m->object = new_object; - vm_page_insert_radixdone(m, new_object); if (vm_page_any_valid(m)) vm_page_dirty(m); Index: sys/vm/vm_radix.h =================================================================== --- sys/vm/vm_radix.h +++ sys/vm/vm_radix.h @@ -366,5 +366,18 @@ return (VM_RADIX_PCTRIE_REPLACE(&rtree->rt_trie, newpage)); } +/* + * Remove the page last visited by the src iterator, change its vm_index to + * dindex, and insert it near the dst iterator. Panic if the src page has been + * removed or if the dst trie already has a dindex page. Return zero on success + * or a non-zero error on memory allocation failure. + */ +static __inline int +vm_radix_transfer(struct pctrie_iter *src, vm_page_t m, + struct pctrie_iter *dst, vm_pindex_t dindex) +{ + return (VM_RADIX_PCTRIE_TRANSFER(src, m, dst, dindex)); +} + #endif /* _KERNEL */ #endif /* !_VM_RADIX_H_ */