Index: sys/kern/subr_pctrie.c =================================================================== --- sys/kern/subr_pctrie.c +++ sys/kern/subr_pctrie.c @@ -539,14 +539,27 @@ enum pctrie_access access) { struct pctrie_node *node; - int slot; + int slot, top; + + top = it->top; + if (top != 0) { + node = it->path[top - 1]; + KASSERT(!powerof2(node->pn_popmap), + ("%s: freed node in iter path", __func__)); + if (!pctrie_keybarr(node, index, &slot)) { + node = pctrie_node_load( + &node->pn_child[slot], smr, access); + if (pctrie_isleaf(node)) + return (node); + } + } /* * Climb the search path to find the lowest node from which to start the * search for a value matching 'index'. */ - while (it->top != 0) { - node = it->path[it->top - 1]; + while (top != 0) { + node = it->path[top - 1]; KASSERT(!powerof2(node->pn_popmap), ("%s: freed node in iter path", __func__)); if (!pctrie_keybarr(node, index, &slot)) { @@ -554,18 +567,19 @@ &node->pn_child[slot], smr, access); break; } - --it->top; + --top; } - if (it->top == 0) + if (top == 0) node = pctrie_root_load(it->ptree, smr, access); /* Seek a node that matches index. */ while (!pctrie_isleaf(node) && !pctrie_keybarr(node, index, &slot)) { - KASSERT(it->top < nitems(it->path), + KASSERT(top < nitems(it->path), ("%s: path overflow in trie %p", __func__, it->ptree)); - it->path[it->top++] = node; + it->path[top++] = node; node = pctrie_node_load(&node->pn_child[slot], smr, access); } + it->top = top; return (node); } @@ -781,6 +795,43 @@ return (pctrie_lookup_ge_node(node, index + 1)); } +static struct pctrie_node * +_pctrie_iter_lookup_gt_node(struct pctrie_iter *it, uint64_t index) +{ + struct pctrie_node *node; + int slot, top; + + top = it->top; + if (top != 0) { + node = it->path[top - 1]; + slot = pctrie_slot(node, index); + if ((node->pn_popmap & (-2 << slot)) != 0) { + /* Return least child with a descendant > index. */ + slot = ffs(node->pn_popmap & (-2 << slot)) - 1; + node = pctrie_node_load(&node->pn_child[slot], NULL, + PCTRIE_LOCKED); + return (node); + } + --top; + } + while (top != 0) { + node = it->path[top - 1]; + slot = pctrie_slot(node, index); + if ((node->pn_popmap & (-2 << slot)) != 0) { + /* Return least child with a descendant > index. */ + slot = ffs(node->pn_popmap & (-2 << slot)) - 1; + node = pctrie_node_load(&node->pn_child[slot], NULL, + PCTRIE_LOCKED); + it->top = top; + return (node); + } + --top; + } + it->top = 0; + return (NULL); + +} + /* * Find first leaf >= index, and fill iter with the path to the parent of that * leaf. Return NULL if there is no such leaf less than limit. @@ -801,21 +852,11 @@ */ if (node == PCTRIE_NULL || *pctrie_toval(node) < index) { /* Climb the path to find a node with a descendant > index. */ - while (it->top != 0) { - node = it->path[it->top - 1]; - slot = pctrie_slot(node, index) + 1; - if ((node->pn_popmap >> slot) != 0) - break; - --it->top; - } - if (it->top == 0) + node = _pctrie_iter_lookup_gt_node(it, index); + if (node == NULL) return (NULL); - - /* Step to the least child with a descendant > index. */ - slot += ffs(node->pn_popmap >> slot) - 1; - node = pctrie_node_load(&node->pn_child[slot], NULL, - PCTRIE_LOCKED); } + /* Descend to the least leaf of the subtrie. */ while (!pctrie_isleaf(node)) { if (it->limit != 0 && node->pn_owner >= it->limit) @@ -938,6 +979,44 @@ return (pctrie_lookup_le_node(node, index - 1)); } +static struct pctrie_node * +_pctrie_iter_lookup_lt_node(struct pctrie_iter *it, uint64_t index) +{ + struct pctrie_node *node; + int slot, top; + + top = it->top; + if (top != 0) { + node = it->path[top - 1]; + slot = pctrie_slot(node, index); + if ((node->pn_popmap & ((1 << slot) - 1)) != 0) { + /* Return greatest child with a descendant < index. */ + slot = ilog2(node->pn_popmap & ((1 << slot) - 1)); + node = pctrie_node_load(&node->pn_child[slot], NULL, + PCTRIE_LOCKED); + return (node); + } + --top; + } + + while (top != 0) { + node = it->path[top - 1]; + slot = pctrie_slot(node, index); + if ((node->pn_popmap & ((1 << slot) - 1)) != 0) { + /* Return greatest child with a descendant < index. */ + slot = ilog2(node->pn_popmap & ((1 << slot) - 1)); + node = pctrie_node_load(&node->pn_child[slot], NULL, + PCTRIE_LOCKED); + it->top = top; + return (node); + } + --top; + } + it->top = 0; + return (NULL); + +} + /* * Find first leaf <= index, and fill iter with the path to the parent of that * leaf. Return NULL if there is no such leaf greater than limit. @@ -958,21 +1037,11 @@ */ if (node == PCTRIE_NULL || *pctrie_toval(node) > index) { /* Climb the path to find a node with a descendant < index. */ - while (it->top != 0) { - node = it->path[it->top - 1]; - slot = pctrie_slot(node, index); - if ((node->pn_popmap & ((1 << slot) - 1)) != 0) - break; - --it->top; - } - if (it->top == 0) + node = _pctrie_iter_lookup_lt_node(it, index); + if (node == NULL) return (NULL); - - /* Step to the greatest child with a descendant < index. */ - slot = ilog2(node->pn_popmap & ((1 << slot) - 1)); - node = pctrie_node_load(&node->pn_child[slot], NULL, - PCTRIE_LOCKED); } + /* Descend to the greatest leaf of the subtrie. */ while (!pctrie_isleaf(node)) { if (it->limit != 0 && it->limit >= Index: sys/vm/vm_kern.c =================================================================== --- sys/vm/vm_kern.c +++ sys/vm/vm_kern.c @@ -648,8 +648,8 @@ pmap_remove(kernel_pmap, addr, addr + size); offset = addr - VM_MIN_KERNEL_ADDRESS; end = offset + size; - VM_OBJECT_WLOCK(object); vm_page_iter_init(&pages, object); + VM_OBJECT_WLOCK(object); m = vm_page_iter_lookup(&pages, atop(offset)); domain = vm_page_domain(m); if (__predict_true((m->oflags & VPO_KMEM_EXEC) == 0)) @@ -660,7 +660,7 @@ m = vm_page_iter_lookup(&pages, atop(offset))) { vm_page_xbusy_claim(m); vm_page_unwire_noq(m); - vm_page_iter_free(&pages); + vm_page_iter_free(m, &pages); } VM_OBJECT_WUNLOCK(object); Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -1702,7 +1702,6 @@ */ vm_page_iter_init(&pages, backing_object); for (p = vm_page_iter_lookup_ge(&pages, 0); p != NULL; p = next) { - next = TAILQ_NEXT(p, listq); new_pindex = p->pindex - backing_offset_index; /* @@ -1997,7 +1996,6 @@ vm_object_pip_add(object, 1); vm_page_iter_limit_init(&pages, object, end); again: - pctrie_iter_reset(&pages); for (p = vm_page_iter_lookup_ge(&pages, start); p != NULL; p = vm_radix_iter_step(&pages)) { /* @@ -2026,6 +2024,7 @@ if (vm_page_tryxbusy(p) == 0) { if (vm_page_busy_sleep(p, "vmopar", 0)) VM_OBJECT_WLOCK(object); + pctrie_iter_reset(&pages); goto again; } if ((options & OBJPR_VALIDONLY) != 0 && vm_page_none_valid(p)) { @@ -2060,7 +2059,7 @@ if ((options & OBJPR_NOTMAPPED) == 0 && object->ref_count != 0 && !vm_page_try_remove_all(p)) goto wired; - vm_page_iter_free(&pages); + vm_page_iter_free(p, &pages); } vm_object_pip_wakeup(object); Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h +++ sys/vm/vm_page.h @@ -602,7 +602,7 @@ void vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m, vm_pindex_t pindex, const char *wmesg, int allocflags); void vm_page_free(vm_page_t m); -void vm_page_iter_free(struct pctrie_iter *); +void vm_page_iter_free(vm_page_t m, struct pctrie_iter *); void vm_page_free_zero(vm_page_t m); void vm_page_activate (vm_page_t); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -1716,11 +1716,8 @@ * Free the current page, as identified by iterator. */ void -vm_page_iter_free(struct pctrie_iter *pages) +vm_page_iter_free(vm_page_t m, struct pctrie_iter *pages) { - vm_page_t m; - - m = vm_radix_iter_page(pages); vm_radix_iter_remove(pages); vm_page_free_object_prep(m); vm_page_xunbusy(m);