Index: sys/kern/subr_pctrie.c =================================================================== --- sys/kern/subr_pctrie.c +++ sys/kern/subr_pctrie.c @@ -583,6 +583,21 @@ return (pctrie_match_value(node, index)); } +/* + * Returns whether a value is stored at a given index value, and sets the + * value, if there is one. + */ +bool +pctrie_iter_find(struct pctrie_iter *it, uint64_t index, uint64_t **found) +{ + struct pctrie_node *node; + + it->index = index; + node = _pctrie_iter_lookup_node(it, index, NULL, PCTRIE_LOCKED); + *found = pctrie_match_value(node, index); + return (*found != NULL); +} + /* * Returns the value stored at a given index value, possibly NULL. */ Index: sys/sys/pctrie.h =================================================================== --- sys/sys/pctrie.h +++ sys/sys/pctrie.h @@ -244,6 +244,18 @@ return (0); \ } \ \ +static __inline __unused bool \ +name##_PCTRIE_ITER_FIND(struct pctrie_iter *it, uint64_t index, \ + struct type **found) \ +{ \ + uint64_t *val; \ + if (!pctrie_iter_find(it, index, &val)) \ + return (false); \ + *found = (struct type *) \ + ((uintptr_t)val - __offsetof(struct type, field)); \ + return (true); \ +} \ + \ static __inline __unused struct type * \ name##_PCTRIE_ITER_LOOKUP(struct pctrie_iter *it, uint64_t index) \ { \ @@ -375,6 +387,8 @@ uint64_t *pctrie_lookup(struct pctrie *ptree, uint64_t key); uint64_t *pctrie_lookup_unlocked(struct pctrie *ptree, uint64_t key, smr_t smr); +bool pctrie_iter_find(struct pctrie_iter *it, uint64_t index, + uint64_t **found); uint64_t *pctrie_iter_lookup(struct pctrie_iter *it, uint64_t index); uint64_t *pctrie_iter_stride(struct pctrie_iter *it, int stride); uint64_t *pctrie_iter_next(struct pctrie_iter *it); Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -1962,8 +1962,8 @@ * found page has pending operations, sleep and restart * the scan. */ - m = vm_page_iter_lookup(&pages, blks.index + i); - if (m != NULL && (m->oflags & VPO_SWAPINPROG) != 0) { + if (vm_radix_iter_found(&pages, blks.index + i, &m) && + (m->oflags & VPO_SWAPINPROG) != 0) { m->oflags |= VPO_SWAPSLEEP; VM_OBJECT_SLEEP(object, &object->handle, PSWP, "swpoff", 0); @@ -2383,11 +2383,10 @@ if (sb->d[i] == SWAPBLK_NONE) continue; swp_pager_update_freerange(&range, sb->d[i]); - if (freed != NULL) { - m = vm_page_iter_lookup(&pages, blks.index + i); - if (m == NULL || vm_page_none_valid(m)) - fc++; - } + if (freed != NULL && + (!vm_radix_iter_found(&pages, blks.index + i, &m) || + vm_page_none_valid(m))) + fc++; sb->d[i] = SWAPBLK_NONE; } if (swp_pager_swblk_empty(sb, 0, start) && @@ -2540,7 +2539,7 @@ VM_OBJECT_ASSERT_RLOCKED(object); vm_page_iter_init(&pages, object); swblk_iter_init_only(&blks, object); - while (((m = vm_page_iter_lookup(&pages, pindex)) != NULL && + while ((vm_radix_iter_found(&pages, pindex, &m) && vm_page_any_valid(m)) || ((sb = swblk_iter_lookup(&blks, pindex)) != NULL && sb->d[pindex % SWAP_META_PAGES] != SWAPBLK_NONE)) @@ -2627,13 +2626,13 @@ * object and we might as well give up now. */ new_pindex = pi - backing_offset_index; - pp = vm_page_iter_lookup(&pages, new_pindex); /* * The valid check here is stable due to object lock being * required to clear valid and initiate paging. */ - if ((pp == NULL || vm_page_none_valid(pp)) && + if ((!vm_radix_iter_found(&pages, new_pindex, &pp) || + vm_page_none_valid(pp)) && !swp_pager_haspage_iter(&blks, new_pindex, NULL, NULL)) break; Index: sys/vm/vm_radix.h =================================================================== --- sys/vm/vm_radix.h +++ sys/vm/vm_radix.h @@ -140,6 +140,19 @@ pctrie_iter_limit_init(pages, &rtree->rt_trie, limit); } +/* + * Finds the value stored at the index, if any. + * Requires that access be externally synchronized by a lock. + * + * If the index is not present, NULL is returned. + */ +static __inline bool +vm_radix_iter_found(struct pctrie_iter *pages, vm_pindex_t index, + vm_page_t *found) +{ + return (VM_RADIX_PCTRIE_ITER_FIND(pages, index, found)); +} + /* * Returns the value stored at the index. * Requires that access be externally synchronized by a lock.