Index: sys/kern/subr_pctrie.c =================================================================== --- sys/kern/subr_pctrie.c +++ sys/kern/subr_pctrie.c @@ -549,7 +549,7 @@ */ while (it->top != 0) { node = it->path[it->top - 1]; - KASSERT(!powerof2(node->pn_popmap), + KASSERT(access == PCTRIE_SMR || !powerof2(node->pn_popmap), ("%s: freed node in iter path", __func__)); if (!pctrie_keybarr(node, index, &slot)) { node = pctrie_node_load( @@ -586,7 +586,8 @@ } /* - * Returns the value stored at a given index value, possibly NULL. + * Returns the value stored at a given index value, possibly NULL, assuming + * access is externally synchronized by a lock. */ uint64_t * pctrie_iter_lookup(struct pctrie_iter *it, uint64_t index) @@ -634,9 +635,8 @@ * Returns the value stored at a fixed offset from the current index value, * possibly NULL. */ -static __always_inline uint64_t * -_pctrie_iter_stride(struct pctrie_iter *it, int stride, smr_t smr, - enum pctrie_access access) +uint64_t * +pctrie_iter_stride(struct pctrie_iter *it, int stride) { uint64_t index = it->index + stride; @@ -647,17 +647,7 @@ if ((index < it->limit) != (it->index < it->limit)) return (NULL); - return (_pctrie_iter_lookup(it, index, smr, access)); -} - -/* - * Returns the value stored at a fixed offset from the current index value, - * possibly NULL. - */ -uint64_t * -pctrie_iter_stride(struct pctrie_iter *it, int stride) -{ - return (_pctrie_iter_stride(it, stride, NULL, PCTRIE_LOCKED)); + return (_pctrie_iter_lookup(it, index, NULL, PCTRIE_LOCKED)); } /* @@ -667,7 +657,7 @@ uint64_t * pctrie_iter_next(struct pctrie_iter *it) { - return (_pctrie_iter_stride(it, 1, NULL, PCTRIE_LOCKED)); + return (pctrie_iter_stride(it, 1)); } /* @@ -677,7 +667,27 @@ uint64_t * pctrie_iter_prev(struct pctrie_iter *it) { - return (_pctrie_iter_stride(it, -1, NULL, PCTRIE_LOCKED)); + return (pctrie_iter_stride(it, -1)); +} + +/* + * Returns the number of contiguous, non-NULL entries read into the value[] + * array, without requiring an external lock. + */ +int +pctrie_lookup_range_unlocked(struct pctrie *ptree, uint64_t index, + uint64_t *value[], int count, smr_t smr) +{ + struct pctrie_iter it; + int i = 0; + + pctrie_iter_init(&it, ptree); + smr_enter(smr); + while (i < count && NULL != + (value[i] = _pctrie_iter_lookup(&it, index + i, smr, PCTRIE_SMR))) + i++; + smr_exit(smr); + return (i); } /* Index: sys/sys/pctrie.h =================================================================== --- sys/sys/pctrie.h +++ sys/sys/pctrie.h @@ -48,6 +48,19 @@ return name##_PCTRIE_VAL2PTR(pctrie_lookup_unlocked(ptree, \ key, (smr))); \ } \ + \ +static __inline __unused int \ +name##_PCTRIE_LOOKUP_RANGE_UNLOCKED(struct pctrie *ptree, uint64_t key, \ + struct type *value[], int count) \ +{ \ + uint64_t *data[count]; \ + \ + count = pctrie_lookup_range_unlocked(ptree, key, data, count, \ + (smr)); \ + for (int i = 0; i < count; i++) \ + value[i] = name##_PCTRIE_VAL2PTR(data[i]); \ + return (count); \ +} \ #ifdef INVARIANTS void pctrie_subtree_lookup_gt_assert(struct pctrie_node *node, @@ -383,6 +396,8 @@ uint64_t *pctrie_lookup(struct pctrie *ptree, uint64_t key); uint64_t *pctrie_lookup_unlocked(struct pctrie *ptree, uint64_t key, smr_t smr); +int pctrie_lookup_range_unlocked(struct pctrie *ptree, + uint64_t index, uint64_t *value[], int count, smr_t smr); uint64_t *pctrie_iter_lookup(struct pctrie_iter *it, uint64_t index); uint64_t *pctrie_iter_stride(struct pctrie_iter *it, int stride); uint64_t *pctrie_iter_next(struct pctrie_iter *it); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -5122,7 +5122,7 @@ { vm_page_t m; int flags; - int i; + int i, num_fetched; KASSERT(count > 0, ("vm_page_grab_pages_unlocked: invalid page count %d", count)); @@ -5134,22 +5134,10 @@ */ flags = allocflags & ~VM_ALLOC_NOBUSY; vm_page_grab_check(flags); - m = NULL; - for (i = 0; i < count; i++, pindex++) { - /* - * We may see a false NULL here because the previous page has - * been removed or just inserted and the list is loaded without - * barriers. Switch to radix to verify. - */ - if (m == NULL || QMD_IS_TRASHED(m) || m->pindex != pindex || - atomic_load_ptr(&m->object) != object) { - /* - * This guarantees the result is instantaneously - * correct. - */ - m = NULL; - } - m = vm_page_acquire_unlocked(object, pindex, m, flags); + num_fetched = vm_radix_lookup_range_unlocked(&object->rtree, pindex, + ma, count); + for (i = 0; i < num_fetched; i++, pindex++) { + m = vm_page_acquire_unlocked(object, pindex, ma[i], flags); if (m == PAGE_NOT_ACQUIRED) return (i); if (m == NULL) @@ -5161,8 +5149,8 @@ } /* m will still be wired or busy according to flags. */ vm_page_grab_release(m, allocflags); + /* vm_page_acquire_unlocked may not return ma[i]. */ ma[i] = m; - m = TAILQ_NEXT(m, listq); } if (i == count || (allocflags & VM_ALLOC_NOCREAT) != 0) return (i); Index: sys/vm/vm_radix.h =================================================================== --- sys/vm/vm_radix.h +++ sys/vm/vm_radix.h @@ -121,6 +121,18 @@ return (VM_RADIX_PCTRIE_LOOKUP_UNLOCKED(&rtree->rt_trie, index)); } +/* + * Returns the number of contiguous, non-NULL pages read into the ma[] + * array, without requiring an external lock. + */ +static __inline int +vm_radix_lookup_range_unlocked(struct vm_radix *rtree, vm_pindex_t index, + vm_page_t ma[], int count) +{ + return (VM_RADIX_PCTRIE_LOOKUP_RANGE_UNLOCKED(&rtree->rt_trie, index, + ma, count)); +} + /* * Initialize an iterator for vm_radix. */