Page MenuHomeFreeBSD
Files F146960248

D47114.id144871.diff
No OneTemporary
Actions

D47114.id144871.diff
View Options

Index: sys/kern/subr_pctrie.c
===================================================================
--- sys/kern/subr_pctrie.c
+++ sys/kern/subr_pctrie.c
@@ -549,7 +549,7 @@
*/
while (it->top != 0) {
node = it->path[it->top - 1];
- KASSERT(!powerof2(node->pn_popmap),
+ KASSERT(access == PCTRIE_SMR || !powerof2(node->pn_popmap),
("%s: freed node in iter path", __func__));
if (!pctrie_keybarr(node, index, &slot)) {
node = pctrie_node_load(
@@ -586,7 +586,8 @@
}
/*
- * Returns the value stored at a given index value, possibly NULL.
+ * Returns the value stored at a given index value, possibly NULL, assuming
+ * access is externally synchronized by a lock.
*/
uint64_t *
pctrie_iter_lookup(struct pctrie_iter *it, uint64_t index)
@@ -634,9 +635,8 @@
* Returns the value stored at a fixed offset from the current index value,
* possibly NULL.
*/
-static __always_inline uint64_t *
-_pctrie_iter_stride(struct pctrie_iter *it, int stride, smr_t smr,
- enum pctrie_access access)
+uint64_t *
+pctrie_iter_stride(struct pctrie_iter *it, int stride)
{
uint64_t index = it->index + stride;
@@ -647,17 +647,7 @@
if ((index < it->limit) != (it->index < it->limit))
return (NULL);
- return (_pctrie_iter_lookup(it, index, smr, access));
-}
-
-/*
- * Returns the value stored at a fixed offset from the current index value,
- * possibly NULL.
- */
-uint64_t *
-pctrie_iter_stride(struct pctrie_iter *it, int stride)
-{
- return (_pctrie_iter_stride(it, stride, NULL, PCTRIE_LOCKED));
+ return (_pctrie_iter_lookup(it, index, NULL, PCTRIE_LOCKED));
}
/*
@@ -667,7 +657,7 @@
uint64_t *
pctrie_iter_next(struct pctrie_iter *it)
{
- return (_pctrie_iter_stride(it, 1, NULL, PCTRIE_LOCKED));
+ return (pctrie_iter_stride(it, 1));
}
/*
@@ -677,7 +667,27 @@
uint64_t *
pctrie_iter_prev(struct pctrie_iter *it)
{
- return (_pctrie_iter_stride(it, -1, NULL, PCTRIE_LOCKED));
+ return (pctrie_iter_stride(it, -1));
+}
+
+/*
+ * Returns the number of contiguous, non-NULL entries read into the value[]
+ * array, without requiring an external lock.
+ */
+int
+pctrie_lookup_range_unlocked(struct pctrie *ptree, uint64_t index,
+ uint64_t *value[], int count, smr_t smr)
+{
+ struct pctrie_iter it;
+ int i = 0;
+
+ pctrie_iter_init(&it, ptree);
+ smr_enter(smr);
+ while (i < count && NULL !=
+ (value[i] = _pctrie_iter_lookup(&it, index + i, smr, PCTRIE_SMR)))
+ i++;
+ smr_exit(smr);
+ return (i);
}
/*
Index: sys/sys/pctrie.h
===================================================================
--- sys/sys/pctrie.h
+++ sys/sys/pctrie.h
@@ -48,6 +48,19 @@
return name##_PCTRIE_VAL2PTR(pctrie_lookup_unlocked(ptree, \
key, (smr))); \
} \
+ \
+static __inline __unused int \
+name##_PCTRIE_LOOKUP_RANGE_UNLOCKED(struct pctrie *ptree, uint64_t key, \
+ struct type *value[], int count) \
+{ \
+ uint64_t *data[count]; \
+ \
+ count = pctrie_lookup_range_unlocked(ptree, key, data, count, \
+ (smr)); \
+ for (int i = 0; i < count; i++) \
+ value[i] = name##_PCTRIE_VAL2PTR(data[i]); \
+ return (count); \
+} \
#ifdef INVARIANTS
void pctrie_subtree_lookup_gt_assert(struct pctrie_node *node,
@@ -383,6 +396,8 @@
uint64_t *pctrie_lookup(struct pctrie *ptree, uint64_t key);
uint64_t *pctrie_lookup_unlocked(struct pctrie *ptree, uint64_t key,
smr_t smr);
+int pctrie_lookup_range_unlocked(struct pctrie *ptree,
+ uint64_t index, uint64_t *value[], int count, smr_t smr);
uint64_t *pctrie_iter_lookup(struct pctrie_iter *it, uint64_t index);
uint64_t *pctrie_iter_stride(struct pctrie_iter *it, int stride);
uint64_t *pctrie_iter_next(struct pctrie_iter *it);
Index: sys/vm/vm_page.c
===================================================================
--- sys/vm/vm_page.c
+++ sys/vm/vm_page.c
@@ -5122,7 +5122,7 @@
{
vm_page_t m;
int flags;
- int i;
+ int i, num_fetched;
KASSERT(count > 0,
("vm_page_grab_pages_unlocked: invalid page count %d", count));
@@ -5134,22 +5134,10 @@
*/
flags = allocflags & ~VM_ALLOC_NOBUSY;
vm_page_grab_check(flags);
- m = NULL;
- for (i = 0; i < count; i++, pindex++) {
- /*
- * We may see a false NULL here because the previous page has
- * been removed or just inserted and the list is loaded without
- * barriers. Switch to radix to verify.
- */
- if (m == NULL || QMD_IS_TRASHED(m) || m->pindex != pindex ||
- atomic_load_ptr(&m->object) != object) {
- /*
- * This guarantees the result is instantaneously
- * correct.
- */
- m = NULL;
- }
- m = vm_page_acquire_unlocked(object, pindex, m, flags);
+ num_fetched = vm_radix_lookup_range_unlocked(&object->rtree, pindex,
+ ma, count);
+ for (i = 0; i < num_fetched; i++, pindex++) {
+ m = vm_page_acquire_unlocked(object, pindex, ma[i], flags);
if (m == PAGE_NOT_ACQUIRED)
return (i);
if (m == NULL)
@@ -5162,7 +5150,6 @@
/* m will still be wired or busy according to flags. */
vm_page_grab_release(m, allocflags);
ma[i] = m;
- m = TAILQ_NEXT(m, listq);
}
if (i == count || (allocflags & VM_ALLOC_NOCREAT) != 0)
return (i);
Index: sys/vm/vm_radix.h
===================================================================
--- sys/vm/vm_radix.h
+++ sys/vm/vm_radix.h
@@ -121,6 +121,18 @@
return (VM_RADIX_PCTRIE_LOOKUP_UNLOCKED(&rtree->rt_trie, index));
}
+/*
+ * Returns the number of contiguous, non-NULL pages read into the ma[]
+ * array, without requiring an external lock.
+ */
+static __inline int
+vm_radix_lookup_range_unlocked(struct vm_radix *rtree, vm_pindex_t index,
+ vm_page_t ma[], int count)
+{
+ return (VM_RADIX_PCTRIE_LOOKUP_RANGE_UNLOCKED(&rtree->rt_trie, index,
+ ma, count));
+}
+
/*
* Initialize an iterator for vm_radix.
*/

File Metadata

Mime Type
text/plain
Expires
Sun, Mar 8, 3:49 AM (19 h, 46 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
29393722
Default Alt Text
D47114.id144871.diff (5 KB)

Event Timeline