Page MenuHomeFreeBSD
Files F157644956

D50253.id155172.diff
No OneTemporary
Actions

D50253.id155172.diff
View Options

diff --git a/sys/kern/subr_pctrie.c b/sys/kern/subr_pctrie.c
--- a/sys/kern/subr_pctrie.c
+++ b/sys/kern/subr_pctrie.c
@@ -612,13 +612,14 @@
* array, starting at index.
*/
static __always_inline int
-_pctrie_lookup_range(struct pctrie *ptree, uint64_t index, uint64_t *value[],
- int count, smr_t smr, enum pctrie_access access)
+_pctrie_lookup_range(struct pctrie *ptree, struct pctrie_node *node,
+ uint64_t index, uint64_t *value[], int count,
+ struct pctrie_node **parent_out, smr_t smr, enum pctrie_access access)
{
- struct pctrie_node *parent, *node;
+ struct pctrie_node *parent;
int base, end, i;
- parent = NULL;
+ parent = node;
for (i = 0; i < count;) {
node = _pctrie_lookup_node(ptree, parent, index + i, &parent,
smr, access);
@@ -641,6 +642,8 @@
if (i < end)
break;
}
+ if (parent_out != NULL)
+ *parent_out = parent;
return (i);
}
@@ -653,8 +656,8 @@
pctrie_lookup_range(struct pctrie *ptree, uint64_t index,
uint64_t *value[], int count)
{
- return (_pctrie_lookup_range(ptree, index, value, count, NULL,
- PCTRIE_LOCKED));
+ return (_pctrie_lookup_range(ptree, NULL, index, value, count, NULL,
+ NULL, PCTRIE_LOCKED));
}
/*
@@ -671,11 +674,25 @@
int res;
smr_enter(smr);
- res = _pctrie_lookup_range(ptree, index, value, count, smr, PCTRIE_SMR);
+ res = _pctrie_lookup_range(ptree, NULL, index, value, count, NULL,
+ smr, PCTRIE_SMR);
smr_exit(smr);
return (res);
}
+/*
+ * Returns the number of contiguous, non-NULL entries read into the value[]
+ * array, starting at index, assuming access is externally synchronized by a
+ * lock. Uses an iterator.
+ */
+int
+pctrie_iter_lookup_range(struct pctrie_iter *it, uint64_t index,
+ uint64_t *value[], int count)
+{
+ return (_pctrie_lookup_range(it->ptree, it->node, index, value, count,
+ &it->node, NULL, PCTRIE_LOCKED));
+}
+
/*
* Find first leaf >= index, and fill iter with the path to the parent of that
* leaf. Return NULL if there is no such leaf less than limit.
diff --git a/sys/sys/pctrie.h b/sys/sys/pctrie.h
--- a/sys/sys/pctrie.h
+++ b/sys/sys/pctrie.h
@@ -217,7 +217,7 @@
\
static __inline __unused int \
name##_PCTRIE_LOOKUP_RANGE(struct pctrie *ptree, uint64_t key, \
- struct type *value[], int count) \
+ struct type *value[], int count) \
{ \
uint64_t **data = (uint64_t **)value; \
\
@@ -227,6 +227,18 @@
return (count); \
} \
\
+static __inline __unused int \
+name##_PCTRIE_ITER_LOOKUP_RANGE(struct pctrie_iter *it, uint64_t key, \
+ struct type *value[], int count) \
+{ \
+ uint64_t **data = (uint64_t **)value; \
+ \
+ count = pctrie_iter_lookup_range(it, key, data, count); \
+ for (int i = 0; i < count; i++) \
+ value[i] = name##_PCTRIE_NZVAL2PTR(data[i]); \
+ return (count); \
+} \
+ \
static __inline __unused struct type * \
name##_PCTRIE_LOOKUP_LE(struct pctrie *ptree, uint64_t key) \
{ \
@@ -399,6 +411,8 @@
uint64_t index, uint64_t *value[], int count);
int pctrie_lookup_range_unlocked(struct pctrie *ptree,
uint64_t index, uint64_t *value[], int count, smr_t smr);
+int pctrie_iter_lookup_range(struct pctrie_iter *it, uint64_t index,
+ uint64_t *value[], int count);
uint64_t *pctrie_iter_lookup(struct pctrie_iter *it, uint64_t index);
uint64_t *pctrie_iter_stride(struct pctrie_iter *it, int stride);
uint64_t *pctrie_iter_next(struct pctrie_iter *it);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -5150,7 +5150,7 @@
struct pctrie_iter pages;
vm_page_t m;
int pflags;
- int i;
+ int ahead, i;
VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0,
@@ -5163,9 +5163,15 @@
i = 0;
vm_page_iter_init(&pages, object);
retrylookup:
+ ahead = 0;
for (; i < count; i++) {
- m = vm_radix_iter_lookup(&pages, pindex + i);
- if (m != NULL) {
+ if (ahead == 0) {
+ ahead = vm_radix_iter_lookup_range(
+ &pages, pindex + i, &ma[i], count - i);
+ }
+ if (ahead > 0) {
+ --ahead;
+ m = ma[i];
if (!vm_page_tryacquire(m, allocflags)) {
if (vm_page_grab_sleep(object, m, pindex + i,
"grbmaw", allocflags, true)) {
@@ -5185,6 +5191,7 @@
break;
goto retrylookup;
}
+ ma[i] = m;
}
if (vm_page_none_valid(m) &&
(allocflags & VM_ALLOC_ZERO) != 0) {
@@ -5193,7 +5200,6 @@
vm_page_valid(m);
}
vm_page_grab_release(m, allocflags);
- ma[i] = m;
}
return (i);
}
diff --git a/sys/vm/vm_radix.h b/sys/vm/vm_radix.h
--- a/sys/vm/vm_radix.h
+++ b/sys/vm/vm_radix.h
@@ -113,6 +113,17 @@
ma, count));
}
+/*
+ * Returns the number of contiguous, non-NULL pages read into the ma[]
+ * array, without requiring an external lock.
+ */
+static __inline int
+vm_radix_iter_lookup_range(struct pctrie_iter *pages, vm_pindex_t index,
+ vm_page_t ma[], int count)
+{
+ return (VM_RADIX_PCTRIE_ITER_LOOKUP_RANGE(pages, index, ma, count));
+}
+
/*
* Initialize an iterator for vm_radix.
*/

File Metadata

Mime Type
text/plain
Expires
Sun, May 24, 4:39 PM (11 m, 31 s)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
33479742
Default Alt Text
D50253.id155172.diff (5 KB)

Event Timeline