Page MenuHomeFreeBSD
Files F162585411

D51162.id157994.diff
No OneTemporary
Actions

D51162.id157994.diff
View Options

Index: sys/vm/vm_fault.c
===================================================================
--- sys/vm/vm_fault.c
+++ sys/vm/vm_fault.c
@@ -2104,6 +2104,7 @@
vm_offset_t vaddr;
vm_page_t dst_m;
vm_page_t src_m;
+ int i, nread, npages;
bool upgrade;
upgrade = src_entry == dst_entry;
@@ -2126,6 +2127,7 @@
src_object = src_entry->object.vm_object;
src_pindex = OFF_TO_IDX(src_entry->offset);
+ npages = atop(dst_entry->end - dst_entry->start);
if (upgrade && (dst_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) {
dst_object = src_object;
@@ -2136,8 +2138,7 @@
* Doesn't actually shadow anything - we copy the pages
* directly.
*/
- dst_object = vm_object_allocate_anon(atop(dst_entry->end -
- dst_entry->start), NULL, NULL, 0);
+ dst_object = vm_object_allocate_anon(npages, NULL, NULL, 0);
#if VM_NRESERVLEVEL > 0
dst_object->flags |= OBJ_COLORED;
dst_object->pg_color = atop(dst_entry->start);
@@ -2174,21 +2175,21 @@
* with the source object, all of its pages must be dirtied,
* regardless of whether they can be written.
*/
+ vm_page_t ma[npages];
vm_page_iter_init(&pages, dst_object);
- for (vaddr = dst_entry->start, dst_pindex = 0;
- vaddr < dst_entry->end;
- vaddr += PAGE_SIZE, dst_pindex++) {
+ for (dst_pindex = 0; dst_pindex < npages;) {
again:
/*
* Find the page in the source object, and copy it in.
* Because the source is wired down, the page will be
* in memory.
*/
- if (src_object != dst_object)
- VM_OBJECT_RLOCK(src_object);
object = src_object;
+ if (object != dst_object)
+ VM_OBJECT_RLOCK(object);
pindex = src_pindex + dst_pindex;
- while ((src_m = vm_page_lookup(object, pindex)) == NULL &&
+ while ((nread = vm_page_lookup_range(object, pindex,
+ &ma[dst_pindex], npages - dst_pindex)) == 0 &&
(backing_object = object->backing_object) != NULL) {
/*
* Unless the source mapping is read-only or
@@ -2209,57 +2210,68 @@
VM_OBJECT_RUNLOCK(object);
object = backing_object;
}
- KASSERT(src_m != NULL, ("vm_fault_copy_entry: page missing"));
+ KASSERT(nread != 0, ("vm_fault_copy_entry: page missing"));
if (object != dst_object) {
/*
* Allocate a page in the destination object.
*/
- pindex = (src_object == dst_object ? src_pindex : 0) +
- dst_pindex;
- dst_m = vm_page_alloc_iter(dst_object, pindex,
- VM_ALLOC_NORMAL, &pages);
- if (dst_m == NULL) {
- VM_OBJECT_WUNLOCK(dst_object);
- VM_OBJECT_RUNLOCK(object);
- vm_wait(dst_object);
- VM_OBJECT_WLOCK(dst_object);
- pctrie_iter_reset(&pages);
- goto again;
- }
+ for (i = 0; i < nread; i++) {
+ pindex = dst_pindex +
+ (src_object == dst_object ? src_pindex : 0);
+ dst_m = vm_page_alloc_iter(dst_object, pindex,
+ VM_ALLOC_NORMAL, &pages);
+ if (dst_m == NULL) {
+ VM_OBJECT_WUNLOCK(dst_object);
+ VM_OBJECT_RUNLOCK(object);
+ vm_wait(dst_object);
+ VM_OBJECT_WLOCK(dst_object);
+ pctrie_iter_reset(&pages);
+ goto again;
+ }
- /*
- * See the comment in vm_fault_cow().
- */
- if (src_object == dst_object &&
- (object->flags & OBJ_ONEMAPPING) == 0)
- pmap_remove_all(src_m);
- pmap_copy_page(src_m, dst_m);
+ /*
+ * See the comment in vm_fault_cow().
+ */
+ src_m = ma[dst_pindex];
+ if (src_object == dst_object &&
+ (object->flags & OBJ_ONEMAPPING) == 0)
+ pmap_remove_all(src_m);
+ pmap_copy_page(src_m, dst_m);
- /*
- * The object lock does not guarantee that "src_m" will
- * transition from invalid to valid, but it does ensure
- * that "src_m" will not transition from valid to
- * invalid.
- */
- dst_m->dirty = dst_m->valid = src_m->valid;
+ /*
+ * The object lock does not guarantee that
+ * "src_m" will transition from invalid to
+ * valid, but it does ensure that "src_m" will
+ * not transition from valid to invalid.
+ */
+ dst_m->dirty = dst_m->valid = src_m->valid;
+ if (upgrade)
+ vm_page_unwire(src_m, PQ_INACTIVE);
+ ma[dst_pindex++] = dst_m;
+ }
VM_OBJECT_RUNLOCK(object);
} else {
- dst_m = src_m;
- if (vm_page_busy_acquire(
- dst_m, VM_ALLOC_WAITFAIL) == 0) {
- pctrie_iter_reset(&pages);
- goto again;
+ for (i = 0; i < nread; i++) {
+ dst_m = ma[dst_pindex];
+ if (vm_page_busy_acquire(
+ dst_m, VM_ALLOC_WAITFAIL) == 0) {
+ pctrie_iter_reset(&pages);
+ goto again;
+ }
+ if (dst_m->pindex >= dst_object->size) {
+ /*
+ * We are upgrading. Index can occur
+ * out of bounds if the object type is
+ * vnode and the file was truncated.
+ */
+ vm_page_xunbusy(dst_m);
+ break;
+ }
+ dst_pindex++;
}
- if (dst_m->pindex >= dst_object->size) {
- /*
- * We are upgrading. Index can occur
- * out of bounds if the object type is
- * vnode and the file was truncated.
- */
- vm_page_xunbusy(dst_m);
+ if (i < nread)
break;
- }
}
/*
@@ -2274,8 +2286,12 @@
* all copies of the wired map entry have similar
* backing pages.
*/
- if (vm_page_all_valid(dst_m)) {
+ for (i = dst_pindex - nread; i < dst_pindex; i++) {
+ dst_m = ma[i];
+ if (!vm_page_all_valid(dst_m))
+ continue;
VM_OBJECT_WUNLOCK(dst_object);
+ vaddr = dst_entry->start + ptoa(i);
pmap_enter(dst_map->pmap, vaddr, dst_m, prot,
access | (upgrade ? PMAP_ENTER_WIRED : 0), 0);
VM_OBJECT_WLOCK(dst_object);
@@ -2284,18 +2300,20 @@
/*
* Mark it no longer busy, and put it on the active list.
*/
- if (upgrade) {
- if (src_m != dst_m) {
- vm_page_unwire(src_m, PQ_INACTIVE);
- vm_page_wire(dst_m);
+ for (i = dst_pindex - nread; i < dst_pindex; i++) {
+ dst_m = ma[i];
+ if (upgrade) {
+ if (object != dst_object)
+ vm_page_wire(dst_m);
+ else {
+ KASSERT(vm_page_wired(dst_m),
+ ("dst_m %p is not wired", dst_m));
+ }
} else {
- KASSERT(vm_page_wired(dst_m),
- ("dst_m %p is not wired", dst_m));
+ vm_page_activate(dst_m);
}
- } else {
- vm_page_activate(dst_m);
+ vm_page_xunbusy(dst_m);
}
- vm_page_xunbusy(dst_m);
}
VM_OBJECT_WUNLOCK(dst_object);
if (upgrade) {
Index: sys/vm/vm_page.h
===================================================================
--- sys/vm/vm_page.h
+++ sys/vm/vm_page.h
@@ -618,6 +618,8 @@
vm_object_t new_object, vm_pindex_t new_pindex);
void vm_page_launder(vm_page_t m);
vm_page_t vm_page_lookup(vm_object_t, vm_pindex_t);
+int vm_page_lookup_range(vm_object_t, vm_pindex_t index,
+ vm_page_t ma[], int count);
vm_page_t vm_page_lookup_unlocked(vm_object_t, vm_pindex_t);
void vm_page_pqbatch_drain(void);
void vm_page_pqbatch_submit(vm_page_t m, uint8_t queue);
Index: sys/vm/vm_page.c
===================================================================
--- sys/vm/vm_page.c
+++ sys/vm/vm_page.c
@@ -1752,6 +1752,22 @@
return (vm_radix_lookup(&object->rtree, pindex));
}
+/*
+ * vm_page_lookup_range:
+ *
+ * Returns number of the pages read into ma, beginning at the given index.
+ *
+ * The object must be locked.
+ */
+int
+vm_page_lookup_range(vm_object_t object, vm_pindex_t pindex,
+ vm_page_t ma[], int count)
+{
+
+ VM_OBJECT_ASSERT_LOCKED(object);
+ return (vm_radix_lookup_range(&object->rtree, pindex, ma, count));
+}
+
/*
* vm_page_iter_init:
*
Index: sys/vm/vm_radix.h
===================================================================
--- sys/vm/vm_radix.h
+++ sys/vm/vm_radix.h
@@ -101,6 +101,20 @@
return (VM_RADIX_PCTRIE_LOOKUP_UNLOCKED(&rtree->rt_trie, index));
}
+/*
+ * Returns the number of contiguous, non-NULL pages read into the ma[]
+ * array.
+ *
+ * Requires that access be externally synchronized by a lock.
+ */
+static __inline int
+vm_radix_lookup_range(struct vm_radix *rtree, vm_pindex_t index,
+ vm_page_t ma[], int count)
+{
+ return (VM_RADIX_PCTRIE_LOOKUP_RANGE(&rtree->rt_trie, index,
+ ma, count));
+}
+
/*
* Returns the number of contiguous, non-NULL pages read into the ma[]
* array, without requiring an external lock.
@@ -115,7 +129,9 @@
/*
* Returns the number of contiguous, non-NULL pages read into the ma[]
- * array, without requiring an external lock.
+ * array.
+ *
+ * Requires that access be externally synchronized by a lock.
*/
static __inline int
vm_radix_iter_lookup_range(struct pctrie_iter *pages, vm_pindex_t index,

File Metadata

Mime Type
text/plain
Expires
Wed, Jul 15, 7:27 PM (5 h, 57 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
35111370
Default Alt Text
D51162.id157994.diff (8 KB)

Event Timeline