diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -127,6 +127,7 @@ vm_prot_t prot; int fault_flags; boolean_t wired; + bool can_read_lock; /* Control state. */ struct timeval oom_start_time; @@ -212,7 +213,7 @@ m = *mp; if (m != NULL) { - VM_OBJECT_ASSERT_WLOCKED(m->object); + VM_OBJECT_ASSERT_LOCKED(m->object); if (!vm_page_wired(m)) vm_page_free(m); else @@ -278,7 +279,7 @@ unlock_and_deallocate(struct faultstate *fs) { - VM_OBJECT_WUNLOCK(fs->object); + VM_OBJECT_UNLOCK(fs->object); fault_deallocate(fs); } @@ -1047,6 +1048,11 @@ { vm_object_t next_object; + if (fs->object == fs->first_object || !fs->can_read_lock) + VM_OBJECT_ASSERT_WLOCKED(fs->object); + else + VM_OBJECT_ASSERT_LOCKED(fs->object); + /* * The requested page does not exist at this object/ * offset. Remove the invalid page from the object, @@ -1067,18 +1073,20 @@ * Move on to the next object. Lock the next object before * unlocking the current one. */ - VM_OBJECT_ASSERT_WLOCKED(fs->object); next_object = fs->object->backing_object; if (next_object == NULL) return (false); MPASS(fs->first_m != NULL); KASSERT(fs->object != next_object, ("object loop %p", next_object)); - VM_OBJECT_WLOCK(next_object); + if (fs->can_read_lock) + VM_OBJECT_RLOCK(next_object); + else + VM_OBJECT_WLOCK(next_object); vm_object_pip_add(next_object, 1); if (fs->object != fs->first_object) vm_object_pip_wakeup(fs->object); fs->pindex += OFF_TO_IDX(fs->object->backing_object_offset); - VM_OBJECT_WUNLOCK(fs->object); + VM_OBJECT_UNLOCK(fs->object); fs->object = next_object; return (true); @@ -1363,7 +1371,7 @@ unlock_map(fs); if (fs->m != vm_page_lookup(fs->object, fs->pindex) || !vm_page_busy_sleep(fs->m, "vmpfw", 0)) - VM_OBJECT_WUNLOCK(fs->object); + VM_OBJECT_UNLOCK(fs->object); VM_CNT_INC(v_intrans); vm_object_deallocate(fs->first_object); } @@ -1382,6 +1390,11 @@ enum fault_status res; bool dead; + if (fs->object == fs->first_object || !fs->can_read_lock) + VM_OBJECT_ASSERT_WLOCKED(fs->object); + else + VM_OBJECT_ASSERT_LOCKED(fs->object); + /* * If the object is marked for imminent termination, we retry * here, since the collapse pass has raced with us. Otherwise, @@ -1412,11 +1425,10 @@ * done. */ if (vm_page_all_valid(fs->m)) { - VM_OBJECT_WUNLOCK(fs->object); + VM_OBJECT_UNLOCK(fs->object); return (FAULT_SOFT); } } - VM_OBJECT_ASSERT_WLOCKED(fs->object); /* * Page is not resident. If the pager might contain the page @@ -1425,6 +1437,12 @@ */ if (fs->m == NULL && (fault_object_needs_getpages(fs->object) || fs->object == fs->first_object)) { + if (!VM_OBJECT_WOWNED(fs->object) && !VM_OBJECT_TRYUPGRADE(fs->object)) { + MPASS(fs->can_read_lock); + fs->can_read_lock = false; + unlock_and_deallocate(fs); + return (FAULT_RESTART); + } res = vm_fault_allocate(fs); if (res != FAULT_CONTINUE) return (res); @@ -1447,10 +1465,14 @@ * prevents simultaneous faults and collapses while * the object lock is dropped. */ - VM_OBJECT_WUNLOCK(fs->object); + VM_OBJECT_UNLOCK(fs->object); res = vm_fault_getpages(fs, behindp, aheadp); - if (res == FAULT_CONTINUE) - VM_OBJECT_WLOCK(fs->object); + if (res == FAULT_CONTINUE) { + if (fs->object == fs->first_object || !fs->can_read_lock) + VM_OBJECT_WLOCK(fs->object); + else + VM_OBJECT_RLOCK(fs->object); + } } else { res = FAULT_CONTINUE; } @@ -1479,6 +1501,7 @@ fs.lookup_still_valid = false; fs.oom_started = false; fs.nera = -1; + fs.can_read_lock = true; faultcount = 0; hardfault = false; @@ -1597,7 +1620,7 @@ unlock_and_deallocate(&fs); return (KERN_OUT_OF_BOUNDS); } - VM_OBJECT_WUNLOCK(fs.object); + VM_OBJECT_UNLOCK(fs.object); vm_fault_zerofill(&fs); /* Don't try to prefault neighboring pages. */ faultcount = 1; diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h --- a/sys/vm/vm_object.h +++ b/sys/vm/vm_object.h @@ -267,6 +267,8 @@ rw_wowned(&(object)->lock) #define VM_OBJECT_WUNLOCK(object) \ rw_wunlock(&(object)->lock) +#define VM_OBJECT_UNLOCK(object) \ + rw_unlock(&(object)->lock) #define VM_OBJECT_DROP(object) \ lock_class_rw.lc_unlock(&(object)->lock.lock_object) #define VM_OBJECT_PICKUP(object, state) \