Index: sys/vm/vm_fault.c =================================================================== --- sys/vm/vm_fault.c +++ sys/vm/vm_fault.c @@ -1052,6 +1052,8 @@ { vm_object_t next_object; + VM_OBJECT_ASSERT_WLOCKED(fs->object); + /* * The requested page does not exist at this object/ * offset. Remove the invalid page from the object, @@ -1072,13 +1074,12 @@ * Move on to the next object. Lock the next object before * unlocking the current one. */ - VM_OBJECT_ASSERT_WLOCKED(fs->object); next_object = fs->object->backing_object; if (next_object == NULL) return (false); MPASS(fs->first_m != NULL); KASSERT(fs->object != next_object, ("object loop %p", next_object)); - VM_OBJECT_WLOCK(next_object); + VM_OBJECT_RLOCK(next_object); vm_object_pip_add(next_object, 1); if (fs->object != fs->first_object) vm_object_pip_wakeup(fs->object); @@ -1348,7 +1349,7 @@ * page except, perhaps, to pmap it. */ static void -vm_fault_busy_sleep(struct faultstate *fs) +vm_fault_busy_sleep_impl(struct faultstate *fs, bool objrlocked) { /* * Reference the page before unlocking and @@ -1363,12 +1364,30 @@ vm_object_pip_wakeup(fs->object); unlock_map(fs); if (fs->m != vm_page_lookup(fs->object, fs->pindex) || - !vm_page_busy_sleep(fs->m, "vmpfw", 0)) - VM_OBJECT_WUNLOCK(fs->object); + !vm_page_busy_sleep(fs->m, "vmpfw", 0)) { + if (objrlocked) + VM_OBJECT_RUNLOCK(fs->object); + else + VM_OBJECT_WUNLOCK(fs->object); + } VM_CNT_INC(v_intrans); vm_object_deallocate(fs->first_object); } +static void +vm_fault_busy_sleep(struct faultstate *fs) +{ + + vm_fault_busy_sleep_impl(fs, false); +} + +static void +vm_fault_busy_sleep_rlocked(struct faultstate *fs) +{ + + vm_fault_busy_sleep_impl(fs, true); +} + /* * Handle page lookup, populate, allocate, page-in for the current * object. @@ -1383,6 +1402,8 @@ enum fault_status res; bool dead; + VM_OBJECT_ASSERT_WLOCKED(fs->object); + /* * If the object is marked for imminent termination, we retry * here, since the collapse pass has raced with us. Otherwise, @@ -1417,7 +1438,6 @@ return (FAULT_SOFT); } } - VM_OBJECT_ASSERT_WLOCKED(fs->object); /* * Page is not resident. If the pager might contain the page @@ -1458,6 +1478,39 @@ return (res); } +/* + * Like the above, but try to get away with a read-locked object. + * This often can be done for backing objects. + */ +static enum fault_status +vm_fault_object_rlocked(struct faultstate *fs) +{ + + VM_OBJECT_ASSERT_RLOCKED(fs->object); + + fs->m = vm_page_lookup(fs->object, fs->pindex); + if (fs->m != NULL && vm_page_all_valid(fs->m)) { + if (!vm_page_tryxbusy(fs->m)) { + vm_fault_busy_sleep_rlocked(fs); + return (FAULT_RESTART); + } + + if (vm_page_all_valid(fs->m)) { + VM_OBJECT_RUNLOCK(fs->object); + return (FAULT_SOFT); + } + + vm_page_xunbusy(fs->m); + fs->m = NULL; + } + + if (!VM_OBJECT_TRYUPGRADE(fs->object)) { + VM_OBJECT_RUNLOCK(fs->object); + VM_OBJECT_WLOCK(fs->object); + } + return (FAULT_CONTINUE); +} + int vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags, vm_page_t *m_hold) @@ -1582,27 +1635,43 @@ case FAULT_CONTINUE: break; default: - panic("vm_fault: Unhandled status %d", res); + panic("vm_fault: Unhandled vm_fault_object status %d", res); } /* - * The page was not found in the current object. Try to - * traverse into a backing object or zero fill if none is - * found. + * The page was not found in the current object. + * Traverse into a backing object if there is one. */ - if (vm_fault_next(&fs)) - continue; - if ((fs.fault_flags & VM_FAULT_NOFILL) != 0) { - if (fs.first_object == fs.object) - fault_page_free(&fs.first_m); - unlock_and_deallocate(&fs); - return (KERN_OUT_OF_BOUNDS); + if (!vm_fault_next(&fs)) { + /* + * No backing object, zero fill if requested. + */ + if ((fs.fault_flags & VM_FAULT_NOFILL) != 0) { + if (fs.first_object == fs.object) + fault_page_free(&fs.first_m); + unlock_and_deallocate(&fs); + return (KERN_OUT_OF_BOUNDS); + } + VM_OBJECT_WUNLOCK(fs.object); + vm_fault_zerofill(&fs); + /* + * Don't try to prefault neighboring pages. + */ + faultcount = 1; + break; + } + + res = vm_fault_object_rlocked(&fs); + switch (res) { + case FAULT_SOFT: + goto found; + case FAULT_RESTART: + goto RetryFault; + case FAULT_CONTINUE: + break; + default: + panic("vm_fault: Unhandled vm_fault_object_rlocked status %d", res); } - VM_OBJECT_WUNLOCK(fs.object); - vm_fault_zerofill(&fs); - /* Don't try to prefault neighboring pages. */ - faultcount = 1; - break; } found: