diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -348,12 +348,32 @@ MPASS(fs->vp == NULL); + /* + * If we fail, vast majority of the time it is because the page is not + * there to begin with. Opportunistically perform the lookup and + * subsequent checks without the object lock, revalidate later. + * + * Note: a busy page can be mapped for read|execute access. + */ + m = vm_page_lookup_unlocked(fs->first_object, fs->first_pindex); + if (m == NULL || !vm_page_all_valid(m) || + ((fs->prot & VM_PROT_WRITE) != 0 && vm_page_busied(m))) { + VM_OBJECT_WLOCK(fs->first_object); + return (FAULT_FAILURE); + } + vaddr = fs->vaddr; + + VM_OBJECT_RLOCK(fs->first_object); vm_object_busy(fs->first_object); - m = vm_page_lookup(fs->first_object, fs->first_pindex); - /* A busy page can be mapped for read|execute access. */ - if (m == NULL || ((fs->prot & VM_PROT_WRITE) != 0 && - vm_page_busied(m)) || !vm_page_all_valid(m)) + /* + * Now that we stabilized the state, revalidate the page is in the shape + * we encountered above. + */ + if (m->object != fs->first_object || m->pindex != fs->first_pindex) + goto fail; + if (!vm_page_all_valid(m) || + ((fs->prot & VM_PROT_WRITE) != 0 && vm_page_busied(m))) goto fail; m_map = m; psind = 0; @@ -405,6 +425,10 @@ return (FAULT_SUCCESS); fail: vm_object_unbusy(fs->first_object); + if (!VM_OBJECT_TRYUPGRADE(fs->first_object)) { + VM_OBJECT_RUNLOCK(fs->first_object); + VM_OBJECT_WLOCK(fs->first_object); + } return (FAULT_FAILURE); } @@ -1556,14 +1580,12 @@ if (fs.vp == NULL /* avoid locked vnode leak */ && (fs.entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) == 0 && (fs.fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0) { - VM_OBJECT_RLOCK(fs.first_object); res = vm_fault_soft_fast(&fs); - if (res == FAULT_SUCCESS) + if (res == FAULT_SUCCESS) { + VM_OBJECT_ASSERT_UNLOCKED(fs.first_object); return (KERN_SUCCESS); - if (!VM_OBJECT_TRYUPGRADE(fs.first_object)) { - VM_OBJECT_RUNLOCK(fs.first_object); - VM_OBJECT_WLOCK(fs.first_object); } + VM_OBJECT_ASSERT_WLOCKED(fs.first_object); } else { VM_OBJECT_WLOCK(fs.first_object); }