Index: sys/vm/vm_fault.c
===================================================================
--- sys/vm/vm_fault.c
+++ sys/vm/vm_fault.c
@@ -151,19 +151,38 @@
     "the page fault handler");
 
 static inline void
-release_page(struct faultstate *fs)
+fault_page_release(vm_page_t *mp)
 {
+	vm_page_t m;
 
-	if (fs->m != NULL) {
+	m = *mp;
+	if (m != NULL) {
 		/*
-		 * fs->m's object lock might not be held, so the page must be
-		 * kept busy until we are done with it.
+		 * We are likely to loop around again and attempt to busy
+		 * this page.  Deactivating it leaves it available for
+		 * pageout while optimizing fault restarts.
 		 */
-		vm_page_lock(fs->m);
-		vm_page_deactivate(fs->m);
-		vm_page_unlock(fs->m);
-		vm_page_xunbusy(fs->m);
-		fs->m = NULL;
+		vm_page_lock(m);
+		vm_page_deactivate(m);
+		vm_page_unlock(m);
+		vm_page_xunbusy(m);
+		*mp = NULL;
+	}
+}
+
+static inline void
+fault_page_free(vm_page_t *mp)
+{
+	vm_page_t m;
+
+	m = *mp;
+	if (m != NULL) {
+		VM_OBJECT_ASSERT_WLOCKED(m->object);
+		if (vm_page_remove(m))
+			vm_page_free(m);
+		else
+			vm_page_xunbusy(m);
+		*mp = NULL;
 	}
 }
 
@@ -191,13 +210,13 @@
 fault_deallocate(struct faultstate *fs)
 {
 
+	fault_page_release(&fs->m);
 	vm_object_pip_wakeup(fs->object);
 	if (fs->object != fs->first_object) {
 		VM_OBJECT_WLOCK(fs->first_object);
-		vm_page_free(fs->first_m);
-		vm_object_pip_wakeup(fs->first_object);
+		fault_page_free(&fs->first_m);
 		VM_OBJECT_WUNLOCK(fs->first_object);
-		fs->first_m = NULL;
+		vm_object_pip_wakeup(fs->first_object);
 	}
 	vm_object_deallocate(fs->first_object);
 	unlock_map(fs);
@@ -678,7 +697,6 @@
 	}
 
 	vhold(vp);
-	release_page(fs);
 	unlock_and_deallocate(fs);
 	error = vget(vp, locked | LK_RETRY | LK_CANRECURSE, curthread);
 	vdrop(vp);
@@ -795,7 +813,7 @@
 
 	fs.lookup_still_valid = true;
 
-	fs.first_m = NULL;
+	fs.m = fs.first_m = NULL;
 
 	/*
 	 * Search for the page at object/offset.
@@ -803,6 +821,8 @@
 	fs.object = fs.first_object;
 	fs.pindex = fs.first_pindex;
 	while (TRUE) {
+		KASSERT(fs.m == NULL,
+		    ("page still set %p at loop start", fs.m));
 		/*
 		 * If the object is marked for imminent termination,
 		 * we retry here, since the collapse pass has raced
@@ -847,23 +867,15 @@
 				 */
 				vm_page_aflag_set(fs.m, PGA_REFERENCED);
 				if (fs.object != fs.first_object) {
-					if (!VM_OBJECT_TRYWLOCK(
-					    fs.first_object)) {
-						VM_OBJECT_WUNLOCK(fs.object);
-						VM_OBJECT_WLOCK(fs.first_object);
-						VM_OBJECT_WLOCK(fs.object);
-					}
-					vm_page_free(fs.first_m);
+					fault_page_release(&fs.first_m);
 					vm_object_pip_wakeup(fs.first_object);
-					VM_OBJECT_WUNLOCK(fs.first_object);
-					fs.first_m = NULL;
 				}
 				unlock_map(&fs);
+				vm_object_pip_wakeup(fs.object);
 				if (fs.m == vm_page_lookup(fs.object,
 				    fs.pindex)) {
 					vm_page_sleep_if_busy(fs.m, "vmpfw");
 				}
-				vm_object_pip_wakeup(fs.object);
 				VM_OBJECT_WUNLOCK(fs.object);
 				VM_CNT_INC(v_intrans);
 				vm_object_deallocate(fs.first_object);
@@ -1112,40 +1124,28 @@
 			 * an error.
 			 */
 			if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) {
-				if (!vm_page_wired(fs.m))
-					vm_page_free(fs.m);
-				else
-					vm_page_xunbusy(fs.m);
-				fs.m = NULL;
+				fault_page_free(&fs.m);
 				unlock_and_deallocate(&fs);
 				return (KERN_OUT_OF_BOUNDS);
 			}
 
-			/*
-			 * The requested page does not exist at this object/
-			 * offset.  Remove the invalid page from the object,
-			 * waking up anyone waiting for it, and continue on to
-			 * the next object.  However, if this is the top-level
-			 * object, we must leave the busy page in place to
-			 * prevent another process from rushing past us, and
-			 * inserting the page in that object at the same time
-			 * that we are.
-			 */
-			if (fs.object != fs.first_object) {
-				if (!vm_page_wired(fs.m))
-					vm_page_free(fs.m);
-				else
-					vm_page_xunbusy(fs.m);
-				fs.m = NULL;
-			}
 		}
 
 		/*
-		 * We get here if the object has default pager (or unwiring) 
-		 * or the pager doesn't have the page.
+		 * The requested page does not exist at this object/
+		 * offset.  Remove the invalid page from the object,
+		 * waking up anyone waiting for it, and continue on to
+		 * the next object.  However, if this is the top-level
+		 * object, we must leave the busy page in place to
+		 * prevent another process from rushing past us, and
+		 * inserting the page in that object at the same time
+		 * that we are.
 		 */
-		if (fs.object == fs.first_object)
+		if (fs.object == fs.first_object) {
 			fs.first_m = fs.m;
+			fs.m = NULL;
+		} else
+			fault_page_free(&fs.m);
 
 		/*
 		 * Move on to the next object.  Lock the next object before
@@ -1163,9 +1163,11 @@
 
 				fs.object = fs.first_object;
 				fs.pindex = fs.first_pindex;
-				fs.m = fs.first_m;
 				VM_OBJECT_WLOCK(fs.object);
 			}
+			MPASS(fs.first_m != NULL);
+			MPASS(fs.m == NULL);
+			fs.m = fs.first_m;
 			fs.first_m = NULL;
 
 			/*
@@ -1182,6 +1184,7 @@
 			faultcount = 1;
 			break;	/* break to PAGE HAS BEEN FOUND */
 		} else {
+			MPASS(fs.first_m != NULL);
 			KASSERT(fs.object != next_object,
 			    ("object loop %p", next_object));
 			VM_OBJECT_WLOCK(next_object);
@@ -1278,7 +1281,7 @@
 				/*
 				 * We no longer need the old page or object.
 				 */
-				release_page(&fs);
+				fault_page_release(&fs.m);
 			}
 			/*
 			 * fs.object != fs.first_object due to above 
@@ -1316,7 +1319,6 @@
 	 */
 	if (!fs.lookup_still_valid) {
 		if (!vm_map_trylock_read(fs.map)) {
-			release_page(&fs);
 			unlock_and_deallocate(&fs);
 			goto RetryFault;
 		}
@@ -1331,7 +1333,6 @@
 			 * pageout will grab it eventually.
 			 */
 			if (result != KERN_SUCCESS) {
-				release_page(&fs);
 				unlock_and_deallocate(&fs);
 
 				/*
@@ -1344,7 +1345,6 @@
 			}
 			if ((retry_object != fs.first_object) ||
 			    (retry_pindex != fs.first_pindex)) {
-				release_page(&fs);
 				unlock_and_deallocate(&fs);
 				goto RetryFault;
 			}
@@ -1360,7 +1360,6 @@
 			prot &= retry_prot;
 			fault_type &= retry_prot;
 			if (prot == 0) {
-				release_page(&fs);
 				unlock_and_deallocate(&fs);
 				goto RetryFault;
 			}
@@ -1421,6 +1420,7 @@
 		vm_page_wire(fs.m);
 	}
 	vm_page_xunbusy(fs.m);
+	fs.m = NULL;
 
 	/*
 	 * Unlock everything, and return
Index: sys/vm/vm_object.c
===================================================================
--- sys/vm/vm_object.c
+++ sys/vm/vm_object.c
@@ -1477,6 +1477,18 @@
 			goto retry;
 		}
 
+		/*
+		 * The page was left invalid.  Likely placed there by
+		 * an incomplete fault.  Just remove and ignore.
+		 */
+		if (vm_page_none_valid(m)) {
+			if (vm_page_remove(m))
+				vm_page_free(m);
+			else
+				vm_page_xunbusy(m);
+			continue;
+		}
+
 		/* vm_page_rename() will dirty the page. */
 		if (vm_page_rename(m, new_object, idx)) {
 			vm_page_xunbusy(m);
@@ -1688,8 +1700,18 @@
 			continue;
 		}
 
-		KASSERT(pp == NULL || !vm_page_none_valid(pp),
-		    ("unbusy invalid page %p", pp));
+		if (pp != NULL && vm_page_none_valid(pp)) {
+			/*
+			 * The page was invalid in the parent.  Likely placed
+			 * there by an incomplete fault.  Just remove and
+			 * ignore.  p can replace it.
+			 */
+			if (vm_page_remove(pp))
+				vm_page_free(pp);
+			else
+				vm_page_xunbusy(pp);
+			pp = NULL;
+		}
 
 		if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL,
 			NULL)) {