Index: sys/vm/swap_pager.c
===================================================================
--- sys/vm/swap_pager.c
+++ sys/vm/swap_pager.c
@@ -321,8 +321,6 @@
 #endif
 }
 
-#define SWM_POP		0x01	/* pop out			*/
-
 static int swap_pager_full = 2;	/* swap space exhaustion (task killing) */
 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
 static struct mtx swbuf_mtx;	/* to sync nsw_wcount_async */
@@ -425,7 +423,7 @@
 static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst,
     vm_pindex_t pindex, vm_pindex_t count);
 static void swp_pager_meta_free_all(vm_object_t);
-static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
+static daddr_t swp_pager_meta_lookup(vm_object_t, vm_pindex_t);
 
 static void
 swp_pager_init_freerange(daddr_t *start, daddr_t *num)
@@ -941,13 +939,13 @@
 {
 	daddr_t dstaddr;
 
-	if (swp_pager_meta_ctl(dstobject, pindex, 0) != SWAPBLK_NONE) {
+	if (swp_pager_meta_lookup(dstobject, pindex) != SWAPBLK_NONE) {
 		/* Caller should destroy the source block. */
 		return (false);
 	}
 
 	/*
-	 * Destination has no swapblk and is not resident, transfer source.
+	 * Transfer source.
 	 * swp_pager_meta_build() can sleep.
 	 */
 	vm_object_pip_add(srcobject, 1);
@@ -1053,7 +1051,7 @@
 	/*
 	 * do we have good backing store at the requested index ?
 	 */
-	blk0 = swp_pager_meta_ctl(object, pindex, 0);
+	blk0 = swp_pager_meta_lookup(object, pindex);
 	if (blk0 == SWAPBLK_NONE) {
 		if (before)
 			*before = 0;
@@ -1069,7 +1067,7 @@
 		for (i = 1; i < SWB_NPAGES; i++) {
 			if (i > pindex)
 				break;
-			blk = swp_pager_meta_ctl(object, pindex - i, 0);
+			blk = swp_pager_meta_lookup(object, pindex - i);
 			if (blk != blk0 - i)
 				break;
 		}
@@ -1081,7 +1079,7 @@
 	 */
 	if (after != NULL) {
 		for (i = 1; i < SWB_NPAGES; i++) {
-			blk = swp_pager_meta_ctl(object, pindex + i, 0);
+			blk = swp_pager_meta_lookup(object, pindex + i);
 			if (blk != blk0 + i)
 				break;
 		}
@@ -1112,11 +1110,35 @@
 static void
 swap_pager_unswapped(vm_page_t m)
 {
-	daddr_t srcaddr;
+	struct swblk *sb;
+	bool locked;
 
-	srcaddr = swp_pager_meta_ctl(m->object, m->pindex, SWM_POP);
-	if (srcaddr != SWAPBLK_NONE)
-		swp_pager_freeswapspace(srcaddr, 1);
+	VM_OBJECT_ASSERT_LOCKED(m->object);
+
+	/*
+	 * The meta data only exists if the object is OBJT_SWAP
+	 * and even then might not be allocated yet.
+	 */
+	if (m->object->type != OBJT_SWAP)
+		return;
+
+	sb = SWAP_PCTRIE_LOOKUP(&m->object->un_pager.swp.swp_blks,
+	    rounddown(m->pindex, SWAP_META_PAGES));
+	if (sb == NULL)
+		return;
+	if (sb->d[m->pindex % SWAP_META_PAGES] == SWAPBLK_NONE)
+		return;
+	locked = VM_OBJECT_WOWNED(m->object);
+	if (!locked && !VM_OBJECT_TRYUPGRADE(m->object))
+		return;
+	sb->d[m->pindex % SWAP_META_PAGES] = SWAPBLK_NONE;
+	if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
+		SWAP_PCTRIE_REMOVE(&m->object->un_pager.swp.swp_blks,
+				   rounddown(m->pindex, SWAP_META_PAGES));
+		uma_zfree(swblk_zone, sb);
+	}
+	if (!locked)
+		VM_OBJECT_LOCK_DOWNGRADE(m->object);
 }
 
 /*
@@ -1210,7 +1232,7 @@
 	vm_object_pip_add(object, count);
 
 	pindex = bm->pindex;
-	blk = swp_pager_meta_ctl(object, pindex, 0);
+	blk = swp_pager_meta_lookup(object, pindex);
 	KASSERT(blk != SWAPBLK_NONE,
 	    ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex));
 
@@ -2110,19 +2132,13 @@
  *	When acting on a busy resident page and paging is in progress, we
  *	have to wait until paging is complete but otherwise can act on the
  *	busy page.
- *
- *	SWM_POP		remove from meta data but do not free it
  */
 static daddr_t
-swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
+swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex)
 {
 	struct swblk *sb;
-	daddr_t r1;
 
-	if ((flags & SWM_POP) != 0)
-		VM_OBJECT_ASSERT_WLOCKED(object);
-	else
-		VM_OBJECT_ASSERT_LOCKED(object);
+	VM_OBJECT_ASSERT_LOCKED(object);
 
 	/*
 	 * The meta data only exists if the object is OBJT_SWAP
@@ -2135,18 +2151,7 @@
 	    rounddown(pindex, SWAP_META_PAGES));
 	if (sb == NULL)
 		return (SWAPBLK_NONE);
-	r1 = sb->d[pindex % SWAP_META_PAGES];
-	if (r1 == SWAPBLK_NONE)
-		return (SWAPBLK_NONE);
-	if ((flags & SWM_POP) != 0) {
-		sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE;
-		if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
-			SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
-			    rounddown(pindex, SWAP_META_PAGES));
-			uma_zfree(swblk_zone, sb);
-		}
-	}
-	return (r1);
+	return (sb->d[pindex % SWAP_META_PAGES]);
 }
 
 /*
Index: sys/vm/vm_fault.c
===================================================================
--- sys/vm/vm_fault.c
+++ sys/vm/vm_fault.c
@@ -273,7 +273,7 @@
 		vm_page_dirty(m);
 	if (!excl)
 		vm_page_unlock(m);
-	else if (need_dirty)
+	if (need_dirty)
 		vm_pager_page_unswapped(m);
 }