Index: sys/vm/swap_pager.c
===================================================================
--- sys/vm/swap_pager.c
+++ sys/vm/swap_pager.c
@@ -949,7 +949,7 @@
 	}
 
 	/*
-	 * Destination has no swapblk and is not resident, transfer source.
+	 * Transfer source.
 	 * swp_pager_meta_build() can sleep.
 	 */
 	vm_object_pip_add(srcobject, 1);
@@ -1117,8 +1117,9 @@
 swap_pager_unswapped(vm_page_t m)
 {
 	struct swblk *sb;
+	bool locked;
 
-	VM_OBJECT_ASSERT_WLOCKED(m->object);
+	VM_OBJECT_ASSERT_LOCKED(m->object);
 
 	/*
 	 * The meta data only exists if the object is OBJT_SWAP
@@ -1133,9 +1134,14 @@
 		return;
 	if (sb->d[m->pindex % SWAP_META_PAGES] == SWAPBLK_NONE)
 		return;
+	locked = VM_OBJECT_WOWNED(m->object);
+	if (!locked && !VM_OBJECT_TRYUPGRADE(m->object))
+		return;
 	swp_pager_freeswapspace(sb->d[m->pindex % SWAP_META_PAGES], 1);
 	sb->d[m->pindex % SWAP_META_PAGES] = SWAPBLK_NONE;
 	swp_pager_free_empty_swblk(m->object, sb);
+	if (!locked)
+		VM_OBJECT_LOCK_DOWNGRADE(m->object);
 }
 
 /*
Index: sys/vm/vm_fault.c
===================================================================
--- sys/vm/vm_fault.c
+++ sys/vm/vm_fault.c
@@ -273,7 +273,7 @@
 		vm_page_dirty(m);
 	if (!excl)
 		vm_page_unlock(m);
-	else if (need_dirty)
+	if (need_dirty)
 		vm_pager_page_unswapped(m);
 }