Index: sys/vm/swap_pager.c
===================================================================
--- sys/vm/swap_pager.c
+++ sys/vm/swap_pager.c
@@ -321,8 +321,6 @@
 #endif
 }
 
-#define SWM_POP		0x01	/* pop out			*/
-
 static int swap_pager_full = 2;	/* swap space exhaustion (task killing) */
 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
 static struct mtx swbuf_mtx;	/* to sync nsw_wcount_async */
@@ -422,8 +420,11 @@
  */
 static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t);
+static void swp_pager_meta_copy_free(vm_object_t src, vm_object_t dst,
+    vm_pindex_t, vm_pindex_t);
 static void swp_pager_meta_free_all(vm_object_t);
-static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
+static daddr_t swp_pager_meta_lookup(vm_object_t, vm_pindex_t);
+static daddr_t swp_pager_meta_remove(vm_object_t, vm_pindex_t);
 
 static void
 swp_pager_init_freerange(daddr_t *start, daddr_t *num)
@@ -933,6 +934,38 @@
 	return (0);
 }
 
+static bool
+swp_pager_copy_source(vm_object_t srcobject, vm_object_t dstobject,
+    vm_pindex_t pindex, daddr_t addr)
+{
+	daddr_t dstaddr;
+
+	if (dstobject == NULL ||
+	    swp_pager_meta_lookup(dstobject, pindex) != SWAPBLK_NONE) {
+		/*
+		 * Destination has valid swapblk or it is represented by a
+		 * resident page.  Caller should destroy the source block.
+		 */
+		return (false);
+	}
+
+	/*
+	 * Destination has no swapblk and is not resident, copy source.
+	 *
+	 * swp_pager_meta_build() can sleep.
+	 */
+	vm_object_pip_add(srcobject, 1);
+	VM_OBJECT_WUNLOCK(srcobject);
+	vm_object_pip_add(dstobject, 1);
+	dstaddr = swp_pager_meta_build(dstobject, pindex, addr);
+	KASSERT(dstaddr == SWAPBLK_NONE,
+	    ("Unexpected destination swapblk"));
+	vm_object_pip_wakeup(dstobject);
+	VM_OBJECT_WLOCK(srcobject);
+	vm_object_pip_wakeup(srcobject);
+	return (true);
+}
+
 /*
  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
  *			and destroy the source.
@@ -956,8 +989,6 @@
 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
     vm_pindex_t offset, int destroysource)
 {
-	vm_pindex_t i;
-	daddr_t dstaddr, n_free, s_free, srcaddr;
 
 	VM_OBJECT_ASSERT_WLOCKED(srcobject);
 	VM_OBJECT_ASSERT_WLOCKED(dstobject);
@@ -984,39 +1015,8 @@
 	/*
 	 * Transfer source to destination.
 	 */
-	swp_pager_init_freerange(&s_free, &n_free);
-	for (i = 0; i < dstobject->size; ++i) {
-		srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP);
-		if (srcaddr == SWAPBLK_NONE)
-			continue;
-		dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
-		if (dstaddr != SWAPBLK_NONE) {
-			/*
-			 * Destination has valid swapblk or it is represented
-			 * by a resident page.  We destroy the source block.
-			 */
-			swp_pager_update_freerange(&s_free, &n_free, srcaddr);
-			continue;
-		}
+	swp_pager_meta_copy_free(srcobject, dstobject, offset, dstobject->size);
 
-		/*
-		 * Destination has no swapblk and is not resident,
-		 * copy source.
-		 *
-		 * swp_pager_meta_build() can sleep.
-		 */
-		vm_object_pip_add(srcobject, 1);
-		VM_OBJECT_WUNLOCK(srcobject);
-		vm_object_pip_add(dstobject, 1);
-		dstaddr = swp_pager_meta_build(dstobject, i, srcaddr);
-		KASSERT(dstaddr == SWAPBLK_NONE,
-		    ("Unexpected destination swapblk"));
-		vm_object_pip_wakeup(dstobject);
-		VM_OBJECT_WLOCK(srcobject);
-		vm_object_pip_wakeup(srcobject);
-	}
-	swp_pager_freeswapspace(s_free, n_free);
-
 	/*
 	 * Free left over swap blocks in source.
 	 *
@@ -1057,7 +1057,7 @@
 	/*
 	 * do we have good backing store at the requested index ?
 	 */
-	blk0 = swp_pager_meta_ctl(object, pindex, 0);
+	blk0 = swp_pager_meta_lookup(object, pindex);
 	if (blk0 == SWAPBLK_NONE) {
 		if (before)
 			*before = 0;
@@ -1073,7 +1073,7 @@
 		for (i = 1; i < SWB_NPAGES; i++) {
 			if (i > pindex)
 				break;
-			blk = swp_pager_meta_ctl(object, pindex - i, 0);
+			blk = swp_pager_meta_lookup(object, pindex - i);
 			if (blk != blk0 - i)
 				break;
 		}
@@ -1085,7 +1085,7 @@
 	 */
 	if (after != NULL) {
 		for (i = 1; i < SWB_NPAGES; i++) {
-			blk = swp_pager_meta_ctl(object, pindex + i, 0);
+			blk = swp_pager_meta_lookup(object, pindex + i);
 			if (blk != blk0 + i)
 				break;
 		}
@@ -1118,7 +1118,7 @@
 {
 	daddr_t srcaddr;
 
-	srcaddr = swp_pager_meta_ctl(m->object, m->pindex, SWM_POP);
+	srcaddr = swp_pager_meta_remove(m->object, m->pindex);
 	if (srcaddr != SWAPBLK_NONE)
 		swp_pager_freeswapspace(srcaddr, 1);
 }
@@ -1214,7 +1214,7 @@
 	vm_object_pip_add(object, count);
 
 	pindex = bm->pindex;
-	blk = swp_pager_meta_ctl(object, pindex, 0);
+	blk = swp_pager_meta_lookup(object, pindex);
 	KASSERT(blk != SWAPBLK_NONE,
 	    ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex));
 
@@ -2003,31 +2003,30 @@
 }
 
 /*
- * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
+ * SWP_PAGER_META_COPY_FREE() - free a range of blocks in the srcobject's swap
+ * metadata, or copy it into dstobject.
  *
  *	The requested range of blocks is freed, with any associated swap
  *	returned to the swap bitmap.
- *
- *	This routine will free swap metadata structures as they are cleaned
- *	out.  This routine does *NOT* operate on swap metadata associated
- *	with resident pages.
  */
 static void
-swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
+swp_pager_meta_copy_free(vm_object_t srcobject, vm_object_t dstobject,
+			 vm_pindex_t pindex, vm_pindex_t count)
 {
 	struct swblk *sb;
 	daddr_t n_free, s_free;
-	vm_pindex_t last;
+	vm_pindex_t offset, last;
 	int i, limit, start;
 
-	VM_OBJECT_ASSERT_WLOCKED(object);
-	if (object->type != OBJT_SWAP || count == 0)
+	VM_OBJECT_ASSERT_WLOCKED(srcobject);
+	if (srcobject->type != OBJT_SWAP || count == 0)
 		return;
 
 	swp_pager_init_freerange(&s_free, &n_free);
+	offset = pindex;
 	last = pindex + count;
 	for (;;) {
-		sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
+		sb = SWAP_PCTRIE_LOOKUP_GE(&srcobject->un_pager.swp.swp_blks,
 		    rounddown(pindex, SWAP_META_PAGES));
 		if (sb == NULL || sb->p >= last)
 			break;
@@ -2037,13 +2036,17 @@
 		for (i = start; i < limit; i++) {
 			if (sb->d[i] == SWAPBLK_NONE)
 				continue;
-			swp_pager_update_freerange(&s_free, &n_free, sb->d[i]);
+			if (!swp_pager_copy_source(srcobject, dstobject, 
+				    sb->p + i - offset, sb->d[i])) {
+				swp_pager_update_freerange(&s_free, &n_free,
+				    sb->d[i]);
+			}
 			sb->d[i] = SWAPBLK_NONE;
 		}
 		pindex = sb->p + SWAP_META_PAGES;
 		if (swp_pager_swblk_empty(sb, 0, start) &&
 		    swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
-			SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
+			SWAP_PCTRIE_REMOVE(&srcobject->un_pager.swp.swp_blks,
 			    sb->p);
 			uma_zfree(swblk_zone, sb);
 		}
@@ -2052,6 +2055,23 @@
 }
 
 /*
+ * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
+ *
+ *	The requested range of blocks is freed, with any associated swap
+ *	returned to the swap bitmap.
+ *
+ *	This routine will free swap metadata structures as they are cleaned
+ *	out.  This routine does *NOT* operate on swap metadata associated
+ *	with resident pages.
+ */
+static void
+swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
+{
+	swp_pager_meta_copy_free(object, NULL, pindex, count);
+}
+
+
+/*
  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
  *
  *	This routine locates and destroys all swap metadata associated with
@@ -2085,28 +2105,55 @@
 }
 
 /*
- * SWP_PAGER_METACTL() -  misc control of swap meta data.
+ * SWP_PAGER_META_LOOKUP() -  look up swap meta data.
  *
- *	This routine is capable of looking up, or removing swapblk
- *	assignments in the swap meta data.  It returns the swapblk being
- *	looked-up, popped, or SWAPBLK_NONE if the block was invalid.
+ *	Look up swapblk assignments in the swap meta data.  Returns the swapblk
+ *	for (pindex = 0; (sb = being looked-up, or SWAPBLK_NONE if the block
+ *	was invalid.
  *
  *	When acting on a busy resident page and paging is in progress, we
  *	have to wait until paging is complete but otherwise can act on the
  *	busy page.
+ */
+static daddr_t
+swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex)
+{
+	struct swblk *sb;
+
+	VM_OBJECT_ASSERT_LOCKED(object);
+
+	/*
+	 * The meta data only exists if the object is OBJT_SWAP
+	 * and even then might not be allocated yet.
+	 */
+	if (object->type != OBJT_SWAP)
+		return (SWAPBLK_NONE);
+
+	sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
+	    rounddown(pindex, SWAP_META_PAGES));
+	if (sb == NULL)
+		return (SWAPBLK_NONE);
+	return (sb->d[pindex % SWAP_META_PAGES]);
+}
+
+/*
+ * SWP_PAGER_META_REMOVE() -  remove swap meta data.
  *
- *	SWM_POP		remove from meta data but do not free it
+ *	This routine is capable of removing swapblk assignments in the swap
+ *	for (pindex = 0; (sb = meta data.  It returns the swapblk being
+ *	removed, or SWAPBLK_NONE if the block was invalid.
+ *
+ *	When acting on a busy resident page and paging is in progress, we
+ *	have to wait until paging is complete but otherwise can act on the
+ *	busy page.
  */
 static daddr_t
-swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
+swp_pager_meta_remove(vm_object_t object, vm_pindex_t pindex)
 {
 	struct swblk *sb;
 	daddr_t r1;
 
-	if ((flags & SWM_POP) != 0)
-		VM_OBJECT_ASSERT_WLOCKED(object);
-	else
-		VM_OBJECT_ASSERT_LOCKED(object);
+	VM_OBJECT_ASSERT_WLOCKED(object);
 
 	/*
 	 * The meta data only exists if the object is OBJT_SWAP
@@ -2122,13 +2169,11 @@
 	r1 = sb->d[pindex % SWAP_META_PAGES];
 	if (r1 == SWAPBLK_NONE)
 		return (SWAPBLK_NONE);
-	if ((flags & SWM_POP) != 0) {
-		sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE;
-		if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
-			SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
-			    rounddown(pindex, SWAP_META_PAGES));
-			uma_zfree(swblk_zone, sb);
-		}
+	sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE;
+	if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
+		SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
+		    rounddown(pindex, SWAP_META_PAGES));
+		uma_zfree(swblk_zone, sb);
 	}
 	return (r1);
 }