Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -407,6 +407,7 @@ static void swp_sizecheck(void); static void swp_pager_async_iodone(struct buf *bp); static bool swp_pager_swblk_empty(struct swblk *sb, int start, int limit); +static void swp_pager_swblk_free_empty(vm_object_t, struct swblk *sb); static int swapongeom(struct vnode *); static int swaponvp(struct thread *, struct vnode *, u_long); static int swapoff_one(struct swdevt *sp, struct ucred *cred); @@ -424,6 +425,8 @@ static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); static void swp_pager_meta_free_all(vm_object_t); static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); +static vm_pindex_t swp_pager_meta_find_least(vm_object_t, vm_pindex_t, + daddr_t*); static void swp_pager_init_freerange(daddr_t *start, daddr_t *num) @@ -985,10 +988,13 @@ * Transfer source to destination. */ swp_pager_init_freerange(&s_free, &n_free); - for (i = 0; i < dstobject->size; ++i) { - srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP); - if (srcaddr == SWAPBLK_NONE) - continue; + + /* + * Search srcobject pindex of corresponding dstobject between 0 and + * dstobject->size to transfer. + */ + for (i = 0; (i = swp_pager_meta_find_least(srcobject, i + offset, + &srcaddr) - offset) < dstobject->size; ++i) { dstaddr = swp_pager_meta_ctl(dstobject, i, 0); if (dstaddr != SWAPBLK_NONE) { /* @@ -1876,6 +1882,19 @@ } return (true); } + +/* + * SWP_PAGER_SWBLK_FREE_EMPTY() - frees if a block is free + */ +static void +swp_pager_swblk_free_empty(vm_object_t object, struct swblk *sb) +{ + + if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { + SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); + uma_zfree(swblk_zone, sb); + } +} /* * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object @@ -1994,11 +2013,8 @@ /* * Free the swblk if we end up with the empty page run. */ - if (swapblk == SWAPBLK_NONE && - swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { - SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi); - uma_zfree(swblk_zone, sb); - } + if (swapblk == SWAPBLK_NONE) + swp_pager_swblk_free_empty(object, sb); return (prev_swapblk); } @@ -2124,11 +2140,7 @@ return (SWAPBLK_NONE); if ((flags & SWM_POP) != 0) { sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE; - if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { - SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, - rounddown(pindex, SWAP_META_PAGES)); - uma_zfree(swblk_zone, sb); - } + swp_pager_swblk_free_empty(object, sb); } return (r1); } @@ -2136,17 +2148,23 @@ /* * Returns the least page index which is greater than or equal to the * parameter pindex and for which there is a swap block allocated. + * If removal argument is passed, the swpblk is removed. * Returns object's size if the object's type is not swap or if there * are no allocated swap blocks for the object after the requested * pindex. */ vm_pindex_t -swap_pager_find_least(vm_object_t object, vm_pindex_t pindex) +swp_pager_meta_find_least(vm_object_t object, vm_pindex_t pindex, + daddr_t *removal) { struct swblk *sb; int i; - VM_OBJECT_ASSERT_LOCKED(object); + if (removal) + VM_OBJECT_ASSERT_WLOCKED(object); + else + VM_OBJECT_ASSERT_LOCKED(object); + if (object->type != OBJT_SWAP) return (object->size); @@ -2155,19 +2173,17 @@ if (sb == NULL) return (object->size); if (sb->p < pindex) { - for (i = pindex % SWAP_META_PAGES; i < SWAP_META_PAGES; i++) { + for (i = pindex % SWAP_META_PAGES; i < SWAP_META_PAGES; i++) if (sb->d[i] != SWAPBLK_NONE) - return (sb->p + i); - } + goto found; sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, roundup(pindex, SWAP_META_PAGES)); if (sb == NULL) return (object->size); } - for (i = 0; i < SWAP_META_PAGES; i++) { + for (i = 0; i < SWAP_META_PAGES; i++) if (sb->d[i] != SWAPBLK_NONE) - return (sb->p + i); - } + goto found; /* * We get here if a swblk is present in the trie but it @@ -2175,6 +2191,27 @@ */ MPASS(0); return (object->size); +found: + if (removal) { + *removal = sb->d[pindex % SWAP_META_PAGES]; + sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE; + swp_pager_swblk_free_empty(object, sb); + } + return (sb->p + i); +} + +/* + * Returns the least page index which is greater than or equal to the + * parameter pindex and for which there is a swap block allocated. + * Returns object's size if the object's type is not swap or if there + * are no allocated swap blocks for the object after the requested + * pindex. + */ +vm_pindex_t +swap_pager_find_least(vm_object_t object, vm_pindex_t pindex) +{ + + return swp_pager_meta_find_least(object, pindex, NULL); } /*