Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -390,6 +390,7 @@ static void swp_sizecheck(void); static void swp_pager_async_iodone(struct buf *bp); +static bool swp_pager_swblk_empty(struct swblk *sb, int start, int limit); static int swapongeom(struct vnode *); static int swaponvp(struct thread *, struct vnode *, u_long); static int swapoff_one(struct swdevt *sp, struct ucred *cred); @@ -397,7 +398,7 @@ /* * Swap bitmap functions */ -static void swp_pager_freeswapspace(daddr_t blk, int npages); +static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages); static daddr_t swp_pager_getswapspace(int npages); /* @@ -768,10 +769,12 @@ * This routine may not sleep. */ static void -swp_pager_freeswapspace(daddr_t blk, int npages) +swp_pager_freeswapspace(daddr_t blk, daddr_t npages) { struct swdevt *sp; + if (npages == 0) + return; mtx_lock(&sw_dev_mtx); TAILQ_FOREACH(sp, &swtailq, sw_list) { if (blk >= sp->sw_first && blk < sp->sw_end) { @@ -1761,6 +1764,22 @@ */ /* + * SWP_PAGER_SWBLK_EMPTY() - is a range of blocks free? + */ +static bool +swp_pager_swblk_empty(struct swblk *sb, int start, int limit) +{ + int i; + + MPASS(start >= 0 && limit <= SWAP_META_PAGES); + for (i = start; i < limit; i++) { + if (sb->d[i] != SWAPBLK_NONE) + return (false); + } + return (true); +} + +/* * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object * * We first convert the object to a swap object if it is a default @@ -1876,16 +1895,10 @@ /* * Free the swblk if we end up with the empty page run. */ - if (swapblk == SWAPBLK_NONE) { - for (i = 0; i < SWAP_META_PAGES; i++) { - if (sb->d[i] != SWAPBLK_NONE) - break; - } - if (i == SWAP_META_PAGES) { - SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, - rdpi); - uma_zfree(swblk_zone, sb); - } + if (swapblk == SWAPBLK_NONE && + swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { + SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi); + uma_zfree(swblk_zone, sb); } } @@ -1903,37 +1916,47 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) { struct swblk *sb; + daddr_t first_free, num_free; vm_pindex_t last; - int i; + int i, limit, start; bool empty; VM_OBJECT_ASSERT_WLOCKED(object); if (object->type != OBJT_SWAP || count == 0) return; - last = pindex + count - 1; + first_free = SWAPBLK_NONE; + num_free = 0; + last = pindex + count; for (;;) { sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, rounddown(pindex, SWAP_META_PAGES)); - if (sb == NULL || sb->p > last) + if (sb == NULL || sb->p >= last) break; - empty = true; - for (i = 0; i < SWAP_META_PAGES; i++) { + start = pindex > sb->p ? pindex - sb->p : 0; + limit = last - sb->p < SWAP_META_PAGES ? last - sb->p : + SWAP_META_PAGES; + for (i = start; i < limit; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; - if (pindex <= sb->p + i && sb->p + i <= last) { - swp_pager_freeswapspace(sb->d[i], 1); - sb->d[i] = SWAPBLK_NONE; - } else - empty = false; + if (first_free + num_free == sb->d[i]) + num_free++; + else { + swp_pager_freeswapspace(first_free, num_free); + first_free = sb->d[i]; + num_free = 1; + } + sb->d[i] = SWAPBLK_NONE; } - pindex = sb->p + SWAP_META_PAGES; - if (empty) { + if (swp_pager_swblk_empty(sb, 0, start) && + swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); uma_zfree(swblk_zone, sb); } + pindex = sb->p + SWAP_META_PAGES; } + swp_pager_freeswapspace(first_free, num_free); } /* @@ -1946,6 +1969,7 @@ swp_pager_meta_free_all(vm_object_t object) { struct swblk *sb; + daddr_t first_free, num_free; vm_pindex_t pindex; int i; @@ -1953,16 +1977,26 @@ if (object->type != OBJT_SWAP) return; + first_free = SWAPBLK_NONE; + num_free = 0; for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( &object->un_pager.swp.swp_blks, pindex)) != NULL;) { pindex = sb->p + SWAP_META_PAGES; for (i = 0; i < SWAP_META_PAGES; i++) { - if (sb->d[i] != SWAPBLK_NONE) - swp_pager_freeswapspace(sb->d[i], 1); + if (sb->d[i] == SWAPBLK_NONE) + continue; + if (first_free + num_free == sb->d[i]) + num_free++; + else { + swp_pager_freeswapspace(first_free, num_free); + first_free = sb->d[i]; + num_free = 1; + } } SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); uma_zfree(swblk_zone, sb); } + swp_pager_freeswapspace(first_free, num_free); } /* @@ -1987,7 +2021,6 @@ { struct swblk *sb; daddr_t r1; - int i; if ((flags & (SWM_FREE | SWM_POP)) != 0) VM_OBJECT_ASSERT_WLOCKED(object); @@ -2010,11 +2043,7 @@ return (SWAPBLK_NONE); if ((flags & (SWM_FREE | SWM_POP)) != 0) { sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE; - for (i = 0; i < SWAP_META_PAGES; i++) { - if (sb->d[i] != SWAPBLK_NONE) - break; - } - if (i == SWAP_META_PAGES) { + if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rounddown(pindex, SWAP_META_PAGES)); uma_zfree(swblk_zone, sb);