Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -403,11 +403,32 @@ /* * Metadata functions */ -static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); +static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); static void swp_pager_meta_free_all(vm_object_t); static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); +static void +swp_pager_init_freerange(daddr_t *start, daddr_t *num) +{ + + *start = SWAPBLK_NONE; + *num = 0; +} + +static void +swp_pager_update_freerange(daddr_t *start, daddr_t *num, daddr_t addr) +{ + + if (*start + *num == addr) { + (*num)++; + } else { + swp_pager_freeswapspace(*start, *num); + *start = addr; + *num = 1; + } +} + static void * swblk_trie_alloc(struct pctrie *ptree) { @@ -861,7 +882,9 @@ int n = 0; daddr_t blk = SWAPBLK_NONE; vm_pindex_t beg = start; /* save start index */ + daddr_t addr, s_free, n_free; + swp_pager_init_freerange(&s_free, &n_free); VM_OBJECT_WLOCK(object); while (size) { if (n == 0) { @@ -875,12 +898,15 @@ } } } - swp_pager_meta_build(object, start, blk); + addr = swp_pager_meta_build(object, start, blk); + if (addr != SWAPBLK_NONE) + swp_pager_update_freerange(&s_free, &n_free, addr); --size; ++start; ++blk; --n; } + swp_pager_freeswapspace(s_free, n_free); swp_pager_meta_free(object, start, n); VM_OBJECT_WUNLOCK(object); return (0); @@ -910,7 +936,7 @@ vm_pindex_t offset, int destroysource) { vm_pindex_t i; - daddr_t dstaddr, first_free, num_free, srcaddr; + daddr_t dstaddr, s_free, n_free, srcaddr; VM_OBJECT_ASSERT_WLOCKED(srcobject); VM_OBJECT_ASSERT_WLOCKED(dstobject); @@ -937,42 +963,37 @@ /* * Transfer source to destination. */ - first_free = SWAPBLK_NONE; - num_free = 0; + swp_pager_init_freerange(&s_free, &n_free); for (i = 0; i < dstobject->size; ++i) { srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP); if (srcaddr == SWAPBLK_NONE) continue; dstaddr = swp_pager_meta_ctl(dstobject, i, 0); - if (dstaddr == SWAPBLK_NONE) { - /* - * Destination has no swapblk and is not resident, - * copy source. - * - * swp_pager_meta_build() can sleep. - */ - vm_object_pip_add(srcobject, 1); - VM_OBJECT_WUNLOCK(srcobject); - vm_object_pip_add(dstobject, 1); - swp_pager_meta_build(dstobject, i, srcaddr); - vm_object_pip_wakeup(dstobject); - VM_OBJECT_WLOCK(srcobject); - vm_object_pip_wakeup(srcobject); - } else { + if (dstaddr != SWAPBLK_NONE) { /* * Destination has valid swapblk or it is represented - * by a resident page. We destroy the sourceblock. + * by a resident page. We destroy the source block. */ - if (first_free + num_free == srcaddr) - num_free++; - else { - swp_pager_freeswapspace(first_free, num_free); - first_free = srcaddr; - num_free = 1; - } + swp_pager_update_freerange(&s_free, &n_free, srcaddr); + continue; } + /* + * Destination has no swapblk and is not resident, + * copy source. + * + * swp_pager_meta_build() can sleep. + */ + vm_object_pip_add(srcobject, 1); + VM_OBJECT_WUNLOCK(srcobject); + vm_object_pip_add(dstobject, 1); + dstaddr = swp_pager_meta_build(dstobject, i, srcaddr); + KASSERT(dstaddr == SWAPBLK_NONE, + ("Unexpected destination swapblk")); + vm_object_pip_wakeup(dstobject); + VM_OBJECT_WLOCK(srcobject); + vm_object_pip_wakeup(srcobject); } - swp_pager_freeswapspace(first_free, num_free); + swp_pager_freeswapspace(s_free, n_free); /* * Free left over swap blocks in source. @@ -1307,7 +1328,9 @@ { int i, n; boolean_t sync; + daddr_t addr, s_free, n_free; + swp_pager_init_freerange(&s_free, &n_free); if (count && ma[0]->object != object) { panic("swap_pager_putpages: object mismatch %p/%p", object, @@ -1322,8 +1345,11 @@ * check for bogus sysops * force sync if not pageout process */ - if (object->type != OBJT_SWAP) - swp_pager_meta_build(object, 0, SWAPBLK_NONE); + if (object->type != OBJT_SWAP) { + addr = swp_pager_meta_build(object, 0, SWAPBLK_NONE); + if (addr != SWAPBLK_NONE) + swp_pager_update_freerange(&s_free, &n_free, addr); + } VM_OBJECT_WUNLOCK(object); n = 0; @@ -1391,11 +1417,11 @@ for (j = 0; j < n; ++j) { vm_page_t mreq = ma[i+j]; - swp_pager_meta_build( - mreq->object, - mreq->pindex, - blk + j - ); + addr = swp_pager_meta_build(mreq->object, mreq->pindex, + blk + j); + if (addr != SWAPBLK_NONE) + swp_pager_update_freerange(&s_free, &n_free, + addr); MPASS(mreq->dirty == VM_PAGE_BITS_ALL); mreq->oflags |= VPO_SWAPINPROG; bp->b_pages[j] = mreq; @@ -1453,6 +1479,7 @@ swp_pager_async_iodone(bp); } VM_OBJECT_WLOCK(object); + swp_pager_freeswapspace(s_free, n_free); } /* @@ -1783,14 +1810,15 @@ * * The specified swapblk is added to the object's swap metadata. If * the swapblk is not valid, it is freed instead. Any previously - * assigned swapblk is freed. + * assigned swapblk is returned. */ -static void +static daddr_t swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk) { static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted; struct swblk *sb, *sb1; vm_pindex_t modpi, rdpi; + daddr_t prev_swapblk; int error, i; VM_OBJECT_ASSERT_WLOCKED(object); @@ -1815,7 +1843,7 @@ sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi); if (sb == NULL) { if (swapblk == SWAPBLK_NONE) - return; + return (SWAPBLK_NONE); for (;;) { sb = uma_zalloc(swblk_zone, M_NOWAIT | (curproc == pageproc ? M_USE_RESERVE : 0)); @@ -1882,9 +1910,8 @@ MPASS(sb->p == rdpi); modpi = pindex % SWAP_META_PAGES; - /* Delete prior contents of metadata. */ - if (sb->d[modpi] != SWAPBLK_NONE) - swp_pager_freeswapspace(sb->d[modpi], 1); + /* Return prior contents of metadata. */ + prev_swapblk = sb->d[modpi]; /* Enter block into metadata. */ sb->d[modpi] = swapblk; @@ -1896,6 +1923,7 @@ SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi); uma_zfree(swblk_zone, sb); } + return (prev_swapblk); } /* @@ -1912,7 +1940,7 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) { struct swblk *sb; - daddr_t first_free, num_free; + daddr_t s_free, n_free; vm_pindex_t last; int i, limit, start; @@ -1920,8 +1948,7 @@ if (object->type != OBJT_SWAP || count == 0) return; - first_free = SWAPBLK_NONE; - num_free = 0; + swp_pager_init_freerange(&s_free, &n_free); last = pindex + count; for (;;) { sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, @@ -1934,13 +1961,7 @@ for (i = start; i < limit; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; - if (first_free + num_free == sb->d[i]) - num_free++; - else { - swp_pager_freeswapspace(first_free, num_free); - first_free = sb->d[i]; - num_free = 1; - } + swp_pager_update_freerange(&s_free, &n_free, sb->d[i]); sb->d[i] = SWAPBLK_NONE; } if (swp_pager_swblk_empty(sb, 0, start) && @@ -1951,7 +1972,7 @@ } pindex = sb->p + SWAP_META_PAGES; } - swp_pager_freeswapspace(first_free, num_free); + swp_pager_freeswapspace(s_free, n_free); } /* @@ -1964,7 +1985,7 @@ swp_pager_meta_free_all(vm_object_t object) { struct swblk *sb; - daddr_t first_free, num_free; + daddr_t s_free, n_free; vm_pindex_t pindex; int i; @@ -1972,26 +1993,19 @@ if (object->type != OBJT_SWAP) return; - first_free = SWAPBLK_NONE; - num_free = 0; + swp_pager_init_freerange(&s_free, &n_free); for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( &object->un_pager.swp.swp_blks, pindex)) != NULL;) { pindex = sb->p + SWAP_META_PAGES; for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; - if (first_free + num_free == sb->d[i]) - num_free++; - else { - swp_pager_freeswapspace(first_free, num_free); - first_free = sb->d[i]; - num_free = 1; - } + swp_pager_update_freerange(&s_free, &n_free, sb->d[i]); } SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); uma_zfree(swblk_zone, sb); } - swp_pager_freeswapspace(first_free, num_free); + swp_pager_freeswapspace(s_free, n_free); } /*