Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -403,11 +403,30 @@ /* * Metadata functions */ -static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); +static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); static void swp_pager_meta_free_all(vm_object_t); static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); +static void +swp_pager_init_freerange(daddr_t *start, daddr_t *num) +{ + *start = SWAPBLK_NONE; + *num = 0; +} + +static void +swp_pager_update_freerange(daddr_t *start, daddr_t *num, daddr_t addr) +{ + if (*start + *num == addr) + (*num)++; + else { + swp_pager_freeswapspace(*start, *num); + *start = addr; + *num = 1; + } +} + static void * swblk_trie_alloc(struct pctrie *ptree) { @@ -861,7 +880,9 @@ int n = 0; daddr_t blk = SWAPBLK_NONE; vm_pindex_t beg = start; /* save start index */ + daddr_t addr, s_free, n_free; + swp_pager_init_freerange(&s_free, &n_free); VM_OBJECT_WLOCK(object); while (size) { if (n == 0) { @@ -875,12 +896,15 @@ } } } - swp_pager_meta_build(object, start, blk); + addr = swp_pager_meta_build(object, start, blk); + if (addr != SWAPBLK_NONE) + swp_pager_update_freerange(&s_free, &n_free, addr); --size; ++start; ++blk; --n; } + swp_pager_freeswapspace(s_free, n_free); swp_pager_meta_free(object, start, n); VM_OBJECT_WUNLOCK(object); return (0); @@ -910,7 +934,7 @@ vm_pindex_t offset, int destroysource) { vm_pindex_t i; - daddr_t dstaddr, first_free, num_free, srcaddr; + daddr_t dstaddr, s_free, n_free, srcaddr; VM_OBJECT_ASSERT_WLOCKED(srcobject); VM_OBJECT_ASSERT_WLOCKED(dstobject); @@ -937,8 +961,7 @@ /* * Transfer source to destination. */ - first_free = SWAPBLK_NONE; - num_free = 0; + swp_pager_init_freerange(&s_free, &n_free); for (i = 0; i < dstobject->size; ++i) { srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP); if (srcaddr == SWAPBLK_NONE) @@ -954,25 +977,21 @@ vm_object_pip_add(srcobject, 1); VM_OBJECT_WUNLOCK(srcobject); vm_object_pip_add(dstobject, 1); - swp_pager_meta_build(dstobject, i, srcaddr); + srcaddr = swp_pager_meta_build(dstobject, i, srcaddr); vm_object_pip_wakeup(dstobject); VM_OBJECT_WLOCK(srcobject); vm_object_pip_wakeup(srcobject); - } else { - /* - * Destination has valid swapblk or it is represented - * by a resident page. We destroy the sourceblock. - */ - if (first_free + num_free == srcaddr) - num_free++; - else { - swp_pager_freeswapspace(first_free, num_free); - first_free = srcaddr; - num_free = 1; - } + if (srcaddr == SWAPBLK_NONE) + continue; + } + /* + * Destination has valid swapblk or it is represented + * by a resident page. We destroy the sourceblock. + */ + swp_pager_update_freerange(&s_free, &n_free, srcaddr); } - swp_pager_freeswapspace(first_free, num_free); + swp_pager_freeswapspace(s_free, n_free); /* * Free left over swap blocks in source. @@ -1305,7 +1324,9 @@ { int i, n; boolean_t sync; + daddr_t addr, s_free, n_free; + swp_pager_init_freerange(&s_free, &n_free); if (count && m[0]->object != object) { panic("swap_pager_putpages: object mismatch %p/%p", object, @@ -1320,8 +1341,11 @@ * check for bogus sysops * force sync if not pageout process */ - if (object->type != OBJT_SWAP) - swp_pager_meta_build(object, 0, SWAPBLK_NONE); + if (object->type != OBJT_SWAP) { + addr = swp_pager_meta_build(object, 0, SWAPBLK_NONE); + if (addr != SWAPBLK_NONE) + swp_pager_update_freerange(&s_free, &n_free, addr); + } VM_OBJECT_WUNLOCK(object); n = 0; @@ -1389,11 +1413,14 @@ for (j = 0; j < n; ++j) { vm_page_t mreq = m[i+j]; - swp_pager_meta_build( + addr = swp_pager_meta_build( mreq->object, mreq->pindex, blk + j ); + if (addr != SWAPBLK_NONE) + swp_pager_update_freerange(&s_free, &n_free, + addr); MPASS(mreq->dirty == VM_PAGE_BITS_ALL); mreq->oflags |= VPO_SWAPINPROG; bp->b_pages[j] = mreq; @@ -1451,6 +1478,7 @@ swp_pager_async_iodone(bp); } VM_OBJECT_WLOCK(object); + swp_pager_freeswapspace(s_free, n_free); } /* @@ -1781,14 +1809,15 @@ * * The specified swapblk is added to the object's swap metadata. If * the swapblk is not valid, it is freed instead. Any previously - * assigned swapblk is freed. + * assigned swapblk is returned. */ -static void +static daddr_t swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk) { static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted; struct swblk *sb, *sb1; vm_pindex_t modpi, rdpi; + daddr_t prev_swapblk; int error, i; VM_OBJECT_ASSERT_WLOCKED(object); @@ -1813,7 +1842,7 @@ sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi); if (sb == NULL) { if (swapblk == SWAPBLK_NONE) - return; + return (SWAPBLK_NONE); for (;;) { sb = uma_zalloc(swblk_zone, M_NOWAIT | (curproc == pageproc ? M_USE_RESERVE : 0)); @@ -1880,9 +1909,8 @@ MPASS(sb->p == rdpi); modpi = pindex % SWAP_META_PAGES; - /* Delete prior contents of metadata. */ - if (sb->d[modpi] != SWAPBLK_NONE) - swp_pager_freeswapspace(sb->d[modpi], 1); + /* Return prior contents of metadata. */ + prev_swapblk = sb->d[modpi]; /* Enter block into metadata. */ sb->d[modpi] = swapblk; @@ -1894,6 +1922,7 @@ SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi); uma_zfree(swblk_zone, sb); } + return (prev_swapblk); } /* @@ -1910,7 +1939,7 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) { struct swblk *sb; - daddr_t first_free, num_free; + daddr_t s_free, n_free; vm_pindex_t last; int i, limit, start; @@ -1918,8 +1947,7 @@ if (object->type != OBJT_SWAP || count == 0) return; - first_free = SWAPBLK_NONE; - num_free = 0; + swp_pager_init_freerange(&s_free, &n_free); last = pindex + count; for (;;) { sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, @@ -1932,13 +1960,7 @@ for (i = start; i < limit; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; - if (first_free + num_free == sb->d[i]) - num_free++; - else { - swp_pager_freeswapspace(first_free, num_free); - first_free = sb->d[i]; - num_free = 1; - } + swp_pager_update_freerange(&s_free, &n_free, sb->d[i]); sb->d[i] = SWAPBLK_NONE; } if (swp_pager_swblk_empty(sb, 0, start) && @@ -1949,7 +1971,7 @@ } pindex = sb->p + SWAP_META_PAGES; } - swp_pager_freeswapspace(first_free, num_free); + swp_pager_freeswapspace(s_free, n_free); } /* @@ -1962,7 +1984,7 @@ swp_pager_meta_free_all(vm_object_t object) { struct swblk *sb; - daddr_t first_free, num_free; + daddr_t s_free, n_free; vm_pindex_t pindex; int i; @@ -1970,26 +1992,19 @@ if (object->type != OBJT_SWAP) return; - first_free = SWAPBLK_NONE; - num_free = 0; + swp_pager_init_freerange(&s_free, &n_free); for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( &object->un_pager.swp.swp_blks, pindex)) != NULL;) { pindex = sb->p + SWAP_META_PAGES; for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; - if (first_free + num_free == sb->d[i]) - num_free++; - else { - swp_pager_freeswapspace(first_free, num_free); - first_free = sb->d[i]; - num_free = 1; - } + swp_pager_update_freerange(&s_free, &n_free, sb->d[i]); } SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); uma_zfree(swblk_zone, sb); } - swp_pager_freeswapspace(first_free, num_free); + swp_pager_freeswapspace(s_free, n_free); } /*