Index: sys/fs/tmpfs/tmpfs_vnops.c =================================================================== --- sys/fs/tmpfs/tmpfs_vnops.c +++ sys/fs/tmpfs/tmpfs_vnops.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -2098,11 +2099,13 @@ static off_t tmpfs_seek_data_locked(vm_object_t obj, off_t noff) { + struct pctrie_iter blks; vm_page_t m; vm_pindex_t p, p_m, p_swp; p = OFF_TO_IDX(noff); m = vm_page_find_least(obj, p); + swblk_iter_init(&blks, obj); /* * Microoptimize the most common case for SEEK_DATA, where @@ -2111,7 +2114,7 @@ if (m != NULL && vm_page_any_valid(m) && m->pindex == p) return (noff); - p_swp = swap_pager_find_least(obj, p); + p_swp = swap_pager_find_least(&blks, p); if (p_swp == p) return (noff); @@ -2139,9 +2142,11 @@ static off_t tmpfs_seek_hole_locked(vm_object_t obj, off_t noff) { + struct pctrie_iter blks; vm_page_t m; vm_pindex_t p, p_swp; + swblk_iter_init(&blks, obj); for (;; noff = tmpfs_seek_next(noff)) { /* * Walk over the largest sequential run of the valid pages. @@ -2156,7 +2161,7 @@ * there is a hole in the swap at the same place. */ p = OFF_TO_IDX(noff); - p_swp = swap_pager_find_least(obj, p); + p_swp = swap_pager_find_least(&blks, p); if (p_swp != p) { noff = IDX_TO_OFF(p); break; Index: sys/vm/swap_pager.h =================================================================== --- sys/vm/swap_pager.h +++ sys/vm/swap_pager.h @@ -40,6 +40,7 @@ #include struct buf; +struct pctrie_iter; struct swdevt; struct thread; typedef void sw_strategy_t(struct buf *, struct swdevt *); @@ -74,7 +75,8 @@ struct xswdev; int swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len); void swap_pager_copy(vm_object_t, vm_object_t, vm_pindex_t, int); -vm_pindex_t swap_pager_find_least(vm_object_t object, vm_pindex_t pindex); +void swblk_iter_init(struct pctrie_iter *blks, vm_object_t object); +vm_pindex_t swap_pager_find_least(struct pctrie_iter *blks, vm_pindex_t pindex); void swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size, vm_size_t *freed); void swap_pager_swap_init(void); Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -531,6 +531,23 @@ PCTRIE_DEFINE(SWAP, swblk, p, swblk_trie_alloc, swblk_trie_free); +void +swblk_iter_init(struct pctrie_iter *blks, vm_object_t object) +{ + VM_OBJECT_ASSERT_LOCKED(object); + MPASS((object->flags & OBJ_SWAP) != 0); + pctrie_iter_init(blks, &object->un_pager.swp.swp_blks); +} + +static void +swblk_iter_limit_init(struct pctrie_iter *blks, vm_object_t object, + vm_pindex_t limit) +{ + VM_OBJECT_ASSERT_LOCKED(object); + MPASS((object->flags & OBJ_SWAP) != 0); + pctrie_iter_limit_init(blks, &object->un_pager.swp.swp_blks, limit); +} + static struct swblk * swblk_lookup(vm_object_t object, vm_pindex_t pindex) { @@ -539,31 +556,29 @@ } static struct swblk * -swblk_start(vm_object_t object, vm_pindex_t pindex) +swblk_start(struct pctrie_iter *blks, vm_pindex_t pindex) { - return (SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, + return (SWAP_PCTRIE_ITER_LOOKUP_GE(blks, rounddown(pindex, SWAP_META_PAGES))); } static struct swblk * -swblk_next(vm_object_t object, struct swblk *sb) +swblk_next(struct pctrie_iter *blks) { - return (swblk_start(object, sb->p + SWAP_META_PAGES)); + return (SWAP_PCTRIE_ITER_JUMP_GE(blks, SWAP_META_PAGES)); } static struct swblk * -swblk_start_limit(vm_object_t object, vm_pindex_t pindex, vm_pindex_t limit) +swblk_restart(struct pctrie_iter *blks, vm_pindex_t pindex) { - struct swblk *sb = swblk_start(object, pindex); - if (sb != NULL && sb->p < limit) - return (sb); - return (NULL); + pctrie_iter_reset(blks); + return (SWAP_PCTRIE_ITER_LOOKUP(blks, pindex)); } -static struct swblk * -swblk_next_limit(vm_object_t object, struct swblk *sb, vm_pindex_t limit) +static void +swblk_remove(struct pctrie_iter *blks) { - return (swblk_start_limit(object, sb->p + SWAP_META_PAGES, limit)); + SWAP_PCTRIE_ITER_REMOVE(blks); } static void @@ -1827,6 +1842,7 @@ u_long swap_pager_swapped_pages(vm_object_t object) { + struct pctrie_iter blks; struct swblk *sb; u_long res; int i; @@ -1837,8 +1853,8 @@ return (0); res = 0; - for (sb = swblk_start(object, 0); sb != NULL; - sb = swblk_next(object, sb)) { + swblk_iter_init(&blks, object); + for (sb = swblk_start(&blks, 0); sb != NULL; sb = swblk_next(&blks)) { for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->d[i] != SWAPBLK_NONE) res++; @@ -1856,6 +1872,7 @@ static void swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object) { + struct pctrie_iter blks; struct page_range range; struct swblk *sb; vm_page_t m; @@ -1869,34 +1886,21 @@ ("%s: Object not swappable", __func__)); pi = 0; - i = 0; swp_pager_init_freerange(&range); - for (;;) { - if (i == 0 && (object->flags & OBJ_DEAD) != 0) { - /* - * Make sure that pending writes finish before - * returning. - */ - vm_object_pip_wait(object, "swpoff"); - swp_pager_meta_free_all(object); - break; - } - +restart: + swblk_iter_init(&blks, object); + sb = swblk_start(&blks, pi); +reloop: + for (i = 0, sb_empty = true, m = NULL; + sb != NULL && !(i == 0 && (object->flags & OBJ_DEAD) != 0); i++) { if (i == SWAP_META_PAGES) { pi = sb->p + SWAP_META_PAGES; if (sb_empty) { - swblk_lookup_remove(object, sb); + swblk_remove(&blks); uma_zfree(swblk_zone, sb); } - i = 0; - } - - if (i == 0) { - sb = swblk_start(object, pi); - if (sb == NULL) - break; - sb_empty = true; - m = NULL; + sb = swblk_next(&blks); + goto reloop; } /* Skip an invalid block. */ @@ -1905,7 +1909,6 @@ if (blk != SWAPBLK_NONE) sb_empty = false; m = NULL; - i++; continue; } @@ -1919,8 +1922,7 @@ m->oflags |= VPO_SWAPSLEEP; VM_OBJECT_SLEEP(object, &object->handle, PSWP, "swpoff", 0); - i = 0; /* Restart scan after object lock dropped. */ - continue; + goto restart; } /* @@ -1929,7 +1931,6 @@ */ if (m != NULL && vm_page_all_valid(m)) { swp_pager_force_dirty(&range, m, &sb->d[i]); - i++; continue; } @@ -1941,8 +1942,10 @@ m = NULL; /* If no page available, repeat this iteration. */ - if (m == NULL) + if (m == NULL) { + i--; continue; + } /* Get the page from swap, mark it dirty, restart the scan. */ vm_object_pip_add(object, 1); @@ -1956,7 +1959,14 @@ ("%s: Page %p not all valid", __func__, m)); swp_pager_force_dirty(&range, m, &sb->d[i]); vm_page_xunbusy(m); - i = 0; /* Restart scan after object lock dropped. */ + goto restart; + } + if (i == 0 && (object->flags & OBJ_DEAD) != 0) { + /* + * Make sure that pending writes finish before returning. + */ + vm_object_pip_wait(object, "swpoff"); + swp_pager_meta_free_all(object); } swp_pager_freeswapspace(&range); } @@ -2197,11 +2207,13 @@ swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject, vm_pindex_t pindex, vm_pindex_t count) { + struct pctrie_iter blks; struct page_range range; struct swblk *sb; daddr_t blk; - vm_pindex_t offset, last; + vm_pindex_t last; int i, limit, start; + bool reset; VM_OBJECT_ASSERT_WLOCKED(srcobject); VM_OBJECT_ASSERT_WLOCKED(dstobject); @@ -2210,18 +2222,18 @@ return; swp_pager_init_freerange(&range); - offset = pindex; last = pindex + count; - sb = swblk_start_limit(srcobject, pindex, last); - start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0; - for (; sb != NULL; - sb = swblk_start_limit(srcobject, pindex, last), start = 0) { + swblk_iter_limit_init(&blks, srcobject, last); + reset = false; + for (sb = swblk_start(&blks, pindex), + start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0; + sb != NULL; sb = swblk_next(&blks), start = 0) { limit = MIN(last - sb->p, SWAP_META_PAGES); for (i = start; i < limit; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; blk = swp_pager_meta_build(dstobject, - sb->p + i - offset, sb->d[i], true); + sb->p + i - pindex, sb->d[i], true); if (blk == sb->d[i]) { /* * Destination has no swapblk and is not @@ -2231,16 +2243,21 @@ */ VM_OBJECT_WUNLOCK(srcobject); swp_pager_meta_build(dstobject, - sb->p + i - offset, sb->d[i], false); + sb->p + i - pindex, sb->d[i], false); VM_OBJECT_WLOCK(srcobject); + reset = true; } else if (blk != SWAPBLK_NONE) swp_pager_update_freerange(&range, sb->d[i]); sb->d[i] = SWAPBLK_NONE; } - pindex = sb->p + SWAP_META_PAGES; + if (reset) { + /* Rebuild search path after losing object lock. */ + reset = false; + swblk_restart(&blks, sb->p); + } if (swp_pager_swblk_empty(sb, 0, start) && swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { - swblk_lookup_remove(srcobject, sb); + swblk_remove(&blks); uma_zfree(swblk_zone, sb); } } @@ -2261,6 +2278,7 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count, vm_size_t *freed) { + struct pctrie_iter blks; struct page_range range; struct swblk *sb; vm_page_t m; @@ -2277,10 +2295,10 @@ swp_pager_init_freerange(&range); last = pindex + count; - sb = swblk_start_limit(object, pindex, last); - start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0; - for (; sb != NULL; - sb = swblk_start_limit(object, pindex, last), start = 0) { + swblk_iter_limit_init(&blks, object, last); + for (sb = swblk_start(&blks, pindex), + start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0; + sb != NULL; sb = swblk_next(&blks), start = 0) { limit = MIN(last - sb->p, SWAP_META_PAGES); for (i = start; i < limit; i++) { if (sb->d[i] == SWAPBLK_NONE) @@ -2295,10 +2313,9 @@ } sb->d[i] = SWAPBLK_NONE; } - pindex = sb->p + SWAP_META_PAGES; if (swp_pager_swblk_empty(sb, 0, start) && swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { - swblk_lookup_remove(object, sb); + swblk_remove(&blks); uma_zfree(swblk_zone, sb); } } @@ -2376,19 +2393,19 @@ * if are no allocated swap blocks for the object after the requested pindex. */ vm_pindex_t -swap_pager_find_least(vm_object_t object, vm_pindex_t pindex) +swap_pager_find_least(struct pctrie_iter *blks, vm_pindex_t pindex) { struct swblk *sb; int i; - if ((sb = swblk_start(object, pindex)) == NULL) + if ((sb = swblk_start(blks, pindex)) == NULL) return (OBJ_MAX_SIZE); if (sb->p < pindex) { for (i = pindex % SWAP_META_PAGES; i < SWAP_META_PAGES; i++) { if (sb->d[i] != SWAPBLK_NONE) return (sb->p + i); } - if ((sb = swblk_next(object, sb)) == NULL) + if ((sb = swblk_next(blks)) == NULL) return (OBJ_MAX_SIZE); } for (i = 0; i < SWAP_META_PAGES; i++) { @@ -2831,6 +2848,7 @@ long vmspace_swap_count(struct vmspace *vmspace) { + struct pctrie_iter blks; vm_map_t map; vm_map_entry_t cur; vm_object_t object; @@ -2853,8 +2871,9 @@ goto unlock; pi = OFF_TO_IDX(cur->offset); e = pi + OFF_TO_IDX(cur->end - cur->start); - for (sb = swblk_start_limit(object, pi, e); - sb != NULL; sb = swblk_next_limit(object, sb, e)) { + swblk_iter_limit_init(&blks, object, e); + for (sb = swblk_start(&blks, pi); + sb != NULL; sb = swblk_next(&blks)) { for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->p + i < e && sb->d[i] != SWAPBLK_NONE) Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -1681,6 +1681,7 @@ static bool vm_object_scan_all_shadowed(vm_object_t object) { + struct pctrie_iter blks; vm_object_t backing_object; vm_page_t p, pp; vm_pindex_t backing_offset_index, new_pindex, pi, ps; @@ -1695,7 +1696,8 @@ pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset); p = vm_page_find_least(backing_object, pi); - ps = swap_pager_find_least(backing_object, pi); + swblk_iter_init(&blks, backing_object); + ps = swap_pager_find_least(&blks, pi); /* * Only check pages inside the parent object's range and @@ -1705,7 +1707,7 @@ if (p != NULL && p->pindex < pi) p = TAILQ_NEXT(p, listq); if (ps < pi) - ps = swap_pager_find_least(backing_object, pi); + ps = swap_pager_find_least(&blks, pi); if (p == NULL && ps >= backing_object->size) break; else if (p == NULL)