Index: sys/fs/tmpfs/tmpfs_vnops.c =================================================================== --- sys/fs/tmpfs/tmpfs_vnops.c +++ sys/fs/tmpfs/tmpfs_vnops.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -2098,11 +2099,13 @@ static off_t tmpfs_seek_data_locked(vm_object_t obj, off_t noff) { + struct pctrie_iter blks; vm_page_t m; vm_pindex_t p, p_m, p_swp; p = OFF_TO_IDX(noff); m = vm_page_find_least(obj, p); + swblk_iter_init(&blks, obj); /* * Microoptimize the most common case for SEEK_DATA, where @@ -2111,7 +2114,7 @@ if (m != NULL && vm_page_any_valid(m) && m->pindex == p) return (noff); - p_swp = swap_pager_find_least(obj, p); + p_swp = swap_pager_find_least(&blks, p); if (p_swp == p) return (noff); @@ -2139,9 +2142,11 @@ static off_t tmpfs_seek_hole_locked(vm_object_t obj, off_t noff) { + struct pctrie_iter blks; vm_page_t m; vm_pindex_t p, p_swp; + swblk_iter_init(&blks, obj); for (;; noff = tmpfs_seek_next(noff)) { /* * Walk over the largest sequential run of the valid pages. @@ -2156,7 +2161,7 @@ * there is a hole in the swap at the same place. */ p = OFF_TO_IDX(noff); - p_swp = swap_pager_find_least(obj, p); + p_swp = swap_pager_find_least(&blks, p); if (p_swp != p) { noff = IDX_TO_OFF(p); break; Index: sys/vm/swap_pager.h =================================================================== --- sys/vm/swap_pager.h +++ sys/vm/swap_pager.h @@ -40,6 +40,7 @@ #include struct buf; +struct pctrie_iter; struct swdevt; struct thread; typedef void sw_strategy_t(struct buf *, struct swdevt *); @@ -74,7 +75,8 @@ struct xswdev; int swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len); void swap_pager_copy(vm_object_t, vm_object_t, vm_pindex_t, int); -vm_pindex_t swap_pager_find_least(vm_object_t object, vm_pindex_t pindex); +void swblk_iter_init(struct pctrie_iter *blks, vm_object_t object); +vm_pindex_t swap_pager_find_least(struct pctrie_iter *blks, vm_pindex_t pindex); void swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size, vm_size_t *freed); void swap_pager_swap_init(void); Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -531,39 +531,47 @@ PCTRIE_DEFINE(SWAP, swblk, p, swblk_trie_alloc, swblk_trie_free); -static struct swblk * -swblk_lookup(vm_object_t object, vm_pindex_t pindex) +void +swblk_iter_init(struct pctrie_iter *blks, vm_object_t object) { - return (SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, - rounddown(pindex, SWAP_META_PAGES))); + VM_OBJECT_ASSERT_LOCKED(object); + MPASS((object->flags & OBJ_SWAP) != 0); + pctrie_iter_init(blks, &object->un_pager.swp.swp_blks); +} + +static void +swblk_iter_limit_init(struct pctrie_iter *blks, vm_object_t object, + vm_pindex_t limit) +{ + VM_OBJECT_ASSERT_LOCKED(object); + MPASS((object->flags & OBJ_SWAP) != 0); + pctrie_iter_limit_init(blks, &object->un_pager.swp.swp_blks, limit); } static struct swblk * -swblk_start(vm_object_t object, vm_pindex_t pindex) +swblk_lookup(vm_object_t object, vm_pindex_t pindex) { - return (SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, + return (SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rounddown(pindex, SWAP_META_PAGES))); } static struct swblk * -swblk_next(vm_object_t object, struct swblk *sb) +swblk_start(struct pctrie_iter *blks, vm_pindex_t pindex) { - return (swblk_start(object, sb->p + SWAP_META_PAGES)); + return (SWAP_PCTRIE_ITER_LOOKUP_GE(blks, + rounddown(pindex, SWAP_META_PAGES))); } static struct swblk * -swblk_start_limit(vm_object_t object, vm_pindex_t pindex, vm_pindex_t limit) +swblk_next(struct pctrie_iter *blks) { - struct swblk *sb = swblk_start(object, pindex); - if (sb != NULL && sb->p < limit) - return (sb); - return (NULL); + return (SWAP_PCTRIE_ITER_JUMP_GE(blks, SWAP_META_PAGES)); } -static struct swblk * -swblk_next_limit(vm_object_t object, struct swblk *sb, vm_pindex_t limit) +static void +swblk_remove(struct pctrie_iter *blks) { - return (swblk_start_limit(object, sb->p + SWAP_META_PAGES, limit)); + SWAP_PCTRIE_ITER_REMOVE(blks); } static void @@ -1827,6 +1835,7 @@ u_long swap_pager_swapped_pages(vm_object_t object) { + struct pctrie_iter blks; struct swblk *sb; u_long res; int i; @@ -1837,8 +1846,8 @@ return (0); res = 0; - for (sb = swblk_start(object, 0); sb != NULL; - sb = swblk_next(object, sb)) { + swblk_iter_init(&blks, object); + for (sb = swblk_start(&blks, 0); sb != NULL; sb = swblk_next(&blks)) { for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->d[i] != SWAPBLK_NONE) res++; @@ -1856,6 +1865,7 @@ static void swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object) { + struct pctrie_iter blks; struct page_range range; struct swblk *sb; vm_page_t m; @@ -1869,34 +1879,21 @@ ("%s: Object not swappable", __func__)); pi = 0; - i = 0; swp_pager_init_freerange(&range); - for (;;) { - if (i == 0 && (object->flags & OBJ_DEAD) != 0) { - /* - * Make sure that pending writes finish before - * returning. - */ - vm_object_pip_wait(object, "swpoff"); - swp_pager_meta_free_all(object); - break; - } - +restart: + swblk_iter_init(&blks, object); + sb = swblk_start(&blks, pi); +reloop: + for (i = 0, sb_empty = true, m = NULL; + sb != NULL && !(i == 0 && (object->flags & OBJ_DEAD) != 0); i++) { if (i == SWAP_META_PAGES) { - pi = sb->p + SWAP_META_PAGES; + pi = blks.index + SWAP_META_PAGES; if (sb_empty) { - swblk_lookup_remove(object, sb); + swblk_remove(&blks); uma_zfree(swblk_zone, sb); } - i = 0; - } - - if (i == 0) { - sb = swblk_start(object, pi); - if (sb == NULL) - break; - sb_empty = true; - m = NULL; + sb = swblk_next(&blks); + goto reloop; } /* Skip an invalid block. */ @@ -1905,7 +1902,6 @@ if (blk != SWAPBLK_NONE) sb_empty = false; m = NULL; - i++; continue; } @@ -1914,13 +1910,12 @@ * page has pending operations, sleep and restart the scan. */ m = m != NULL ? vm_page_next(m) : - vm_page_lookup(object, sb->p + i); + vm_page_lookup(object, blks.index + i); if (m != NULL && (m->oflags & VPO_SWAPINPROG) != 0) { m->oflags |= VPO_SWAPSLEEP; VM_OBJECT_SLEEP(object, &object->handle, PSWP, "swpoff", 0); - i = 0; /* Restart scan after object lock dropped. */ - continue; + goto restart; } /* @@ -1929,20 +1924,21 @@ */ if (m != NULL && vm_page_all_valid(m)) { swp_pager_force_dirty(&range, m, &sb->d[i]); - i++; continue; } /* Is there a page we can acquire or allocate? */ if (m == NULL) { - m = vm_page_alloc(object, sb->p + i, + m = vm_page_alloc(object, blks.index + i, VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL); } else if (!vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL)) m = NULL; /* If no page available, repeat this iteration. */ - if (m == NULL) + if (m == NULL) { + i--; continue; + } /* Get the page from swap, mark it dirty, restart the scan. */ vm_object_pip_add(object, 1); @@ -1956,7 +1952,14 @@ ("%s: Page %p not all valid", __func__, m)); swp_pager_force_dirty(&range, m, &sb->d[i]); vm_page_xunbusy(m); - i = 0; /* Restart scan after object lock dropped. */ + goto restart; + } + if (i == 0 && (object->flags & OBJ_DEAD) != 0) { + /* + * Make sure that pending writes finish before returning. + */ + vm_object_pip_wait(object, "swpoff"); + swp_pager_meta_free_all(object); } swp_pager_freeswapspace(&range); } @@ -2197,13 +2200,12 @@ swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject, vm_pindex_t pindex, vm_pindex_t count) { + struct pctrie_iter blks; struct page_range range; struct swblk *sb; daddr_t blk, d[SWAP_META_PAGES]; - vm_pindex_t offset, last; + vm_pindex_t last; int d_mask, i, limit, start; - _Static_assert(8 * sizeof(d_mask) >= SWAP_META_PAGES, - "d_mask not big enough"); VM_OBJECT_ASSERT_WLOCKED(srcobject); VM_OBJECT_ASSERT_WLOCKED(dstobject); @@ -2213,20 +2215,16 @@ swp_pager_init_freerange(&range); d_mask = 0; - offset = pindex; last = pindex + count; - sb = swblk_start_limit(srcobject, pindex, last); - start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0; - for (; sb != NULL; - sb = swblk_start_limit(srcobject, pindex, last), start = 0) { - pindex = sb->p; - MPASS(d_mask == 0); - limit = MIN(last - pindex, SWAP_META_PAGES); + swblk_iter_limit_init(&blks, srcobject, last); + for (sb = swblk_start(&blks, pindex), start = pindex % SWAP_META_PAGES; + sb != NULL; sb = swblk_next(&blks), start = 0) { + limit = MIN(last - blks.index, SWAP_META_PAGES); for (i = start; i < limit; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; blk = swp_pager_meta_build(dstobject, - pindex + i - offset, sb->d[i], true); + blks.index + i - pindex, sb->d[i], true); if (blk == sb->d[i]) { /* * Failed memory allocation stopped transfer; @@ -2241,7 +2239,7 @@ } if (swp_pager_swblk_empty(sb, 0, start) && swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { - swblk_lookup_remove(srcobject, sb); + swblk_remove(&blks); uma_zfree(swblk_zone, sb); } if (d_mask != 0) { @@ -2250,12 +2248,12 @@ do { i = ffs(d_mask) - 1; swp_pager_meta_build(dstobject, - pindex + i - offset, d[i], false); + blks.index + i - pindex, d[i], false); d_mask &= ~(1 << i); } while (d_mask != 0); VM_OBJECT_WLOCK(srcobject); + pctrie_iter_reset(&blks); } - pindex += SWAP_META_PAGES; } swp_pager_freeswapspace(&range); } @@ -2274,6 +2272,7 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count, vm_size_t *freed) { + struct pctrie_iter blks; struct page_range range; struct swblk *sb; vm_page_t m; @@ -2290,28 +2289,27 @@ swp_pager_init_freerange(&range); last = pindex + count; - sb = swblk_start_limit(object, pindex, last); - start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0; - for (; sb != NULL; - sb = swblk_start_limit(object, pindex, last), start = 0) { - limit = MIN(last - sb->p, SWAP_META_PAGES); + swblk_iter_limit_init(&blks, object, last); + for (sb = swblk_start(&blks, pindex), start = pindex % SWAP_META_PAGES; + sb != NULL; sb = swblk_next(&blks), start = 0) { + limit = MIN(last - blks.index, SWAP_META_PAGES); for (i = start; i < limit; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; swp_pager_update_freerange(&range, sb->d[i]); if (freed != NULL) { - m = (m != NULL && m->pindex == sb->p + i - 1) ? + m = (m != NULL && + m->pindex == blks.index + i - 1) ? vm_page_next(m) : - vm_page_lookup(object, sb->p + i); + vm_page_lookup(object, blks.index + i); if (m == NULL || vm_page_none_valid(m)) fc++; } sb->d[i] = SWAPBLK_NONE; } - pindex = sb->p + SWAP_META_PAGES; if (swp_pager_swblk_empty(sb, 0, start) && swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { - swblk_lookup_remove(object, sb); + swblk_remove(&blks); uma_zfree(swblk_zone, sb); } } @@ -2389,24 +2387,24 @@ * if are no allocated swap blocks for the object after the requested pindex. */ vm_pindex_t -swap_pager_find_least(vm_object_t object, vm_pindex_t pindex) +swap_pager_find_least(struct pctrie_iter *blks, vm_pindex_t pindex) { struct swblk *sb; int i; - if ((sb = swblk_start(object, pindex)) == NULL) + if ((sb = swblk_start(blks, pindex)) == NULL) return (OBJ_MAX_SIZE); - if (sb->p < pindex) { + if (blks->index < pindex) { for (i = pindex % SWAP_META_PAGES; i < SWAP_META_PAGES; i++) { if (sb->d[i] != SWAPBLK_NONE) - return (sb->p + i); + return (blks->index + i); } - if ((sb = swblk_next(object, sb)) == NULL) + if ((sb = swblk_next(blks)) == NULL) return (OBJ_MAX_SIZE); } for (i = 0; i < SWAP_META_PAGES; i++) { if (sb->d[i] != SWAPBLK_NONE) - return (sb->p + i); + return (blks->index + i); } /* @@ -2844,13 +2842,14 @@ long vmspace_swap_count(struct vmspace *vmspace) { + struct pctrie_iter blks; vm_map_t map; vm_map_entry_t cur; vm_object_t object; struct swblk *sb; vm_pindex_t e, pi; long count; - int i; + int i, limit, start; map = &vmspace->vm_map; count = 0; @@ -2866,11 +2865,12 @@ goto unlock; pi = OFF_TO_IDX(cur->offset); e = pi + OFF_TO_IDX(cur->end - cur->start); - for (sb = swblk_start_limit(object, pi, e); - sb != NULL; sb = swblk_next_limit(object, sb, e)) { - for (i = 0; i < SWAP_META_PAGES; i++) { - if (sb->p + i < e && - sb->d[i] != SWAPBLK_NONE) + swblk_iter_limit_init(&blks, object, e); + for (sb = swblk_start(&blks, pi), start = pi % SWAP_META_PAGES; + sb != NULL; sb = swblk_next(&blks), start = 0) { + limit = MIN(e - blks.index, SWAP_META_PAGES); + for (i = start; i < limit; i++) { + if (sb->d[i] != SWAPBLK_NONE) count++; } } Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -1681,6 +1681,7 @@ static bool vm_object_scan_all_shadowed(vm_object_t object) { + struct pctrie_iter blks; vm_object_t backing_object; vm_page_t p, pp; vm_pindex_t backing_offset_index, new_pindex, pi, ps; @@ -1695,7 +1696,8 @@ pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset); p = vm_page_find_least(backing_object, pi); - ps = swap_pager_find_least(backing_object, pi); + swblk_iter_init(&blks, backing_object); + ps = swap_pager_find_least(&blks, pi); /* * Only check pages inside the parent object's range and @@ -1705,7 +1707,7 @@ if (p != NULL && p->pindex < pi) p = TAILQ_NEXT(p, listq); if (ps < pi) - ps = swap_pager_find_least(backing_object, pi); + ps = swap_pager_find_least(&blks, pi); if (p == NULL && ps >= backing_object->size) break; else if (p == NULL)