Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -1651,8 +1651,74 @@ } /* + * SWP_PAGER_FIND_CONTINUOUS() finds continuous swap pages + * + * Returns the number of continuous pages that start at "i"-th index + * of "sb" swap block up to the "max" pages. + */ +static int +swp_pager_find_continuous(vm_object_t object, struct swblk *sb, int i, int max) +{ + daddr_t expected, start; + int end; + + if (sb == NULL || i < 0 || i >= SWAP_META_PAGES || + max <= 0 || sb->d[i] == SWAPBLK_NONE) + return (0); + + start = sb->d[i]; + end = MIN(SWAP_META_PAGES - i, max); + for (expected = start + 1, i++;; i = 0) { + for (; i < end; i++, expected++) { + if (sb->d[i] != expected) + return (expected - start); + } + if (i == max) + return (expected - start); + sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, + sb->p + SWAP_META_PAGES); + if (sb == NULL) + return (expected - start); + max -= SWAP_META_PAGES; + end = MIN(SWAP_META_PAGES, max); + } +} + +static void +swp_pager_force_dirty(vm_page_t m) +{ + + vm_object_pip_wakeup(m->object); + vm_page_dirty(m); +#ifdef INVARIANTS + vm_page_lock(m); + if (!vm_page_wired(m) && m->queue == PQ_NONE) + panic("page %p is neither wired nor queued", m); + vm_page_unlock(m); +#endif + vm_page_xunbusy(m); + vm_pager_page_unswapped(m); +} + +static void +swp_pager_force_launder(vm_page_t m) +{ + + vm_object_pip_wakeup(m->object); + vm_page_dirty(m); + vm_page_lock(m); + vm_page_launder(m); + vm_page_unlock(m); + vm_page_xunbusy(m); + vm_pager_page_unswapped(m); +} + +/* * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in * + * Returns the number of pages that are paged in. The maximum number of + * pages this function can page in at a time is SWB_NPAGES. + * * This routine dissociates the page at the given index within an object * from its backing store, paging it in if it does not reside in memory. * If the page is paged in, it is marked dirty and placed in the laundry @@ -1663,40 +1729,70 @@ * We also attempt to swap in all other pages in the swap block. * However, we only guarantee that the one at the specified index is * paged in. - * - * XXX - The code to page the whole block in doesn't work, so we - * revert to the one-by-one behavior for now. Sigh. */ -static inline void -swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex) +static int +swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex, int npages) { - vm_page_t m; + vm_page_t ma[npages]; + int i, j; - vm_object_pip_add(object, 1); - m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); - if (m->valid == VM_PAGE_BITS_ALL) { - vm_object_pip_wakeup(object); - vm_page_dirty(m); -#ifdef INVARIANTS - vm_page_lock(m); - if (!vm_page_wired(m) && m->queue == PQ_NONE) - panic("page %p is neither wired nor queued", m); - vm_page_unlock(m); -#endif - vm_page_xunbusy(m); - vm_pager_page_unswapped(m); - return; + npages = vm_page_grab_pages(object, pindex, VM_ALLOC_NORMAL, ma, + npages); + vm_object_pip_add(object, npages); + for (i = j = 0;; i++) { + if (i < npages && ma[i]->valid != VM_PAGE_BITS_ALL) + continue; + if (j < i && swap_pager_getpages(object, &ma[j], i - j, NULL, + NULL) != VM_PAGER_OK) + panic("swp_pager_force_pagein: read from swap failed " + "for page %p", ma[i]); + while (j < i) + swp_pager_force_launder(ma[j++]); + if (i == npages) + break; + swp_pager_force_dirty(ma[j++]); } + return (npages); +} - if (swap_pager_getpages(object, &m, 1, NULL, NULL) != VM_PAGER_OK) - panic("swap_pager_force_pagein: read from swap failed");/*XXX*/ - vm_object_pip_wakeup(object); - vm_page_dirty(m); - vm_page_lock(m); - vm_page_launder(m); - vm_page_unlock(m); - vm_page_xunbusy(m); - vm_pager_page_unswapped(m); +/* + * swap_pager_swapoff_object: + * + * Page in all of the pages that have been paged out for an object + * from a swap device. + */ +static void +swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object) +{ + struct swblk *sb; + vm_pindex_t pi; + int i, n; + int maxpages; + + for (pi = 0, i = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( + &object->un_pager.swp.swp_blks, pi)) != NULL; + i -= SWAP_META_PAGES) { + pi = sb->p + SWAP_META_PAGES; + /* + * A single swp_pager_force_pagein() call can + * page-in more than pages on a sb->d[] array. + * If last page-in handled more than a sb->d[] block, + * we skip blocks and advance to the next one. + */ + while (i < SWAP_META_PAGES) { + if (swp_pager_isondev(sb->d[i], sp)) { + maxpages = swp_pager_find_continuous(object, + sb, i, SWB_NPAGES); + n = swp_pager_force_pagein(object, sb->p + i, + maxpages); +if (maxpages != n) + printf("page-in max-maxpages=%d v.s. n=%d\n", maxpages, n); + i += n; + } else + i++; + } + } + KASSERT(i == 0, ("Paged-in %d more pages than we had known", i)); } /* @@ -1712,10 +1808,8 @@ static void swap_pager_swapoff(struct swdevt *sp) { - struct swblk *sb; vm_object_t object; - vm_pindex_t pi; - int i, retries; + int retries; sx_assert(&swdev_syscall_lock, SA_XLOCKED); @@ -1745,17 +1839,7 @@ if (object->type != OBJT_SWAP) goto next_obj; - for (pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( - &object->un_pager.swp.swp_blks, pi)) != NULL; ) { - pi = sb->p + SWAP_META_PAGES; - for (i = 0; i < SWAP_META_PAGES; i++) { - if (sb->d[i] == SWAPBLK_NONE) - continue; - if (swp_pager_isondev(sb->d[i], sp)) - swp_pager_force_pagein(object, - sb->p + i); - } - } + swap_pager_swapoff_object(sp, object); next_obj: VM_OBJECT_WUNLOCK(object); mtx_lock(&vm_object_list_mtx);