Changeset View
Standalone View
sys/vm/swap_pager.c
Show First 20 Lines • Show All 1,182 Lines • ▼ Show 20 Lines | |||||
* caller may optionally specify that additional pages preceding and | * caller may optionally specify that additional pages preceding and | ||||
* succeeding the specified range be paged in. The number of such pages | * succeeding the specified range be paged in. The number of such pages | ||||
* is returned in the "rbehind" and "rahead" parameters, and they will | * is returned in the "rbehind" and "rahead" parameters, and they will | ||||
* be in the inactive queue upon return. | * be in the inactive queue upon return. | ||||
* | * | ||||
* The pages in "ma" must be busied and will remain busied upon return. | * The pages in "ma" must be busied and will remain busied upon return. | ||||
*/ | */ | ||||
static int | static int | ||||
swap_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int *rbehind, | swap_pager_getpages_locked(vm_object_t object, vm_page_t *ma, int count, | ||||
int *rahead) | int *rbehind, int *rahead) | ||||
{ | { | ||||
struct buf *bp; | struct buf *bp; | ||||
vm_page_t bm, mpred, msucc, p; | vm_page_t bm, mpred, msucc, p; | ||||
vm_pindex_t pindex; | vm_pindex_t pindex; | ||||
daddr_t blk; | daddr_t blk; | ||||
int i, maxahead, maxbehind, reqcount; | int i, maxahead, maxbehind, reqcount; | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
reqcount = count; | reqcount = count; | ||||
KASSERT(object->type == OBJT_SWAP, | KASSERT(object->type == OBJT_SWAP, | ||||
("%s: object not swappable", __func__)); | ("%s: object not swappable", __func__)); | ||||
if (!swap_pager_haspage(object, ma[0]->pindex, &maxbehind, &maxahead)) { | if (!swap_pager_haspage(object, ma[0]->pindex, &maxbehind, &maxahead)) { | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
return (VM_PAGER_FAIL); | return (VM_PAGER_FAIL); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 138 Lines • ▼ Show 20 Lines | "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n", | ||||
/* | /* | ||||
* A final note: in a low swap situation, we cannot deallocate swap | * A final note: in a low swap situation, we cannot deallocate swap | ||||
* and mark a page dirty here because the caller is likely to mark | * and mark a page dirty here because the caller is likely to mark | ||||
* the page clean when we return, causing the page to possibly revert | * the page clean when we return, causing the page to possibly revert | ||||
* to all-zero's later. | * to all-zero's later. | ||||
*/ | */ | ||||
} | } | ||||
static int | |||||
swap_pager_getpages(vm_object_t object, vm_page_t *ma, int count, | |||||
int *rbehind, int *rahead) | |||||
{ | |||||
VM_OBJECT_WLOCK(object); | |||||
return (swap_pager_getpages_locked(object, ma, count, rbehind, rahead)); | |||||
} | |||||
/* | /* | ||||
* swap_pager_getpages_async(): | * swap_pager_getpages_async(): | ||||
* | * | ||||
* Right now this is emulation of asynchronous operation on top of | * Right now this is emulation of asynchronous operation on top of | ||||
* swap_pager_getpages(). | * swap_pager_getpages(). | ||||
*/ | */ | ||||
static int | static int | ||||
swap_pager_getpages_async(vm_object_t object, vm_page_t *ma, int count, | swap_pager_getpages_async(vm_object_t object, vm_page_t *ma, int count, | ||||
▲ Show 20 Lines • Show All 344 Lines • ▼ Show 20 Lines | swap_pager_nswapdev(void) | ||||
return (nswapdev); | return (nswapdev); | ||||
} | } | ||||
static void | static void | ||||
swp_pager_force_dirty(vm_page_t m) | swp_pager_force_dirty(vm_page_t m) | ||||
{ | { | ||||
vm_page_dirty(m); | vm_page_dirty(m); | ||||
#ifdef INVARIANTS | |||||
if (!vm_page_wired(m) && m->a.queue == PQ_NONE) | |||||
panic("page %p is neither wired nor queued", m); | |||||
#endif | |||||
vm_page_xunbusy(m); | vm_page_xunbusy(m); | ||||
swap_pager_unswapped(m); | swap_pager_unswapped(m); | ||||
} | |||||
static void | |||||
swp_pager_force_launder(vm_page_t m) | |||||
{ | |||||
vm_page_dirty(m); | |||||
vm_page_launder(m); | vm_page_launder(m); | ||||
kib: Isn't it better to set unswapped before xunbusy, even if it is not strictly needed now ? | |||||
vm_page_xunbusy(m); | |||||
swap_pager_unswapped(m); | |||||
} | } | ||||
/* | /* | ||||
* SWP_PAGER_FORCE_PAGEIN() - force swap blocks to be paged in | |||||
* | |||||
* This routine dissociates pages starting at the given index within an | |||||
* object from their backing store, paging them in if they do not reside | |||||
* in memory. Pages that are paged in are marked dirty and placed in the | |||||
* laundry queue. Pages are marked dirty because they no longer have | |||||
* backing store. They are placed in the laundry queue because they have | |||||
* not been accessed recently. Otherwise, they would already reside in | |||||
* memory. | |||||
*/ | |||||
static void | |||||
swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex, int npages) | |||||
{ | |||||
vm_page_t ma[npages]; | |||||
int i, j; | |||||
KASSERT(npages > 0, ("%s: No pages", __func__)); | |||||
KASSERT(npages <= MAXPHYS / PAGE_SIZE, | |||||
("%s: Too many pages: %d", __func__, npages)); | |||||
KASSERT(object->type == OBJT_SWAP, | |||||
("%s: Object not swappable", __func__)); | |||||
vm_object_pip_add(object, npages); | |||||
vm_page_grab_pages(object, pindex, VM_ALLOC_NORMAL, ma, npages); | |||||
for (i = j = 0;; i++) { | |||||
/* Count nonresident pages, to page-in all at once. */ | |||||
if (i < npages && ma[i]->valid != VM_PAGE_BITS_ALL) | |||||
continue; | |||||
if (j < i) { | |||||
VM_OBJECT_WUNLOCK(object); | |||||
/* Page-in nonresident pages. Mark for laundering. */ | |||||
if (swap_pager_getpages(object, &ma[j], i - j, NULL, | |||||
NULL) != VM_PAGER_OK) | |||||
panic("%s: read from swap failed", __func__); | |||||
VM_OBJECT_WLOCK(object); | |||||
do { | |||||
swp_pager_force_launder(ma[j]); | |||||
} while (++j < i); | |||||
} | |||||
if (i == npages) | |||||
break; | |||||
/* Mark dirty a resident page. */ | |||||
swp_pager_force_dirty(ma[j++]); | |||||
} | |||||
vm_object_pip_wakeupn(object, npages); | |||||
} | |||||
/* | |||||
* swap_pager_swapoff_object: | * swap_pager_swapoff_object: | ||||
* | * | ||||
* Page in all of the pages that have been paged out for an object | * Page in all of the pages that have been paged out for an object | ||||
* to a swap device. | * to a swap device. | ||||
*/ | */ | ||||
static void | static void | ||||
swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object) | swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object) | ||||
{ | { | ||||
struct swblk *sb; | struct swblk *sb; | ||||
vm_pindex_t pi, s_pindex; | vm_page_t m, ma[SWAP_META_PAGES]; | ||||
daddr_t blk, n_blks, s_blk; | vm_pindex_t pi; | ||||
int i; | daddr_t blk; | ||||
int i, j, npages, nv, rahead, rv; | |||||
KASSERT(object->type == OBJT_SWAP, | KASSERT(object->type == OBJT_SWAP, | ||||
("%s: Object not swappable", __func__)); | ("%s: Object not swappable", __func__)); | ||||
n_blks = 0; | |||||
for (pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( | for (pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( | ||||
&object->un_pager.swp.swp_blks, pi)) != NULL; ) { | &object->un_pager.swp.swp_blks, pi)) != NULL; ) { | ||||
if ((object->flags & OBJ_DEAD) != 0) { | |||||
swp_pager_meta_free_all(object); | |||||
break; | |||||
} | |||||
for (i = 0; i < SWAP_META_PAGES; i++) { | for (i = 0; i < SWAP_META_PAGES; i++) { | ||||
blk = sb->d[i]; | |||||
if (!swp_pager_isondev(blk, sp)) | |||||
blk = SWAPBLK_NONE; | |||||
/* | /* | ||||
* If there are no blocks/pages accumulated, start a new | * Count the number of contiguous valid blocks. | ||||
* accumulation here. | |||||
*/ | */ | ||||
if (n_blks == 0) { | for (nv = 0; nv < SWAP_META_PAGES - i; nv++) { | ||||
if (blk != SWAPBLK_NONE) { | blk = sb->d[i + nv]; | ||||
s_blk = blk; | if (!swp_pager_isondev(blk, sp) || | ||||
s_pindex = sb->p + i; | blk == SWAPBLK_NONE) | ||||
n_blks = 1; | break; | ||||
} | } | ||||
if (nv == 0) | |||||
continue; | continue; | ||||
} | |||||
/* | /* | ||||
* If the accumulation can be extended without breaking | * Grab the page for the first valid block. If the page | ||||
* the sequence of consecutive blocks and pages that | * is valid, mark it dirty and free its swap block. Try | ||||
* swp_pager_force_pagein() depends on, do so. | * to batch this operation since it may cause sp to be | ||||
* freed, meaning that we must restart the scan. | |||||
* Invalid pages will be revisited in future iterations. | |||||
*/ | */ | ||||
if (n_blks < MAXPHYS / PAGE_SIZE && | m = vm_page_grab(object, sb->p + i, VM_ALLOC_NORMAL | | ||||
jeffUnsubmitted Not Done Inline Actionsvm_page_grab_valid() supports a prefetch count that is validated with has_pages. You could likely use it here and simplify this block. You also can test the valid bit with only the object lock and optimize this further by looping with vm_page_next(). I would do a lookup and iteration with vm_page_lookup() and vm_page_next() and only use vm_page_grab_valid() when no page was present. If you don't do this I will do so to fix my kstack problem. I would drop a comment documenting that the valid test depends on the object lock as I have done elsewhere. You will have to add SLEEPFAIL support to grab_valid but that is trivial. I just didn't have a use case before. jeff: vm_page_grab_valid() supports a prefetch count that is validated with has_pages. You could… | |||||
markjAuthorUnsubmitted Done Inline ActionsAfter thinking it over some more, I'm not sure that we can avoid busying valid pages here. I think that is the only way we synchronize with a concurrent putpages. Once putpages has allocated swap blocks and submitted I/O to a device, swapoff should wait for that to finish before proceeding. GEOM does some tracking of in-flight requests and should wait for them to drain before destroying a swap device, so that may be sufficient after all, but I need to spend some time to verify that. markj: After thinking it over some more, I'm not sure that we can avoid busying valid pages here. I… | |||||
s_blk + n_blks == blk && | VM_ALLOC_WAITFAIL); | ||||
s_pindex + n_blks == sb->p + i) { | if (m == NULL) | ||||
++n_blks; | break; | ||||
continue; | if (vm_page_all_valid(m)) { | ||||
npages = vm_page_grab_pages(object, | |||||
m->pindex + 1, | |||||
VM_ALLOC_NOCREAT | VM_ALLOC_NOWAIT, ma, | |||||
nv - 1); | |||||
swp_pager_force_dirty(m); | |||||
for (j = 0; j < npages; j++) { | |||||
if (vm_page_all_valid(ma[j])) | |||||
swp_pager_force_dirty(ma[j]); | |||||
else | |||||
vm_page_xunbusy(ma[j]); | |||||
Not Done Inline ActionsIt may not be required by style but I really prefer { } here. jeff: It may not be required by style but I really prefer { } here. | |||||
} | } | ||||
break; | |||||
} | |||||
vm_object_pip_add(object, 1); | |||||
rahead = SWAP_META_PAGES; | |||||
rv = swap_pager_getpages_locked(object, &m, 1, NULL, | |||||
&rahead); | |||||
if (rv != VM_PAGER_OK) | |||||
panic("%s: read from swap failed: %d", | |||||
__func__, rv); | |||||
vm_object_pip_wakeupn(object, 1); | |||||
VM_OBJECT_WLOCK(object); | |||||
vm_page_xunbusy(m); | |||||
/* | /* | ||||
* The sequence of consecutive blocks and pages cannot | * The object lock was dropped so we must restart the | ||||
* be extended, so page them all in here. Then, | * scan of this swap block. Pages paged in during this | ||||
* because doing so involves releasing and reacquiring | * iteration will be marked dirty in a future iteration. | ||||
* a lock that protects the swap block pctrie, do not | |||||
* rely on the current swap block. Break this loop and | |||||
* re-fetch the same pindex from the pctrie again. | |||||
*/ | */ | ||||
swp_pager_force_pagein(object, s_pindex, n_blks); | |||||
n_blks = 0; | |||||
break; | break; | ||||
} | } | ||||
if (i == SWAP_META_PAGES) | if (i == SWAP_META_PAGES) | ||||
pi = sb->p + SWAP_META_PAGES; | pi = sb->p + SWAP_META_PAGES; | ||||
} | } | ||||
if (n_blks > 0) | |||||
swp_pager_force_pagein(object, s_pindex, n_blks); | |||||
} | } | ||||
/* | /* | ||||
* swap_pager_swapoff: | * swap_pager_swapoff: | ||||
* | * | ||||
* Page in all of the pages that have been paged out to the | * Page in all of the pages that have been paged out to the | ||||
* given device. The corresponding blocks in the bitmap must be | * given device. The corresponding blocks in the bitmap must be | ||||
* marked as allocated and the device must be flagged SW_CLOSING. | * marked as allocated and the device must be flagged SW_CLOSING. | ||||
▲ Show 20 Lines • Show All 1,251 Lines • Show Last 20 Lines |
Isn't it better to set unswapped before xunbusy, even if it is not strictly needed now ?