Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/swap_pager.c
Show First 20 Lines • Show All 430 Lines • ▼ Show 20 Lines | static boolean_t | ||||
swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after); | swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after); | ||||
static void swap_pager_init(void); | static void swap_pager_init(void); | ||||
static void swap_pager_unswapped(vm_page_t); | static void swap_pager_unswapped(vm_page_t); | ||||
static void swap_pager_swapoff(struct swdevt *sp); | static void swap_pager_swapoff(struct swdevt *sp); | ||||
static void swap_pager_update_writecount(vm_object_t object, | static void swap_pager_update_writecount(vm_object_t object, | ||||
vm_offset_t start, vm_offset_t end); | vm_offset_t start, vm_offset_t end); | ||||
static void swap_pager_release_writecount(vm_object_t object, | static void swap_pager_release_writecount(vm_object_t object, | ||||
vm_offset_t start, vm_offset_t end); | vm_offset_t start, vm_offset_t end); | ||||
static void swap_pager_freespace(vm_object_t object, vm_pindex_t start, | static void swap_pager_freespace_pgo(vm_object_t object, vm_pindex_t start, | ||||
vm_size_t size); | vm_size_t size); | ||||
const struct pagerops swappagerops = { | const struct pagerops swappagerops = { | ||||
.pgo_kvme_type = KVME_TYPE_SWAP, | .pgo_kvme_type = KVME_TYPE_SWAP, | ||||
.pgo_init = swap_pager_init, /* early system initialization of pager */ | .pgo_init = swap_pager_init, /* early system initialization of pager */ | ||||
.pgo_alloc = swap_pager_alloc, /* allocate an OBJT_SWAP object */ | .pgo_alloc = swap_pager_alloc, /* allocate an OBJT_SWAP object */ | ||||
.pgo_dealloc = swap_pager_dealloc, /* deallocate an OBJT_SWAP object */ | .pgo_dealloc = swap_pager_dealloc, /* deallocate an OBJT_SWAP object */ | ||||
.pgo_getpages = swap_pager_getpages, /* pagein */ | .pgo_getpages = swap_pager_getpages, /* pagein */ | ||||
.pgo_getpages_async = swap_pager_getpages_async, /* pagein (async) */ | .pgo_getpages_async = swap_pager_getpages_async, /* pagein (async) */ | ||||
.pgo_putpages = swap_pager_putpages, /* pageout */ | .pgo_putpages = swap_pager_putpages, /* pageout */ | ||||
.pgo_haspage = swap_pager_haspage, /* get backing store status for page */ | .pgo_haspage = swap_pager_haspage, /* get backing store status for page */ | ||||
.pgo_pageunswapped = swap_pager_unswapped, /* remove swap related to page */ | .pgo_pageunswapped = swap_pager_unswapped, /* remove swap related to page */ | ||||
.pgo_update_writecount = swap_pager_update_writecount, | .pgo_update_writecount = swap_pager_update_writecount, | ||||
.pgo_release_writecount = swap_pager_release_writecount, | .pgo_release_writecount = swap_pager_release_writecount, | ||||
.pgo_freespace = swap_pager_freespace, | .pgo_freespace = swap_pager_freespace_pgo, | ||||
}; | }; | ||||
/* | /* | ||||
* swap_*() routines are externally accessible. swp_*() routines are | * swap_*() routines are externally accessible. swp_*() routines are | ||||
* internal. | * internal. | ||||
*/ | */ | ||||
static int nswap_lowat = 128; /* in pages, swap_pager_almost_full warn */ | static int nswap_lowat = 128; /* in pages, swap_pager_almost_full warn */ | ||||
static int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */ | static int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */ | ||||
Show All 15 Lines | |||||
*/ | */ | ||||
static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages); | static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages); | ||||
static daddr_t swp_pager_getswapspace(int *npages); | static daddr_t swp_pager_getswapspace(int *npages); | ||||
/* | /* | ||||
* Metadata functions | * Metadata functions | ||||
*/ | */ | ||||
static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); | static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); | ||||
static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); | static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_size_t, | ||||
vm_size_t *); | |||||
static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst, | static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst, | ||||
vm_pindex_t pindex, vm_pindex_t count); | vm_pindex_t pindex, vm_pindex_t count, vm_size_t *freed); | ||||
static void swp_pager_meta_free_all(vm_object_t); | static void swp_pager_meta_free_all(vm_object_t); | ||||
static daddr_t swp_pager_meta_lookup(vm_object_t, vm_pindex_t); | static daddr_t swp_pager_meta_lookup(vm_object_t, vm_pindex_t); | ||||
static void | static void | ||||
swp_pager_init_freerange(daddr_t *start, daddr_t *num) | swp_pager_init_freerange(daddr_t *start, daddr_t *num) | ||||
{ | { | ||||
*start = SWAPBLK_NONE; | *start = SWAPBLK_NONE; | ||||
▲ Show 20 Lines • Show All 478 Lines • ▼ Show 20 Lines | |||||
* This routine removes swapblk assignments from swap metadata. | * This routine removes swapblk assignments from swap metadata. | ||||
* | * | ||||
* The external callers of this routine typically have already destroyed | * The external callers of this routine typically have already destroyed | ||||
* or renamed vm_page_t's associated with this range in the object so | * or renamed vm_page_t's associated with this range in the object so | ||||
* we should be ok. | * we should be ok. | ||||
* | * | ||||
* The object must be locked. | * The object must be locked. | ||||
*/ | */ | ||||
void | |||||
swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size, | |||||
vm_size_t *freed) | |||||
{ | |||||
MPASS((object->flags & OBJ_SWAP) != 0); | |||||
swp_pager_meta_free(object, start, size, freed); | |||||
} | |||||
static void | static void | ||||
swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size) | swap_pager_freespace_pgo(vm_object_t object, vm_pindex_t start, vm_size_t size) | ||||
{ | { | ||||
MPASS((object->flags & OBJ_SWAP) != 0); | |||||
swp_pager_meta_free(object, start, size); | swp_pager_meta_free(object, start, size, NULL); | ||||
} | } | ||||
/* | /* | ||||
* SWAP_PAGER_RESERVE() - reserve swap blocks in object | * SWAP_PAGER_RESERVE() - reserve swap blocks in object | ||||
* | * | ||||
* Assigns swap blocks to the specified range within the object. The | * Assigns swap blocks to the specified range within the object. The | ||||
* swap blocks are not zeroed. Any previous swap assignment is destroyed. | * swap blocks are not zeroed. Any previous swap assignment is destroyed. | ||||
* | * | ||||
* Returns 0 on success, -1 on failure. | * Returns 0 on success, -1 on failure. | ||||
*/ | */ | ||||
int | int | ||||
swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_pindex_t size) | swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_pindex_t size) | ||||
{ | { | ||||
daddr_t addr, blk, n_free, s_free; | daddr_t addr, blk, n_free, s_free; | ||||
vm_pindex_t i, j; | vm_pindex_t i, j; | ||||
int n; | int n; | ||||
swp_pager_init_freerange(&s_free, &n_free); | swp_pager_init_freerange(&s_free, &n_free); | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
for (i = 0; i < size; i += n) { | for (i = 0; i < size; i += n) { | ||||
n = MIN(size - i, INT_MAX); | n = MIN(size - i, INT_MAX); | ||||
blk = swp_pager_getswapspace(&n); | blk = swp_pager_getswapspace(&n); | ||||
if (blk == SWAPBLK_NONE) { | if (blk == SWAPBLK_NONE) { | ||||
swp_pager_meta_free(object, start, i); | swp_pager_meta_free(object, start, i, NULL); | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
return (-1); | return (-1); | ||||
} | } | ||||
for (j = 0; j < n; ++j) { | for (j = 0; j < n; ++j) { | ||||
addr = swp_pager_meta_build(object, | addr = swp_pager_meta_build(object, | ||||
start + i + j, blk + j); | start + i + j, blk + j); | ||||
if (addr != SWAPBLK_NONE) | if (addr != SWAPBLK_NONE) | ||||
swp_pager_update_freerange(&s_free, &n_free, | swp_pager_update_freerange(&s_free, &n_free, | ||||
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines | if (destroysource && (srcobject->flags & OBJ_ANON) == 0 && | ||||
sx_xunlock(&sw_alloc_sx); | sx_xunlock(&sw_alloc_sx); | ||||
VM_OBJECT_WLOCK(dstobject); | VM_OBJECT_WLOCK(dstobject); | ||||
VM_OBJECT_WLOCK(srcobject); | VM_OBJECT_WLOCK(srcobject); | ||||
} | } | ||||
/* | /* | ||||
* Transfer source to destination. | * Transfer source to destination. | ||||
*/ | */ | ||||
swp_pager_meta_transfer(srcobject, dstobject, offset, dstobject->size); | swp_pager_meta_transfer(srcobject, dstobject, offset, dstobject->size, | ||||
NULL); | |||||
/* | /* | ||||
* Free left over swap blocks in source. | * Free left over swap blocks in source. | ||||
*/ | */ | ||||
if (destroysource) | if (destroysource) | ||||
swp_pager_meta_free_all(srcobject); | swp_pager_meta_free_all(srcobject); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 1,006 Lines • ▼ Show 20 Lines | |||||
* SWP_PAGER_META_TRANSFER() - free a range of blocks in the srcobject's swap | * SWP_PAGER_META_TRANSFER() - free a range of blocks in the srcobject's swap | ||||
* metadata, or transfer it into dstobject. | * metadata, or transfer it into dstobject. | ||||
* | * | ||||
* This routine will free swap metadata structures as they are cleaned | * This routine will free swap metadata structures as they are cleaned | ||||
* out. | * out. | ||||
*/ | */ | ||||
static void | static void | ||||
swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject, | swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject, | ||||
vm_pindex_t pindex, vm_pindex_t count) | vm_pindex_t pindex, vm_pindex_t count, vm_size_t *moved) | ||||
{ | { | ||||
struct swblk *sb; | struct swblk *sb; | ||||
vm_page_t m; | |||||
daddr_t n_free, s_free; | daddr_t n_free, s_free; | ||||
vm_pindex_t offset, last; | vm_pindex_t offset, last; | ||||
vm_size_t mc; | |||||
int i, limit, start; | int i, limit, start; | ||||
VM_OBJECT_ASSERT_WLOCKED(srcobject); | VM_OBJECT_ASSERT_WLOCKED(srcobject); | ||||
MPASS(moved == NULL || dstobject == NULL); | |||||
mc = 0; | |||||
m = NULL; | |||||
if (count == 0 || pctrie_is_empty(&srcobject->un_pager.swp.swp_blks)) | if (count == 0 || pctrie_is_empty(&srcobject->un_pager.swp.swp_blks)) | ||||
return; | goto out; | ||||
swp_pager_init_freerange(&s_free, &n_free); | swp_pager_init_freerange(&s_free, &n_free); | ||||
offset = pindex; | offset = pindex; | ||||
last = pindex + count; | last = pindex + count; | ||||
for (;;) { | for (;;) { | ||||
sb = SWAP_PCTRIE_LOOKUP_GE(&srcobject->un_pager.swp.swp_blks, | sb = SWAP_PCTRIE_LOOKUP_GE(&srcobject->un_pager.swp.swp_blks, | ||||
rounddown(pindex, SWAP_META_PAGES)); | rounddown(pindex, SWAP_META_PAGES)); | ||||
if (sb == NULL || sb->p >= last) | if (sb == NULL || sb->p >= last) | ||||
break; | break; | ||||
start = pindex > sb->p ? pindex - sb->p : 0; | start = pindex > sb->p ? pindex - sb->p : 0; | ||||
limit = last - sb->p < SWAP_META_PAGES ? last - sb->p : | limit = last - sb->p < SWAP_META_PAGES ? last - sb->p : | ||||
SWAP_META_PAGES; | SWAP_META_PAGES; | ||||
for (i = start; i < limit; i++) { | for (i = start; i < limit; i++) { | ||||
if (sb->d[i] == SWAPBLK_NONE) | if (sb->d[i] == SWAPBLK_NONE) | ||||
continue; | continue; | ||||
if (dstobject == NULL || | if (dstobject == NULL || | ||||
!swp_pager_xfer_source(srcobject, dstobject, | !swp_pager_xfer_source(srcobject, dstobject, | ||||
sb->p + i - offset, sb->d[i])) { | sb->p + i - offset, sb->d[i])) { | ||||
swp_pager_update_freerange(&s_free, &n_free, | swp_pager_update_freerange(&s_free, &n_free, | ||||
sb->d[i]); | sb->d[i]); | ||||
} | } | ||||
if (moved != NULL) { | |||||
if (m != NULL && m->pindex != pindex + i - 1) | |||||
m = NULL; | |||||
m = m != NULL ? vm_page_next(m) : | |||||
vm_page_lookup(srcobject, pindex + i); | |||||
if (m == NULL || vm_page_none_valid(m)) | |||||
mc++; | |||||
} | |||||
sb->d[i] = SWAPBLK_NONE; | sb->d[i] = SWAPBLK_NONE; | ||||
} | } | ||||
pindex = sb->p + SWAP_META_PAGES; | pindex = sb->p + SWAP_META_PAGES; | ||||
if (swp_pager_swblk_empty(sb, 0, start) && | if (swp_pager_swblk_empty(sb, 0, start) && | ||||
swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { | swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { | ||||
SWAP_PCTRIE_REMOVE(&srcobject->un_pager.swp.swp_blks, | SWAP_PCTRIE_REMOVE(&srcobject->un_pager.swp.swp_blks, | ||||
sb->p); | sb->p); | ||||
uma_zfree(swblk_zone, sb); | uma_zfree(swblk_zone, sb); | ||||
} | } | ||||
} | } | ||||
swp_pager_freeswapspace(s_free, n_free); | swp_pager_freeswapspace(s_free, n_free); | ||||
out: | |||||
if (moved != NULL) | |||||
*moved = mc; | |||||
} | } | ||||
/* | /* | ||||
* SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata | * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata | ||||
* | * | ||||
* The requested range of blocks is freed, with any associated swap | * The requested range of blocks is freed, with any associated swap | ||||
* returned to the swap bitmap. | * returned to the swap bitmap. | ||||
* | * | ||||
* This routine will free swap metadata structures as they are cleaned | * This routine will free swap metadata structures as they are cleaned | ||||
* out. This routine does *NOT* operate on swap metadata associated | * out. This routine does *NOT* operate on swap metadata associated | ||||
* with resident pages. | * with resident pages. | ||||
*/ | */ | ||||
static void | static void | ||||
swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) | swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count, | ||||
vm_size_t *freed) | |||||
{ | { | ||||
swp_pager_meta_transfer(object, NULL, pindex, count); | swp_pager_meta_transfer(object, NULL, pindex, count, freed); | ||||
} | } | ||||
/* | /* | ||||
* SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object | * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object | ||||
* | * | ||||
* This routine locates and destroys all swap metadata associated with | * This routine locates and destroys all swap metadata associated with | ||||
* an object. | * an object. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 948 Lines • Show Last 20 Lines |