Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/swap_pager.c
Show First 20 Lines • Show All 315 Lines • ▼ Show 20 Lines | swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred) | ||||
if (prev < pdecr) | if (prev < pdecr) | ||||
printf("negative vmsize for uid = %d\n", uip->ui_uid); | printf("negative vmsize for uid = %d\n", uip->ui_uid); | ||||
#ifdef RACCT | #ifdef RACCT | ||||
if (racct_enable) | if (racct_enable) | ||||
racct_sub_cred(cred, RACCT_SWAP, decr); | racct_sub_cred(cred, RACCT_SWAP, decr); | ||||
#endif | #endif | ||||
} | } | ||||
#define SWM_POP 0x01 /* pop out */ | |||||
static int swap_pager_full = 2; /* swap space exhaustion (task killing) */ | static int swap_pager_full = 2; /* swap space exhaustion (task killing) */ | ||||
static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/ | static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/ | ||||
static struct mtx swbuf_mtx; /* to sync nsw_wcount_async */ | static struct mtx swbuf_mtx; /* to sync nsw_wcount_async */ | ||||
static int nsw_wcount_async; /* limit async write buffers */ | static int nsw_wcount_async; /* limit async write buffers */ | ||||
static int nsw_wcount_async_max;/* assigned maximum */ | static int nsw_wcount_async_max;/* assigned maximum */ | ||||
static int nsw_cluster_max; /* maximum VOP I/O allowed */ | static int nsw_cluster_max; /* maximum VOP I/O allowed */ | ||||
static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS); | static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS); | ||||
▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Metadata functions | * Metadata functions | ||||
*/ | */ | ||||
static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); | static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); | ||||
static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); | static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); | ||||
static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst, | static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst, | ||||
vm_pindex_t pindex, vm_pindex_t count); | vm_pindex_t pindex, vm_pindex_t count); | ||||
static void swp_pager_meta_free_all(vm_object_t); | static void swp_pager_meta_free_all(vm_object_t); | ||||
static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); | static daddr_t swp_pager_meta_lookup(vm_object_t, vm_pindex_t); | ||||
static void | static void | ||||
swp_pager_init_freerange(daddr_t *start, daddr_t *num) | swp_pager_init_freerange(daddr_t *start, daddr_t *num) | ||||
{ | { | ||||
*start = SWAPBLK_NONE; | *start = SWAPBLK_NONE; | ||||
*num = 0; | *num = 0; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 499 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static bool | static bool | ||||
swp_pager_xfer_source(vm_object_t srcobject, vm_object_t dstobject, | swp_pager_xfer_source(vm_object_t srcobject, vm_object_t dstobject, | ||||
vm_pindex_t pindex, daddr_t addr) | vm_pindex_t pindex, daddr_t addr) | ||||
{ | { | ||||
daddr_t dstaddr; | daddr_t dstaddr; | ||||
if (swp_pager_meta_ctl(dstobject, pindex, 0) != SWAPBLK_NONE) { | if (swp_pager_meta_lookup(dstobject, pindex) != SWAPBLK_NONE) { | ||||
/* Caller should destroy the source block. */ | /* Caller should destroy the source block. */ | ||||
return (false); | return (false); | ||||
} | } | ||||
/* | /* | ||||
* Destination has no swapblk and is not resident, transfer source. | * Transfer source. | ||||
* swp_pager_meta_build() can sleep. | * swp_pager_meta_build() can sleep. | ||||
*/ | */ | ||||
vm_object_pip_add(srcobject, 1); | vm_object_pip_add(srcobject, 1); | ||||
VM_OBJECT_WUNLOCK(srcobject); | VM_OBJECT_WUNLOCK(srcobject); | ||||
vm_object_pip_add(dstobject, 1); | vm_object_pip_add(dstobject, 1); | ||||
dstaddr = swp_pager_meta_build(dstobject, pindex, addr); | dstaddr = swp_pager_meta_build(dstobject, pindex, addr); | ||||
KASSERT(dstaddr == SWAPBLK_NONE, | KASSERT(dstaddr == SWAPBLK_NONE, | ||||
("Unexpected destination swapblk")); | ("Unexpected destination swapblk")); | ||||
▲ Show 20 Lines • Show All 89 Lines • ▼ Show 20 Lines | swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, | ||||
daddr_t blk, blk0; | daddr_t blk, blk0; | ||||
int i; | int i; | ||||
VM_OBJECT_ASSERT_LOCKED(object); | VM_OBJECT_ASSERT_LOCKED(object); | ||||
/* | /* | ||||
* do we have good backing store at the requested index ? | * do we have good backing store at the requested index ? | ||||
*/ | */ | ||||
blk0 = swp_pager_meta_ctl(object, pindex, 0); | blk0 = swp_pager_meta_lookup(object, pindex); | ||||
if (blk0 == SWAPBLK_NONE) { | if (blk0 == SWAPBLK_NONE) { | ||||
if (before) | if (before) | ||||
*before = 0; | *before = 0; | ||||
if (after) | if (after) | ||||
*after = 0; | *after = 0; | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
/* | /* | ||||
* find backwards-looking contiguous good backing store | * find backwards-looking contiguous good backing store | ||||
*/ | */ | ||||
if (before != NULL) { | if (before != NULL) { | ||||
for (i = 1; i < SWB_NPAGES; i++) { | for (i = 1; i < SWB_NPAGES; i++) { | ||||
if (i > pindex) | if (i > pindex) | ||||
break; | break; | ||||
blk = swp_pager_meta_ctl(object, pindex - i, 0); | blk = swp_pager_meta_lookup(object, pindex - i); | ||||
if (blk != blk0 - i) | if (blk != blk0 - i) | ||||
break; | break; | ||||
} | } | ||||
*before = i - 1; | *before = i - 1; | ||||
} | } | ||||
/* | /* | ||||
* find forward-looking contiguous good backing store | * find forward-looking contiguous good backing store | ||||
*/ | */ | ||||
if (after != NULL) { | if (after != NULL) { | ||||
for (i = 1; i < SWB_NPAGES; i++) { | for (i = 1; i < SWB_NPAGES; i++) { | ||||
blk = swp_pager_meta_ctl(object, pindex + i, 0); | blk = swp_pager_meta_lookup(object, pindex + i); | ||||
if (blk != blk0 + i) | if (blk != blk0 + i) | ||||
break; | break; | ||||
} | } | ||||
*after = i - 1; | *after = i - 1; | ||||
} | } | ||||
return (TRUE); | return (TRUE); | ||||
} | } | ||||
Show All 14 Lines | |||||
* | * | ||||
* This routine may not sleep. | * This routine may not sleep. | ||||
* | * | ||||
* The object containing the page must be locked. | * The object containing the page must be locked. | ||||
*/ | */ | ||||
static void | static void | ||||
swap_pager_unswapped(vm_page_t m) | swap_pager_unswapped(vm_page_t m) | ||||
{ | { | ||||
daddr_t srcaddr; | struct swblk *sb; | ||||
bool locked; | |||||
srcaddr = swp_pager_meta_ctl(m->object, m->pindex, SWM_POP); | VM_OBJECT_ASSERT_LOCKED(m->object); | ||||
if (srcaddr != SWAPBLK_NONE) | |||||
swp_pager_freeswapspace(srcaddr, 1); | /* | ||||
* The meta data only exists if the object is OBJT_SWAP | |||||
* and even then might not be allocated yet. | |||||
*/ | |||||
if (m->object->type != OBJT_SWAP) | |||||
return; | |||||
sb = SWAP_PCTRIE_LOOKUP(&m->object->un_pager.swp.swp_blks, | |||||
rounddown(m->pindex, SWAP_META_PAGES)); | |||||
if (sb == NULL) | |||||
return; | |||||
if (sb->d[m->pindex % SWAP_META_PAGES] == SWAPBLK_NONE) | |||||
return; | |||||
locked = VM_OBJECT_WOWNED(m->object); | |||||
if (!locked && !VM_OBJECT_TRYUPGRADE(m->object)) | |||||
return; | |||||
sb->d[m->pindex % SWAP_META_PAGES] = SWAPBLK_NONE; | |||||
ota_j.email.ne.jp: We need
We also need
swp_pager_freeswapspace(sb->d[m->pindex % SWAP_META_PAGES, 1);
before… | |||||
if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { | |||||
SWAP_PCTRIE_REMOVE(&m->object->un_pager.swp.swp_blks, | |||||
rounddown(m->pindex, SWAP_META_PAGES)); | |||||
uma_zfree(swblk_zone, sb); | |||||
} | } | ||||
if (!locked) | |||||
VM_OBJECT_LOCK_DOWNGRADE(m->object); | |||||
Not Done Inline ActionsGiven swp_pager_meta_ctl() with SWM_POP calls VM_OBJECT_ASSERT_WLOCKED, I expected VM_OBJECT_ASSERT_WLOCKED. ota_j.email.ne.jp: Given swp_pager_meta_ctl() with SWM_POP calls VM_OBJECT_ASSERT_WLOCKED, I expected… | |||||
} | |||||
/* | /* | ||||
* swap_pager_getpages() - bring pages in from swap | * swap_pager_getpages() - bring pages in from swap | ||||
* | * | ||||
* Attempt to page in the pages in array "ma" of length "count". The | * Attempt to page in the pages in array "ma" of length "count". The | ||||
* caller may optionally specify that additional pages preceding and | * caller may optionally specify that additional pages preceding and | ||||
* succeeding the specified range be paged in. The number of such pages | * succeeding the specified range be paged in. The number of such pages | ||||
* is returned in the "rbehind" and "rahead" parameters, and they will | * is returned in the "rbehind" and "rahead" parameters, and they will | ||||
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines | swap_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int *rbehind, | ||||
if (rbehind != NULL) | if (rbehind != NULL) | ||||
count += *rbehind; | count += *rbehind; | ||||
if (rahead != NULL) | if (rahead != NULL) | ||||
count += *rahead; | count += *rahead; | ||||
vm_object_pip_add(object, count); | vm_object_pip_add(object, count); | ||||
pindex = bm->pindex; | pindex = bm->pindex; | ||||
blk = swp_pager_meta_ctl(object, pindex, 0); | blk = swp_pager_meta_lookup(object, pindex); | ||||
KASSERT(blk != SWAPBLK_NONE, | KASSERT(blk != SWAPBLK_NONE, | ||||
("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex)); | ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex)); | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
bp = uma_zalloc(swrbuf_zone, M_WAITOK); | bp = uma_zalloc(swrbuf_zone, M_WAITOK); | ||||
/* Pages cannot leave the object while busy. */ | /* Pages cannot leave the object while busy. */ | ||||
for (i = 0, p = bm; i < count; i++, p = TAILQ_NEXT(p, listq)) { | for (i = 0, p = bm; i < count; i++, p = TAILQ_NEXT(p, listq)) { | ||||
MPASS(p->pindex == bm->pindex + i); | MPASS(p->pindex == bm->pindex + i); | ||||
▲ Show 20 Lines • Show All 883 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* This routine is capable of looking up, or removing swapblk | * This routine is capable of looking up, or removing swapblk | ||||
* assignments in the swap meta data. It returns the swapblk being | * assignments in the swap meta data. It returns the swapblk being | ||||
* looked-up, popped, or SWAPBLK_NONE if the block was invalid. | * looked-up, popped, or SWAPBLK_NONE if the block was invalid. | ||||
* | * | ||||
* When acting on a busy resident page and paging is in progress, we | * When acting on a busy resident page and paging is in progress, we | ||||
* have to wait until paging is complete but otherwise can act on the | * have to wait until paging is complete but otherwise can act on the | ||||
* busy page. | * busy page. | ||||
* | |||||
* SWM_POP remove from meta data but do not free it | |||||
*/ | */ | ||||
static daddr_t | static daddr_t | ||||
swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags) | swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex) | ||||
{ | { | ||||
struct swblk *sb; | struct swblk *sb; | ||||
daddr_t r1; | |||||
if ((flags & SWM_POP) != 0) | |||||
VM_OBJECT_ASSERT_WLOCKED(object); | |||||
else | |||||
VM_OBJECT_ASSERT_LOCKED(object); | VM_OBJECT_ASSERT_LOCKED(object); | ||||
/* | /* | ||||
* The meta data only exists if the object is OBJT_SWAP | * The meta data only exists if the object is OBJT_SWAP | ||||
* and even then might not be allocated yet. | * and even then might not be allocated yet. | ||||
*/ | */ | ||||
if (object->type != OBJT_SWAP) | if (object->type != OBJT_SWAP) | ||||
Not Done Inline ActionsIs this check still needed after the rest of the changes ? kib: Is this check still needed after the rest of the changes ? | |||||
return (SWAPBLK_NONE); | return (SWAPBLK_NONE); | ||||
sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, | sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, | ||||
rounddown(pindex, SWAP_META_PAGES)); | rounddown(pindex, SWAP_META_PAGES)); | ||||
if (sb == NULL) | if (sb == NULL) | ||||
return (SWAPBLK_NONE); | return (SWAPBLK_NONE); | ||||
r1 = sb->d[pindex % SWAP_META_PAGES]; | return (sb->d[pindex % SWAP_META_PAGES]); | ||||
if (r1 == SWAPBLK_NONE) | |||||
return (SWAPBLK_NONE); | |||||
if ((flags & SWM_POP) != 0) { | |||||
sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE; | |||||
if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { | |||||
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, | |||||
rounddown(pindex, SWAP_META_PAGES)); | |||||
uma_zfree(swblk_zone, sb); | |||||
} | |||||
} | |||||
return (r1); | |||||
} | } | ||||
/* | /* | ||||
* Returns the least page index which is greater than or equal to the | * Returns the least page index which is greater than or equal to the | ||||
* parameter pindex and for which there is a swap block allocated. | * parameter pindex and for which there is a swap block allocated. | ||||
* Returns object's size if the object's type is not swap or if there | * Returns object's size if the object's type is not swap or if there | ||||
* are no allocated swap blocks for the object after the requested | * are no allocated swap blocks for the object after the requested | ||||
* pindex. | * pindex. | ||||
▲ Show 20 Lines • Show All 880 Lines • Show Last 20 Lines |
We need
We also need
swp_pager_freeswapspace(sb->d[m->pindex % SWAP_META_PAGES, 1);
before SWAPBLK_NONE assignment.
swp_pager_freeswapspace calls blist_free while swp_pager_free_empty_swblk calls uma_zfree.