Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/swap_pager.c
Show First 20 Lines • Show All 318 Lines • ▼ Show 20 Lines | if (racct_enable) | ||||
racct_sub_cred(cred, RACCT_SWAP, decr); | racct_sub_cred(cred, RACCT_SWAP, decr); | ||||
#endif | #endif | ||||
} | } | ||||
#define SWM_POP 0x01 /* pop out */ | #define SWM_POP 0x01 /* pop out */ | ||||
static int swap_pager_full = 2; /* swap space exhaustion (task killing) */ | static int swap_pager_full = 2; /* swap space exhaustion (task killing) */ | ||||
static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/ | static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/ | ||||
static int nsw_rcount; /* free read buffers */ | static struct mtx swbuf_mtx; /* to sync nsw_wcount_async */ | ||||
static int nsw_wcount_sync; /* limit write buffers / synchronous */ | static int nsw_wcount_async; /* limit async write buffers */ | ||||
static int nsw_wcount_async; /* limit write buffers / asynchronous */ | |||||
static int nsw_wcount_async_max;/* assigned maximum */ | static int nsw_wcount_async_max;/* assigned maximum */ | ||||
static int nsw_cluster_max; /* maximum VOP I/O allowed */ | static int nsw_cluster_max; /* maximum VOP I/O allowed */ | ||||
static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS); | static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS); | ||||
SYSCTL_PROC(_vm, OID_AUTO, swap_async_max, CTLTYPE_INT | CTLFLAG_RW | | SYSCTL_PROC(_vm, OID_AUTO, swap_async_max, CTLTYPE_INT | CTLFLAG_RW | | ||||
CTLFLAG_MPSAFE, NULL, 0, sysctl_swap_async_max, "I", | CTLFLAG_MPSAFE, NULL, 0, sysctl_swap_async_max, "I", | ||||
"Maximum running async swap ops"); | "Maximum running async swap ops"); | ||||
static int sysctl_swap_fragmentation(SYSCTL_HANDLER_ARGS); | static int sysctl_swap_fragmentation(SYSCTL_HANDLER_ARGS); | ||||
Show All 9 Lines | |||||
*/ | */ | ||||
#define NOBJLISTS 8 | #define NOBJLISTS 8 | ||||
#define NOBJLIST(handle) \ | #define NOBJLIST(handle) \ | ||||
(&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)]) | (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)]) | ||||
static struct pagerlst swap_pager_object_list[NOBJLISTS]; | static struct pagerlst swap_pager_object_list[NOBJLISTS]; | ||||
static uma_zone_t swwbuf_zone; | |||||
static uma_zone_t swrbuf_zone; | |||||
static uma_zone_t swblk_zone; | static uma_zone_t swblk_zone; | ||||
static uma_zone_t swpctrie_zone; | static uma_zone_t swpctrie_zone; | ||||
/* | /* | ||||
* pagerops for OBJT_SWAP - "swap pager". Some ops are also global procedure | * pagerops for OBJT_SWAP - "swap pager". Some ops are also global procedure | ||||
* calls hooked from other parts of the VM system and do not appear here. | * calls hooked from other parts of the VM system and do not appear here. | ||||
* (see vm/swap_pager.h). | * (see vm/swap_pager.h). | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 171 Lines • ▼ Show 20 Lines | swap_pager_swap_init(void) | ||||
* with three or four active swap devices since the system does not | * with three or four active swap devices since the system does not | ||||
* typically have to pageout at extreme bandwidths. We will want | * typically have to pageout at extreme bandwidths. We will want | ||||
* at least 2 per swap devices, and 4 is a pretty good value if you | * at least 2 per swap devices, and 4 is a pretty good value if you | ||||
* have one NFS swap device due to the command/ack latency over NFS. | * have one NFS swap device due to the command/ack latency over NFS. | ||||
* So it all works out pretty well. | * So it all works out pretty well. | ||||
*/ | */ | ||||
nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER); | nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER); | ||||
mtx_lock(&pbuf_mtx); | |||||
nsw_rcount = (nswbuf + 1) / 2; | |||||
nsw_wcount_sync = (nswbuf + 3) / 4; | |||||
nsw_wcount_async = 4; | nsw_wcount_async = 4; | ||||
nsw_wcount_async_max = nsw_wcount_async; | nsw_wcount_async_max = nsw_wcount_async; | ||||
mtx_unlock(&pbuf_mtx); | mtx_init(&swbuf_mtx, "async swbuf mutex", NULL, MTX_DEF); | ||||
swwbuf_zone = uma_zsecond_create("swwbuf", pbuf_ctor, pbuf_dtor, | |||||
pbuf_init, NULL, pbuf_zone); | |||||
uma_zone_set_max(swwbuf_zone, (nswbuf + 3) / 4); | |||||
swrbuf_zone = uma_zsecond_create("swrbuf", pbuf_ctor, pbuf_dtor, | |||||
pbuf_init, NULL, pbuf_zone); | |||||
uma_zone_set_max(swrbuf_zone, (nswbuf + 1) / 2); | |||||
/* | /* | ||||
* Initialize our zone, guessing on the number we need based | * Initialize our zone, guessing on the number we need based | ||||
* on the number of pages in the system. | * on the number of pages in the system. | ||||
*/ | */ | ||||
n = vm_cnt.v_page_count / 2; | n = vm_cnt.v_page_count / 2; | ||||
if (maxswzone && n > maxswzone / sizeof(struct swblk)) | if (maxswzone && n > maxswzone / sizeof(struct swblk)) | ||||
n = maxswzone / sizeof(struct swblk); | n = maxswzone / sizeof(struct swblk); | ||||
swpctrie_zone = uma_zcreate("swpctrie", pctrie_node_size(), NULL, NULL, | swpctrie_zone = uma_zcreate("swpctrie", pctrie_node_size(), NULL, NULL, | ||||
▲ Show 20 Lines • Show All 643 Lines • ▼ Show 20 Lines | swap_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int *rbehind, | ||||
vm_object_pip_add(object, count); | vm_object_pip_add(object, count); | ||||
pindex = bm->pindex; | pindex = bm->pindex; | ||||
blk = swp_pager_meta_ctl(object, pindex, 0); | blk = swp_pager_meta_ctl(object, pindex, 0); | ||||
KASSERT(blk != SWAPBLK_NONE, | KASSERT(blk != SWAPBLK_NONE, | ||||
("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex)); | ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex)); | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
bp = getpbuf(&nsw_rcount); | bp = uma_zalloc(swrbuf_zone, M_WAITOK); | ||||
/* Pages cannot leave the object while busy. */ | /* Pages cannot leave the object while busy. */ | ||||
for (i = 0, p = bm; i < count; i++, p = TAILQ_NEXT(p, listq)) { | for (i = 0, p = bm; i < count; i++, p = TAILQ_NEXT(p, listq)) { | ||||
MPASS(p->pindex == bm->pindex + i); | MPASS(p->pindex == bm->pindex + i); | ||||
bp->b_pages[i] = p; | bp->b_pages[i] = p; | ||||
} | } | ||||
bp->b_flags |= B_PAGING; | bp->b_flags |= B_PAGING; | ||||
bp->b_iocmd = BIO_READ; | bp->b_iocmd = BIO_READ; | ||||
▲ Show 20 Lines • Show All 184 Lines • ▼ Show 20 Lines | if (blk == SWAPBLK_NONE) { | ||||
rtvals[i+j] = VM_PAGER_FAIL; | rtvals[i+j] = VM_PAGER_FAIL; | ||||
continue; | continue; | ||||
} | } | ||||
/* | /* | ||||
* All I/O parameters have been satisfied, build the I/O | * All I/O parameters have been satisfied, build the I/O | ||||
* request and assign the swap space. | * request and assign the swap space. | ||||
*/ | */ | ||||
if (sync == TRUE) { | if (sync != TRUE) { | ||||
bp = getpbuf(&nsw_wcount_sync); | mtx_lock(&swbuf_mtx); | ||||
} else { | while (nsw_wcount_async == 0) | ||||
bp = getpbuf(&nsw_wcount_async); | msleep(&nsw_wcount_async, &swbuf_mtx, PVM, | ||||
bp->b_flags = B_ASYNC; | "swbufa", 0); | ||||
nsw_wcount_async--; | |||||
mtx_unlock(&swbuf_mtx); | |||||
} | } | ||||
bp = uma_zalloc(swwbuf_zone, M_WAITOK); | |||||
if (sync != TRUE) | |||||
bp->b_flags = B_ASYNC; | |||||
bp->b_flags |= B_PAGING; | bp->b_flags |= B_PAGING; | ||||
bp->b_iocmd = BIO_WRITE; | bp->b_iocmd = BIO_WRITE; | ||||
bp->b_rcred = crhold(thread0.td_ucred); | bp->b_rcred = crhold(thread0.td_ucred); | ||||
bp->b_wcred = crhold(thread0.td_ucred); | bp->b_wcred = crhold(thread0.td_ucred); | ||||
bp->b_bcount = PAGE_SIZE * n; | bp->b_bcount = PAGE_SIZE * n; | ||||
bp->b_bufsize = PAGE_SIZE * n; | bp->b_bufsize = PAGE_SIZE * n; | ||||
bp->b_blkno = blk; | bp->b_blkno = blk; | ||||
▲ Show 20 Lines • Show All 206 Lines • ▼ Show 20 Lines | swp_pager_async_iodone(struct buf *bp) | ||||
*/ | */ | ||||
if (bp->b_vp) { | if (bp->b_vp) { | ||||
bp->b_vp = NULL; | bp->b_vp = NULL; | ||||
bp->b_bufobj = NULL; | bp->b_bufobj = NULL; | ||||
} | } | ||||
/* | /* | ||||
* release the physical I/O buffer | * release the physical I/O buffer | ||||
*/ | */ | ||||
relpbuf( | if (bp->b_flags & B_ASYNC) { | ||||
bp, | mtx_lock(&swbuf_mtx); | ||||
((bp->b_iocmd == BIO_READ) ? &nsw_rcount : | if (++nsw_wcount_async == 1) | ||||
((bp->b_flags & B_ASYNC) ? | wakeup(&nsw_wcount_async); | ||||
&nsw_wcount_async : | mtx_unlock(&swbuf_mtx); | ||||
&nsw_wcount_sync | |||||
) | |||||
) | |||||
); | |||||
} | } | ||||
uma_zfree((bp->b_iocmd == BIO_READ) ? swrbuf_zone : swwbuf_zone, bp); | |||||
} | |||||
int | int | ||||
swap_pager_nswapdev(void) | swap_pager_nswapdev(void) | ||||
{ | { | ||||
return (nswapdev); | return (nswapdev); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 967 Lines • ▼ Show 20 Lines | swapgeom_done(struct bio *bp2) | ||||
bp = bp2->bio_caller2; | bp = bp2->bio_caller2; | ||||
cp = bp2->bio_from; | cp = bp2->bio_from; | ||||
bp->b_ioflags = bp2->bio_flags; | bp->b_ioflags = bp2->bio_flags; | ||||
if (bp2->bio_error) | if (bp2->bio_error) | ||||
bp->b_ioflags |= BIO_ERROR; | bp->b_ioflags |= BIO_ERROR; | ||||
bp->b_resid = bp->b_bcount - bp2->bio_completed; | bp->b_resid = bp->b_bcount - bp2->bio_completed; | ||||
bp->b_error = bp2->bio_error; | bp->b_error = bp2->bio_error; | ||||
bp->b_caller1 = NULL; | |||||
bufdone(bp); | bufdone(bp); | ||||
sp = bp2->bio_caller1; | sp = bp2->bio_caller1; | ||||
mtx_lock(&sw_dev_mtx); | mtx_lock(&sw_dev_mtx); | ||||
swapgeom_release(cp, sp); | swapgeom_release(cp, sp); | ||||
mtx_unlock(&sw_dev_mtx); | mtx_unlock(&sw_dev_mtx); | ||||
g_destroy_bio(bp2); | g_destroy_bio(bp2); | ||||
} | } | ||||
Show All 23 Lines | if (bio == NULL) { | ||||
swapgeom_release(cp, sp); | swapgeom_release(cp, sp); | ||||
mtx_unlock(&sw_dev_mtx); | mtx_unlock(&sw_dev_mtx); | ||||
bp->b_error = ENOMEM; | bp->b_error = ENOMEM; | ||||
bp->b_ioflags |= BIO_ERROR; | bp->b_ioflags |= BIO_ERROR; | ||||
bufdone(bp); | bufdone(bp); | ||||
return; | return; | ||||
} | } | ||||
bp->b_caller1 = bio; | |||||
bio->bio_caller1 = sp; | bio->bio_caller1 = sp; | ||||
bio->bio_caller2 = bp; | bio->bio_caller2 = bp; | ||||
bio->bio_cmd = bp->b_iocmd; | bio->bio_cmd = bp->b_iocmd; | ||||
bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE; | bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE; | ||||
bio->bio_length = bp->b_bcount; | bio->bio_length = bp->b_bcount; | ||||
bio->bio_done = swapgeom_done; | bio->bio_done = swapgeom_done; | ||||
if (!buf_mapped(bp)) { | if (!buf_mapped(bp)) { | ||||
bio->bio_ma = bp->b_pages; | bio->bio_ma = bp->b_pages; | ||||
▲ Show 20 Lines • Show All 198 Lines • ▼ Show 20 Lines | sysctl_swap_async_max(SYSCTL_HANDLER_ARGS) | ||||
new = nsw_wcount_async_max; | new = nsw_wcount_async_max; | ||||
error = sysctl_handle_int(oidp, &new, 0, req); | error = sysctl_handle_int(oidp, &new, 0, req); | ||||
if (error != 0 || req->newptr == NULL) | if (error != 0 || req->newptr == NULL) | ||||
return (error); | return (error); | ||||
if (new > nswbuf / 2 || new < 1) | if (new > nswbuf / 2 || new < 1) | ||||
return (EINVAL); | return (EINVAL); | ||||
mtx_lock(&pbuf_mtx); | mtx_lock(&swbuf_mtx); | ||||
while (nsw_wcount_async_max != new) { | while (nsw_wcount_async_max != new) { | ||||
/* | /* | ||||
* Adjust difference. If the current async count is too low, | * Adjust difference. If the current async count is too low, | ||||
* we will need to sqeeze our update slowly in. Sleep with a | * we will need to sqeeze our update slowly in. Sleep with a | ||||
* higher priority than getpbuf() to finish faster. | * higher priority than getpbuf() to finish faster. | ||||
*/ | */ | ||||
n = new - nsw_wcount_async_max; | n = new - nsw_wcount_async_max; | ||||
if (nsw_wcount_async + n >= 0) { | if (nsw_wcount_async + n >= 0) { | ||||
nsw_wcount_async += n; | nsw_wcount_async += n; | ||||
nsw_wcount_async_max += n; | nsw_wcount_async_max += n; | ||||
wakeup(&nsw_wcount_async); | wakeup(&nsw_wcount_async); | ||||
} else { | } else { | ||||
nsw_wcount_async_max -= nsw_wcount_async; | nsw_wcount_async_max -= nsw_wcount_async; | ||||
nsw_wcount_async = 0; | nsw_wcount_async = 0; | ||||
msleep(&nsw_wcount_async, &pbuf_mtx, PSWP, | msleep(&nsw_wcount_async, &swbuf_mtx, PSWP, | ||||
"swpsysctl", 0); | "swpsysctl", 0); | ||||
} | } | ||||
} | } | ||||
mtx_unlock(&pbuf_mtx); | mtx_unlock(&swbuf_mtx); | ||||
return (0); | return (0); | ||||
} | } |