Differential D22437 Diff 64561 head/sys/vm/swap_pager.c

Changeset View

Standalone View

head/sys/vm/swap_pager.c

Show First 20 Lines • Show All 315 Lines • ▼ Show 20 Lines	swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
if (prev < pdecr)		if (prev < pdecr)
printf("negative vmsize for uid = %d\n", uip->ui_uid);		printf("negative vmsize for uid = %d\n", uip->ui_uid);
#ifdef RACCT		#ifdef RACCT
if (racct_enable)		if (racct_enable)
racct_sub_cred(cred, RACCT_SWAP, decr);		racct_sub_cred(cred, RACCT_SWAP, decr);
#endif		#endif
}		}

#define SWM_POP 0x01 /* pop out */

static int swap_pager_full = 2; /* swap space exhaustion (task killing) */		static int swap_pager_full = 2; /* swap space exhaustion (task killing) */
static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/		static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
static struct mtx swbuf_mtx; /* to sync nsw_wcount_async */		static struct mtx swbuf_mtx; /* to sync nsw_wcount_async */
static int nsw_wcount_async; /* limit async write buffers */		static int nsw_wcount_async; /* limit async write buffers */
static int nsw_wcount_async_max;/* assigned maximum */		static int nsw_wcount_async_max;/* assigned maximum */
static int nsw_cluster_max; /* maximum VOP I/O allowed */		static int nsw_cluster_max; /* maximum VOP I/O allowed */

static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS);		static int sysctl_swap_async_max(SYSCTL_HANDLER_ARGS);
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Lines
/*		/*
* Metadata functions		* Metadata functions
*/		*/
static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);		static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t);		static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t);
static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst,		static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst,
vm_pindex_t pindex, vm_pindex_t count);		vm_pindex_t pindex, vm_pindex_t count);
static void swp_pager_meta_free_all(vm_object_t);		static void swp_pager_meta_free_all(vm_object_t);
static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);		static daddr_t swp_pager_meta_lookup(vm_object_t, vm_pindex_t);

static void		static void
swp_pager_init_freerange(daddr_t start, daddr_t num)		swp_pager_init_freerange(daddr_t start, daddr_t num)
{		{

*start = SWAPBLK_NONE;		*start = SWAPBLK_NONE;
*num = 0;		*num = 0;
}		}
▲ Show 20 Lines • Show All 499 Lines • ▼ Show 20 Lines
}		}

static bool		static bool
swp_pager_xfer_source(vm_object_t srcobject, vm_object_t dstobject,		swp_pager_xfer_source(vm_object_t srcobject, vm_object_t dstobject,
vm_pindex_t pindex, daddr_t addr)		vm_pindex_t pindex, daddr_t addr)
{		{
daddr_t dstaddr;		daddr_t dstaddr;

if (swp_pager_meta_ctl(dstobject, pindex, 0) != SWAPBLK_NONE) {		KASSERT(srcobject->type == OBJT_SWAP,
		("%s: Srcobject not swappable", __func__));
		if (dstobject->type == OBJT_SWAP &&
		swp_pager_meta_lookup(dstobject, pindex) != SWAPBLK_NONE) {
/* Caller should destroy the source block. */		/* Caller should destroy the source block. */
return (false);		return (false);
}		}

/*		/*
* Destination has no swapblk and is not resident, transfer source.		* Destination has no swapblk and is not resident, transfer source.
* swp_pager_meta_build() can sleep.		* swp_pager_meta_build() can sleep.
*/		*/
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Lines
static boolean_t		static boolean_t
swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,		swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
int *after)		int *after)
{		{
daddr_t blk, blk0;		daddr_t blk, blk0;
int i;		int i;

VM_OBJECT_ASSERT_LOCKED(object);		VM_OBJECT_ASSERT_LOCKED(object);
		KASSERT(object->type == OBJT_SWAP,
		("%s: object not swappable", __func__));

/*		/*
* do we have good backing store at the requested index ?		* do we have good backing store at the requested index ?
*/		*/
blk0 = swp_pager_meta_ctl(object, pindex, 0);		blk0 = swp_pager_meta_lookup(object, pindex);
if (blk0 == SWAPBLK_NONE) {		if (blk0 == SWAPBLK_NONE) {
if (before)		if (before)
*before = 0;		*before = 0;
if (after)		if (after)
*after = 0;		*after = 0;
return (FALSE);		return (FALSE);
}		}

/*		/*
* find backwards-looking contiguous good backing store		* find backwards-looking contiguous good backing store
*/		*/
if (before != NULL) {		if (before != NULL) {
for (i = 1; i < SWB_NPAGES; i++) {		for (i = 1; i < SWB_NPAGES; i++) {
if (i > pindex)		if (i > pindex)
break;		break;
blk = swp_pager_meta_ctl(object, pindex - i, 0);		blk = swp_pager_meta_lookup(object, pindex - i);
if (blk != blk0 - i)		if (blk != blk0 - i)
break;		break;
}		}
*before = i - 1;		*before = i - 1;
}		}

/*		/*
* find forward-looking contiguous good backing store		* find forward-looking contiguous good backing store
*/		*/
if (after != NULL) {		if (after != NULL) {
for (i = 1; i < SWB_NPAGES; i++) {		for (i = 1; i < SWB_NPAGES; i++) {
blk = swp_pager_meta_ctl(object, pindex + i, 0);		blk = swp_pager_meta_lookup(object, pindex + i);
if (blk != blk0 + i)		if (blk != blk0 + i)
break;		break;
}		}
*after = i - 1;		*after = i - 1;
}		}
return (TRUE);		return (TRUE);
}		}

Show All 14 Lines
*		*
* This routine may not sleep.		* This routine may not sleep.
*		*
* The object containing the page must be locked.		* The object containing the page must be locked.
*/		*/
static void		static void
swap_pager_unswapped(vm_page_t m)		swap_pager_unswapped(vm_page_t m)
{		{
daddr_t srcaddr;		struct swblk *sb;

srcaddr = swp_pager_meta_ctl(m->object, m->pindex, SWM_POP);		VM_OBJECT_ASSERT_WLOCKED(m->object);
if (srcaddr != SWAPBLK_NONE)
swp_pager_freeswapspace(srcaddr, 1);		/*
		* The meta data only exists if the object is OBJT_SWAP
		* and even then might not be allocated yet.
		*/
		KASSERT(m->object->type == OBJT_SWAP,
		("Free object not swappable"));

		sb = SWAP_PCTRIE_LOOKUP(&m->object->un_pager.swp.swp_blks,
		rounddown(m->pindex, SWAP_META_PAGES));
		if (sb == NULL)
		return;
		if (sb->d[m->pindex % SWAP_META_PAGES] == SWAPBLK_NONE)
		return;
		swp_pager_freeswapspace(sb->d[m->pindex % SWAP_META_PAGES], 1);
		sb->d[m->pindex % SWAP_META_PAGES] = SWAPBLK_NONE;
		swp_pager_free_empty_swblk(m->object, sb);
}		}

/*		/*
* swap_pager_getpages() - bring pages in from swap		* swap_pager_getpages() - bring pages in from swap
*		*
* Attempt to page in the pages in array "ma" of length "count". The		* Attempt to page in the pages in array "ma" of length "count". The
* caller may optionally specify that additional pages preceding and		* caller may optionally specify that additional pages preceding and
* succeeding the specified range be paged in. The number of such pages		* succeeding the specified range be paged in. The number of such pages
Show All 18 Lines	swap_pager_getpages(vm_object_t object, vm_page_t ma, int count, int rbehind,
* Determine the final number of read-behind pages and		* Determine the final number of read-behind pages and
* allocate them BEFORE releasing the object lock. Otherwise,		* allocate them BEFORE releasing the object lock. Otherwise,
* there can be a problematic race with vm_object_split().		* there can be a problematic race with vm_object_split().
* Specifically, vm_object_split() might first transfer pages		* Specifically, vm_object_split() might first transfer pages
* that precede ma[0] in the current object to a new object,		* that precede ma[0] in the current object to a new object,
* and then this function incorrectly recreates those pages as		* and then this function incorrectly recreates those pages as
* read-behind pages in the current object.		* read-behind pages in the current object.
*/		*/
		KASSERT(object->type == OBJT_SWAP,
		("%s: object not swappable", __func__));
if (!swap_pager_haspage(object, ma[0]->pindex, &maxbehind, &maxahead))		if (!swap_pager_haspage(object, ma[0]->pindex, &maxbehind, &maxahead))
return (VM_PAGER_FAIL);		return (VM_PAGER_FAIL);

/*		/*
* Clip the readahead and readbehind ranges to exclude resident pages.		* Clip the readahead and readbehind ranges to exclude resident pages.
*/		*/
if (rahead != NULL) {		if (rahead != NULL) {
KASSERT(reqcount - 1 <= maxahead,		KASSERT(reqcount - 1 <= maxahead,
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines	swap_pager_getpages(vm_object_t object, vm_page_t ma, int count, int rbehind,
if (rbehind != NULL)		if (rbehind != NULL)
count += *rbehind;		count += *rbehind;
if (rahead != NULL)		if (rahead != NULL)
count += *rahead;		count += *rahead;

vm_object_pip_add(object, count);		vm_object_pip_add(object, count);

pindex = bm->pindex;		pindex = bm->pindex;
blk = swp_pager_meta_ctl(object, pindex, 0);		blk = swp_pager_meta_lookup(object, pindex);
KASSERT(blk != SWAPBLK_NONE,		KASSERT(blk != SWAPBLK_NONE,
("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex));		("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex));

VM_OBJECT_WUNLOCK(object);		VM_OBJECT_WUNLOCK(object);
bp = uma_zalloc(swrbuf_zone, M_WAITOK);		bp = uma_zalloc(swrbuf_zone, M_WAITOK);
/* Pages cannot leave the object while busy. */		/* Pages cannot leave the object while busy. */
for (i = 0, p = bm; i < count; i++, p = TAILQ_NEXT(p, listq)) {		for (i = 0, p = bm; i < count; i++, p = TAILQ_NEXT(p, listq)) {
MPASS(p->pindex == bm->pindex + i);		MPASS(p->pindex == bm->pindex + i);
▲ Show 20 Lines • Show All 460 Lines • ▼ Show 20 Lines
swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex, int npages)		swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex, int npages)
{		{
vm_page_t ma[npages];		vm_page_t ma[npages];
int i, j;		int i, j;

KASSERT(npages > 0, ("%s: No pages", __func__));		KASSERT(npages > 0, ("%s: No pages", __func__));
KASSERT(npages <= MAXPHYS / PAGE_SIZE,		KASSERT(npages <= MAXPHYS / PAGE_SIZE,
("%s: Too many pages: %d", __func__, npages));		("%s: Too many pages: %d", __func__, npages));
		KASSERT(object->type == OBJT_SWAP,
		("%s: Object not swappable", __func__));
vm_object_pip_add(object, npages);		vm_object_pip_add(object, npages);
vm_page_grab_pages(object, pindex, VM_ALLOC_NORMAL, ma, npages);		vm_page_grab_pages(object, pindex, VM_ALLOC_NORMAL, ma, npages);
for (i = j = 0;; i++) {		for (i = j = 0;; i++) {
/* Count nonresident pages, to page-in all at once. */		/* Count nonresident pages, to page-in all at once. */
if (i < npages && ma[i]->valid != VM_PAGE_BITS_ALL)		if (i < npages && ma[i]->valid != VM_PAGE_BITS_ALL)
continue;		continue;
if (j < i) {		if (j < i) {
/* Page-in nonresident pages. Mark for laundering. */		/* Page-in nonresident pages. Mark for laundering. */
Show All 21 Lines
static void		static void
swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object)		swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object)
{		{
struct swblk *sb;		struct swblk *sb;
vm_pindex_t pi, s_pindex;		vm_pindex_t pi, s_pindex;
daddr_t blk, n_blks, s_blk;		daddr_t blk, n_blks, s_blk;
int i;		int i;

		KASSERT(object->type == OBJT_SWAP,
		("%s: Object not swappable", __func__));
n_blks = 0;		n_blks = 0;
for (pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(		for (pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
&object->un_pager.swp.swp_blks, pi)) != NULL; ) {		&object->un_pager.swp.swp_blks, pi)) != NULL; ) {
for (i = 0; i < SWAP_META_PAGES; i++) {		for (i = 0; i < SWAP_META_PAGES; i++) {
blk = sb->d[i];		blk = sb->d[i];
if (!swp_pager_isondev(blk, sp))		if (!swp_pager_isondev(blk, sp))
blk = SWAPBLK_NONE;		blk = SWAPBLK_NONE;

▲ Show 20 Lines • Show All 382 Lines • ▼ Show 20 Lines
*		*
* This routine is capable of looking up, or removing swapblk		* This routine is capable of looking up, or removing swapblk
* assignments in the swap meta data. It returns the swapblk being		* assignments in the swap meta data. It returns the swapblk being
* looked-up, popped, or SWAPBLK_NONE if the block was invalid.		* looked-up, popped, or SWAPBLK_NONE if the block was invalid.
*		*
* When acting on a busy resident page and paging is in progress, we		* When acting on a busy resident page and paging is in progress, we
* have to wait until paging is complete but otherwise can act on the		* have to wait until paging is complete but otherwise can act on the
* busy page.		* busy page.
*
* SWM_POP remove from meta data but do not free it
*/		*/
static daddr_t		static daddr_t
swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)		swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex)
{		{
struct swblk *sb;		struct swblk *sb;
daddr_t r1;

if ((flags & SWM_POP) != 0)
VM_OBJECT_ASSERT_WLOCKED(object);
else
VM_OBJECT_ASSERT_LOCKED(object);		VM_OBJECT_ASSERT_LOCKED(object);

/*		/*
* The meta data only exists if the object is OBJT_SWAP		* The meta data only exists if the object is OBJT_SWAP
* and even then might not be allocated yet.		* and even then might not be allocated yet.
*/		*/
if (object->type != OBJT_SWAP)		KASSERT(object->type == OBJT_SWAP,
return (SWAPBLK_NONE);		("Lookup object not swappable"));

sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,		sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
rounddown(pindex, SWAP_META_PAGES));		rounddown(pindex, SWAP_META_PAGES));
if (sb == NULL)		if (sb == NULL)
return (SWAPBLK_NONE);		return (SWAPBLK_NONE);
r1 = sb->d[pindex % SWAP_META_PAGES];		return (sb->d[pindex % SWAP_META_PAGES]);
if (r1 == SWAPBLK_NONE)
return (SWAPBLK_NONE);
if ((flags & SWM_POP) != 0) {
sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE;
swp_pager_free_empty_swblk(object, sb);
}
return (r1);
}		}

/*		/*
* Returns the least page index which is greater than or equal to the		* Returns the least page index which is greater than or equal to the
* parameter pindex and for which there is a swap block allocated.		* parameter pindex and for which there is a swap block allocated.
* Returns object's size if the object's type is not swap or if there		* Returns object's size if the object's type is not swap or if there
* are no allocated swap blocks for the object after the requested		* are no allocated swap blocks for the object after the requested
* pindex.		* pindex.
▲ Show 20 Lines • Show All 880 Lines • Show Last 20 Lines