Differential D13707 Diff 46391 head/sys/vm/swap_pager.c

Changeset View

Standalone View

head/sys/vm/swap_pager.c

Show First 20 Lines • Show All 397 Lines • ▼ Show 20 Lines
* Swap bitmap functions		* Swap bitmap functions
*/		*/
static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages);		static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages);
static daddr_t swp_pager_getswapspace(int npages);		static daddr_t swp_pager_getswapspace(int npages);

/*		/*
* Metadata functions		* Metadata functions
*/		*/
static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);		static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t);		static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t);
static void swp_pager_meta_free_all(vm_object_t);		static void swp_pager_meta_free_all(vm_object_t);
static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);		static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);

		static void
		swp_pager_init_freerange(daddr_t start, daddr_t num)
		{

		*start = SWAPBLK_NONE;
		*num = 0;
		}

		static void
		swp_pager_update_freerange(daddr_t start, daddr_t num, daddr_t addr)
		{

		if (start + num == addr) {
		(*num)++;
		} else {
		swp_pager_freeswapspace(start, num);
		*start = addr;
		*num = 1;
		}
		}

static void *		static void *
swblk_trie_alloc(struct pctrie *ptree)		swblk_trie_alloc(struct pctrie *ptree)
{		{

return (uma_zalloc(swpctrie_zone, M_NOWAIT \| (curproc == pageproc ?		return (uma_zalloc(swpctrie_zone, M_NOWAIT \| (curproc == pageproc ?
M_USE_RESERVE : 0)));		M_USE_RESERVE : 0)));
}		}

▲ Show 20 Lines • Show All 437 Lines • ▼ Show 20 Lines
* Returns 0 on success, -1 on failure.		* Returns 0 on success, -1 on failure.
*/		*/
int		int
swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)		swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
{		{
int n = 0;		int n = 0;
daddr_t blk = SWAPBLK_NONE;		daddr_t blk = SWAPBLK_NONE;
vm_pindex_t beg = start; /* save start index */		vm_pindex_t beg = start; /* save start index */
		daddr_t addr, n_free, s_free;

		swp_pager_init_freerange(&s_free, &n_free);
VM_OBJECT_WLOCK(object);		VM_OBJECT_WLOCK(object);
while (size) {		while (size) {
if (n == 0) {		if (n == 0) {
n = BLIST_MAX_ALLOC;		n = BLIST_MAX_ALLOC;
while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {		while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
n >>= 1;		n >>= 1;
if (n == 0) {		if (n == 0) {
swp_pager_meta_free(object, beg, start - beg);		swp_pager_meta_free(object, beg, start - beg);
VM_OBJECT_WUNLOCK(object);		VM_OBJECT_WUNLOCK(object);
return (-1);		return (-1);
}		}
}		}
}		}
swp_pager_meta_build(object, start, blk);		addr = swp_pager_meta_build(object, start, blk);
		if (addr != SWAPBLK_NONE)
		swp_pager_update_freerange(&s_free, &n_free, addr);
--size;		--size;
++start;		++start;
++blk;		++blk;
--n;		--n;
}		}
		swp_pager_freeswapspace(s_free, n_free);
swp_pager_meta_free(object, start, n);		swp_pager_meta_free(object, start, n);
VM_OBJECT_WUNLOCK(object);		VM_OBJECT_WUNLOCK(object);
return (0);		return (0);
}		}

/*		/*
* SWAP_PAGER_COPY() - copy blocks from source pager to destination pager		* SWAP_PAGER_COPY() - copy blocks from source pager to destination pager
* and destroy the source.		* and destroy the source.
Show All 13 Lines
* The source and destination objects must be locked.		* The source and destination objects must be locked.
* Both object locks may temporarily be released.		* Both object locks may temporarily be released.
*/		*/
void		void
swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,		swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
vm_pindex_t offset, int destroysource)		vm_pindex_t offset, int destroysource)
{		{
vm_pindex_t i;		vm_pindex_t i;
daddr_t dstaddr, first_free, num_free, srcaddr;		daddr_t dstaddr, n_free, s_free, srcaddr;

VM_OBJECT_ASSERT_WLOCKED(srcobject);		VM_OBJECT_ASSERT_WLOCKED(srcobject);
VM_OBJECT_ASSERT_WLOCKED(dstobject);		VM_OBJECT_ASSERT_WLOCKED(dstobject);

/*		/*
* If destroysource is set, we remove the source object from the		* If destroysource is set, we remove the source object from the
* swap_pager internal queue now.		* swap_pager internal queue now.
*/		*/
Show All 10 Lines	if (destroysource && srcobject->handle != NULL) {
vm_object_pip_wakeup(dstobject);		vm_object_pip_wakeup(dstobject);
VM_OBJECT_WLOCK(srcobject);		VM_OBJECT_WLOCK(srcobject);
vm_object_pip_wakeup(srcobject);		vm_object_pip_wakeup(srcobject);
}		}

/*		/*
* Transfer source to destination.		* Transfer source to destination.
*/		*/
first_free = SWAPBLK_NONE;		swp_pager_init_freerange(&s_free, &n_free);
num_free = 0;
for (i = 0; i < dstobject->size; ++i) {		for (i = 0; i < dstobject->size; ++i) {
srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP);		srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP);
if (srcaddr == SWAPBLK_NONE)		if (srcaddr == SWAPBLK_NONE)
continue;		continue;
dstaddr = swp_pager_meta_ctl(dstobject, i, 0);		dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
if (dstaddr == SWAPBLK_NONE) {		if (dstaddr != SWAPBLK_NONE) {
/*		/*
		* Destination has valid swapblk or it is represented
		* by a resident page. We destroy the source block.
		*/
		swp_pager_update_freerange(&s_free, &n_free, srcaddr);
		continue;
		}

		/*
* Destination has no swapblk and is not resident,		* Destination has no swapblk and is not resident,
* copy source.		* copy source.
*		*
* swp_pager_meta_build() can sleep.		* swp_pager_meta_build() can sleep.
*/		*/
vm_object_pip_add(srcobject, 1);		vm_object_pip_add(srcobject, 1);
VM_OBJECT_WUNLOCK(srcobject);		VM_OBJECT_WUNLOCK(srcobject);
vm_object_pip_add(dstobject, 1);		vm_object_pip_add(dstobject, 1);
swp_pager_meta_build(dstobject, i, srcaddr);		dstaddr = swp_pager_meta_build(dstobject, i, srcaddr);
		KASSERT(dstaddr == SWAPBLK_NONE,
		("Unexpected destination swapblk"));
vm_object_pip_wakeup(dstobject);		vm_object_pip_wakeup(dstobject);
VM_OBJECT_WLOCK(srcobject);		VM_OBJECT_WLOCK(srcobject);
vm_object_pip_wakeup(srcobject);		vm_object_pip_wakeup(srcobject);
} else {
/*
* Destination has valid swapblk or it is represented
* by a resident page. We destroy the sourceblock.
*/
if (first_free + num_free == srcaddr)
num_free++;
else {
swp_pager_freeswapspace(first_free, num_free);
first_free = srcaddr;
num_free = 1;
}		}
}		swp_pager_freeswapspace(s_free, n_free);
}
swp_pager_freeswapspace(first_free, num_free);

/*		/*
* Free left over swap blocks in source.		* Free left over swap blocks in source.
*		*
* We have to revert the type to OBJT_DEFAULT so we do not accidentally		* We have to revert the type to OBJT_DEFAULT so we do not accidentally
* double-remove the object from the swap queues.		* double-remove the object from the swap queues.
*/		*/
if (destroysource) {		if (destroysource) {
▲ Show 20 Lines • Show All 318 Lines • ▼ Show 20 Lines
* We need to unbusy the rest on I/O completion.		* We need to unbusy the rest on I/O completion.
*/		*/
static void		static void
swap_pager_putpages(vm_object_t object, vm_page_t *ma, int count,		swap_pager_putpages(vm_object_t object, vm_page_t *ma, int count,
int flags, int *rtvals)		int flags, int *rtvals)
{		{
int i, n;		int i, n;
boolean_t sync;		boolean_t sync;
		daddr_t addr, n_free, s_free;

		swp_pager_init_freerange(&s_free, &n_free);
if (count && ma[0]->object != object) {		if (count && ma[0]->object != object) {
panic("swap_pager_putpages: object mismatch %p/%p",		panic("swap_pager_putpages: object mismatch %p/%p",
object,		object,
ma[0]->object		ma[0]->object
);		);
}		}

/*		/*
* Step 1		* Step 1
*		*
* Turn object into OBJT_SWAP		* Turn object into OBJT_SWAP
* check for bogus sysops		* check for bogus sysops
* force sync if not pageout process		* force sync if not pageout process
*/		*/
if (object->type != OBJT_SWAP)		if (object->type != OBJT_SWAP) {
swp_pager_meta_build(object, 0, SWAPBLK_NONE);		addr = swp_pager_meta_build(object, 0, SWAPBLK_NONE);
		KASSERT(addr == SWAPBLK_NONE,
		("unexpected object swap block"));
		}
VM_OBJECT_WUNLOCK(object);		VM_OBJECT_WUNLOCK(object);

n = 0;		n = 0;
if (curproc != pageproc)		if (curproc != pageproc)
sync = TRUE;		sync = TRUE;
else		else
sync = (flags & VM_PAGER_PUT_SYNC) != 0;		sync = (flags & VM_PAGER_PUT_SYNC) != 0;

▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines	for (i = 0; i < count; i += n) {
bp->b_bcount = PAGE_SIZE * n;		bp->b_bcount = PAGE_SIZE * n;
bp->b_bufsize = PAGE_SIZE * n;		bp->b_bufsize = PAGE_SIZE * n;
bp->b_blkno = blk;		bp->b_blkno = blk;

VM_OBJECT_WLOCK(object);		VM_OBJECT_WLOCK(object);
for (j = 0; j < n; ++j) {		for (j = 0; j < n; ++j) {
vm_page_t mreq = ma[i+j];		vm_page_t mreq = ma[i+j];

swp_pager_meta_build(		addr = swp_pager_meta_build(mreq->object, mreq->pindex,
mreq->object,		blk + j);
mreq->pindex,		if (addr != SWAPBLK_NONE)
blk + j		swp_pager_update_freerange(&s_free, &n_free,
);		addr);
MPASS(mreq->dirty == VM_PAGE_BITS_ALL);		MPASS(mreq->dirty == VM_PAGE_BITS_ALL);
mreq->oflags \|= VPO_SWAPINPROG;		mreq->oflags \|= VPO_SWAPINPROG;
bp->b_pages[j] = mreq;		bp->b_pages[j] = mreq;
}		}
VM_OBJECT_WUNLOCK(object);		VM_OBJECT_WUNLOCK(object);
bp->b_npages = n;		bp->b_npages = n;
/*		/*
* Must set dirty range for NFS to work.		* Must set dirty range for NFS to work.
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines	for (i = 0; i < count; i += n) {

/*		/*
* Now that we are through with the bp, we can call the		* Now that we are through with the bp, we can call the
* normal async completion, which frees everything up.		* normal async completion, which frees everything up.
*/		*/
swp_pager_async_iodone(bp);		swp_pager_async_iodone(bp);
}		}
VM_OBJECT_WLOCK(object);		VM_OBJECT_WLOCK(object);
		swp_pager_freeswapspace(s_free, n_free);
}		}

/*		/*
* swp_pager_async_iodone:		* swp_pager_async_iodone:
*		*
* Completion routine for asynchronous reads and writes from/to swap.		* Completion routine for asynchronous reads and writes from/to swap.
* Also called manually by synchronous code to finish up a bp.		* Also called manually by synchronous code to finish up a bp.
*		*
▲ Show 20 Lines • Show All 314 Lines • ▼ Show 20 Lines
/*		/*
* SWP_PAGER_META_BUILD() - add swap block to swap meta data for object		* SWP_PAGER_META_BUILD() - add swap block to swap meta data for object
*		*
* We first convert the object to a swap object if it is a default		* We first convert the object to a swap object if it is a default
* object.		* object.
*		*
* The specified swapblk is added to the object's swap metadata. If		* The specified swapblk is added to the object's swap metadata. If
* the swapblk is not valid, it is freed instead. Any previously		* the swapblk is not valid, it is freed instead. Any previously
* assigned swapblk is freed.		* assigned swapblk is returned.
*/		*/
static void		static daddr_t
swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)		swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
{		{
static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted;		static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted;
struct swblk sb, sb1;		struct swblk sb, sb1;
vm_pindex_t modpi, rdpi;		vm_pindex_t modpi, rdpi;
		daddr_t prev_swapblk;
int error, i;		int error, i;

VM_OBJECT_ASSERT_WLOCKED(object);		VM_OBJECT_ASSERT_WLOCKED(object);

/*		/*
* Convert default object to swap object if necessary		* Convert default object to swap object if necessary
*/		*/
if (object->type != OBJT_SWAP) {		if (object->type != OBJT_SWAP) {
pctrie_init(&object->un_pager.swp.swp_blks);		pctrie_init(&object->un_pager.swp.swp_blks);

/*		/*
* Ensure that swap_pager_swapoff()'s iteration over		* Ensure that swap_pager_swapoff()'s iteration over
* object_list does not see a garbage pctrie.		* object_list does not see a garbage pctrie.
*/		*/
atomic_thread_fence_rel();		atomic_thread_fence_rel();

object->type = OBJT_SWAP;		object->type = OBJT_SWAP;
KASSERT(object->handle == NULL, ("default pager with handle"));		KASSERT(object->handle == NULL, ("default pager with handle"));
}		}

rdpi = rounddown(pindex, SWAP_META_PAGES);		rdpi = rounddown(pindex, SWAP_META_PAGES);
sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi);		sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi);
if (sb == NULL) {		if (sb == NULL) {
if (swapblk == SWAPBLK_NONE)		if (swapblk == SWAPBLK_NONE)
return;		return (SWAPBLK_NONE);
for (;;) {		for (;;) {
sb = uma_zalloc(swblk_zone, M_NOWAIT \| (curproc ==		sb = uma_zalloc(swblk_zone, M_NOWAIT \| (curproc ==
pageproc ? M_USE_RESERVE : 0));		pageproc ? M_USE_RESERVE : 0));
if (sb != NULL) {		if (sb != NULL) {
sb->p = rdpi;		sb->p = rdpi;
for (i = 0; i < SWAP_META_PAGES; i++)		for (i = 0; i < SWAP_META_PAGES; i++)
sb->d[i] = SWAPBLK_NONE;		sb->d[i] = SWAPBLK_NONE;
if (atomic_cmpset_int(&swblk_zone_exhausted,		if (atomic_cmpset_int(&swblk_zone_exhausted,
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines	for (;;) {
goto allocated;		goto allocated;
}		}
}		}
}		}
allocated:		allocated:
MPASS(sb->p == rdpi);		MPASS(sb->p == rdpi);

modpi = pindex % SWAP_META_PAGES;		modpi = pindex % SWAP_META_PAGES;
/* Delete prior contents of metadata. */		/* Return prior contents of metadata. */
if (sb->d[modpi] != SWAPBLK_NONE)		prev_swapblk = sb->d[modpi];
swp_pager_freeswapspace(sb->d[modpi], 1);
/* Enter block into metadata. */		/* Enter block into metadata. */
sb->d[modpi] = swapblk;		sb->d[modpi] = swapblk;

/*		/*
* Free the swblk if we end up with the empty page run.		* Free the swblk if we end up with the empty page run.
*/		*/
if (swapblk == SWAPBLK_NONE &&		if (swapblk == SWAPBLK_NONE &&
swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {		swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi);		SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi);
uma_zfree(swblk_zone, sb);		uma_zfree(swblk_zone, sb);
}		}
		return (prev_swapblk);
}		}

/*		/*
* SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata		* SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
*		*
* The requested range of blocks is freed, with any associated swap		* The requested range of blocks is freed, with any associated swap
* returned to the swap bitmap.		* returned to the swap bitmap.
*		*
* This routine will free swap metadata structures as they are cleaned		* This routine will free swap metadata structures as they are cleaned
* out. This routine does NOT operate on swap metadata associated		* out. This routine does NOT operate on swap metadata associated
* with resident pages.		* with resident pages.
*/		*/
static void		static void
swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)		swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
{		{
struct swblk *sb;		struct swblk *sb;
daddr_t first_free, num_free;		daddr_t n_free, s_free;
vm_pindex_t last;		vm_pindex_t last;
int i, limit, start;		int i, limit, start;

VM_OBJECT_ASSERT_WLOCKED(object);		VM_OBJECT_ASSERT_WLOCKED(object);
if (object->type != OBJT_SWAP \|\| count == 0)		if (object->type != OBJT_SWAP \|\| count == 0)
return;		return;

first_free = SWAPBLK_NONE;		swp_pager_init_freerange(&s_free, &n_free);
num_free = 0;
last = pindex + count;		last = pindex + count;
for (;;) {		for (;;) {
sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,		sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
rounddown(pindex, SWAP_META_PAGES));		rounddown(pindex, SWAP_META_PAGES));
if (sb == NULL \|\| sb->p >= last)		if (sb == NULL \|\| sb->p >= last)
break;		break;
start = pindex > sb->p ? pindex - sb->p : 0;		start = pindex > sb->p ? pindex - sb->p : 0;
limit = last - sb->p < SWAP_META_PAGES ? last - sb->p :		limit = last - sb->p < SWAP_META_PAGES ? last - sb->p :
SWAP_META_PAGES;		SWAP_META_PAGES;
for (i = start; i < limit; i++) {		for (i = start; i < limit; i++) {
if (sb->d[i] == SWAPBLK_NONE)		if (sb->d[i] == SWAPBLK_NONE)
continue;		continue;
if (first_free + num_free == sb->d[i])		swp_pager_update_freerange(&s_free, &n_free, sb->d[i]);
num_free++;
else {
swp_pager_freeswapspace(first_free, num_free);
first_free = sb->d[i];
num_free = 1;
}
sb->d[i] = SWAPBLK_NONE;		sb->d[i] = SWAPBLK_NONE;
}		}
if (swp_pager_swblk_empty(sb, 0, start) &&		if (swp_pager_swblk_empty(sb, 0, start) &&
swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {		swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,		SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
sb->p);		sb->p);
uma_zfree(swblk_zone, sb);		uma_zfree(swblk_zone, sb);
}		}
pindex = sb->p + SWAP_META_PAGES;		pindex = sb->p + SWAP_META_PAGES;
}		}
swp_pager_freeswapspace(first_free, num_free);		swp_pager_freeswapspace(s_free, n_free);
}		}

/*		/*
* SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object		* SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
*		*
* This routine locates and destroys all swap metadata associated with		* This routine locates and destroys all swap metadata associated with
* an object.		* an object.
*/		*/
static void		static void
swp_pager_meta_free_all(vm_object_t object)		swp_pager_meta_free_all(vm_object_t object)
{		{
struct swblk *sb;		struct swblk *sb;
daddr_t first_free, num_free;		daddr_t n_free, s_free;
vm_pindex_t pindex;		vm_pindex_t pindex;
int i;		int i;

VM_OBJECT_ASSERT_WLOCKED(object);		VM_OBJECT_ASSERT_WLOCKED(object);
if (object->type != OBJT_SWAP)		if (object->type != OBJT_SWAP)
return;		return;

first_free = SWAPBLK_NONE;		swp_pager_init_freerange(&s_free, &n_free);
num_free = 0;
for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(		for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
&object->un_pager.swp.swp_blks, pindex)) != NULL;) {		&object->un_pager.swp.swp_blks, pindex)) != NULL;) {
pindex = sb->p + SWAP_META_PAGES;		pindex = sb->p + SWAP_META_PAGES;
for (i = 0; i < SWAP_META_PAGES; i++) {		for (i = 0; i < SWAP_META_PAGES; i++) {
if (sb->d[i] == SWAPBLK_NONE)		if (sb->d[i] == SWAPBLK_NONE)
continue;		continue;
if (first_free + num_free == sb->d[i])		swp_pager_update_freerange(&s_free, &n_free, sb->d[i]);
num_free++;
else {
swp_pager_freeswapspace(first_free, num_free);
first_free = sb->d[i];
num_free = 1;
}		}
}
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p);		SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p);
uma_zfree(swblk_zone, sb);		uma_zfree(swblk_zone, sb);
}		}
swp_pager_freeswapspace(first_free, num_free);		swp_pager_freeswapspace(s_free, n_free);
}		}

/*		/*
* SWP_PAGER_METACTL() - misc control of swap meta data.		* SWP_PAGER_METACTL() - misc control of swap meta data.
*		*
* This routine is capable of looking up, or removing swapblk		* This routine is capable of looking up, or removing swapblk
* assignments in the swap meta data. It returns the swapblk being		* assignments in the swap meta data. It returns the swapblk being
* looked-up, popped, or SWAPBLK_NONE if the block was invalid.		* looked-up, popped, or SWAPBLK_NONE if the block was invalid.
▲ Show 20 Lines • Show All 881 Lines • Show Last 20 Lines