Differential D22280 Diff 64185 head/sys/vm/swap_pager.c

Changeset View

Standalone View

head/sys/vm/swap_pager.c

Show First 20 Lines • Show All 416 Lines • ▼ Show 20 Lines
static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages);		static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages);
static daddr_t swp_pager_getswapspace(int *npages, int limit);		static daddr_t swp_pager_getswapspace(int *npages, int limit);

/*		/*
* Metadata functions		* Metadata functions
*/		*/
static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);		static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t);		static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t);
		static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst,
		vm_pindex_t pindex, vm_pindex_t count);
static void swp_pager_meta_free_all(vm_object_t);		static void swp_pager_meta_free_all(vm_object_t);
static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);		static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);

static void		static void
swp_pager_init_freerange(daddr_t start, daddr_t num)		swp_pager_init_freerange(daddr_t start, daddr_t num)
{		{

*start = SWAPBLK_NONE;		*start = SWAPBLK_NONE;
▲ Show 20 Lines • Show All 495 Lines • ▼ Show 20 Lines	for (j = 0; j < n; ++j) {
addr);		addr);
}		}
}		}
swp_pager_freeswapspace(s_free, n_free);		swp_pager_freeswapspace(s_free, n_free);
VM_OBJECT_WUNLOCK(object);		VM_OBJECT_WUNLOCK(object);
return (0);		return (0);
}		}

		static bool
		swp_pager_xfer_source(vm_object_t srcobject, vm_object_t dstobject,
		vm_pindex_t pindex, daddr_t addr)
		{
		daddr_t dstaddr;

		if (swp_pager_meta_ctl(dstobject, pindex, 0) != SWAPBLK_NONE) {
		/* Caller should destroy the source block. */
		return (false);
		}

/*		/*
		* Destination has no swapblk and is not resident, transfer source.
		* swp_pager_meta_build() can sleep.
		*/
		vm_object_pip_add(srcobject, 1);
		VM_OBJECT_WUNLOCK(srcobject);
		vm_object_pip_add(dstobject, 1);
		dstaddr = swp_pager_meta_build(dstobject, pindex, addr);
		KASSERT(dstaddr == SWAPBLK_NONE,
		("Unexpected destination swapblk"));
		vm_object_pip_wakeup(dstobject);
		VM_OBJECT_WLOCK(srcobject);
		vm_object_pip_wakeup(srcobject);
		return (true);
		}

		/*
* SWAP_PAGER_COPY() - copy blocks from source pager to destination pager		* SWAP_PAGER_COPY() - copy blocks from source pager to destination pager
* and destroy the source.		* and destroy the source.
*		*
* Copy any valid swapblks from the source to the destination. In		* Copy any valid swapblks from the source to the destination. In
* cases where both the source and destination have a valid swapblk,		* cases where both the source and destination have a valid swapblk,
* we keep the destination's.		* we keep the destination's.
*		*
* This routine is allowed to sleep. It may sleep allocating metadata		* This routine is allowed to sleep. It may sleep allocating metadata
* indirectly through swp_pager_meta_build() or if paging is still in		* indirectly through swp_pager_meta_build() or if paging is still in
* progress on the source.		* progress on the source.
*		*
* The source object contains no vm_page_t's (which is just as well)		* The source object contains no vm_page_t's (which is just as well)
*		*
* The source object is of type OBJT_SWAP.		* The source object is of type OBJT_SWAP.
*		*
* The source and destination objects must be locked.		* The source and destination objects must be locked.
* Both object locks may temporarily be released.		* Both object locks may temporarily be released.
*/		*/
void		void
swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,		swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
vm_pindex_t offset, int destroysource)		vm_pindex_t offset, int destroysource)
{		{
vm_pindex_t i;
daddr_t dstaddr, n_free, s_free, srcaddr;

VM_OBJECT_ASSERT_WLOCKED(srcobject);		VM_OBJECT_ASSERT_WLOCKED(srcobject);
VM_OBJECT_ASSERT_WLOCKED(dstobject);		VM_OBJECT_ASSERT_WLOCKED(dstobject);

/*		/*
* If destroysource is set, we remove the source object from the		* If destroysource is set, we remove the source object from the
* swap_pager internal queue now.		* swap_pager internal queue now.
*/		*/
Show All 10 Lines	if (destroysource && srcobject->handle != NULL) {
vm_object_pip_wakeup(dstobject);		vm_object_pip_wakeup(dstobject);
VM_OBJECT_WLOCK(srcobject);		VM_OBJECT_WLOCK(srcobject);
vm_object_pip_wakeup(srcobject);		vm_object_pip_wakeup(srcobject);
}		}

/*		/*
* Transfer source to destination.		* Transfer source to destination.
*/		*/
swp_pager_init_freerange(&s_free, &n_free);		swp_pager_meta_transfer(srcobject, dstobject, offset, dstobject->size);
for (i = 0; i < dstobject->size; ++i) {
srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP);
if (srcaddr == SWAPBLK_NONE)
continue;
dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
if (dstaddr != SWAPBLK_NONE) {
/*
* Destination has valid swapblk or it is represented
* by a resident page. We destroy the source block.
*/
swp_pager_update_freerange(&s_free, &n_free, srcaddr);
continue;
}

/*		/*
* Destination has no swapblk and is not resident,
* copy source.
*
* swp_pager_meta_build() can sleep.
*/
vm_object_pip_add(srcobject, 1);
VM_OBJECT_WUNLOCK(srcobject);
vm_object_pip_add(dstobject, 1);
dstaddr = swp_pager_meta_build(dstobject, i, srcaddr);
KASSERT(dstaddr == SWAPBLK_NONE,
("Unexpected destination swapblk"));
vm_object_pip_wakeup(dstobject);
VM_OBJECT_WLOCK(srcobject);
vm_object_pip_wakeup(srcobject);
}
swp_pager_freeswapspace(s_free, n_free);

/*
* Free left over swap blocks in source.		* Free left over swap blocks in source.
*		*
* We have to revert the type to OBJT_DEFAULT so we do not accidentally		* We have to revert the type to OBJT_DEFAULT so we do not accidentally
* double-remove the object from the swap queues.		* double-remove the object from the swap queues.
*/		*/
if (destroysource) {		if (destroysource) {
swp_pager_meta_free_all(srcobject);		swp_pager_meta_free_all(srcobject);
/*		/*
▲ Show 20 Lines • Show All 969 Lines • ▼ Show 20 Lines	if (swapblk == SWAPBLK_NONE &&
swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {		swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi);		SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi);
uma_zfree(swblk_zone, sb);		uma_zfree(swblk_zone, sb);
}		}
return (prev_swapblk);		return (prev_swapblk);
}		}

/*		/*
* SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata		* SWP_PAGER_META_TRANSFER() - free a range of blocks in the srcobject's swap
		* metadata, or transfer it into dstobject.
*		*
* The requested range of blocks is freed, with any associated swap
* returned to the swap bitmap.
*
* This routine will free swap metadata structures as they are cleaned		* This routine will free swap metadata structures as they are cleaned
* out. This routine does NOT operate on swap metadata associated		* out.
* with resident pages.
*/		*/
static void		static void
swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)		swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject,
		vm_pindex_t pindex, vm_pindex_t count)
{		{
struct swblk *sb;		struct swblk *sb;
daddr_t n_free, s_free;		daddr_t n_free, s_free;
vm_pindex_t last;		vm_pindex_t offset, last;
int i, limit, start;		int i, limit, start;

VM_OBJECT_ASSERT_WLOCKED(object);		VM_OBJECT_ASSERT_WLOCKED(srcobject);
if (object->type != OBJT_SWAP \|\| count == 0)		if (srcobject->type != OBJT_SWAP \|\| count == 0)
return;		return;

swp_pager_init_freerange(&s_free, &n_free);		swp_pager_init_freerange(&s_free, &n_free);
		offset = pindex;
last = pindex + count;		last = pindex + count;
for (;;) {		for (;;) {
sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,		sb = SWAP_PCTRIE_LOOKUP_GE(&srcobject->un_pager.swp.swp_blks,
rounddown(pindex, SWAP_META_PAGES));		rounddown(pindex, SWAP_META_PAGES));
if (sb == NULL \|\| sb->p >= last)		if (sb == NULL \|\| sb->p >= last)
break;		break;
start = pindex > sb->p ? pindex - sb->p : 0;		start = pindex > sb->p ? pindex - sb->p : 0;
limit = last - sb->p < SWAP_META_PAGES ? last - sb->p :		limit = last - sb->p < SWAP_META_PAGES ? last - sb->p :
SWAP_META_PAGES;		SWAP_META_PAGES;
for (i = start; i < limit; i++) {		for (i = start; i < limit; i++) {
if (sb->d[i] == SWAPBLK_NONE)		if (sb->d[i] == SWAPBLK_NONE)
continue;		continue;
swp_pager_update_freerange(&s_free, &n_free, sb->d[i]);		if (dstobject == NULL \|\|
		!swp_pager_xfer_source(srcobject, dstobject,
		sb->p + i - offset, sb->d[i])) {
		swp_pager_update_freerange(&s_free, &n_free,
		sb->d[i]);
		}
sb->d[i] = SWAPBLK_NONE;		sb->d[i] = SWAPBLK_NONE;
}		}
pindex = sb->p + SWAP_META_PAGES;		pindex = sb->p + SWAP_META_PAGES;
if (swp_pager_swblk_empty(sb, 0, start) &&		if (swp_pager_swblk_empty(sb, 0, start) &&
swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {		swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,		SWAP_PCTRIE_REMOVE(&srcobject->un_pager.swp.swp_blks,
sb->p);		sb->p);
uma_zfree(swblk_zone, sb);		uma_zfree(swblk_zone, sb);
}		}
}		}
swp_pager_freeswapspace(s_free, n_free);		swp_pager_freeswapspace(s_free, n_free);
		}

		/*
		* SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
		*
		* The requested range of blocks is freed, with any associated swap
		* returned to the swap bitmap.
		*
		* This routine will free swap metadata structures as they are cleaned
		* out. This routine does NOT operate on swap metadata associated
		* with resident pages.
		*/
		static void
		swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
		{
		swp_pager_meta_transfer(object, NULL, pindex, count);
}		}

/*		/*
* SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object		* SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
*		*
* This routine locates and destroys all swap metadata associated with		* This routine locates and destroys all swap metadata associated with
* an object.		* an object.
*/		*/
▲ Show 20 Lines • Show All 962 Lines • Show Last 20 Lines