Differential D22882 Diff 66066 head/sys/vm/vm_pageout.c

Changeset View

Standalone View

head/sys/vm/vm_pageout.c

Show First 20 Lines • Show All 312 Lines • ▼ Show 20 Lines
{		{

if (ss->bq.bq_cnt == 0)		if (ss->bq.bq_cnt == 0)
vm_pageout_collect_batch(ss, dequeue);		vm_pageout_collect_batch(ss, dequeue);
return (vm_batchqueue_pop(&ss->bq));		return (vm_batchqueue_pop(&ss->bq));
}		}

/*		/*
		* Determine whether processing of a page should be deferred and ensure that any
		* outstanding queue operations are processed.
		*/
		static __always_inline bool
		vm_pageout_defer(vm_page_t m, const uint8_t queue, const bool enqueued)
		{
		vm_page_astate_t as;

		as = vm_page_astate_load(m);
		if (__predict_false(as.queue != queue \|\|
		((as.flags & PGA_ENQUEUED) != 0) != enqueued))
		return (true);
		if ((as.flags & PGA_QUEUE_OP_MASK) != 0) {
		vm_page_pqbatch_submit(m, queue);
		return (true);
		}
		return (false);
		}

		/*
* Scan for pages at adjacent offsets within the given page's object that are		* Scan for pages at adjacent offsets within the given page's object that are
* eligible for laundering, form a cluster of these pages and the given page,		* eligible for laundering, form a cluster of these pages and the given page,
* and launder that cluster.		* and launder that cluster.
*/		*/
static int		static int
vm_pageout_cluster(vm_page_t m)		vm_pageout_cluster(vm_page_t m)
{		{
vm_object_t object;		vm_object_t object;
▲ Show 20 Lines • Show All 421 Lines • ▼ Show 20 Lines	scan:
while (launder > 0 && (m = vm_pageout_next(&ss, false)) != NULL) {		while (launder > 0 && (m = vm_pageout_next(&ss, false)) != NULL) {
if (__predict_false((m->flags & PG_MARKER) != 0))		if (__predict_false((m->flags & PG_MARKER) != 0))
continue;		continue;

vm_page_change_lock(m, &mtx);		vm_page_change_lock(m, &mtx);

recheck:		recheck:
/*		/*
* The page may have been disassociated from the queue		* Don't touch a page that was removed from the queue after the
* or even freed while locks were dropped. We thus must be		* page queue lock was released. Otherwise, ensure that any
* careful whenever modifying page state. Once the object lock		* pending queue operations, such as dequeues for wired pages,
* has been acquired, we have a stable reference to the page.		* are handled.
*/		*/
if (vm_page_queue(m) != queue)		if (vm_pageout_defer(m, queue, true))
continue;		continue;

/*
* A requeue was requested, so this page gets a second
* chance.
*/
if ((m->a.flags & PGA_REQUEUE) != 0) {
vm_page_pqbatch_submit(m, queue);
continue;
}

/*
* Wired pages may not be freed. Complete their removal
* from the queue now to avoid needless revisits during
* future scans. This check is racy and must be reverified once
* we hold the object lock and have verified that the page
* is not busy.
*/
if (vm_page_wired(m)) {
vm_page_dequeue_deferred(m);
continue;
}

if (object != m->object) {		if (object != m->object) {
if (object != NULL)		if (object != NULL)
VM_OBJECT_WUNLOCK(object);		VM_OBJECT_WUNLOCK(object);

/*		/*
* A page's object pointer may be set to NULL before		* A page's object pointer may be set to NULL before
* the object lock is acquired.		* the object lock is acquired.
*/		*/
Show All 13 Lines	if (__predict_false(m->object == NULL))
continue;		continue;
KASSERT(m->object == object, ("page %p does not belong to %p",		KASSERT(m->object == object, ("page %p does not belong to %p",
m, object));		m, object));

if (vm_page_tryxbusy(m) == 0)		if (vm_page_tryxbusy(m) == 0)
continue;		continue;

/*		/*
* Re-check for wirings now that we hold the object lock and		* Check for wirings now that we hold the object lock and have
* have verified that the page is unbusied. If the page is		* verified that the page is unbusied. If the page is mapped,
* mapped, it may still be wired by pmap lookups. The call to		* it may still be wired by pmap lookups. The call to
* vm_page_try_remove_all() below atomically checks for such		* vm_page_try_remove_all() below atomically checks for such
* wirings and removes mappings. If the page is unmapped, the		* wirings and removes mappings. If the page is unmapped, the
* wire count is guaranteed not to increase.		* wire count is guaranteed not to increase.
*/		*/
if (__predict_false(vm_page_wired(m))) {		if (__predict_false(vm_page_wired(m))) {
vm_page_dequeue_deferred(m);		vm_page_dequeue_deferred(m);
goto skip_page;		goto skip_page;
}		}
▲ Show 20 Lines • Show All 427 Lines • ▼ Show 20 Lines	if (__predict_false(m == &vmd->vmd_clock[1])) {
goto act_scan;		goto act_scan;
}		}
if (__predict_false((m->flags & PG_MARKER) != 0))		if (__predict_false((m->flags & PG_MARKER) != 0))
continue;		continue;

vm_page_change_lock(m, &mtx);		vm_page_change_lock(m, &mtx);

/*		/*
* The page may have been disassociated from the queue		* Don't touch a page that was removed from the queue after the
* or even freed while locks were dropped. We thus must be		* page queue lock was released. Otherwise, ensure that any
* careful whenever modifying page state. Once the object lock		* pending queue operations, such as dequeues for wired pages,
* has been acquired, we have a stable reference to the page.		* are handled.
*/		*/
if (vm_page_queue(m) != PQ_ACTIVE)		if (vm_pageout_defer(m, PQ_ACTIVE, true))
continue;		continue;

/*		/*
* Wired pages are dequeued lazily.
*/
if (vm_page_wired(m)) {
vm_page_dequeue_deferred(m);
continue;
}

/*
* A page's object pointer may be set to NULL before		* A page's object pointer may be set to NULL before
* the object lock is acquired.		* the object lock is acquired.
*/		*/
object = (vm_object_t)atomic_load_ptr(&m->object);		object = (vm_object_t)atomic_load_ptr(&m->object);
if (__predict_false(object == NULL))		if (__predict_false(object == NULL))
/*		/*
* The page has been removed from its object.		* The page has been removed from its object.
*/		*/
▲ Show 20 Lines • Show All 223 Lines • ▼ Show 20 Lines	vm_pageout_scan_inactive(struct vm_domain *vmd, int shortage,
while (page_shortage > 0 && (m = vm_pageout_next(&ss, true)) != NULL) {		while (page_shortage > 0 && (m = vm_pageout_next(&ss, true)) != NULL) {
KASSERT((m->flags & PG_MARKER) == 0,		KASSERT((m->flags & PG_MARKER) == 0,
("marker page %p was dequeued", m));		("marker page %p was dequeued", m));

vm_page_change_lock(m, &mtx);		vm_page_change_lock(m, &mtx);

recheck:		recheck:
/*		/*
* The page may have been disassociated from the queue		* Don't touch a page that was removed from the queue after the
* or even freed while locks were dropped. We thus must be		* page queue lock was released. Otherwise, ensure that any
* careful whenever modifying page state. Once the object lock		* pending queue operations, such as dequeues for wired pages,
* has been acquired, we have a stable reference to the page.		* are handled.
*/		*/
old = vm_page_astate_load(m);		if (vm_pageout_defer(m, PQ_INACTIVE, false))
if (old.queue != PQ_INACTIVE \|\|
(old.flags & PGA_QUEUE_STATE_MASK) != 0)
continue;		continue;

/*
* Wired pages may not be freed. Complete their removal
* from the queue now to avoid needless revisits during
* future scans. This check is racy and must be reverified once
* we hold the object lock and have verified that the page
* is not busy.
*/
if (vm_page_wired(m)) {
vm_page_dequeue_deferred(m);
continue;
}

if (object != m->object) {		if (object != m->object) {
if (object != NULL)		if (object != NULL)
VM_OBJECT_WUNLOCK(object);		VM_OBJECT_WUNLOCK(object);

/*		/*
* A page's object pointer may be set to NULL before		* A page's object pointer may be set to NULL before
* the object lock is acquired.		* the object lock is acquired.
▲ Show 20 Lines • Show All 762 Lines • Show Last 20 Lines