Page MenuHomeFreeBSD
Differential D22885 Diff 66067 head/sys/vm/vm_pageout.c

Changeset View

Standalone View

View Options

head/sys/vm/vm_pageout.c

Show First 20 Lines • Show All 390 Lines • ▼ Show 20 Lines if (vm_page_wired(p)) {
break; break;
} }
vm_page_test_dirty(p); vm_page_test_dirty(p);
if (p->dirty == 0) { if (p->dirty == 0) {
ib = 0; ib = 0;
vm_page_xunbusy(p); vm_page_xunbusy(p);
break; break;
} }
vm_page_lock(p);
if (!vm_page_in_laundry(p) || !vm_page_try_remove_write(p)) { if (!vm_page_in_laundry(p) || !vm_page_try_remove_write(p)) {
vm_page_unlock(p);
vm_page_xunbusy(p); vm_page_xunbusy(p);
ib = 0; ib = 0;
break; break;
} }
vm_page_unlock(p);
mc[--page_base] = pb = p; mc[--page_base] = pb = p;
++pageout_count; ++pageout_count;
++ib; ++ib;
/* /*
* We are at an alignment boundary. Stop here, and switch * We are at an alignment boundary. Stop here, and switch
* directions. Do not clear ib. * directions. Do not clear ib.
*/ */
Show All 9 Lines if (vm_page_wired(p)) {
vm_page_xunbusy(p); vm_page_xunbusy(p);
break; break;
} }
vm_page_test_dirty(p); vm_page_test_dirty(p);
if (p->dirty == 0) { if (p->dirty == 0) {
vm_page_xunbusy(p); vm_page_xunbusy(p);
break; break;
} }
vm_page_lock(p);
if (!vm_page_in_laundry(p) || !vm_page_try_remove_write(p)) { if (!vm_page_in_laundry(p) || !vm_page_try_remove_write(p)) {
vm_page_unlock(p);
vm_page_xunbusy(p); vm_page_xunbusy(p);
break; break;
} }
vm_page_unlock(p);
mc[page_base + pageout_count] = ps = p; mc[page_base + pageout_count] = ps = p;
++pageout_count; ++pageout_count;
++is; ++is;
} }
/* /*
* If we exhausted our forward scan, continue with the reverse scan * If we exhausted our forward scan, continue with the reverse scan
* when possible, even past an alignment boundary. This catches * when possible, even past an alignment boundary. This catches
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Linesvm_pageout_flush(vm_page_t *mc, int count, int flags, int mreq, int *prunlen,
for (i = 0; i < count; i++) { for (i = 0; i < count; i++) {
vm_page_t mt = mc[i]; vm_page_t mt = mc[i];
KASSERT(pageout_status[i] == VM_PAGER_PEND || KASSERT(pageout_status[i] == VM_PAGER_PEND ||
!pmap_page_is_write_mapped(mt), !pmap_page_is_write_mapped(mt),
("vm_pageout_flush: page %p is not write protected", mt)); ("vm_pageout_flush: page %p is not write protected", mt));
switch (pageout_status[i]) { switch (pageout_status[i]) {
case VM_PAGER_OK: case VM_PAGER_OK:
vm_page_lock(mt); /*
* The page may have moved since laundering started, in
* which case it should be left alone.
*/
if (vm_page_in_laundry(mt)) if (vm_page_in_laundry(mt))
vm_page_deactivate_noreuse(mt); vm_page_deactivate_noreuse(mt);
vm_page_unlock(mt);
/* FALLTHROUGH */ /* FALLTHROUGH */
case VM_PAGER_PEND: case VM_PAGER_PEND:
numpagedout++; numpagedout++;
break; break;
case VM_PAGER_BAD: case VM_PAGER_BAD:
/* /*
* The page is outside the object's range. We pretend * The page is outside the object's range. We pretend
* that the page out worked and clean the page, so the * that the page out worked and clean the page, so the
* changes will be lost if the page is reclaimed by * changes will be lost if the page is reclaimed by
* the page daemon. * the page daemon.
*/ */
vm_page_undirty(mt); vm_page_undirty(mt);
vm_page_lock(mt);
if (vm_page_in_laundry(mt)) if (vm_page_in_laundry(mt))
vm_page_deactivate_noreuse(mt); vm_page_deactivate_noreuse(mt);
vm_page_unlock(mt);
break; break;
case VM_PAGER_ERROR: case VM_PAGER_ERROR:
case VM_PAGER_FAIL: case VM_PAGER_FAIL:
/* /*
* If the page couldn't be paged out to swap because the * If the page couldn't be paged out to swap because the
* pager wasn't able to find space, place the page in * pager wasn't able to find space, place the page in
* the PQ_UNSWAPPABLE holding queue. This is an * the PQ_UNSWAPPABLE holding queue. This is an
* optimization that prevents the page daemon from * optimization that prevents the page daemon from
* wasting CPU cycles on pages that cannot be reclaimed * wasting CPU cycles on pages that cannot be reclaimed
* becase no swap device is configured. * becase no swap device is configured.
* *
* Otherwise, reactivate the page so that it doesn't * Otherwise, reactivate the page so that it doesn't
* clog the laundry and inactive queues. (We will try * clog the laundry and inactive queues. (We will try
* paging it out again later.) * paging it out again later.)
*/ */
vm_page_lock(mt);
if (object->type == OBJT_SWAP && if (object->type == OBJT_SWAP &&
pageout_status[i] == VM_PAGER_FAIL) { pageout_status[i] == VM_PAGER_FAIL) {
vm_page_unswappable(mt); vm_page_unswappable(mt);
numpagedout++; numpagedout++;
} else } else
vm_page_activate(mt); vm_page_activate(mt);
vm_page_unlock(mt);
if (eio != NULL && i >= mreq && i - mreq < runlen) if (eio != NULL && i >= mreq && i - mreq < runlen)
*eio = TRUE; *eio = TRUE;
break; break;
case VM_PAGER_AGAIN: case VM_PAGER_AGAIN:
if (i >= mreq && i - mreq < runlen) if (i >= mreq && i - mreq < runlen)
runlen = i - mreq; runlen = i - mreq;
break; break;
} }
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines
vm_pageout_clean(vm_page_t m, int *numpagedout) vm_pageout_clean(vm_page_t m, int *numpagedout)
{ {
struct vnode *vp; struct vnode *vp;
struct mount *mp; struct mount *mp;
vm_object_t object; vm_object_t object;
vm_pindex_t pindex; vm_pindex_t pindex;
int error, lockmode; int error, lockmode;
vm_page_assert_locked(m);
object = m->object; object = m->object;
VM_OBJECT_ASSERT_WLOCKED(object); VM_OBJECT_ASSERT_WLOCKED(object);
error = 0; error = 0;
vp = NULL; vp = NULL;
mp = NULL; mp = NULL;
/* /*
* The object is already known NOT to be dead. It * The object is already known NOT to be dead. It
* is possible for the vget() to block the whole * is possible for the vget() to block the whole
* pageout daemon, but the new low-memory handling * pageout daemon, but the new low-memory handling
* code should prevent it. * code should prevent it.
* *
* We can't wait forever for the vnode lock, we might * We can't wait forever for the vnode lock, we might
* deadlock due to a vn_read() getting stuck in * deadlock due to a vn_read() getting stuck in
* vm_wait while holding this vnode. We skip the * vm_wait while holding this vnode. We skip the
* vnode if we can't get it in a reasonable amount * vnode if we can't get it in a reasonable amount
* of time. * of time.
*/ */
if (object->type == OBJT_VNODE) { if (object->type == OBJT_VNODE) {
vm_page_unlock(m);
vm_page_xunbusy(m); vm_page_xunbusy(m);
vp = object->handle; vp = object->handle;
if (vp->v_type == VREG && if (vp->v_type == VREG &&
vn_start_write(vp, &mp, V_NOWAIT) != 0) { vn_start_write(vp, &mp, V_NOWAIT) != 0) {
mp = NULL; mp = NULL;
error = EDEADLK; error = EDEADLK;
goto unlock_all; goto unlock_all;
} }
Show All 14 Lines if (object->type == OBJT_VNODE) {
/* /*
* Ensure that the object and vnode were not disassociated * Ensure that the object and vnode were not disassociated
* while locks were dropped. * while locks were dropped.
*/ */
if (vp->v_object != object) { if (vp->v_object != object) {
error = ENOENT; error = ENOENT;
goto unlock_all; goto unlock_all;
} }
vm_page_lock(m);
/* /*
* While the object and page were unlocked, the page * While the object was unlocked, the page may have been:
* may have been:
* (1) moved to a different queue, * (1) moved to a different queue,
* (2) reallocated to a different object, * (2) reallocated to a different object,
* (3) reallocated to a different offset, or * (3) reallocated to a different offset, or
* (4) cleaned. * (4) cleaned.
*/ */
if (!vm_page_in_laundry(m) || m->object != object || if (!vm_page_in_laundry(m) || m->object != object ||
m->pindex != pindex || m->dirty == 0) { m->pindex != pindex || m->dirty == 0) {
vm_page_unlock(m);
error = ENXIO; error = ENXIO;
goto unlock_all; goto unlock_all;
} }
/* /*
* The page may have been busied while the object and page * The page may have been busied while the object lock was
* locks were released. * released.
*/ */
if (vm_page_tryxbusy(m) == 0) { if (vm_page_tryxbusy(m) == 0) {
vm_page_unlock(m);
error = EBUSY; error = EBUSY;
goto unlock_all; goto unlock_all;
} }
} }
/* /*
* Remove all writeable mappings, failing if the page is wired. * Remove all writeable mappings, failing if the page is wired.
*/ */
if (!vm_page_try_remove_write(m)) { if (!vm_page_try_remove_write(m)) {
vm_page_xunbusy(m); vm_page_xunbusy(m);
vm_page_unlock(m);
error = EBUSY; error = EBUSY;
goto unlock_all; goto unlock_all;
} }
vm_page_unlock(m);
/* /*
* If a page is dirty, then it is either being washed * If a page is dirty, then it is either being washed
* (but not yet cleaned) or it is still in the * (but not yet cleaned) or it is still in the
* laundry. If it is still in the laundry, then we * laundry. If it is still in the laundry, then we
* start the cleaning operation. * start the cleaning operation.
*/ */
if ((*numpagedout = vm_pageout_cluster(m)) == 0) if ((*numpagedout = vm_pageout_cluster(m)) == 0)
error = EIO; error = EIO;
unlock_all: unlock_all:
VM_OBJECT_WUNLOCK(object); VM_OBJECT_WUNLOCK(object);
unlock_mp: unlock_mp:
vm_page_lock_assert(m, MA_NOTOWNED);
if (mp != NULL) { if (mp != NULL) {
if (vp != NULL) if (vp != NULL)
vput(vp); vput(vp);
vm_object_deallocate(object); vm_object_deallocate(object);
vn_finished_write(mp); vn_finished_write(mp);
} }
return (error); return (error);
} }
/* /*
* Attempt to launder the specified number of pages. * Attempt to launder the specified number of pages.
* *
* Returns the number of pages successfully laundered. * Returns the number of pages successfully laundered.
*/ */
static int static int
vm_pageout_launder(struct vm_domain *vmd, int launder, bool in_shortfall) vm_pageout_launder(struct vm_domain *vmd, int launder, bool in_shortfall)
{ {
struct scan_state ss; struct scan_state ss;
struct vm_pagequeue *pq; struct vm_pagequeue *pq;
struct mtx *mtx;
vm_object_t object; vm_object_t object;
vm_page_t m, marker; vm_page_t m, marker;
vm_page_astate_t new, old; vm_page_astate_t new, old;
int act_delta, error, numpagedout, queue, refs, starting_target; int act_delta, error, numpagedout, queue, refs, starting_target;
int vnodes_skipped; int vnodes_skipped;
bool pageout_ok; bool pageout_ok;
mtx = NULL;
object = NULL; object = NULL;
starting_target = launder; starting_target = launder;
vnodes_skipped = 0; vnodes_skipped = 0;
/* /*
* Scan the laundry queues for pages eligible to be laundered. We stop * Scan the laundry queues for pages eligible to be laundered. We stop
* once the target number of dirty pages have been laundered, or once * once the target number of dirty pages have been laundered, or once
* we've reached the end of the queue. A single iteration of this loop * we've reached the end of the queue. A single iteration of this loop
Show All 11 Linesscan:
marker = &vmd->vmd_markers[queue]; marker = &vmd->vmd_markers[queue];
pq = &vmd->vmd_pagequeues[queue]; pq = &vmd->vmd_pagequeues[queue];
vm_pagequeue_lock(pq); vm_pagequeue_lock(pq);
vm_pageout_init_scan(&ss, pq, marker, NULL, pq->pq_cnt); vm_pageout_init_scan(&ss, pq, marker, NULL, pq->pq_cnt);
while (launder > 0 && (m = vm_pageout_next(&ss, false)) != NULL) { while (launder > 0 && (m = vm_pageout_next(&ss, false)) != NULL) {
if (__predict_false((m->flags & PG_MARKER) != 0)) if (__predict_false((m->flags & PG_MARKER) != 0))
continue; continue;
vm_page_change_lock(m, &mtx);
recheck:
/* /*
* Don't touch a page that was removed from the queue after the * Don't touch a page that was removed from the queue after the
* page queue lock was released. Otherwise, ensure that any * page queue lock was released. Otherwise, ensure that any
* pending queue operations, such as dequeues for wired pages, * pending queue operations, such as dequeues for wired pages,
* are handled. * are handled.
*/ */
if (vm_pageout_defer(m, queue, true)) if (vm_pageout_defer(m, queue, true))
continue; continue;
if (object != m->object) {
if (object != NULL)
VM_OBJECT_WUNLOCK(object);
/* /*
* A page's object pointer may be set to NULL before * Lock the page's object.
* the object lock is acquired.
*/ */
if (object == NULL || object != m->object) {
if (object != NULL)
VM_OBJECT_WUNLOCK(object);
object = (vm_object_t)atomic_load_ptr(&m->object); object = (vm_object_t)atomic_load_ptr(&m->object);
if (object != NULL && !VM_OBJECT_TRYWLOCK(object)) { if (__predict_false(object == NULL))
mtx_unlock(mtx); /* The page is being freed by another thread. */
continue;
/* Depends on type-stability. */ /* Depends on type-stability. */
VM_OBJECT_WLOCK(object); VM_OBJECT_WLOCK(object);
mtx_lock(mtx); if (__predict_false(m->object != object)) {
goto recheck; VM_OBJECT_WUNLOCK(object);
object = NULL;
continue;
} }
} }
if (__predict_false(m->object == NULL))
/*
* The page has been removed from its object.
*/
continue;
KASSERT(m->object == object, ("page %p does not belong to %p",
m, object));
if (vm_page_tryxbusy(m) == 0) if (vm_page_tryxbusy(m) == 0)
continue; continue;
/* /*
* Check for wirings now that we hold the object lock and have * Check for wirings now that we hold the object lock and have
* verified that the page is unbusied. If the page is mapped, * exclusively busied the page. If the page is mapped, it may
* it may still be wired by pmap lookups. The call to * still be wired by pmap lookups. The call to
* vm_page_try_remove_all() below atomically checks for such * vm_page_try_remove_all() below atomically checks for such
* wirings and removes mappings. If the page is unmapped, the * wirings and removes mappings. If the page is unmapped, the
* wire count is guaranteed not to increase. * wire count is guaranteed not to increase after this check.
*/ */
if (__predict_false(vm_page_wired(m))) { if (__predict_false(vm_page_wired(m)))
vm_page_dequeue_deferred(m);
goto skip_page; goto skip_page;
}
/* /*
* Invalid pages can be easily freed. They cannot be * Invalid pages can be easily freed. They cannot be
* mapped; vm_page_free() asserts this. * mapped; vm_page_free() asserts this.
*/ */
if (vm_page_none_valid(m)) if (vm_page_none_valid(m))
goto free_page; goto free_page;
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines while (launder > 0 && (m = vm_pageout_next(&ss, false)) != NULL) {
* If the page appears to be clean at the machine-independent * If the page appears to be clean at the machine-independent
* layer, then remove all of its mappings from the pmap in * layer, then remove all of its mappings from the pmap in
* anticipation of freeing it. If, however, any of the page's * anticipation of freeing it. If, however, any of the page's
* mappings allow write access, then the page may still be * mappings allow write access, then the page may still be
* modified until the last of those mappings are removed. * modified until the last of those mappings are removed.
*/ */
if (object->ref_count != 0) { if (object->ref_count != 0) {
vm_page_test_dirty(m); vm_page_test_dirty(m);
if (m->dirty == 0 && !vm_page_try_remove_all(m)) { if (m->dirty == 0 && !vm_page_try_remove_all(m))
vm_page_dequeue_deferred(m);
goto skip_page; goto skip_page;
} }
}
/* /*
* Clean pages are freed, and dirty pages are paged out unless * Clean pages are freed, and dirty pages are paged out unless
* they belong to a dead object. Requeueing dirty pages from * they belong to a dead object. Requeueing dirty pages from
* dead objects is pointless, as they are being paged out and * dead objects is pointless, as they are being paged out and
* freed by the thread that destroyed the object. * freed by the thread that destroyed the object.
*/ */
if (m->dirty == 0) { if (m->dirty == 0) {
free_page: free_page:
/*
* Now we are guaranteed that no other threads are
* manipulating the page, check for a last-second
* reference.
*/
if (vm_pageout_defer(m, queue, true))
goto skip_page;
vm_page_free(m); vm_page_free(m);
VM_CNT_INC(v_dfree); VM_CNT_INC(v_dfree);
} else if ((object->flags & OBJ_DEAD) == 0) { } else if ((object->flags & OBJ_DEAD) == 0) {
if (object->type != OBJT_SWAP && if (object->type != OBJT_SWAP &&
object->type != OBJT_DEFAULT) object->type != OBJT_DEFAULT)
pageout_ok = true; pageout_ok = true;
else if (disable_swap_pageouts) else if (disable_swap_pageouts)
pageout_ok = false; pageout_ok = false;
Show All 20 Linesfree_page:
error = vm_pageout_clean(m, &numpagedout); error = vm_pageout_clean(m, &numpagedout);
if (error == 0) { if (error == 0) {
launder -= numpagedout; launder -= numpagedout;
ss.scanned += numpagedout; ss.scanned += numpagedout;
} else if (error == EDEADLK) { } else if (error == EDEADLK) {
pageout_lock_miss++; pageout_lock_miss++;
vnodes_skipped++; vnodes_skipped++;
} }
mtx = NULL;
object = NULL; object = NULL;
} else { } else {
skip_page: skip_page:
vm_page_xunbusy(m); vm_page_xunbusy(m);
} }
} }
if (mtx != NULL) {
mtx_unlock(mtx);
mtx = NULL;
}
if (object != NULL) { if (object != NULL) {
VM_OBJECT_WUNLOCK(object); VM_OBJECT_WUNLOCK(object);
object = NULL; object = NULL;
} }
vm_pagequeue_lock(pq); vm_pagequeue_lock(pq);
vm_pageout_end_scan(&ss); vm_pageout_end_scan(&ss);
vm_pagequeue_unlock(pq); vm_pagequeue_unlock(pq);
▲ Show 20 Lines • Show All 220 Lines • ▼ Show 20 Lines
/* /*
* Scan the active queue. If there is no shortage of inactive pages, scan a * Scan the active queue. If there is no shortage of inactive pages, scan a
* small portion of the queue in order to maintain quasi-LRU. * small portion of the queue in order to maintain quasi-LRU.
*/ */
static void static void
vm_pageout_scan_active(struct vm_domain *vmd, int page_shortage) vm_pageout_scan_active(struct vm_domain *vmd, int page_shortage)
{ {
struct scan_state ss; struct scan_state ss;
struct mtx *mtx;
vm_object_t object; vm_object_t object;
vm_page_t m, marker; vm_page_t m, marker;
struct vm_pagequeue *pq; struct vm_pagequeue *pq;
vm_page_astate_t old, new; vm_page_astate_t old, new;
long min_scan; long min_scan;
int act_delta, max_scan, ps_delta, refs, scan_tick; int act_delta, max_scan, ps_delta, refs, scan_tick;
uint8_t nqueue; uint8_t nqueue;
Show All 24 Linesvm_pageout_scan_active(struct vm_domain *vmd, int page_shortage)
* implement the CLOCK algorithm. To keep the implementation of the * implement the CLOCK algorithm. To keep the implementation of the
* enqueue operation consistent for all page queues, we use two hands, * enqueue operation consistent for all page queues, we use two hands,
* represented by marker pages. Scans begin at the first hand, which * represented by marker pages. Scans begin at the first hand, which
* precedes the second hand in the queue. When the two hands meet, * precedes the second hand in the queue. When the two hands meet,
* they are moved back to the head and tail of the queue, respectively, * they are moved back to the head and tail of the queue, respectively,
* and scanning resumes. * and scanning resumes.
*/ */
max_scan = page_shortage > 0 ? pq->pq_cnt : min_scan; max_scan = page_shortage > 0 ? pq->pq_cnt : min_scan;
mtx = NULL;
act_scan: act_scan:
vm_pageout_init_scan(&ss, pq, marker, &vmd->vmd_clock[0], max_scan); vm_pageout_init_scan(&ss, pq, marker, &vmd->vmd_clock[0], max_scan);
while ((m = vm_pageout_next(&ss, false)) != NULL) { while ((m = vm_pageout_next(&ss, false)) != NULL) {
if (__predict_false(m == &vmd->vmd_clock[1])) { if (__predict_false(m == &vmd->vmd_clock[1])) {
vm_pagequeue_lock(pq); vm_pagequeue_lock(pq);
TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q); TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q);
TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[1], plinks.q); TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[1], plinks.q);
TAILQ_INSERT_HEAD(&pq->pq_pl, &vmd->vmd_clock[0], TAILQ_INSERT_HEAD(&pq->pq_pl, &vmd->vmd_clock[0],
plinks.q); plinks.q);
TAILQ_INSERT_TAIL(&pq->pq_pl, &vmd->vmd_clock[1], TAILQ_INSERT_TAIL(&pq->pq_pl, &vmd->vmd_clock[1],
plinks.q); plinks.q);
max_scan -= ss.scanned; max_scan -= ss.scanned;
vm_pageout_end_scan(&ss); vm_pageout_end_scan(&ss);
goto act_scan; goto act_scan;
} }
if (__predict_false((m->flags & PG_MARKER) != 0)) if (__predict_false((m->flags & PG_MARKER) != 0))
continue; continue;
vm_page_change_lock(m, &mtx);
/* /*
* Don't touch a page that was removed from the queue after the * Don't touch a page that was removed from the queue after the
* page queue lock was released. Otherwise, ensure that any * page queue lock was released. Otherwise, ensure that any
* pending queue operations, such as dequeues for wired pages, * pending queue operations, such as dequeues for wired pages,
* are handled. * are handled.
*/ */
if (vm_pageout_defer(m, PQ_ACTIVE, true)) if (vm_pageout_defer(m, PQ_ACTIVE, true))
continue; continue;
▲ Show 20 Lines • Show All 117 Lines • ▼ Show 20 Lines do {
new.flags |= PGA_REQUEUE; new.flags |= PGA_REQUEUE;
new.queue = nqueue; new.queue = nqueue;
} }
} while (!vm_page_pqstate_commit(m, &old, new)); } while (!vm_page_pqstate_commit(m, &old, new));
page_shortage -= ps_delta; page_shortage -= ps_delta;
} }
if (mtx != NULL) {
mtx_unlock(mtx);
mtx = NULL;
}
vm_pagequeue_lock(pq); vm_pagequeue_lock(pq);
TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q); TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q);
TAILQ_INSERT_AFTER(&pq->pq_pl, marker, &vmd->vmd_clock[0], plinks.q); TAILQ_INSERT_AFTER(&pq->pq_pl, marker, &vmd->vmd_clock[0], plinks.q);
vm_pageout_end_scan(&ss); vm_pageout_end_scan(&ss);
vm_pagequeue_unlock(pq); vm_pagequeue_unlock(pq);
} }
static int static int
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines
* Returns true if the shortage was addressed. * Returns true if the shortage was addressed.
*/ */
static int static int
vm_pageout_scan_inactive(struct vm_domain *vmd, int shortage, vm_pageout_scan_inactive(struct vm_domain *vmd, int shortage,
int *addl_shortage) int *addl_shortage)
{ {
struct scan_state ss; struct scan_state ss;
struct vm_batchqueue rq; struct vm_batchqueue rq;
struct mtx *mtx;
vm_page_t m, marker; vm_page_t m, marker;
struct vm_pagequeue *pq; struct vm_pagequeue *pq;
vm_object_t object; vm_object_t object;
vm_page_astate_t old, new; vm_page_astate_t old, new;
int act_delta, addl_page_shortage, deficit, page_shortage, refs; int act_delta, addl_page_shortage, deficit, page_shortage, refs;
int starting_page_shortage; int starting_page_shortage;
/* /*
* The addl_page_shortage is an estimate of the number of temporarily * The addl_page_shortage is an estimate of the number of temporarily
* stuck pages in the inactive queue. In other words, the * stuck pages in the inactive queue. In other words, the
* number of pages from the inactive count that should be * number of pages from the inactive count that should be
* discounted in setting the target for the active queue scan. * discounted in setting the target for the active queue scan.
*/ */
addl_page_shortage = 0; addl_page_shortage = 0;
/* /*
* vmd_pageout_deficit counts the number of pages requested in * vmd_pageout_deficit counts the number of pages requested in
* allocations that failed because of a free page shortage. We assume * allocations that failed because of a free page shortage. We assume
* that the allocations will be reattempted and thus include the deficit * that the allocations will be reattempted and thus include the deficit
* in our scan target. * in our scan target.
*/ */
deficit = atomic_readandclear_int(&vmd->vmd_pageout_deficit); deficit = atomic_readandclear_int(&vmd->vmd_pageout_deficit);
starting_page_shortage = page_shortage = shortage + deficit; starting_page_shortage = page_shortage = shortage + deficit;
mtx = NULL;
object = NULL; object = NULL;
vm_batchqueue_init(&rq); vm_batchqueue_init(&rq);
/* /*
* Start scanning the inactive queue for pages that we can free. The * Start scanning the inactive queue for pages that we can free. The
* scan will stop when we reach the target or we have scanned the * scan will stop when we reach the target or we have scanned the
* entire queue. (Note that m->a.act_count is not used to make * entire queue. (Note that m->a.act_count is not used to make
* decisions for the inactive queue, only for the active queue.) * decisions for the inactive queue, only for the active queue.)
*/ */
marker = &vmd->vmd_markers[PQ_INACTIVE]; marker = &vmd->vmd_markers[PQ_INACTIVE];
pq = &vmd->vmd_pagequeues[PQ_INACTIVE]; pq = &vmd->vmd_pagequeues[PQ_INACTIVE];
vm_pagequeue_lock(pq); vm_pagequeue_lock(pq);
vm_pageout_init_scan(&ss, pq, marker, NULL, pq->pq_cnt); vm_pageout_init_scan(&ss, pq, marker, NULL, pq->pq_cnt);
while (page_shortage > 0 && (m = vm_pageout_next(&ss, true)) != NULL) { while (page_shortage > 0 && (m = vm_pageout_next(&ss, true)) != NULL) {
KASSERT((m->flags & PG_MARKER) == 0, KASSERT((m->flags & PG_MARKER) == 0,
("marker page %p was dequeued", m)); ("marker page %p was dequeued", m));
vm_page_change_lock(m, &mtx);
recheck:
/* /*
* Don't touch a page that was removed from the queue after the * Don't touch a page that was removed from the queue after the
* page queue lock was released. Otherwise, ensure that any * page queue lock was released. Otherwise, ensure that any
* pending queue operations, such as dequeues for wired pages, * pending queue operations, such as dequeues for wired pages,
* are handled. * are handled.
*/ */
if (vm_pageout_defer(m, PQ_INACTIVE, false)) if (vm_pageout_defer(m, PQ_INACTIVE, false))
continue; continue;
if (object != m->object) {
if (object != NULL)
VM_OBJECT_WUNLOCK(object);
/* /*
* A page's object pointer may be set to NULL before * Lock the page's object.
* the object lock is acquired.
*/ */
if (object == NULL || object != m->object) {
if (object != NULL)
VM_OBJECT_WUNLOCK(object);
object = (vm_object_t)atomic_load_ptr(&m->object); object = (vm_object_t)atomic_load_ptr(&m->object);
if (object != NULL && !VM_OBJECT_TRYWLOCK(object)) { if (__predict_false(object == NULL))
mtx_unlock(mtx); /* The page is being freed by another thread. */
continue;
/* Depends on type-stability. */ /* Depends on type-stability. */
VM_OBJECT_WLOCK(object); VM_OBJECT_WLOCK(object);
mtx_lock(mtx); if (__predict_false(m->object != object)) {
goto recheck; VM_OBJECT_WUNLOCK(object);
object = NULL;
goto reinsert;
} }
} }
if (__predict_false(m->object == NULL))
/*
* The page has been removed from its object.
*/
continue;
KASSERT(m->object == object, ("page %p does not belong to %p",
m, object));
if (vm_page_tryxbusy(m) == 0) { if (vm_page_tryxbusy(m) == 0) {
/* /*
* Don't mess with busy pages. Leave them at * Don't mess with busy pages. Leave them at
* the front of the queue. Most likely, they * the front of the queue. Most likely, they
* are being paged out and will leave the * are being paged out and will leave the
* queue shortly after the scan finishes. So, * queue shortly after the scan finishes. So,
* they ought to be discounted from the * they ought to be discounted from the
* inactive count. * inactive count.
*/ */
addl_page_shortage++; addl_page_shortage++;
goto reinsert; goto reinsert;
} }
/* Deferred free of swap space. */ /* Deferred free of swap space. */
if ((m->a.flags & PGA_SWAP_FREE) != 0) if ((m->a.flags & PGA_SWAP_FREE) != 0)
vm_pager_page_unswapped(m); vm_pager_page_unswapped(m);
/* /*
* Re-check for wirings now that we hold the object lock and * Check for wirings now that we hold the object lock and have
* have verified that the page is unbusied. If the page is * exclusively busied the page. If the page is mapped, it may
* mapped, it may still be wired by pmap lookups. The call to * still be wired by pmap lookups. The call to
* vm_page_try_remove_all() below atomically checks for such * vm_page_try_remove_all() below atomically checks for such
* wirings and removes mappings. If the page is unmapped, the * wirings and removes mappings. If the page is unmapped, the
* wire count is guaranteed not to increase. * wire count is guaranteed not to increase after this check.
*/ */
if (__predict_false(vm_page_wired(m))) { if (__predict_false(vm_page_wired(m)))
vm_page_dequeue_deferred(m);
goto skip_page; goto skip_page;
}
/* /*
* Invalid pages can be easily freed. They cannot be * Invalid pages can be easily freed. They cannot be
* mapped, vm_page_free() asserts this. * mapped, vm_page_free() asserts this.
*/ */
if (vm_page_none_valid(m)) if (vm_page_none_valid(m))
goto free_page; goto free_page;
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines while (page_shortage > 0 && (m = vm_pageout_next(&ss, true)) != NULL) {
* If the page appears to be clean at the machine-independent * If the page appears to be clean at the machine-independent
* layer, then remove all of its mappings from the pmap in * layer, then remove all of its mappings from the pmap in
* anticipation of freeing it. If, however, any of the page's * anticipation of freeing it. If, however, any of the page's
* mappings allow write access, then the page may still be * mappings allow write access, then the page may still be
* modified until the last of those mappings are removed. * modified until the last of those mappings are removed.
*/ */
if (object->ref_count != 0) { if (object->ref_count != 0) {
vm_page_test_dirty(m); vm_page_test_dirty(m);
if (m->dirty == 0 && !vm_page_try_remove_all(m)) { if (m->dirty == 0 && !vm_page_try_remove_all(m))
vm_page_dequeue_deferred(m);
goto skip_page; goto skip_page;
} }
}
/* /*
* Clean pages can be freed, but dirty pages must be sent back * Clean pages can be freed, but dirty pages must be sent back
* to the laundry, unless they belong to a dead object. * to the laundry, unless they belong to a dead object.
* Requeueing dirty pages from dead objects is pointless, as * Requeueing dirty pages from dead objects is pointless, as
* they are being paged out and freed by the thread that * they are being paged out and freed by the thread that
* destroyed the object. * destroyed the object.
*/ */
if (m->dirty == 0) { if (m->dirty == 0) {
free_page: free_page:
/* /*
* Because we dequeued the page and have already * Now we are guaranteed that no other threads are
* checked for concurrent dequeue and enqueue * manipulating the page, check for a last-second
* requests, we can safely disassociate the page * reference that would save it from doom.
* from the inactive queue.
*/ */
KASSERT((m->a.flags & PGA_QUEUE_STATE_MASK) == 0, if (vm_pageout_defer(m, PQ_INACTIVE, false))
("page %p has queue state", m)); goto skip_page;
/*
* Because we dequeued the page and have already checked
* for pending dequeue and enqueue requests, we can
* safely disassociate the page from the inactive queue
* without holding the queue lock.
*/
m->a.queue = PQ_NONE; m->a.queue = PQ_NONE;
vm_page_free(m); vm_page_free(m);
page_shortage--; page_shortage--;
continue; continue;
} }
if ((object->flags & OBJ_DEAD) == 0) if ((object->flags & OBJ_DEAD) == 0)
vm_page_launder(m); vm_page_launder(m);
skip_page: skip_page:
vm_page_xunbusy(m); vm_page_xunbusy(m);
continue; continue;
reinsert: reinsert:
vm_pageout_reinsert_inactive(&ss, &rq, m); vm_pageout_reinsert_inactive(&ss, &rq, m);
} }
if (mtx != NULL)
mtx_unlock(mtx);
if (object != NULL) if (object != NULL)
VM_OBJECT_WUNLOCK(object); VM_OBJECT_WUNLOCK(object);
vm_pageout_reinsert_inactive(&ss, &rq, NULL); vm_pageout_reinsert_inactive(&ss, &rq, NULL);
vm_pageout_reinsert_inactive(&ss, &ss.bq, NULL); vm_pageout_reinsert_inactive(&ss, &ss.bq, NULL);
vm_pagequeue_lock(pq); vm_pagequeue_lock(pq);
vm_pageout_end_scan(&ss); vm_pageout_end_scan(&ss);
vm_pagequeue_unlock(pq); vm_pagequeue_unlock(pq);
▲ Show 20 Lines • Show All 602 Lines • Show Last 20 Lines