Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_pageout.c
Show First 20 Lines • Show All 1,189 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
vm_pageout_scan_active(struct vm_domain *vmd, int page_shortage) | vm_pageout_scan_active(struct vm_domain *vmd, int page_shortage) | ||||
{ | { | ||||
struct scan_state ss; | struct scan_state ss; | ||||
struct mtx *mtx; | struct mtx *mtx; | ||||
vm_object_t object; | vm_object_t object; | ||||
vm_page_t m, marker; | vm_page_t m, marker; | ||||
struct vm_pagequeue *pq; | struct vm_pagequeue *pq; | ||||
vm_page_astate_t old, new; | |||||
long min_scan; | long min_scan; | ||||
int act_delta, max_scan, scan_tick; | int act_delta, max_scan, ps_delta, refs, scan_tick; | ||||
uint8_t nqueue; | |||||
marker = &vmd->vmd_markers[PQ_ACTIVE]; | marker = &vmd->vmd_markers[PQ_ACTIVE]; | ||||
pq = &vmd->vmd_pagequeues[PQ_ACTIVE]; | pq = &vmd->vmd_pagequeues[PQ_ACTIVE]; | ||||
vm_pagequeue_lock(pq); | vm_pagequeue_lock(pq); | ||||
/* | /* | ||||
* If we're just idle polling attempt to visit every | * If we're just idle polling attempt to visit every | ||||
* active page within 'update_period' seconds. | * active page within 'update_period' seconds. | ||||
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | while ((m = vm_pageout_next(&ss, false)) != NULL) { | ||||
*/ | */ | ||||
object = (vm_object_t)atomic_load_ptr(&m->object); | object = (vm_object_t)atomic_load_ptr(&m->object); | ||||
if (__predict_false(object == NULL)) | if (__predict_false(object == NULL)) | ||||
/* | /* | ||||
* The page has been removed from its object. | * The page has been removed from its object. | ||||
*/ | */ | ||||
continue; | continue; | ||||
/* Deferred free of swap space. */ | |||||
if ((m->a.flags & PGA_SWAP_FREE) != 0 && | |||||
VM_OBJECT_TRYWLOCK(object)) { | |||||
if (m->object == object) | |||||
vm_pager_page_unswapped(m); | |||||
VM_OBJECT_WUNLOCK(object); | |||||
} | |||||
/* | /* | ||||
* Check to see "how much" the page has been used. | * Check to see "how much" the page has been used. | ||||
* | * | ||||
* Test PGA_REFERENCED after calling pmap_ts_referenced() so | * Test PGA_REFERENCED after calling pmap_ts_referenced() so | ||||
* that a reference from a concurrently destroyed mapping is | * that a reference from a concurrently destroyed mapping is | ||||
* observed here and now. | * observed here and now. | ||||
* | * | ||||
* Perform an unsynchronized object ref count check. While | * Perform an unsynchronized object ref count check. While | ||||
* the page lock ensures that the page is not reallocated to | * the page lock ensures that the page is not reallocated to | ||||
* another object, in particular, one with unmanaged mappings | * another object, in particular, one with unmanaged mappings | ||||
* that cannot support pmap_ts_referenced(), two races are, | * that cannot support pmap_ts_referenced(), two races are, | ||||
* nonetheless, possible: | * nonetheless, possible: | ||||
* 1) The count was transitioning to zero, but we saw a non- | * 1) The count was transitioning to zero, but we saw a non- | ||||
* zero value. pmap_ts_referenced() will return zero | * zero value. pmap_ts_referenced() will return zero | ||||
* because the page is not mapped. | * because the page is not mapped. | ||||
* 2) The count was transitioning to one, but we saw zero. | * 2) The count was transitioning to one, but we saw zero. | ||||
* This race delays the detection of a new reference. At | * This race delays the detection of a new reference. At | ||||
* worst, we will deactivate and reactivate the page. | * worst, we will deactivate and reactivate the page. | ||||
*/ | */ | ||||
if (object->ref_count != 0) | refs = object->ref_count != 0 ? pmap_ts_referenced(m) : 0; | ||||
act_delta = pmap_ts_referenced(m); | |||||
else | old = vm_page_astate_load(m); | ||||
act_delta = 0; | do { | ||||
if ((m->a.flags & PGA_REFERENCED) != 0) { | if (old.queue == PQ_NONE || | ||||
vm_page_aflag_clear(m, PGA_REFERENCED); | (old.flags & PGA_DEQUEUE) != 0) { | ||||
act_delta++; | /* | ||||
* Something has moved the page out of the | |||||
* page queues. Don't touch it. | |||||
*/ | |||||
break; | |||||
} | } | ||||
/* Deferred free of swap space. */ | new = old; | ||||
if ((m->a.flags & PGA_SWAP_FREE) != 0 && | act_delta = refs; | ||||
VM_OBJECT_TRYWLOCK(object)) { | if ((old.flags & PGA_REFERENCED) != 0) { | ||||
if (m->object == object) | new.flags &= ~PGA_REFERENCED; | ||||
vm_pager_page_unswapped(m); | act_delta++; | ||||
VM_OBJECT_WUNLOCK(object); | |||||
} | } | ||||
/* | /* | ||||
* Advance or decay the act_count based on recent usage. | * Advance or decay the act_count based on recent usage. | ||||
*/ | */ | ||||
if (act_delta != 0) { | if (act_delta != 0) { | ||||
m->a.act_count += ACT_ADVANCE + act_delta; | new.act_count += ACT_ADVANCE + act_delta; | ||||
if (m->a.act_count > ACT_MAX) | if (new.act_count > ACT_MAX) | ||||
m->a.act_count = ACT_MAX; | new.act_count = ACT_MAX; | ||||
} else | } else { | ||||
m->a.act_count -= min(m->a.act_count, ACT_DECLINE); | new.act_count -= min(new.act_count, | ||||
ACT_DECLINE); | |||||
} | |||||
if (m->a.act_count == 0) { | if (new.act_count > 0) { | ||||
/* | /* | ||||
* When not short for inactive pages, let dirty pages go | * Adjust the activation count and keep the page | ||||
* through the inactive queue before moving to the | * in the active queue. The count might be left | ||||
* laundry queues. This gives them some extra time to | * unchanged if it is saturated. The page may | ||||
* be reactivated, potentially avoiding an expensive | * have been moved to a different queue since we | ||||
* pageout. However, during a page shortage, the | * started the scan, in which case we move it | ||||
* inactive queue is necessarily small, and so dirty | * back. | ||||
* pages would only spend a trivial amount of time in | |||||
* the inactive queue. Therefore, we might as well | |||||
* place them directly in the laundry queue to reduce | |||||
* queuing overhead. | |||||
*/ | */ | ||||
if (page_shortage <= 0) { | ps_delta = 0; | ||||
vm_page_swapqueue(m, PQ_ACTIVE, PQ_INACTIVE); | if (old.queue != PQ_ACTIVE) { | ||||
old.queue = PQ_ACTIVE; | |||||
old.flags |= PGA_REQUEUE; | |||||
} | |||||
} else { | } else { | ||||
/* | /* | ||||
* When not short for inactive pages, let dirty | |||||
* pages go through the inactive queue before | |||||
* moving to the laundry queue. This gives them | |||||
* some extra time to be reactivated, | |||||
* potentially avoiding an expensive pageout. | |||||
* However, during a page shortage, the inactive | |||||
* queue is necessarily small, and so dirty | |||||
* pages would only spend a trivial amount of | |||||
* time in the inactive queue. Therefore, we | |||||
* might as well place them directly in the | |||||
* laundry queue to reduce queuing overhead. | |||||
* | |||||
* Calling vm_page_test_dirty() here would | * Calling vm_page_test_dirty() here would | ||||
* require acquisition of the object's write | * require acquisition of the object's write | ||||
* lock. However, during a page shortage, | * lock. However, during a page shortage, | ||||
* directing dirty pages into the laundry | * directing dirty pages into the laundry queue | ||||
* queue is only an optimization and not a | * is only an optimization and not a | ||||
* requirement. Therefore, we simply rely on | * requirement. Therefore, we simply rely on | ||||
* the opportunistic updates to the page's | * the opportunistic updates to the page's dirty | ||||
* dirty field by the pmap. | * field by the pmap. | ||||
*/ | */ | ||||
if (m->dirty == 0) { | if (page_shortage <= 0) { | ||||
vm_page_swapqueue(m, PQ_ACTIVE, | nqueue = PQ_INACTIVE; | ||||
PQ_INACTIVE); | ps_delta = 0; | ||||
page_shortage -= | } else if (m->dirty == 0) { | ||||
act_scan_laundry_weight; | nqueue = PQ_INACTIVE; | ||||
ps_delta = act_scan_laundry_weight; | |||||
} else { | } else { | ||||
vm_page_swapqueue(m, PQ_ACTIVE, | nqueue = PQ_LAUNDRY; | ||||
PQ_LAUNDRY); | ps_delta = 1; | ||||
page_shortage--; | |||||
} | } | ||||
new.flags |= PGA_REQUEUE; | |||||
new.queue = nqueue; | |||||
} | } | ||||
} | } while (!vm_page_pqstate_commit(m, &old, new)); | ||||
page_shortage -= ps_delta; | |||||
} | } | ||||
if (mtx != NULL) { | if (mtx != NULL) { | ||||
mtx_unlock(mtx); | mtx_unlock(mtx); | ||||
mtx = NULL; | mtx = NULL; | ||||
} | } | ||||
vm_pagequeue_lock(pq); | vm_pagequeue_lock(pq); | ||||
TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q); | TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q); | ||||
TAILQ_INSERT_AFTER(&pq->pq_pl, marker, &vmd->vmd_clock[0], plinks.q); | TAILQ_INSERT_AFTER(&pq->pq_pl, marker, &vmd->vmd_clock[0], plinks.q); | ||||
▲ Show 20 Lines • Show All 903 Lines • Show Last 20 Lines |