Index: vm/vm_pageout.c =================================================================== --- vm/vm_pageout.c +++ vm/vm_pageout.c @@ -123,7 +123,7 @@ static void vm_pageout_init(void); static int vm_pageout_clean(vm_page_t m, int *numpagedout); static int vm_pageout_cluster(vm_page_t m); -static bool vm_pageout_scan(struct vm_domain *vmd, int pass); +static bool vm_pageout_scan(struct vm_domain *vmd); static void vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage, int starting_page_shortage); @@ -1120,7 +1120,7 @@ * queue scan to meet the target. */ static bool -vm_pageout_scan(struct vm_domain *vmd, int pass) +vm_pageout_scan(struct vm_domain *vmd) { vm_page_t m, next; struct vm_pagequeue *pq; @@ -1131,10 +1131,30 @@ boolean_t queue_locked; /* + * The addl_page_shortage is the number of temporarily stuck pages in + * the inactive queue. In other words, the number of pages from the + * inactive count that should be discounted in setting the target for + * the active queue scan. + */ + addl_page_shortage = 0; + + /* + * Calculate the number of pages that we want to free. This number + * can be negative if many pages are freed between the wakeup call to + * the page daemon and this calculation. + */ + if (vm_paging_needed(vm_cnt.v_free_count)) { + deficit = atomic_readandclear_int(&vm_pageout_deficit); + page_shortage = vm_paging_target() + deficit; + } else + page_shortage = deficit = 0; + starting_page_shortage = page_shortage; + + /* * If we need to reclaim memory ask kernel caches to return * some. We rate limit to avoid thrashing. */ - if (vmd == &vm_dom[0] && pass > 0 && + if (vmd == &vm_dom[0] && page_shortage > 0 && (time_uptime - lowmem_uptime) >= lowmem_period) { /* * Decrease registered cache sizes. @@ -1141,6 +1161,7 @@ */ SDT_PROBE0(vm, , , vm__lowmem_scan); EVENTHANDLER_INVOKE(vm_lowmem, VM_LOW_PAGES); + /* * We do this explicitly after the caches have been * drained above. @@ -1150,26 +1171,6 @@ } /* - * The addl_page_shortage is the number of temporarily - * stuck pages in the inactive queue. In other words, the - * number of pages from the inactive count that should be - * discounted in setting the target for the active queue scan. - */ - addl_page_shortage = 0; - - /* - * Calculate the number of pages that we want to free. This number - * can be negative if many pages are freed between the wakeup call to - * the page daemon and this calculation. - */ - if (pass > 0) { - deficit = atomic_readandclear_int(&vm_pageout_deficit); - page_shortage = vm_paging_target() + deficit; - } else - page_shortage = deficit = 0; - starting_page_shortage = page_shortage; - - /* * Start scanning the inactive queue for pages that we can free. The * scan will stop when we reach the target or we have scanned the * entire queue. (Note that m->act_count is not used to make @@ -1527,7 +1528,7 @@ vm_page_unlock(m); } vm_pagequeue_unlock(pq); - if (pass > 0) + if (starting_page_shortage > 0) vm_swapout_run_idle(); return (page_shortage <= 0); } @@ -1757,12 +1758,11 @@ vm_pageout_worker(void *arg) { struct vm_domain *domain; - int domidx, pass; + int domidx; bool target_met; domidx = (uintptr_t)arg; domain = &vm_dom[domidx]; - pass = 0; target_met = true; /* @@ -1785,16 +1785,16 @@ mtx_lock(&vm_page_queue_free_mtx); /* - * Generally, after a level >= 1 scan, if there are enough - * free pages to wakeup the waiters, then they are already - * awake. A call to vm_page_free() during the scan awakened - * them. However, in the following case, this wakeup serves - * to bound the amount of time that a thread might wait. - * Suppose a thread's call to vm_page_alloc() fails, but - * before that thread calls VM_WAIT, enough pages are freed by - * other threads to alleviate the free page shortage. The - * thread will, nonetheless, wait until another page is freed - * or this wakeup is performed. + * Generally, if there are enough free pages to wakeup the + * waiters, then they are already awake. A call to + * vm_page_free() during the scan awakened them. However, in + * the following case, this wakeup serves to bound the amount + * of time that a thread might wait. Suppose that a thread's + * call to vm_page_alloc() fails, but before that thread calls + * VM_WAIT, enough pages are freed by other threads to + * alleviate the free page shortage. The thread will, + * nonetheless, wait until another page is freed or this + * wakeup is performed. */ if (vm_pages_needed && !vm_page_count_min()) { vm_pages_needed = false; @@ -1802,52 +1802,38 @@ } /* - * Do not clear vm_pageout_wanted until we reach our free page - * target. Otherwise, we may be awakened over and over again, - * wasting CPU time. + * What was the result of the scan? */ - if (vm_pageout_wanted && target_met) - vm_pageout_wanted = false; - - /* - * Might the page daemon receive a wakeup call? - */ - if (vm_pageout_wanted) { + if (!target_met) { /* - * No. Either vm_pageout_wanted was set by another - * thread during the previous scan, which must have - * been a level 0 scan, or vm_pageout_wanted was - * already set and the scan failed to free enough - * pages. If we haven't yet performed a level >= 1 - * (page reclamation) scan, then increase the level - * and scan again now. Otherwise, sleep a bit and - * try again later. + * The scan failed to free enough pages. Sleep a bit, + * and try again later. */ mtx_unlock(&vm_page_queue_free_mtx); - if (pass >= 1) - pause("pwait", hz / VM_INACT_SCAN_RATE); - pass++; + pause("pwait", hz / VM_INACT_SCAN_RATE); + } else if (vm_pages_needed) { + /* + * The scan succeeded in meeting its target. However, + * due to concurrent page allocations, there are still + * threads sleeping in VM_WAIT. Start a new scan + * immediately. + */ + mtx_unlock(&vm_page_queue_free_mtx); } else { /* - * Yes. If threads are still sleeping in VM_WAIT - * then we immediately start a new scan. Otherwise, - * sleep until the next wakeup or until pages need to - * have their reference stats updated. + * There are no threads sleeping in VM_WAIT. Sleep + * until the next wakeup or until pages need to have + * their reference stats updated. */ - if (vm_pages_needed) { - mtx_unlock(&vm_page_queue_free_mtx); - if (pass == 0) - pass++; - } else if (mtx_sleep(&vm_pageout_wanted, - &vm_page_queue_free_mtx, PDROP | PVM, "psleep", + vm_pageout_wanted = false; + if (mtx_sleep(&vm_pageout_wanted, + &vm_page_queue_free_mtx, PDROP | PVM, "psleep", hz) == 0) { VM_CNT_INC(v_pdwakeups); - pass = 1; - } else - pass = 0; + } } - target_met = vm_pageout_scan(domain, pass); + target_met = vm_pageout_scan(domain); } }