Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -2661,15 +2661,9 @@ msleep(&vm_pageout_pages_needed, &vm_page_queue_free_mtx, PDROP | PSWP, "VMWait", 0); } else { - if (__predict_false(pageproc == NULL)) + if (pageproc == NULL) panic("vm_wait in early boot"); - if (!vm_pageout_wanted) { - vm_pageout_wanted = true; - wakeup(&vm_pageout_wanted); - } - vm_pages_needed = true; - msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PVM, - "vmwait", 0); + pagedaemon_wait(PVM, "vmwait"); } } @@ -2699,7 +2693,6 @@ atomic_add_int(&vm_pageout_deficit, max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1)); - pagedaemon_wakeup(); if (req & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) { if (object != NULL) VM_OBJECT_WUNLOCK(object); @@ -2708,8 +2701,10 @@ VM_OBJECT_WLOCK(object); if (req & VM_ALLOC_WAITOK) return (EAGAIN); - } else + } else { mtx_unlock(&vm_page_queue_free_mtx); + pagedaemon_wakeup(); + } return (0); } @@ -2728,13 +2723,7 @@ { mtx_lock(&vm_page_queue_free_mtx); - if (!vm_pageout_wanted) { - vm_pageout_wanted = true; - wakeup(&vm_pageout_wanted); - } - vm_pages_needed = true; - msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PUSER, - "pfault", 0); + pagedaemon_wait(PUSER, "pfault"); } struct vm_pagequeue * Index: sys/vm/vm_pageout.h =================================================================== --- sys/vm/vm_pageout.h +++ sys/vm/vm_pageout.h @@ -96,11 +96,12 @@ * Signal pageout-daemon and wait for it. */ -extern void pagedaemon_wakeup(void); +void pagedaemon_wait(int pri, const char *wmesg); +void pagedaemon_wakeup(void); #define VM_WAIT vm_wait() #define VM_WAITPFAULT vm_waitpfault() -extern void vm_wait(void); -extern void vm_waitpfault(void); +void vm_wait(void); +void vm_waitpfault(void); #ifdef _KERNEL int vm_pageout_flush(vm_page_t *, int, int, int, int *, boolean_t *); Index: sys/vm/vm_pageout.c =================================================================== --- sys/vm/vm_pageout.c +++ sys/vm/vm_pageout.c @@ -1825,10 +1825,15 @@ pass++; } else { /* - * Yes. Sleep until pages need to be reclaimed or + * Yes. If threads are still sleeping in VM_WAIT + * because the previous scan didn't free enough pages, + * then we immediately start a new scan. Otherwise, + * sleep until the next wakeup or until pages need to * have their reference stats updated. */ - if (mtx_sleep(&vm_pageout_wanted, + if (vm_pages_needed) { + mtx_unlock(&vm_page_queue_free_mtx); + } else if (mtx_sleep(&vm_pageout_wanted, &vm_page_queue_free_mtx, PDROP | PVM, "psleep", hz) == 0) { VM_CNT_INC(v_pdwakeups); @@ -1936,17 +1941,41 @@ } /* - * Unless the free page queue lock is held by the caller, this function - * should be regarded as advisory. Specifically, the caller should - * not msleep() on &vm_cnt.v_free_count following this function unless - * the free page queue lock is held until the msleep() is performed. + * Perform an advisory wakeup of the page daemon. */ void pagedaemon_wakeup(void) { + mtx_assert(&vm_page_queue_free_mtx, MA_NOTOWNED); + if (!vm_pageout_wanted && curthread->td_proc != pageproc) { vm_pageout_wanted = true; wakeup(&vm_pageout_wanted); } } + +/* + * Wake up the page daemon and wait for it to reclaim free pages. + * + * This function returns with the free queues mutex unlocked. + */ +void +pagedaemon_wait(int pri, const char *wmesg) +{ + + MPASS(curthread->td_proc != pageproc); + mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); + + /* + * vm_pageout_wanted may have been set by an advisory wakeup, but if the + * page daemon is running on a CPU, the wakeup will have been lost. + * Thus, deliver a potentially spurious wakeup to ensure that the page + * daemon has been notified of the shortage. + */ + if (!vm_pageout_wanted || !vm_pages_needed) + wakeup(&vm_pageout_wanted); + vm_pageout_wanted = vm_pages_needed = true; + msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | pri, + wmesg, 0); +}