if_vtnet: move IFFLAGS ioctl handler to separate function

[bryanv]

Committed in dc9029d863dcc48efebb6a31a25553a7459132aa

sys/dev/virtio/network/if_vtnet.c

 
 /*
  * System initialization
  */
 
 /* the kernel process "vm_pageout"*/
 static void vm_pageout(void);
 static void vm_pageout_init(void);
-static int vm_pageout_clean(vm_page_t m);
+static int vm_pageout_clean(vm_page_t m, int *numpagedout);
 static int vm_pageout_cluster(vm_page_t m);
 static bool vm_pageout_scan(struct vm_domain *vmd, int pass);
 static void vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
     int starting_page_shortage);
 
 SYSINIT(pagedaemon_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, vm_pageout_init,
     NULL);
 

 static struct kproc_desc vm_kp = {
         "vmdaemon",
         vm_daemon,
         &vmproc
 };
 SYSINIT(vmdaemon, SI_SUB_KTHREAD_VM, SI_ORDER_FIRST, kproc_start, &vm_kp);
 #endif
 
+/* Pagedaemon activity rates, in subdivisions of one second. */
+#define VM_LAUNDER_RATE         10
+#define VM_INACT_SCAN_RATE      2
 
 int vm_pageout_deficit;         /* Estimated number of pages deficit */
 u_int vm_pageout_wakeup_thresh;
 static int vm_pageout_oom_seq = 12;
 bool vm_pageout_wanted;         /* Event on which pageout daemon sleeps */
 bool vm_pages_needed;           /* Are threads waiting for free pages? */
 
+/* Pending request for dirty page laundering. */
+static enum {
+        VM_LAUNDRY_IDLE,
+        VM_LAUNDRY_BACKGROUND,
+        VM_LAUNDRY_SHORTFALL
+} vm_laundry_request = VM_LAUNDRY_IDLE;
+
 #if !defined(NO_SWAPPING)
 static int vm_pageout_req_swapout;      /* XXX */
 static int vm_daemon_needed;
 static struct mtx vm_daemon_mtx;
 /* Allow for use by vm_pageout before vm_daemon is initialized. */
 MTX_SYSINIT(vm_daemon, &vm_daemon_mtx, "vm daemon", MTX_DEF);
 #endif
-static int vm_max_launder = 32;
 static int vm_pageout_update_period;
-static int defer_swap_pageouts;
 static int disable_swap_pageouts;
 static int lowmem_period = 10;
 static time_t lowmem_uptime;
 
 #if defined(NO_SWAPPING)
 static int vm_swap_enabled = 0;
 static int vm_swap_idle_enabled = 0;
 #else
 static int vm_swap_enabled = 1;
 static int vm_swap_idle_enabled = 0;
 #endif
 
 static int vm_panic_on_oom = 0;
 
 SYSCTL_INT(_vm, OID_AUTO, panic_on_oom,
         CTLFLAG_RWTUN, &vm_panic_on_oom, 0,
         "panic on out of memory instead of killing the largest process");
 
 SYSCTL_INT(_vm, OID_AUTO, pageout_wakeup_thresh,
         CTLFLAG_RW, &vm_pageout_wakeup_thresh, 0,
         "free page threshold for waking up the pageout daemon");
 
-SYSCTL_INT(_vm, OID_AUTO, max_launder,
-        CTLFLAG_RW, &vm_max_launder, 0, "Limit dirty flushes in pageout");
-
 SYSCTL_INT(_vm, OID_AUTO, pageout_update_period,
         CTLFLAG_RW, &vm_pageout_update_period, 0,
         "Maximum active LRU update period");
   
 SYSCTL_INT(_vm, OID_AUTO, lowmem_period, CTLFLAG_RW, &lowmem_period, 0,
         "Low memory callback period");
 
 #if defined(NO_SWAPPING)
 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
         CTLFLAG_RD, &vm_swap_enabled, 0, "Enable entire process swapout");
 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
         CTLFLAG_RD, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
 #else
 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
         CTLFLAG_RW, &vm_swap_enabled, 0, "Enable entire process swapout");
 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
         CTLFLAG_RW, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
 #endif
 
-SYSCTL_INT(_vm, OID_AUTO, defer_swapspace_pageouts,
-        CTLFLAG_RW, &defer_swap_pageouts, 0, "Give preference to dirty pages in mem");
-
 SYSCTL_INT(_vm, OID_AUTO, disable_swapspace_pageouts,
         CTLFLAG_RW, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages");
 
 static int pageout_lock_miss;
 SYSCTL_INT(_vm, OID_AUTO, pageout_lock_miss,
         CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout");
 
 SYSCTL_INT(_vm, OID_AUTO, pageout_oom_seq,
         CTLFLAG_RW, &vm_pageout_oom_seq, 0,
         "back-to-back calls to oom detector to start OOM");
 
+static int act_scan_laundry_weight = 3;
+SYSCTL_INT(_vm, OID_AUTO, act_scan_laundry_weight, CTLFLAG_RW,
+    &act_scan_laundry_weight, 0,
+    "weight given to clean vs. dirty pages in active queue scans");
+
+static u_int vm_background_launder_target;
+SYSCTL_UINT(_vm, OID_AUTO, background_launder_target, CTLFLAG_RW,
+    &vm_background_launder_target, 0,
+    "background laundering target, in pages");
+
+static u_int vm_background_launder_rate = 4096;
+SYSCTL_UINT(_vm, OID_AUTO, background_launder_rate, CTLFLAG_RW,
+    &vm_background_launder_rate, 0,
+    "background laundering rate, in kilobytes per second");
+
+static u_int vm_background_launder_max = 20 * 1024;
+SYSCTL_UINT(_vm, OID_AUTO, background_launder_max, CTLFLAG_RW,
+    &vm_background_launder_max, 0, "background laundering cap, in kilobytes");
+
 #define VM_PAGEOUT_PAGE_COUNT 16
 int vm_pageout_page_count = VM_PAGEOUT_PAGE_COUNT;
 
 int vm_page_max_wired;          /* XXX max # of wired pages system-wide */
 SYSCTL_INT(_vm, OID_AUTO, max_wired,
         CTLFLAG_RW, &vm_page_max_wired, 0, "System-wide limit to wired page count");
 
+static u_int isqrt(u_int num);
 static boolean_t vm_pageout_fallback_object_lock(vm_page_t, vm_page_t *);
+static int vm_pageout_launder(struct vm_domain *vmd, int launder,
+    bool in_shortfall);
+static void vm_pageout_laundry_worker(void *arg);
 #if !defined(NO_SWAPPING)
 static void vm_pageout_map_deactivate_pages(vm_map_t, long);
 static void vm_pageout_object_deactivate_pages(pmap_t, vm_object_t, long);
 static void vm_req_vmdaemon(int req);
 #endif
 static boolean_t vm_pageout_page_lock(vm_page_t, vm_page_t *);
 
 /*
vm_pageout_cluster(vm_page_t m)
         mc[vm_pageout_page_count] = pb = ps = m;
         pageout_count = 1;
         page_base = vm_pageout_page_count;
         ib = 1;
         is = 1;
 
         /*
          * We can cluster only if the page is not clean, busy, or held, and
-         * the page is inactive.
+         * the page is in the laundry queue.
          *
          * During heavy mmap/modification loads the pageout
          * daemon can really fragment the underlying file
          * due to flushing pages out of order and not trying to
          * align the clusters (which leaves sporadic out-of-order
          * holes).  To solve this problem we do the reverse scan
          * first and attempt to align our cluster, then do a 
          * forward scan if room remains.
if ((p = vm_page_prev(pb)) == NULL || vm_page_busied(p)) {
                         break;
                 }
                 vm_page_test_dirty(p);
                 if (p->dirty == 0) {
                         ib = 0;
                         break;
                 }
                 vm_page_lock(p);
-                if (p->queue != PQ_INACTIVE ||
+                if (!vm_page_in_laundry(p) ||
                     p->hold_count != 0) {        /* may be undergoing I/O */
                         vm_page_unlock(p);
                         ib = 0;
                         break;
                 }
                 vm_page_unlock(p);
                 mc[--page_base] = pb = p;
                 ++pageout_count;
more:
         while (pageout_count < vm_pageout_page_count && 
             pindex + is < object->size) {
                 if ((p = vm_page_next(ps)) == NULL || vm_page_busied(p))
                         break;
                 vm_page_test_dirty(p);
                 if (p->dirty == 0)
                         break;
                 vm_page_lock(p);
-                if (p->queue != PQ_INACTIVE ||
+                if (!vm_page_in_laundry(p) ||
                     p->hold_count != 0) {        /* may be undergoing I/O */
                         vm_page_unlock(p);
                         break;
                 }
                 vm_page_unlock(p);
                 mc[page_base + pageout_count] = ps = p;
                 ++pageout_count;
                 ++is;
vm_pageout_flush(vm_page_t *mc, int count, int flags, int mreq, int *prunlen,
         for (i = 0; i < count; i++) {
                 vm_page_t mt = mc[i];
 
                 KASSERT(pageout_status[i] == VM_PAGER_PEND ||
                     !pmap_page_is_write_mapped(mt),
                     ("vm_pageout_flush: page %p is not write protected", mt));
                 switch (pageout_status[i]) {
                 case VM_PAGER_OK:
+                        vm_page_lock(mt);
+                        if (vm_page_in_laundry(mt))
+                                vm_page_deactivate_noreuse(mt);
+                        vm_page_unlock(mt);
+                        /* FALLTHROUGH */
                 case VM_PAGER_PEND:
                         numpagedout++;
                         break;
                 case VM_PAGER_BAD:
                         /*
-                         * Page outside of range of object. Right now we
-                         * essentially lose the changes by pretending it
-                         * worked.
+                         * The page is outside the object's range.  We pretend
+                         * that the page out worked and clean the page, so the
+                         * changes will be lost if the page is reclaimed by
+                         * the page daemon.
                          */
                         vm_page_undirty(mt);
+                        vm_page_lock(mt);
+                        if (vm_page_in_laundry(mt))
+                                vm_page_deactivate_noreuse(mt);
+                        vm_page_unlock(mt);
                         break;
                 case VM_PAGER_ERROR:
                 case VM_PAGER_FAIL:
                         /*
-                         * If page couldn't be paged out, then reactivate the
-                         * page so it doesn't clog the inactive list.  (We
-                         * will try paging out it again later).
+                         * If the page couldn't be paged out, then reactivate
+                         * it so that it doesn't clog the laundry and inactive
+                         * queues.  (We will try paging it out again later).
                          */
                         vm_page_lock(mt);
                         vm_page_activate(mt);
                         vm_page_unlock(mt);
                         if (eio != NULL && i >= mreq && i - mreq < runlen)
                                 *eio = TRUE;
                         break;
                 case VM_PAGER_AGAIN:
TAILQ_FOREACH(p, &object->memq, listq) {
                                 continue;
                         }
                         act_delta = pmap_ts_referenced(p);
                         if ((p->aflags & PGA_REFERENCED) != 0) {
                                 if (act_delta == 0)
                                         act_delta = 1;
                                 vm_page_aflag_clear(p, PGA_REFERENCED);
                         }
-                        if (p->queue != PQ_ACTIVE && act_delta != 0) {
+                        if (!vm_page_active(p) && act_delta != 0) {
                                 vm_page_activate(p);
                                 p->act_count += act_delta;
-                        } else if (p->queue == PQ_ACTIVE) {
+                        } else if (vm_page_active(p)) {
                                 if (act_delta == 0) {
                                         p->act_count -= min(p->act_count,
                                             ACT_DECLINE);
                                         if (!remove_mode && p->act_count == 0) {
                                                 pmap_remove_all(p);
                                                 vm_page_deactivate(p);
                                         } else
                                                 vm_page_requeue(p);
                                 } else {
                                         vm_page_activate(p);
                                         if (p->act_count < ACT_MAX -
                                             ACT_ADVANCE)
                                                 p->act_count += ACT_ADVANCE;
                                         vm_page_requeue(p);
                                 }
-                        } else if (p->queue == PQ_INACTIVE)
+                        } else if (vm_page_inactive(p))
                                 pmap_remove_all(p);
                         vm_page_unlock(p);
                 }
                 if ((backing_object = object->backing_object) == NULL)
                         goto unlock_return;
                 VM_OBJECT_RLOCK(backing_object);
                 if (object != first_object)
                         VM_OBJECT_RUNLOCK(object);

  * Attempt to acquire all of the necessary locks to launder a page and
  * then call through the clustering layer to PUTPAGES.  Wait a short
  * time for a vnode lock.
  *
  * Requires the page and object lock on entry, releases both before return.
  * Returns 0 on success and an errno otherwise.
  */
 static int
-vm_pageout_clean(vm_page_t m)
+vm_pageout_clean(vm_page_t m, int *numpagedout)
 {
         struct vnode *vp;
         struct mount *mp;
         vm_object_t object;
         vm_pindex_t pindex;
         int error, lockmode;
 
         vm_page_assert_locked(m);
if (object->type == OBJT_VNODE) {
                 /*
                  * While the object and page were unlocked, the page
                  * may have been:
                  * (1) moved to a different queue,
                  * (2) reallocated to a different object,
                  * (3) reallocated to a different offset, or
                  * (4) cleaned.
                  */
-                if (m->queue != PQ_INACTIVE || m->object != object ||
+                if (!vm_page_in_laundry(m) || m->object != object ||
                     m->pindex != pindex || m->dirty == 0) {
                         vm_page_unlock(m);
                         error = ENXIO;
                         goto unlock_all;
                 }
 
                 /*
                  * The page may have been busied or held while the object
                  * and page locks were released.
                  */
                 if (vm_page_busied(m) || m->hold_count != 0) {
                         vm_page_unlock(m);
                         error = EBUSY;
                         goto unlock_all;
                 }
         }
 
         /*
          * If a page is dirty, then it is either being washed
          * (but not yet cleaned) or it is still in the
          * laundry.  If it is still in the laundry, then we
          * start the cleaning operation. 
          */
-        if (vm_pageout_cluster(m) == 0)
+        if ((*numpagedout = vm_pageout_cluster(m)) == 0)
                 error = EIO;
 
 unlock_all:
         VM_OBJECT_WUNLOCK(object);
 
 unlock_mp:
         vm_page_lock_assert(m, MA_NOTOWNED);
         if (mp != NULL) {
                 if (vp != NULL)
                         vput(vp);
                 vm_object_deallocate(object);
                 vn_finished_write(mp);
         }
 
         return (error);
 }
 
 /*
+ * Attempt to launder the specified number of pages.
+ *
+ * Returns the number of pages successfully laundered.
+ */
+static int
+vm_pageout_launder(struct vm_domain *vmd, int launder, bool in_shortfall)
+{
+        struct vm_pagequeue *pq;
+        vm_object_t object;
+        vm_page_t m, next;
+        int act_delta, error, maxscan, numpagedout, starting_target;
+        int vnodes_skipped;
+        bool pageout_ok, queue_locked;
+
+        starting_target = launder;
+        vnodes_skipped = 0;
+
+        /*
+         * Scan the laundry queue for pages eligible to be laundered.  We stop
+         * once the target number of dirty pages have been laundered, or once
+         * we've reached the end of the queue.  A single iteration of this loop
+         * may cause more than one page to be laundered because of clustering.
+         *
+         * maxscan ensures that we don't re-examine requeued pages.  Any
+         * additional pages written as part of a cluster are subtracted from
+         * maxscan since they must be taken from the laundry queue.
+         */
+        pq = &vmd->vmd_pagequeues[PQ_LAUNDRY];
+        maxscan = pq->pq_cnt;
+
+        vm_pagequeue_lock(pq);
+        queue_locked = true;
+        for (m = TAILQ_FIRST(&pq->pq_pl);
+            m != NULL && maxscan-- > 0 && launder > 0;
+            m = next) {
+                vm_pagequeue_assert_locked(pq);
+                KASSERT(queue_locked, ("unlocked laundry queue"));
+                KASSERT(vm_page_in_laundry(m),
+                    ("page %p has an inconsistent queue", m));
+                next = TAILQ_NEXT(m, plinks.q);
+                if ((m->flags & PG_MARKER) != 0)
+                        continue;
+                KASSERT((m->flags & PG_FICTITIOUS) == 0,
+                    ("PG_FICTITIOUS page %p cannot be in laundry queue", m));
+                KASSERT((m->oflags & VPO_UNMANAGED) == 0,
+                    ("VPO_UNMANAGED page %p cannot be in laundry queue", m));
+                if (!vm_pageout_page_lock(m, &next) || m->hold_count != 0) {
+                        vm_page_unlock(m);
+                        continue;
+                }
+                object = m->object;
+                if ((!VM_OBJECT_TRYWLOCK(object) &&
+                    (!vm_pageout_fallback_object_lock(m, &next) ||
+                    m->hold_count != 0)) || vm_page_busied(m)) {
+                        VM_OBJECT_WUNLOCK(object);
+                        vm_page_unlock(m);
+                        continue;
+                }
+
+                /*
+                 * Unlock the laundry queue, invalidating the 'next' pointer.
+                 * Use a marker to remember our place in the laundry queue.
+                 */
+                TAILQ_INSERT_AFTER(&pq->pq_pl, m, &vmd->vmd_laundry_marker,
+                    plinks.q);
+                vm_pagequeue_unlock(pq);
+                queue_locked = false;
+
+                /*
+                 * Invalid pages can be easily freed.  They cannot be
+                 * mapped; vm_page_free() asserts this.
+                 */
+                if (m->valid == 0)
+                        goto free_page;
+
+                /*
+                 * If the page has been referenced and the object is not dead,
+                 * reactivate or requeue the page depending on whether the
+                 * object is mapped.
+                 */
+                if ((m->aflags & PGA_REFERENCED) != 0) {
+                        vm_page_aflag_clear(m, PGA_REFERENCED);
+                        act_delta = 1;
+                } else
+                        act_delta = 0;
+                if (object->ref_count != 0)
+                        act_delta += pmap_ts_referenced(m);
+                else {
+                        KASSERT(!pmap_page_is_mapped(m),
+                            ("page %p is mapped", m));
+                }
+                if (act_delta != 0) {
+                        if (object->ref_count != 0) {
+                                PCPU_INC(cnt.v_reactivated);
+                                vm_page_activate(m);
+
+                                /*
+                                 * Increase the activation count if the page
+                                 * was referenced while in the laundry queue.
+                                 * This makes it less likely that the page will
+                                 * be returned prematurely to the inactive
+                                 * queue.
+                                 */
+                                m->act_count += act_delta + ACT_ADVANCE;
+
+                                /*
+                                 * If this was a background laundering, count
+                                 * activated pages towards our target.  The
+                                 * purpose of background laundering is to ensure
+                                 * that pages are eventually cycled through the
+                                 * laundry queue, and an activation is a valid
+                                 * way out.
+                                 */
+                                if (!in_shortfall)
+                                        launder--;
+                                goto drop_page;
+                        } else if ((object->flags & OBJ_DEAD) == 0)
+                                goto requeue_page;
+                }
+
+                /*
+                 * If the page appears to be clean at the machine-independent
+                 * layer, then remove all of its mappings from the pmap in
+                 * anticipation of freeing it.  If, however, any of the page's
+                 * mappings allow write access, then the page may still be
+                 * modified until the last of those mappings are removed.
+                 */
+                if (object->ref_count != 0) {
+                        vm_page_test_dirty(m);
+                        if (m->dirty == 0)
+                                pmap_remove_all(m);
+                }
+
+                /*
+                 * Clean pages are freed, and dirty pages are paged out unless
+                 * they belong to a dead object.  Requeueing dirty pages from
+                 * dead objects is pointless, as they are being paged out and
+                 * freed by the thread that destroyed the object.
+                 */
+                if (m->dirty == 0) {
+free_page:
+                        vm_page_free(m);
+                        PCPU_INC(cnt.v_dfree);
+                } else if ((object->flags & OBJ_DEAD) == 0) {
+                        if (object->type != OBJT_SWAP &&
+                            object->type != OBJT_DEFAULT)
+                                pageout_ok = true;
+                        else if (disable_swap_pageouts)
+                                pageout_ok = false;
+                        else
+                                pageout_ok = true;
+                        if (!pageout_ok) {
+requeue_page:
+                                vm_pagequeue_lock(pq);
+                                queue_locked = true;
+                                vm_page_requeue_locked(m);
+                                goto drop_page;
+                        }
+
+                        /*
+                         * Form a cluster with adjacent, dirty pages from the
+                         * same object, and page out that entire cluster.
+                         *
+                         * The adjacent, dirty pages must also be in the
+                         * laundry.  However, their mappings are not checked
+                         * for new references.  Consequently, a recently
+                         * referenced page may be paged out.  However, that
+                         * page will not be prematurely reclaimed.  After page
+                         * out, the page will be placed in the inactive queue,
+                         * where any new references will be detected and the
+                         * page reactivated.
+                         */
+                        error = vm_pageout_clean(m, &numpagedout);
+                        if (error == 0) {
+                                launder -= numpagedout;
+                                maxscan -= numpagedout - 1;
+                        } else if (error == EDEADLK) {
+                                pageout_lock_miss++;
+                                vnodes_skipped++;
+                        }
+                        goto relock_queue;
+                }
+drop_page:
+                vm_page_unlock(m);
+                VM_OBJECT_WUNLOCK(object);
+relock_queue:
+                if (!queue_locked) {
+                        vm_pagequeue_lock(pq);
+                        queue_locked = true;
+                }
+                next = TAILQ_NEXT(&vmd->vmd_laundry_marker, plinks.q);
+                TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_laundry_marker, plinks.q);
+        }
+        vm_pagequeue_unlock(pq);
+
+        /*
+         * Wakeup the sync daemon if we skipped a vnode in a writeable object
+         * and we didn't launder enough pages.
+         */
+        if (vnodes_skipped > 0 && launder > 0)
+                (void)speedup_syncer();
+
+        return (starting_target - launder);
+}
+
+/*
+ * Compute the integer square root.
+ */
+static u_int
+isqrt(u_int num)
+{
+        u_int bit, root, tmp;
+
+        bit = 1u << ((NBBY * sizeof(u_int)) - 2);
+        while (bit > num)
+                bit >>= 2;
+        root = 0;
+        while (bit != 0) {
+                tmp = root + bit;
+                root >>= 1;
+                if (num >= tmp) {
+                        num -= tmp;
+                        root += bit;
+                }
+                bit >>= 2;
+        }
+        return (root);
+}
+
+/*
+ * Perform the work of the laundry thread: periodically wake up and determine
+ * whether any pages need to be laundered.  If so, determine the number of pages
+ * that need to be laundered, and launder them.
+ */
+static void
+vm_pageout_laundry_worker(void *arg)
+{
+        struct vm_domain *domain;
+        struct vm_pagequeue *pq;
+        uint64_t nclean, ndirty;
+        u_int last_launder, wakeups;
+        int domidx, last_target, launder, shortfall, shortfall_cycle, target;
+        bool in_shortfall;
+
+        domidx = (uintptr_t)arg;
+        domain = &vm_dom[domidx];
+        pq = &domain->vmd_pagequeues[PQ_LAUNDRY];
+        KASSERT(domain->vmd_segs != 0, ("domain without segments"));
+        vm_pageout_init_marker(&domain->vmd_laundry_marker, PQ_LAUNDRY);
+
+        shortfall = 0;
+        in_shortfall = false;
+        shortfall_cycle = 0;
+        target = 0;
+        last_launder = 0;
+
+        /*
+         * The pageout laundry worker is never done, so loop forever.
+         */
+        for (;;) {
+                KASSERT(target >= 0, ("negative target %d", target));
+                KASSERT(shortfall_cycle >= 0,
+                    ("negative cycle %d", shortfall_cycle));
+                launder = 0;
+                wakeups = VM_METER_PCPU_CNT(v_pdwakeups);
+
+                /*
+                 * First determine whether we need to launder pages to meet a
+                 * shortage of free pages.
+                 */
+                if (shortfall > 0) {
+                        in_shortfall = true;
+                        shortfall_cycle = VM_LAUNDER_RATE / VM_INACT_SCAN_RATE;
+                        target = shortfall;
+                } else if (!in_shortfall)
+                        goto trybackground;
+                else if (shortfall_cycle == 0 || vm_laundry_target() <= 0) {
+                        /*
+                         * We recently entered shortfall and began laundering
+                         * pages.  If we have completed that laundering run
+                         * (and we are no longer in shortfall) or we have met
+                         * our laundry target through other activity, then we
+                         * can stop laundering pages.
+                         */
+                        in_shortfall = false;
+                        target = 0;
+                        goto trybackground;
+                }
+                last_launder = wakeups;
+                launder = target / shortfall_cycle--;
+                goto dolaundry;
+
+                /*
+                 * There's no immediate need to launder any pages; see if we
+                 * meet the conditions to perform background laundering:
+                 *
+                 * 1. The ratio of dirty to clean inactive pages exceeds the
+                 *    background laundering threshold and the pagedaemon has
+                 *    been woken up to reclaim pages since our last
+                 *    laundering, or
+                 * 2. we haven't yet reached the target of the current
+                 *    background laundering run.
+                 *
+                 * The background laundering threshold is not a constant.
+                 * Instead, it is a slowly growing function of the number of
+                 * page daemon wakeups since the last laundering.  Thus, as the
+                 * ratio of dirty to clean inactive pages grows, the amount of
+                 * memory pressure required to trigger laundering decreases.
+                 */
+trybackground:
+                nclean = vm_cnt.v_inactive_count + vm_cnt.v_free_count;
+                ndirty = vm_cnt.v_laundry_count;
+                if (target == 0 && wakeups != last_launder &&
+                    ndirty * isqrt(wakeups - last_launder) >= nclean) {
+                        target = vm_background_launder_target;
+                }
+
+                /*
+                 * We have a non-zero background laundering target.  If we've
+                 * laundered up to our maximum without observing a page daemon
+                 * wakeup, just stop.  This is a safety belt that ensures we
+                 * don't launder an excessive amount if memory pressure is low
+                 * and the ratio of dirty to clean pages is large.  Otherwise,
+                 * proceed at the background laundering rate.
+                 */
+                if (target > 0) {
+                        if (wakeups != last_launder) {
+                                last_launder = wakeups;
+                                last_target = target;
+                        } else if (last_target - target >=
+                            vm_background_launder_max * PAGE_SIZE / 1024) {
+                                target = 0;
+                        }
+                        launder = vm_background_launder_rate * PAGE_SIZE / 1024;
+                        launder /= VM_LAUNDER_RATE;
+                        if (launder > target)
+                                launder = target;
+                }
+
+dolaundry:
+                if (launder > 0) {
+                        /*
+                         * Because of I/O clustering, the number of laundered
+                         * pages could exceed "target" by the maximum size of
+                         * a cluster minus one. 
+                         */
+                        target -= min(vm_pageout_launder(domain, launder,
+                            in_shortfall), target);
+                        pause("laundp", hz / VM_LAUNDER_RATE);
+                }
+
+                /*
+                 * If we're not currently laundering pages and the page daemon
+                 * hasn't posted a new request, sleep until the page daemon
+                 * kicks us.
+                 */
+                vm_pagequeue_lock(pq);
+                if (target == 0 && vm_laundry_request == VM_LAUNDRY_IDLE)
+                        (void)mtx_sleep(&vm_laundry_request,
+                            vm_pagequeue_lockptr(pq), PVM, "launds", 0);
+
+                /*
+                 * If the pagedaemon has indicated that it's in shortfall, start
+                 * a shortfall laundering unless we're already in the middle of
+                 * one.  This may preempt a background laundering.
+                 */
+                if (vm_laundry_request == VM_LAUNDRY_SHORTFALL &&
+                    (!in_shortfall || shortfall_cycle == 0)) {
+                        shortfall = vm_laundry_target() + vm_pageout_deficit;
+                        target = 0;
+                } else
+                        shortfall = 0;
+
+                if (target == 0)
+                        vm_laundry_request = VM_LAUNDRY_IDLE;
+                vm_pagequeue_unlock(pq);
+        }
+}
+
+/*
  *      vm_pageout_scan does the dirty work for the pageout daemon.
  *
- *      pass 0 - Update active LRU/deactivate pages
- *      pass 1 - Free inactive pages
- *      pass 2 - Launder dirty pages
+ *      pass == 0: Update active LRU/deactivate pages
+ *      pass >= 1: Free inactive pages
  *
  * Returns true if pass was zero or enough pages were freed by the inactive
  * queue scan to meet the target.
  */
 static bool
 vm_pageout_scan(struct vm_domain *vmd, int pass)
 {
         vm_page_t m, next;
         struct vm_pagequeue *pq;
         vm_object_t object;
         long min_scan;
-        int act_delta, addl_page_shortage, deficit, error, inactq_shortage;
-        int maxlaunder, maxscan, page_shortage, scan_tick, scanned;
-        int starting_page_shortage, vnodes_skipped;
-        boolean_t pageout_ok, queue_locked;
+        int act_delta, addl_page_shortage, deficit, inactq_shortage, maxscan;
+        int page_shortage, scan_tick, scanned, starting_page_shortage;
+        boolean_t queue_locked;
 
         /*
          * If we need to reclaim memory ask kernel caches to return
          * some.  We rate limit to avoid thrashing.
          */
         if (vmd == &vm_dom[0] && pass > 0 &&
             (time_uptime - lowmem_uptime) >= lowmem_period) {
                 /*
vm_pageout_scan(struct vm_domain *vmd, int pass)
         if (pass > 0) {
                 deficit = atomic_readandclear_int(&vm_pageout_deficit);
                 page_shortage = vm_paging_target() + deficit;
         } else
                 page_shortage = deficit = 0;
         starting_page_shortage = page_shortage;
 
         /*
-         * maxlaunder limits the number of dirty pages we flush per scan.
-         * For most systems a smaller value (16 or 32) is more robust under
-         * extreme memory and disk pressure because any unnecessary writes
-         * to disk can result in extreme performance degredation.  However,
-         * systems with excessive dirty pages (especially when MAP_NOSYNC is
-         * used) will die horribly with limited laundering.  If the pageout
-         * daemon cannot clean enough pages in the first pass, we let it go
-         * all out in succeeding passes.
-         */
-        if ((maxlaunder = vm_max_launder) <= 1)
-                maxlaunder = 1;
-        if (pass > 1)
-                maxlaunder = 10000;
-
-        vnodes_skipped = 0;
-
-        /*
          * Start scanning the inactive queue for pages that we can free.  The
          * scan will stop when we reach the target or we have scanned the
          * entire queue.  (Note that m->act_count is not used to make
          * decisions for the inactive queue, only for the active queue.)
          */
         pq = &vmd->vmd_pagequeues[PQ_INACTIVE];
         maxscan = pq->pq_cnt;
         vm_pagequeue_lock(pq);
         queue_locked = TRUE;
         for (m = TAILQ_FIRST(&pq->pq_pl);
              m != NULL && maxscan-- > 0 && page_shortage > 0;
              m = next) {
                 vm_pagequeue_assert_locked(pq);
                 KASSERT(queue_locked, ("unlocked inactive queue"));
-                KASSERT(m->queue == PQ_INACTIVE, ("Inactive queue %p", m));
+                KASSERT(vm_page_inactive(m), ("Inactive queue %p", m));
 
                 PCPU_INC(cnt.v_pdpages);
                 next = TAILQ_NEXT(m, plinks.q);
 
                 /*
                  * skip marker pages
                  */
                 if (m->flags & PG_MARKER)
unlock_object:
                         VM_OBJECT_WUNLOCK(object);
 unlock_page:
                         vm_page_unlock(m);
                         continue;
                 }
                 KASSERT(m->hold_count == 0, ("Held page %p", m));
 
                 /*
-                 * We unlock the inactive page queue, invalidating the
-                 * 'next' pointer.  Use our marker to remember our
-                 * place.
+                 * Dequeue the inactive page and unlock the inactive page
+                 * queue, invalidating the 'next' pointer.  Dequeueing the
+                 * page here avoids a later reacquisition (and release) of
+                 * the inactive page queue lock when vm_page_activate(),
+                 * vm_page_free(), or vm_page_launder() is called.  Use a
+                 * marker to remember our place in the inactive queue.
                  */
                 TAILQ_INSERT_AFTER(&pq->pq_pl, m, &vmd->vmd_marker, plinks.q);
+                vm_page_dequeue_locked(m);
                 vm_pagequeue_unlock(pq);
                 queue_locked = FALSE;
 
                 /*
                  * Invalid pages can be easily freed. They cannot be
                  * mapped, vm_page_free() asserts this.
                  */
                 if (m->valid == 0)
unlock_page:
                 if (object->ref_count != 0) {
                         act_delta += pmap_ts_referenced(m);
                 } else {
                         KASSERT(!pmap_page_is_mapped(m),
                             ("vm_pageout_scan: page %p is mapped", m));
                 }
                 if (act_delta != 0) {
                         if (object->ref_count != 0) {
+                                PCPU_INC(cnt.v_reactivated);
                                 vm_page_activate(m);
 
                                 /*
                                  * Increase the activation count if the page
                                  * was referenced while in the inactive queue.
                                  * This makes it less likely that the page will
                                  * be returned prematurely to the inactive
                                  * queue.
                                  */
                                 m->act_count += act_delta + ACT_ADVANCE;
                                 goto drop_page;
-                        } else if ((object->flags & OBJ_DEAD) == 0)
-                                goto requeue_page;
+                        } else if ((object->flags & OBJ_DEAD) == 0) {
+                                vm_pagequeue_lock(pq);
+                                queue_locked = TRUE;
+                                m->queue = PQ_INACTIVE;
+                                TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
+                                vm_pagequeue_cnt_inc(pq);
+                                goto drop_page;
                         }
+                }
 
                 /*
                  * If the page appears to be clean at the machine-independent
                  * layer, then remove all of its mappings from the pmap in
                  * anticipation of freeing it.  If, however, any of the page's
                  * mappings allow write access, then the page may still be
                  * modified until the last of those mappings are removed.
                  */
                 if (object->ref_count != 0) {
                         vm_page_test_dirty(m);
                         if (m->dirty == 0)
                                 pmap_remove_all(m);
                 }
 
-                if (m->dirty == 0) {
-                        /*
-                         * Clean pages can be freed.
-                         */
+                 * Clean pages can be freed, but dirty pages must be sent back
+                 * to the laundry, unless they belong to a dead object.
+                 * Requeueing dirty pages from dead objects is pointless, as
+                 * they are being paged out and freed by the thread that
+                 * destroyed the object.
+                if (m->dirty == 0) {
 free_page:
                         vm_page_free(m);
                         PCPU_INC(cnt.v_dfree);
                         --page_shortage;
-                } else if ((object->flags & OBJ_DEAD) != 0) {
-                        /*
-                         * Leave dirty pages from dead objects at the front of
-                         * the queue.  They are being paged out and freed by
-                         * the thread that destroyed the object.  They will
-                         * leave the queue shortly after the scan finishes, so 
-                         * they should be discounted from the inactive count.
-                         */
-                        addl_page_shortage++;
-                } else if ((m->flags & PG_WINATCFLS) == 0 && pass < 2) {
-                        /*
-                         * Dirty pages need to be paged out, but flushing
-                         * a page is extremely expensive versus freeing
-                         * a clean page.  Rather then artificially limiting
-                         * the number of pages we can flush, we instead give
-                         * dirty pages extra priority on the inactive queue
-                         * by forcing them to be cycled through the queue
-                         * twice before being flushed, after which the
-                         * (now clean) page will cycle through once more
-                         * before being freed.  This significantly extends
-                         * the thrash point for a heavily loaded machine.
-                         */
-                        m->flags |= PG_WINATCFLS;
-requeue_page:
-                        vm_pagequeue_lock(pq);
-                        queue_locked = TRUE;
-                        vm_page_requeue_locked(m);
-                } else if (maxlaunder > 0) {
-                        /*
-                         * We always want to try to flush some dirty pages if
-                         * we encounter them, to keep the system stable.
-                         * Normally this number is small, but under extreme
-                         * pressure where there are insufficient clean pages
-                         * on the inactive queue, we may have to go all out.
-                         */
-
-                        if (object->type != OBJT_SWAP &&
-                            object->type != OBJT_DEFAULT)
-                                pageout_ok = TRUE;
-                        else if (disable_swap_pageouts)
-                                pageout_ok = FALSE;
-                        else if (defer_swap_pageouts)
-                                pageout_ok = vm_page_count_min();
-                        else
-                                pageout_ok = TRUE;
-                        if (!pageout_ok)
-                                goto requeue_page;
-                        error = vm_pageout_clean(m);
-                        /*
-                         * Decrement page_shortage on success to account for
-                         * the (future) cleaned page.  Otherwise we could wind
-                         * up laundering or cleaning too many pages.
-                         */
-                        if (error == 0) {
-                                page_shortage--;
-                                maxlaunder--;
-                        } else if (error == EDEADLK) {
-                                pageout_lock_miss++;
-                                vnodes_skipped++;
-                        } else if (error == EBUSY) {
-                                addl_page_shortage++;
-                        }
-                        vm_page_lock_assert(m, MA_NOTOWNED);
-                        goto relock_queue;
-                }
+                } else if ((object->flags & OBJ_DEAD) == 0)
+                        vm_page_launder(m);
 drop_page:
                 vm_page_unlock(m);
                 VM_OBJECT_WUNLOCK(object);
-relock_queue:
                 if (!queue_locked) {
                         vm_pagequeue_lock(pq);
                         queue_locked = TRUE;
                 }
                 next = TAILQ_NEXT(&vmd->vmd_marker, plinks.q);
                 TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_marker, plinks.q);
         }
         vm_pagequeue_unlock(pq);
 
+        /*
+         * Wake up the laundry thread so that it can perform any needed
+         * laundering.  If we didn't meet our target, we're in shortfall and
+         * need to launder more aggressively.
+         */
+        if (vm_laundry_request == VM_LAUNDRY_IDLE &&
+            starting_page_shortage > 0) {
+                pq = &vm_dom[0].vmd_pagequeues[PQ_LAUNDRY];
+                vm_pagequeue_lock(pq);
+                if (page_shortage > 0) {
+                        vm_laundry_request = VM_LAUNDRY_SHORTFALL;
+                        PCPU_INC(cnt.v_pdshortfalls);
+                } else if (vm_laundry_request != VM_LAUNDRY_SHORTFALL)
+                        vm_laundry_request = VM_LAUNDRY_BACKGROUND;
+                wakeup(&vm_laundry_request);
+                vm_pagequeue_unlock(pq);
+        }
+
 #if !defined(NO_SWAPPING)
         /*
          * Wakeup the swapout daemon if we didn't free the targeted number of
          * pages.
          */
         if (vm_swap_enabled && page_shortage > 0)
                 vm_req_vmdaemon(VM_SWAP_NORMAL);
 #endif
 
         /*
-         * Wakeup the sync daemon if we skipped a vnode in a writeable object
-         * and we didn't free enough pages.
-         */
-        if (vnodes_skipped > 0 && page_shortage > vm_cnt.v_free_target -
-            vm_cnt.v_free_min)
-                (void)speedup_syncer();
-
-        /*
          * If the inactive queue scan fails repeatedly to meet its
          * target, kill the largest process.
          */
         vm_pageout_mightbe_oom(vmd, page_shortage, starting_page_shortage);
 
         /*
          * Compute the number of pages we want to try to move from the
-         * active queue to the inactive queue.
+         * active queue to either the inactive or laundry queue.
+         *
+         * When scanning active pages, we make clean pages count more heavily
+         * towards the page shortage than dirty pages.  This is because dirty
+         * pages must be laundered before they can be reused and thus have less
+         * utility when attempting to quickly alleviate a shortage.  However,
+         * this weighting also causes the scan to deactivate dirty pages more
+         * more aggressively, improving the effectiveness of clustering and
+         * ensuring that they can eventually be reused.
          */
-        inactq_shortage = vm_cnt.v_inactive_target - vm_cnt.v_inactive_count +
+        inactq_shortage = vm_cnt.v_inactive_target - (vm_cnt.v_inactive_count +
+            vm_cnt.v_laundry_count / act_scan_laundry_weight) +
             vm_paging_target() + deficit + addl_page_shortage;
+        page_shortage *= act_scan_laundry_weight;
 
         pq = &vmd->vmd_pagequeues[PQ_ACTIVE];
         vm_pagequeue_lock(pq);
         maxscan = pq->pq_cnt;
 
         /*
          * If we're just idle polling attempt to visit every
          * active page within 'update_period' seconds.
for (m = TAILQ_FIRST(&pq->pq_pl), scanned = 0; m != NULL && (scanned <
                 if (act_delta != 0) {
                         m->act_count += ACT_ADVANCE + act_delta;
                         if (m->act_count > ACT_MAX)
                                 m->act_count = ACT_MAX;
                 } else
                         m->act_count -= min(m->act_count, ACT_DECLINE);
 
                 /*
-                 * Move this page to the tail of the active or inactive
+                 * Move this page to the tail of the active, inactive or laundry
                  * queue depending on usage.
                  */
                 if (m->act_count == 0) {
                         /* Dequeue to avoid later lock recursion. */
                         vm_page_dequeue_locked(m);
+
+                        /*
+                         * When not short for inactive pages, let dirty pages go
+                         * through the inactive queue before moving to the
+                         * laundry queues.  This gives them some extra time to
+                         * be reactivated, potentially avoiding an expensive
+                         * pageout.  During a page shortage, the inactive queue
+                         * is necessarily small, so we may move dirty pages
+                         * directly to the laundry queue.
+                         */
+                        if (inactq_shortage <= 0)
                                 vm_page_deactivate(m);
+                        else {
+                                /*
+                                 * Calling vm_page_test_dirty() here would
+                                 * require acquisition of the object's write
+                                 * lock.  However, during a page shortage,
+                                 * directing dirty pages into the laundry
+                                 * queue is only an optimization and not a
+                                 * requirement.  Therefore, we simply rely on
+                                 * the opportunistic updates to the page's
+                                 * dirty field by the pmap.
+                                 */
+                                if (m->dirty == 0) {
+                                        vm_page_deactivate(m);
+                                        inactq_shortage -=
+                                            act_scan_laundry_weight;
+                                } else {
+                                        vm_page_launder(m);
                                         inactq_shortage--;
+                                }
+                        }
                 } else
                         vm_page_requeue_locked(m);
                 vm_page_unlock(m);
         }
         vm_pagequeue_unlock(pq);
 #if !defined(NO_SWAPPING)
         /*
          * Idle process swapout -- run once per second when we are reclaiming
while (TRUE) {
                  * Might the page daemon receive a wakeup call?
                  */
                 if (vm_pageout_wanted) {
                         /*
                          * No.  Either vm_pageout_wanted was set by another
                          * thread during the previous scan, which must have
                          * been a level 0 scan, or vm_pageout_wanted was
                          * already set and the scan failed to free enough
-                         * pages.  If we haven't yet performed a level >= 2
-                         * scan (unlimited dirty cleaning), then upgrade the
-                         * level and scan again now.  Otherwise, sleep a bit
-                         * and try again later.
+                         * pages.  If we haven't yet performed a level >= 1
+                         * (page reclamation) scan, then increase the level
+                         * and scan again now.  Otherwise, sleep a bit and
+                         * try again later.
                          */
                         mtx_unlock(&vm_page_queue_free_mtx);
-                        if (pass > 1)
-                                pause("psleep", hz / 2);
+                        if (pass >= 1)
+                                pause("psleep", hz / VM_INACT_SCAN_RATE);
                         pass++;
                 } else {
                         /*
                          * Yes.  Sleep until pages need to be reclaimed or
                          * have their reference stats updated.
                          */
                         if (mtx_sleep(&vm_pageout_wanted,
                             &vm_page_queue_free_mtx, PDROP | PVM, "psleep",
vm_pageout_init(void)
          * case paging behaviors with stale active LRU.
          */
         if (vm_pageout_update_period == 0)
                 vm_pageout_update_period = 600;
 
         /* XXX does not really belong here */
         if (vm_page_max_wired == 0)
                 vm_page_max_wired = vm_cnt.v_free_count / 3;
+
+        /*
+         * Target amount of memory to move out of the laundry queue during a
+         * background laundering.  This is proportional to the amount of system
+         * memory.
+         */
+        vm_background_launder_target = (vm_cnt.v_free_target -
+            vm_cnt.v_free_min) / 10;
 }
 
 /*
  *     vm_pageout is the high level pageout daemon.
  */
 static void
 vm_pageout(void)
 {
         int error;
 #ifdef VM_NUMA_ALLOC
         int i;
 #endif
 
         swap_pager_swap_init();
+        error = kthread_add(vm_pageout_laundry_worker, NULL, curproc, NULL,
+            0, 0, "laundry: dom0");
+        if (error != 0)
+                panic("starting laundry for domain 0, error %d", error);
 #ifdef VM_NUMA_ALLOC
         for (i = 1; i < vm_ndomains; i++) {
                 error = kthread_add(vm_pageout_worker, (void *)(uintptr_t)i,
                     curproc, NULL, 0, 0, "dom%d", i);
                 if (error != 0) {
                         panic("starting pageout for domain %d, error %d\n",
                             i, error);
                 }