Index: sys/vm/vm_meter.c =================================================================== --- sys/vm/vm_meter.c +++ sys/vm/vm_meter.c @@ -495,6 +495,48 @@ CTLTYPE_U64 | CTLFLAG_MPSAFE | CTLFLAG_RD, NULL, 0, sysctl_vm_pdpages, "QU", "Pages analyzed by pagedaemon"); +#define VM_DOMAIN_PAGEQUEUE_STATS_INIT(oid, pq, prefix, name) \ + SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, \ + prefix, CTLFLAG_RD, &(pq)->pq_cnt, 0, name " pages"); \ + SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, \ + prefix "pdpgs", CTLFLAG_RD, &(pq)->pq_pdpages, 0, \ + name "pages scanned by the page daemon"); \ + +static void +vm_domain_pagequeue_stats_init(struct sysctl_oid *parent, + struct vm_pagequeue *pq, const char *name) +{ + struct sysctl_oid *oid; + char descr[128]; + + snprintf(descr, sizeof(descr), "Counters for the %s queue", name); + oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(parent), OID_AUTO, + name, CTLFLAG_RD, NULL, descr); + + SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "pages", CTLFLAG_RD, &pq->pq_cnt, 0, + "Number of pages in queue"); + + SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "pdpgs", CTLFLAG_RD, &pq->pq_pdpages, 0, + "Pages scanned by the page daemon"); + SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "xbusy", CTLFLAG_RD, &pq->pq_pdxbusy, 0, + "Exclusively busy pages encountered during a queue scan"); + SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "deferred", CTLFLAG_RD, &pq->pq_pddeferred, 0, + "Page scans deferred due to pending queue state updates"); + SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "wired", CTLFLAG_RD, &pq->pq_pdwired, 0, + "Page scans skipped due to page wirings"); + SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "reactivated", CTLFLAG_RD, &pq->pq_pdreactivated, 0, + "Pages reactivated due to references"); + SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, + "requeued", CTLFLAG_RD, &pq->pq_pdrequeued, 0, + "Pages requeued due to references"); +} + static void vm_domain_stats_init(struct vm_domain *vmd, struct sysctl_oid *parent) { @@ -504,37 +546,19 @@ vmd->vmd_name, CTLFLAG_RD, NULL, ""); oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(vmd->vmd_oid), OID_AUTO, "stats", CTLFLAG_RD, NULL, ""); + + vm_domain_pagequeue_stats_init(oid, &vmd->vmd_pagequeues[PQ_ACTIVE], + "active"); + vm_domain_pagequeue_stats_init(oid, &vmd->vmd_pagequeues[PQ_INACTIVE], + "inactive"); + vm_domain_pagequeue_stats_init(oid, &vmd->vmd_pagequeues[PQ_LAUNDRY], + "laundry"); + vm_domain_pagequeue_stats_init(oid, &vmd->vmd_pagequeues[PQ_UNSWAPPABLE], + "unswappable"); + SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "free_count", CTLFLAG_RD, &vmd->vmd_free_count, 0, "Free pages"); - SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, - "active", CTLFLAG_RD, &vmd->vmd_pagequeues[PQ_ACTIVE].pq_cnt, 0, - "Active pages"); - SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, - "actpdpgs", CTLFLAG_RD, - &vmd->vmd_pagequeues[PQ_ACTIVE].pq_pdpages, 0, - "Active pages scanned by the page daemon"); - SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, - "inactive", CTLFLAG_RD, &vmd->vmd_pagequeues[PQ_INACTIVE].pq_cnt, 0, - "Inactive pages"); - SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, - "inactpdpgs", CTLFLAG_RD, - &vmd->vmd_pagequeues[PQ_INACTIVE].pq_pdpages, 0, - "Inactive pages scanned by the page daemon"); - SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, - "laundry", CTLFLAG_RD, &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_cnt, 0, - "laundry pages"); - SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, - "laundpdpgs", CTLFLAG_RD, - &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_pdpages, 0, - "Laundry pages scanned by the page daemon"); - SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "unswappable", - CTLFLAG_RD, &vmd->vmd_pagequeues[PQ_UNSWAPPABLE].pq_cnt, 0, - "Unswappable pages"); - SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, - "unswppdpgs", CTLFLAG_RD, - &vmd->vmd_pagequeues[PQ_UNSWAPPABLE].pq_pdpages, 0, - "Unswappable pages scanned by the page daemon"); SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "inactive_target", CTLFLAG_RD, &vmd->vmd_inactive_target, 0, "Target inactive pages"); Index: sys/vm/vm_pageout.c =================================================================== --- sys/vm/vm_pageout.c +++ sys/vm/vm_pageout.c @@ -208,8 +208,13 @@ struct vm_batchqueue bq; struct vm_pagequeue *pq; vm_page_t marker; - int maxscan; - int scanned; + int maxscan; /* Limit on number of pages to scan. */ + int scanned; /* Pages scanned so far. */ + int xbusy; /* Skipped due to xbusy. */ + int deferred; /* Skipped due to pending queue ops. */ + int wired; /* Skipped due to wirings. */ + int reactivated; /* Moved to the active queue. */ + int requeued; /* Given a second chance. */ }; static void @@ -227,11 +232,11 @@ TAILQ_INSERT_AFTER(&pq->pq_pl, after, marker, plinks.q); vm_page_aflag_set(marker, PGA_ENQUEUED); + memset(ss, 0, sizeof(*ss)); vm_batchqueue_init(&ss->bq); ss->pq = pq; ss->marker = marker; ss->maxscan = maxscan; - ss->scanned = 0; vm_pagequeue_unlock(pq); } @@ -248,6 +253,13 @@ TAILQ_REMOVE(&pq->pq_pl, ss->marker, plinks.q); vm_page_aflag_clear(ss->marker, PGA_ENQUEUED); pq->pq_pdpages += ss->scanned; + pq->pq_pdxbusy += ss->xbusy; + pq->pq_pddeferred += ss->deferred; + pq->pq_pdwired += ss->wired; + pq->pq_pdreactivated += ss->reactivated; + pq->pq_pdrequeued += ss->requeued; + + VM_CNT_ADD(v_reactivated, ss->reactivated); } /* @@ -322,21 +334,36 @@ * outstanding queue operations are processed. */ static __always_inline bool -vm_pageout_defer(vm_page_t m, const uint8_t queue, const bool enqueued) +vm_pageout_defer(struct scan_state *ss, vm_page_t m, const uint8_t queue, + const bool enqueued) { vm_page_astate_t as; as = vm_page_astate_load(m); if (__predict_false(as.queue != queue || - ((as.flags & PGA_ENQUEUED) != 0) != enqueued)) + ((as.flags & PGA_ENQUEUED) != 0) != enqueued)) { + ss->deferred++; return (true); + } if ((as.flags & PGA_QUEUE_OP_MASK) != 0) { vm_page_pqbatch_submit(m, queue); + ss->deferred++; return (true); } return (false); } +static __always_inline bool +vm_pageout_page_xbusy(struct scan_state *ss, vm_page_t m) +{ + + if (!vm_page_tryxbusy(m)) { + ss->xbusy++; + return (false); + } + return (true); +} + /* * Scan for pages at adjacent offsets within the given page's object that are * eligible for laundering, form a cluster of these pages and the given page, @@ -758,7 +785,7 @@ * pending queue operations, such as dequeues for wired pages, * are handled. */ - if (vm_pageout_defer(m, queue, true)) + if (vm_pageout_defer(&ss, m, queue, true)) continue; /* @@ -781,7 +808,7 @@ } } - if (vm_page_tryxbusy(m) == 0) + if (!vm_pageout_page_xbusy(&ss, m)) continue; /* @@ -792,8 +819,10 @@ * wirings and removes mappings. If the page is unmapped, the * wire count is guaranteed not to increase after this check. */ - if (__predict_false(vm_page_wired(m))) + if (__predict_false(vm_page_wired(m))) { + ss.wired++; goto skip_page; + } /* * Invalid pages can be easily freed. They cannot be @@ -811,8 +840,10 @@ * so, discarding any references collected by * pmap_ts_referenced(). */ - if (__predict_false(_vm_page_queue(old) == PQ_NONE)) + if (__predict_false(_vm_page_queue(old) == PQ_NONE)) { + ss.deferred++; goto skip_page; + } new = old; act_delta = refs; @@ -850,12 +881,13 @@ */ if (!in_shortfall) launder--; - VM_CNT_INC(v_reactivated); + ss.reactivated++; goto skip_page; } else if ((object->flags & OBJ_DEAD) == 0) { new.flags |= PGA_REQUEUE; if (!vm_page_pqstate_commit(m, &old, new)) continue; + ss.requeued++; goto skip_page; } break; @@ -870,8 +902,10 @@ */ if (object->ref_count != 0) { vm_page_test_dirty(m); - if (m->dirty == 0 && !vm_page_try_remove_all(m)) + if (m->dirty == 0 && !vm_page_try_remove_all(m)) { + ss.wired++; goto skip_page; + } } /* @@ -887,7 +921,7 @@ * manipulating the page, check for a last-second * reference. */ - if (vm_pageout_defer(m, queue, true)) + if (vm_pageout_defer(&ss, m, queue, true)) goto skip_page; vm_page_free(m); VM_CNT_INC(v_dfree); @@ -1231,7 +1265,7 @@ * pending queue operations, such as dequeues for wired pages, * are handled. */ - if (vm_pageout_defer(m, PQ_ACTIVE, true)) + if (vm_pageout_defer(&ss, m, PQ_ACTIVE, true)) continue; /* @@ -1472,7 +1506,7 @@ * pending queue operations, such as dequeues for wired pages, * are handled. */ - if (vm_pageout_defer(m, PQ_INACTIVE, false)) + if (vm_pageout_defer(&ss, m, PQ_INACTIVE, false)) continue; /* @@ -1495,7 +1529,7 @@ } } - if (vm_page_tryxbusy(m) == 0) { + if (!vm_pageout_page_xbusy(&ss, m)) { /* * Don't mess with busy pages. Leave them at * the front of the queue. Most likely, they @@ -1520,8 +1554,10 @@ * wirings and removes mappings. If the page is unmapped, the * wire count is guaranteed not to increase after this check. */ - if (__predict_false(vm_page_wired(m))) + if (__predict_false(vm_page_wired(m))) { + ss.wired++; goto skip_page; + } /* * Invalid pages can be easily freed. They cannot be @@ -1539,8 +1575,10 @@ * so, discarding any references collected by * pmap_ts_referenced(). */ - if (__predict_false(_vm_page_queue(old) == PQ_NONE)) + if (__predict_false(_vm_page_queue(old) == PQ_NONE)) { + ss.deferred++; goto skip_page; + } new = old; act_delta = refs; @@ -1569,13 +1607,14 @@ if (!vm_page_pqstate_commit(m, &old, new)) continue; - VM_CNT_INC(v_reactivated); + ss.reactivated++; goto skip_page; } else if ((object->flags & OBJ_DEAD) == 0) { new.queue = PQ_INACTIVE; new.flags |= PGA_REQUEUE; if (!vm_page_pqstate_commit(m, &old, new)) continue; + ss.requeued++; goto skip_page; } break; @@ -1590,8 +1629,10 @@ */ if (object->ref_count != 0) { vm_page_test_dirty(m); - if (m->dirty == 0 && !vm_page_try_remove_all(m)) + if (m->dirty == 0 && !vm_page_try_remove_all(m)) { + ss.wired++; goto skip_page; + } } /* @@ -1608,7 +1649,7 @@ * manipulating the page, check for a last-second * reference that would save it from doom. */ - if (vm_pageout_defer(m, PQ_INACTIVE, false)) + if (vm_pageout_defer(&ss, m, PQ_INACTIVE, false)) goto skip_page; /* Index: sys/vm/vm_pagequeue.h =================================================================== --- sys/vm/vm_pagequeue.h +++ sys/vm/vm_pagequeue.h @@ -71,7 +71,13 @@ struct pglist pq_pl; int pq_cnt; const char * const pq_name; + uint64_t pq_pdpages; + uint64_t pq_pdxbusy; + uint64_t pq_pddeferred; + uint64_t pq_pdwired; + uint64_t pq_pdreactivated; + uint64_t pq_pdrequeued; } __aligned(CACHE_LINE_SIZE); #ifndef VM_BATCHQUEUE_SIZE