Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h +++ sys/vm/vm_page.h @@ -109,6 +109,22 @@ * contains the dirty field. In the machine-independent layer, * the implementation of read-modify-write operations on the * field is encapsulated in vm_page_clear_dirty_mask(). + * + * Two types of page structure references exist: holds and wires. + * Holds are short-term references which prevent the page from being freed + * and reused for another purpose. However, it is legal to call + * vm_page_free() on a held page, in which case the page will be removed + * from its paging queue (if any) and its object (if any). Wires are + * stronger references which indicate that the page's contents are in use. + * It is not legal to call vm_page_free() on a wired page. When releasing + * the last wiring of a page, that page may optionally be inserted at the + * tail of a paging queue. If the page already belongs to that queue, it + * is requeued to preserve LRU. Both holds and wires prevent pages from + * being examined by the page daemon. + * + * The wire count field of the page structure has a different meaning for + * page table pages on some architectures: it records the number of valid + * entries in the page. */ #if PAGE_SIZE == 4096 Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -2705,9 +2705,7 @@ if (queue != PQ_NONE) vm_page_dequeue(m); vm_page_enqueue(PQ_ACTIVE, m); - } else - KASSERT(queue == PQ_NONE, - ("vm_page_activate: wired page %p is queued", m)); + } } else { if (m->act_count < ACT_INIT) m->act_count = ACT_INIT; @@ -2825,9 +2823,7 @@ /* * vm_page_wire: * - * Mark this page as wired down by yet - * another map, removing it from paging queues - * as necessary. + * Mark this page as wired down by yet another map. * * If the page is fictitious, then its wire count must remain one. * @@ -2837,11 +2833,6 @@ vm_page_wire(vm_page_t m) { - /* - * Only bump the wire statistics if the page is not already wired, - * and only unqueue the page if it is on some queue (if it is unmanaged - * it is already off the queues). - */ vm_page_lock_assert(m, MA_OWNED); if ((m->flags & PG_FICTITIOUS) != 0) { KASSERT(m->wire_count == 1, @@ -2853,7 +2844,6 @@ KASSERT((m->oflags & VPO_UNMANAGED) == 0 || m->queue == PQ_NONE, ("vm_page_wire: unmanaged page %p is queued", m)); - vm_page_remque(m); atomic_add_int(&vm_cnt.v_wire_count, 1); } m->wire_count++; @@ -2890,18 +2880,24 @@ ("vm_page_unwire: fictitious page %p's wire count isn't one", m)); return (FALSE); } - if (m->wire_count > 0) { - m->wire_count--; - if (m->wire_count == 0) { - atomic_subtract_int(&vm_cnt.v_wire_count, 1); - if ((m->oflags & VPO_UNMANAGED) == 0 && - m->object != NULL && queue != PQ_NONE) + if (m->wire_count == 0) + panic("vm_page_unwire: page %p's wire count is zero", m); + + m->wire_count--; + if (m->wire_count == 0) { + atomic_subtract_int(&vm_cnt.v_wire_count, 1); + if ((m->oflags & VPO_UNMANAGED) == 0 && m->object != NULL && + queue != PQ_NONE) { + if (m->queue == queue) + vm_page_requeue(m); + else { + vm_page_remque(m); vm_page_enqueue(queue, m); - return (TRUE); - } else - return (FALSE); + } + } + return (TRUE); } else - panic("vm_page_unwire: page %p's wire count is zero", m); + return (FALSE); } /* Index: sys/vm/vm_pageout.c =================================================================== --- sys/vm/vm_pageout.c +++ sys/vm/vm_pageout.c @@ -397,11 +397,9 @@ VM_OBJECT_ASSERT_WLOCKED(object); pindex = m->pindex; - /* - * We can't clean the page if it is busy or held. - */ vm_page_assert_unbusied(m); KASSERT(m->hold_count == 0, ("page %p is held", m)); + KASSERT(m->wire_count == 0, ("page %p is wired", m)); vm_page_unlock(m); mc[vm_pageout_page_count] = pb = ps = m; @@ -439,7 +437,7 @@ } vm_page_lock(p); if (!vm_page_in_laundry(p) || - p->hold_count != 0) { /* may be undergoing I/O */ + p->hold_count != 0 || p->wire_count != 0) { vm_page_unlock(p); ib = 0; break; @@ -465,7 +463,7 @@ break; vm_page_lock(p); if (!vm_page_in_laundry(p) || - p->hold_count != 0) { /* may be undergoing I/O */ + p->hold_count != 0 || p->wire_count != 0) { vm_page_unlock(p); break; } @@ -870,7 +868,8 @@ * The page may have been busied or held while the object * and page locks were released. */ - if (vm_page_busied(m) || m->hold_count != 0) { + if (vm_page_busied(m) || m->hold_count != 0 || + m->wire_count != 0) { vm_page_unlock(m); error = EBUSY; goto unlock_all; @@ -959,10 +958,27 @@ vm_page_unlock(m); continue; } + if (m->wire_count != 0) { + vm_page_dequeue_locked(m); + vm_page_unlock(m); + continue; + } object = m->object; - if ((!VM_OBJECT_TRYWLOCK(object) && - (!vm_pageout_fallback_object_lock(m, &next) || - m->hold_count != 0)) || vm_page_busied(m)) { + if (!VM_OBJECT_TRYWLOCK(object)) { + if (!vm_pageout_fallback_object_lock(m, &next) || + m->hold_count != 0) { + VM_OBJECT_WUNLOCK(object); + vm_page_unlock(m); + continue; + } + if (m->wire_count != 0) { + VM_OBJECT_WUNLOCK(object); + vm_page_dequeue_locked(m); + vm_page_unlock(m); + continue; + } + } + if (vm_page_busied(m)) { VM_OBJECT_WUNLOCK(object); vm_page_unlock(m); continue; @@ -1399,7 +1415,16 @@ */ if (!vm_pageout_page_lock(m, &next)) goto unlock_page; - else if (m->hold_count != 0) { + else if (m->wire_count != 0) { + /* + * Wired pages may not be freed, and unwiring a queued + * page will cause it to be requeued. Thus, remove them + * from the queue now to avoid unnecessary revisits. + */ + vm_page_dequeue_locked(m); + addl_page_shortage++; + goto unlock_page; + } else if (m->hold_count != 0) { /* * Held pages are essentially stuck in the * queue. So, they ought to be discounted @@ -1414,7 +1439,11 @@ if (!VM_OBJECT_TRYWLOCK(object)) { if (!vm_pageout_fallback_object_lock(m, &next)) goto unlock_object; - else if (m->hold_count != 0) { + else if (m->wire_count != 0) { + vm_page_dequeue_locked(m); + addl_page_shortage++; + goto unlock_object; + } else if (m->hold_count != 0) { addl_page_shortage++; goto unlock_object; } @@ -1436,6 +1465,7 @@ continue; } KASSERT(m->hold_count == 0, ("Held page %p", m)); + KASSERT(m->wire_count == 0, ("Wired page %p", m)); /* * Dequeue the inactive page and unlock the inactive page @@ -1637,6 +1667,15 @@ VM_CNT_INC(v_pdpages); /* + * Wired pages are dequeued lazily. + */ + if (m->wire_count != 0) { + vm_page_dequeue_locked(m); + vm_page_unlock(m); + continue; + } + + /* * Check to see "how much" the page has been used. */ if ((m->aflags & PGA_REFERENCED) != 0) {