Page MenuHomeFreeBSD
Files F161411638

D22885.id65832.diff
No OneTemporary
Actions

D22885.id65832.diff
View Options

Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c
===================================================================
--- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c
+++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c
@@ -1761,12 +1761,10 @@
bcopy((char *)db->db_data + bufoff, va, PAGESIZE);
zfs_unmap_page(sf);
vm_page_valid(m);
- vm_page_lock(m);
if ((m->busy_lock & VPB_BIT_WAITERS) != 0)
vm_page_activate(m);
else
vm_page_deactivate(m);
- vm_page_unlock(m);
vm_page_sunbusy(m);
}
*rbehind = i;
@@ -1884,12 +1882,10 @@
}
zfs_unmap_page(sf);
vm_page_valid(m);
- vm_page_lock(m);
if ((m->busy_lock & VPB_BIT_WAITERS) != 0)
vm_page_activate(m);
else
vm_page_deactivate(m);
- vm_page_unlock(m);
vm_page_sunbusy(m);
}
*rahead = i;
Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
===================================================================
--- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
+++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
@@ -545,9 +545,7 @@
zfs_vmobject_wlock(obj);
if (error == 0) {
vm_page_valid(pp);
- vm_page_lock(pp);
vm_page_activate(pp);
- vm_page_unlock(pp);
}
vm_page_sunbusy(pp);
if (error != 0 && !vm_page_wired(pp) &&
Index: sys/dev/md/md.c
===================================================================
--- sys/dev/md/md.c
+++ sys/dev/md/md.c
@@ -1145,12 +1145,8 @@
}
if (m != NULL) {
vm_page_xunbusy(m);
- vm_page_lock(m);
- if (vm_page_active(m))
- vm_page_reference(m);
- else
- vm_page_activate(m);
- vm_page_unlock(m);
+ vm_page_reference(m);
+ vm_page_activate(m);
}
/* Actions on further pages start at offset 0 */
Index: sys/fs/tmpfs/tmpfs_subr.c
===================================================================
--- sys/fs/tmpfs/tmpfs_subr.c
+++ sys/fs/tmpfs/tmpfs_subr.c
@@ -1490,9 +1490,7 @@
* current operation is not regarded
* as an access.
*/
- vm_page_lock(m);
vm_page_launder(m);
- vm_page_unlock(m);
} else {
vm_page_free(m);
if (ignerr)
Index: sys/vm/sg_pager.c
===================================================================
--- sys/vm/sg_pager.c
+++ sys/vm/sg_pager.c
@@ -194,9 +194,7 @@
VM_OBJECT_WLOCK(object);
TAILQ_INSERT_TAIL(&object->un_pager.sgp.sgp_pglist, page, plinks.q);
vm_page_replace_checked(page, object, offset, m[0]);
- vm_page_lock(m[0]);
vm_page_free(m[0]);
- vm_page_unlock(m[0]);
m[0] = page;
vm_page_valid(page);
Index: sys/vm/swap_pager.c
===================================================================
--- sys/vm/swap_pager.c
+++ sys/vm/swap_pager.c
@@ -1630,10 +1630,9 @@
* then finish the I/O.
*/
MPASS(m->dirty == VM_PAGE_BITS_ALL);
+
/* PQ_UNSWAPPABLE? */
- vm_page_lock(m);
vm_page_activate(m);
- vm_page_unlock(m);
vm_page_sunbusy(m);
}
} else if (bp->b_iocmd == BIO_READ) {
@@ -1668,9 +1667,7 @@
("swp_pager_async_iodone: page %p is not write"
" protected", m));
vm_page_undirty(m);
- vm_page_lock(m);
vm_page_deactivate_noreuse(m);
- vm_page_unlock(m);
vm_page_sunbusy(m);
}
}
@@ -1732,9 +1729,7 @@
{
vm_page_dirty(m);
- vm_page_lock(m);
vm_page_launder(m);
- vm_page_unlock(m);
vm_page_xunbusy(m);
swap_pager_unswapped(m);
}
Index: sys/vm/vm_fault.c
===================================================================
--- sys/vm/vm_fault.c
+++ sys/vm/vm_fault.c
@@ -162,9 +162,7 @@
* this page. Deactivating it leaves it available for
* pageout while optimizing fault restarts.
*/
- vm_page_lock(m);
vm_page_deactivate(m);
- vm_page_unlock(m);
vm_page_xunbusy(m);
*mp = NULL;
}
@@ -395,9 +393,7 @@
for (pidx = first, m = vm_page_lookup(object, pidx);
pidx <= last; pidx++, m = vm_page_next(m)) {
vm_fault_populate_check_page(m);
- vm_page_lock(m);
vm_page_deactivate(m);
- vm_page_unlock(m);
vm_page_xunbusy(m);
}
}
@@ -1387,13 +1383,10 @@
* If the page is not wired down, then put it where the pageout daemon
* can find it.
*/
- if ((fault_flags & VM_FAULT_WIRE) != 0) {
+ if ((fault_flags & VM_FAULT_WIRE) != 0)
vm_page_wire(fs.m);
- } else {
- vm_page_lock(fs.m);
+ else
vm_page_activate(fs.m);
- vm_page_unlock(fs.m);
- }
if (m_hold != NULL) {
*m_hold = fs.m;
vm_page_wire(fs.m);
@@ -1493,12 +1486,13 @@
* Typically, at this point, prefetched pages
* are still in the inactive queue. Only
* pages that triggered page faults are in the
- * active queue.
+ * active queue. The test for whether the page
+ * is in the inactive queue is racy; in the
+ * worst case we will requeue the page
+ * unnecessarily.
*/
- vm_page_lock(m);
if (!vm_page_inactive(m))
vm_page_deactivate(m);
- vm_page_unlock(m);
}
}
}
@@ -1873,9 +1867,7 @@
("dst_m %p is not wired", dst_m));
}
} else {
- vm_page_lock(dst_m);
vm_page_activate(dst_m);
- vm_page_unlock(dst_m);
}
vm_page_xunbusy(dst_m);
}
Index: sys/vm/vm_object.c
===================================================================
--- sys/vm/vm_object.c
+++ sys/vm/vm_object.c
@@ -1293,9 +1293,7 @@
vm_page_busy_sleep(tm, "madvpo", false);
goto relookup;
}
- vm_page_lock(tm);
vm_page_advise(tm, advice);
- vm_page_unlock(tm);
vm_page_xunbusy(tm);
vm_object_madvise_freespace(tobject, advice, tm->pindex, 1);
next_pindex:
Index: sys/vm/vm_page.h
===================================================================
--- sys/vm/vm_page.h
+++ sys/vm/vm_page.h
@@ -803,12 +803,6 @@
{
uint32_t *addr, val;
- /*
- * The PGA_REFERENCED flag can only be cleared if the page is locked.
- */
- if ((bits & PGA_REFERENCED) != 0)
- vm_page_assert_locked(m);
-
/*
* Access the whole 32-bit word containing the aflags field with an
* atomic update. Parallel non-atomic updates to the other fields
@@ -899,8 +893,6 @@
{
vm_page_astate_t as;
- vm_page_assert_locked(m);
-
as = vm_page_astate_load(m);
if ((as.flags & PGA_DEQUEUE) != 0)
return (PQ_NONE);
Index: sys/vm/vm_page.c
===================================================================
--- sys/vm/vm_page.c
+++ sys/vm/vm_page.c
@@ -1361,12 +1361,10 @@
* have shown that deactivating the page is usually the best choice,
* unless the page is wanted by another thread.
*/
- vm_page_lock(m);
if ((m->busy_lock & VPB_BIT_WAITERS) != 0)
vm_page_activate(m);
else
vm_page_deactivate(m);
- vm_page_unlock(m);
vm_page_xunbusy_unchecked(m);
}
@@ -3589,7 +3587,6 @@
vm_page_enqueue(vm_page_t m, uint8_t queue)
{
- vm_page_assert_locked(m);
KASSERT(m->a.queue == PQ_NONE &&
(m->a.flags & PGA_QUEUE_STATE_MASK) == 0,
("%s: page %p is already enqueued", __func__, m));
@@ -4012,7 +4009,6 @@
vm_page_unswappable(vm_page_t m)
{
- vm_page_assert_locked(m);
KASSERT(!vm_page_wired(m) && (m->oflags & VPO_UNMANAGED) == 0,
("page %p already unswappable", m));
@@ -4114,9 +4110,7 @@
m->dirty == 0 && !vm_page_busied(m)) {
vm_page_free(m);
} else {
- vm_page_lock(m);
vm_page_release_toq(m, PQ_INACTIVE, flags != 0);
- vm_page_unlock(m);
}
}
}
@@ -4185,7 +4179,6 @@
vm_page_advise(vm_page_t m, int advice)
{
- vm_page_assert_locked(m);
VM_OBJECT_ASSERT_WLOCKED(m->object);
if (advice == MADV_FREE)
/*
@@ -4201,15 +4194,15 @@
return;
}
+ if (advice != MADV_FREE && m->dirty == 0 && pmap_is_modified(m))
+ vm_page_dirty(m);
+
/*
* Clear any references to the page. Otherwise, the page daemon will
* immediately reactivate the page.
*/
vm_page_aflag_clear(m, PGA_REFERENCED);
- if (advice != MADV_FREE && m->dirty == 0 && pmap_is_modified(m))
- vm_page_dirty(m);
-
/*
* Place clean pages near the head of the inactive queue rather than
* the tail, thus defeating the queue's LRU operation and ensuring that
Index: sys/vm/vm_pageout.c
===================================================================
--- sys/vm/vm_pageout.c
+++ sys/vm/vm_pageout.c
@@ -376,14 +376,11 @@
vm_page_xunbusy(p);
break;
}
- vm_page_lock(p);
if (!vm_page_in_laundry(p) || !vm_page_try_remove_write(p)) {
- vm_page_unlock(p);
vm_page_xunbusy(p);
ib = 0;
break;
}
- vm_page_unlock(p);
mc[--page_base] = pb = p;
++pageout_count;
++ib;
@@ -409,13 +406,10 @@
vm_page_xunbusy(p);
break;
}
- vm_page_lock(p);
if (!vm_page_in_laundry(p) || !vm_page_try_remove_write(p)) {
- vm_page_unlock(p);
vm_page_xunbusy(p);
break;
}
- vm_page_unlock(p);
mc[page_base + pageout_count] = ps = p;
++pageout_count;
++is;
@@ -491,10 +485,12 @@
("vm_pageout_flush: page %p is not write protected", mt));
switch (pageout_status[i]) {
case VM_PAGER_OK:
- vm_page_lock(mt);
+ /*
+ * The page may have moved since laundering started, in
+ * which case it should be left alone.
+ */
if (vm_page_in_laundry(mt))
vm_page_deactivate_noreuse(mt);
- vm_page_unlock(mt);
/* FALLTHROUGH */
case VM_PAGER_PEND:
numpagedout++;
@@ -507,10 +503,8 @@
* the page daemon.
*/
vm_page_undirty(mt);
- vm_page_lock(mt);
if (vm_page_in_laundry(mt))
vm_page_deactivate_noreuse(mt);
- vm_page_unlock(mt);
break;
case VM_PAGER_ERROR:
case VM_PAGER_FAIL:
@@ -526,14 +520,12 @@
* clog the laundry and inactive queues. (We will try
* paging it out again later.)
*/
- vm_page_lock(mt);
if (object->type == OBJT_SWAP &&
pageout_status[i] == VM_PAGER_FAIL) {
vm_page_unswappable(mt);
numpagedout++;
} else
vm_page_activate(mt);
- vm_page_unlock(mt);
if (eio != NULL && i >= mreq && i - mreq < runlen)
*eio = TRUE;
break;
@@ -591,7 +583,6 @@
vm_pindex_t pindex;
int error, lockmode;
- vm_page_assert_locked(m);
object = m->object;
VM_OBJECT_ASSERT_WLOCKED(object);
error = 0;
@@ -611,7 +602,6 @@
* of time.
*/
if (object->type == OBJT_VNODE) {
- vm_page_unlock(m);
vm_page_xunbusy(m);
vp = object->handle;
if (vp->v_type == VREG &&
@@ -642,11 +632,9 @@
error = ENOENT;
goto unlock_all;
}
- vm_page_lock(m);
/*
- * While the object and page were unlocked, the page
- * may have been:
+ * While the object was unlocked, the page may have been:
* (1) moved to a different queue,
* (2) reallocated to a different object,
* (3) reallocated to a different offset, or
@@ -654,17 +642,15 @@
*/
if (!vm_page_in_laundry(m) || m->object != object ||
m->pindex != pindex || m->dirty == 0) {
- vm_page_unlock(m);
error = ENXIO;
goto unlock_all;
}
/*
- * The page may have been busied while the object and page
- * locks were released.
+ * The page may have been busied while the object lock was
+ * released.
*/
if (vm_page_tryxbusy(m) == 0) {
- vm_page_unlock(m);
error = EBUSY;
goto unlock_all;
}
@@ -675,11 +661,9 @@
*/
if (!vm_page_try_remove_write(m)) {
vm_page_xunbusy(m);
- vm_page_unlock(m);
error = EBUSY;
goto unlock_all;
}
- vm_page_unlock(m);
/*
* If a page is dirty, then it is either being washed
@@ -694,7 +678,6 @@
VM_OBJECT_WUNLOCK(object);
unlock_mp:
- vm_page_lock_assert(m, MA_NOTOWNED);
if (mp != NULL) {
if (vp != NULL)
vput(vp);
@@ -715,7 +698,6 @@
{
struct scan_state ss;
struct vm_pagequeue *pq;
- struct mtx *mtx;
vm_object_t object;
vm_page_t m, marker;
vm_page_astate_t new, old;
@@ -723,7 +705,6 @@
int vnodes_skipped;
bool pageout_ok;
- mtx = NULL;
object = NULL;
starting_target = launder;
vnodes_skipped = 0;
@@ -751,9 +732,6 @@
if (__predict_false((m->flags & PG_MARKER) != 0))
continue;
- vm_page_change_lock(m, &mtx);
-
-recheck:
/*
* Don't touch a page that was removed from the queue after the
* page queue lock was released. Otherwise, ensure that any
@@ -768,38 +746,33 @@
continue;
}
- if (object != m->object) {
+ /*
+ * Lock the page's object.
+ */
+ if (object == NULL || object != m->object) {
if (object != NULL)
VM_OBJECT_WUNLOCK(object);
-
- /*
- * A page's object pointer may be set to NULL before
- * the object lock is acquired.
- */
object = (vm_object_t)atomic_load_ptr(&m->object);
- if (object != NULL && !VM_OBJECT_TRYWLOCK(object)) {
- mtx_unlock(mtx);
- /* Depends on type-stability. */
- VM_OBJECT_WLOCK(object);
- mtx_lock(mtx);
- goto recheck;
+ if (__predict_false(object == NULL))
+ /* The page is being freed by another thread. */
+ continue;
+
+ /* Depends on type-stability. */
+ VM_OBJECT_WLOCK(object);
+ if (__predict_false(m->object != object)) {
+ VM_OBJECT_WUNLOCK(object);
+ object = NULL;
+ continue;
}
}
- if (__predict_false(m->object == NULL))
- /*
- * The page has been removed from its object.
- */
- continue;
- KASSERT(m->object == object, ("page %p does not belong to %p",
- m, object));
if (vm_page_tryxbusy(m) == 0)
continue;
/*
* Check for wirings now that we hold the object lock and have
- * verified that the page is unbusied. If the page is mapped,
- * it may still be wired by pmap lookups. The call to
+ * exclusively busied the page. If the page is mapped, it may
+ * still be wired by pmap lookups. The call to
* vm_page_try_remove_all() below atomically checks for such
* wirings and removes mappings. If the page is unmapped, the
* wire count is guaranteed not to increase.
@@ -889,6 +862,18 @@
}
}
+ /*
+ * Now we are guaranteed that no other threads are manipulating
+ * the page, check for a last-second reference.
+ */
+ old = vm_page_astate_load(m);
+ if (old.queue != queue || (old.flags & PGA_ENQUEUED) == 0)
+ goto skip_page;
+ if ((old.flags & PGA_QUEUE_OP_MASK) != 0) {
+ vm_page_pqbatch_submit(m, queue);
+ goto skip_page;
+ }
+
/*
* Clean pages are freed, and dirty pages are paged out unless
* they belong to a dead object. Requeueing dirty pages from
@@ -933,17 +918,12 @@
pageout_lock_miss++;
vnodes_skipped++;
}
- mtx = NULL;
object = NULL;
} else {
skip_page:
vm_page_xunbusy(m);
}
}
- if (mtx != NULL) {
- mtx_unlock(mtx);
- mtx = NULL;
- }
if (object != NULL) {
VM_OBJECT_WUNLOCK(object);
object = NULL;
@@ -1180,7 +1160,6 @@
vm_pageout_scan_active(struct vm_domain *vmd, int page_shortage)
{
struct scan_state ss;
- struct mtx *mtx;
vm_object_t object;
vm_page_t m, marker;
struct vm_pagequeue *pq;
@@ -1221,7 +1200,6 @@
* and scanning resumes.
*/
max_scan = page_shortage > 0 ? pq->pq_cnt : min_scan;
- mtx = NULL;
act_scan:
vm_pageout_init_scan(&ss, pq, marker, &vmd->vmd_clock[0], max_scan);
while ((m = vm_pageout_next(&ss, false)) != NULL) {
@@ -1240,8 +1218,6 @@
if (__predict_false((m->flags & PG_MARKER) != 0))
continue;
- vm_page_change_lock(m, &mtx);
-
/*
* Don't touch a page that was removed from the queue after the
* page queue lock was released. Otherwise, ensure that any
@@ -1381,10 +1357,6 @@
page_shortage -= ps_delta;
}
- if (mtx != NULL) {
- mtx_unlock(mtx);
- mtx = NULL;
- }
vm_pagequeue_lock(pq);
TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q);
TAILQ_INSERT_AFTER(&pq->pq_pl, marker, &vmd->vmd_clock[0], plinks.q);
@@ -1450,7 +1422,6 @@
{
struct scan_state ss;
struct vm_batchqueue rq;
- struct mtx *mtx;
vm_page_t m, marker;
struct vm_pagequeue *pq;
vm_object_t object;
@@ -1475,7 +1446,6 @@
deficit = atomic_readandclear_int(&vmd->vmd_pageout_deficit);
starting_page_shortage = page_shortage = shortage + deficit;
- mtx = NULL;
object = NULL;
vm_batchqueue_init(&rq);
@@ -1493,9 +1463,6 @@
KASSERT((m->flags & PG_MARKER) == 0,
("marker page %p was dequeued", m));
- vm_page_change_lock(m, &mtx);
-
-recheck:
/*
* Don't touch a page that was removed from the queue after the
* page queue lock was released. Otherwise, ensure that any
@@ -1510,30 +1477,25 @@
continue;
}
- if (object != m->object) {
+ /*
+ * Lock the page's object.
+ */
+ if (object == NULL || object != m->object) {
if (object != NULL)
VM_OBJECT_WUNLOCK(object);
-
- /*
- * A page's object pointer may be set to NULL before
- * the object lock is acquired.
- */
object = (vm_object_t)atomic_load_ptr(&m->object);
- if (object != NULL && !VM_OBJECT_TRYWLOCK(object)) {
- mtx_unlock(mtx);
- /* Depends on type-stability. */
- VM_OBJECT_WLOCK(object);
- mtx_lock(mtx);
- goto recheck;
+ if (__predict_false(object == NULL))
+ /* The page is being freed by another thread. */
+ continue;
+
+ /* Depends on type-stability. */
+ VM_OBJECT_WLOCK(object);
+ if (__predict_false(m->object != object)) {
+ VM_OBJECT_WUNLOCK(object);
+ object = NULL;
+ goto reinsert;
}
}
- if (__predict_false(m->object == NULL))
- /*
- * The page has been removed from its object.
- */
- continue;
- KASSERT(m->object == object, ("page %p does not belong to %p",
- m, object));
if (vm_page_tryxbusy(m) == 0) {
/*
@@ -1553,9 +1515,9 @@
vm_pager_page_unswapped(m);
/*
- * Re-check for wirings now that we hold the object lock and
- * have verified that the page is unbusied. If the page is
- * mapped, it may still be wired by pmap lookups. The call to
+ * Check for wirings now that we hold the object lock and have
+ * exclusively busied the page. If the page is mapped, it may
+ * still be wired by pmap lookups. The call to
* vm_page_try_remove_all() below atomically checks for such
* wirings and removes mappings. If the page is unmapped, the
* wire count is guaranteed not to increase.
@@ -1637,6 +1599,20 @@
}
}
+ /*
+ * Now we are guaranteed that no other threads are manipulating
+ * the page, check for a last-second reference that would save
+ * it from doom.
+ */
+ old = vm_page_astate_load(m);
+ if (__predict_false(old.queue != PQ_INACTIVE ||
+ (old.flags & PGA_ENQUEUED) != 0))
+ goto skip_page;
+ if (__predict_false((old.flags & PGA_QUEUE_OP_MASK) != 0)) {
+ vm_page_pqbatch_submit(m, PQ_INACTIVE);
+ goto skip_page;
+ }
+
/*
* Clean pages can be freed, but dirty pages must be sent back
* to the laundry, unless they belong to a dead object.
@@ -1647,13 +1623,11 @@
if (m->dirty == 0) {
free_page:
/*
- * Because we dequeued the page and have already
- * checked for concurrent dequeue and enqueue
- * requests, we can safely disassociate the page
- * from the inactive queue.
+ * Because we dequeued the page and have already checked
+ * for pending dequeue and enqueue requests, we can
+ * safely disassociate the page from the inactive queue
+ * without holding the queue lock.
*/
- KASSERT((m->a.flags & PGA_QUEUE_STATE_MASK) == 0,
- ("page %p has queue state", m));
m->a.queue = PQ_NONE;
vm_page_free(m);
page_shortage--;
@@ -1667,8 +1641,6 @@
reinsert:
vm_pageout_reinsert_inactive(&ss, &rq, m);
}
- if (mtx != NULL)
- mtx_unlock(mtx);
if (object != NULL)
VM_OBJECT_WUNLOCK(object);
vm_pageout_reinsert_inactive(&ss, &rq, NULL);

File Metadata

Mime Type
text/plain
Expires
Sat, Jul 4, 1:00 PM (2 h, 49 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
34670595
Default Alt Text
D22885.id65832.diff (18 KB)

Event Timeline