D21548.id.diff
No OneTemporary
Actions

Size

28 KB

Referenced Files

None

Subscribers

None

D21548.id.diff
View Options

	Index: head/sys/dev/xen/gntdev/gntdev.c
	===================================================================
	--- head/sys/dev/xen/gntdev/gntdev.c
	+++ head/sys/dev/xen/gntdev/gntdev.c
	@@ -835,9 +835,9 @@
	*mres = NULL;
	}

	+ vm_page_busy_acquire(page, 0);
	vm_page_insert(page, object, pidx);
	page->valid = VM_PAGE_BITS_ALL;
	- vm_page_xbusy(page);
	*mres = page;
	return (VM_PAGER_OK);
	}
	Index: head/sys/dev/xen/privcmd/privcmd.c
	===================================================================
	--- head/sys/dev/xen/privcmd/privcmd.c
	+++ head/sys/dev/xen/privcmd/privcmd.c
	@@ -178,9 +178,9 @@
	*mres = NULL;
	}

	+ vm_page_busy_acquire(page, 0);
	vm_page_insert(page, object, pidx);
	page->valid = VM_PAGE_BITS_ALL;
	- vm_page_xbusy(page);
	*mres = page;
	return (VM_PAGER_OK);
	}
	Index: head/sys/fs/tmpfs/tmpfs_subr.c
	===================================================================
	--- head/sys/fs/tmpfs/tmpfs_subr.c
	+++ head/sys/fs/tmpfs/tmpfs_subr.c
	@@ -1406,10 +1406,8 @@
	if (base != 0) {
	idx = OFF_TO_IDX(newsize);
	retry:
	- m = vm_page_lookup(uobj, idx);
	+ m = vm_page_grab(uobj, idx, VM_ALLOC_NOCREAT);
	if (m != NULL) {
	- if (vm_page_sleep_if_busy(m, "tmfssz"))
	- goto retry;
	MPASS(m->valid == VM_PAGE_BITS_ALL);
	} else if (vm_pager_has_page(uobj, idx, NULL, NULL)) {
	m = vm_page_alloc(uobj, idx, VM_ALLOC_NORMAL \|
	@@ -1430,7 +1428,6 @@
	vm_page_lock(m);
	vm_page_launder(m);
	vm_page_unlock(m);
	- vm_page_xunbusy(m);
	} else {
	vm_page_free(m);
	if (ignerr)
	@@ -1444,6 +1441,7 @@
	if (m != NULL) {
	pmap_zero_page_area(m, base, PAGE_SIZE - base);
	vm_page_dirty(m);
	+ vm_page_xunbusy(m);
	vm_pager_page_unswapped(m);
	}
	}
	Index: head/sys/kern/kern_exec.c
	===================================================================
	--- head/sys/kern/kern_exec.c
	+++ head/sys/kern/kern_exec.c
	@@ -976,10 +976,14 @@
	#if VM_NRESERVLEVEL > 0
	vm_object_color(object, 0);
	#endif
	+retry:
	ma[0] = vm_page_grab(object, 0, VM_ALLOC_NORMAL \| VM_ALLOC_NOBUSY \|
	VM_ALLOC_WIRED);
	if (ma[0]->valid != VM_PAGE_BITS_ALL) {
	- vm_page_xbusy(ma[0]);
	+ if (vm_page_busy_acquire(ma[0], VM_ALLOC_WAITFAIL) == 0) {
	+ vm_page_unwire_noq(ma[0]);
	+ goto retry;
	+ }
	if (!vm_pager_has_page(object, 0, NULL, &after)) {
	if (vm_page_unwire_noq(ma[0]))
	vm_page_free(ma[0]);
	Index: head/sys/kern/uipc_shm.c
	===================================================================
	--- head/sys/kern/uipc_shm.c
	+++ head/sys/kern/uipc_shm.c
	@@ -457,10 +457,9 @@
	if (base != 0) {
	idx = OFF_TO_IDX(length);
	retry:
	- m = vm_page_lookup(object, idx);
	+ m = vm_page_grab(object, idx, VM_ALLOC_NOCREAT);
	if (m != NULL) {
	- if (vm_page_sleep_if_busy(m, "shmtrc"))
	- goto retry;
	+ MPASS(m->valid == VM_PAGE_BITS_ALL);
	} else if (vm_pager_has_page(object, idx, NULL, NULL)) {
	m = vm_page_alloc(object, idx,
	VM_ALLOC_NORMAL \| VM_ALLOC_WAITFAIL);
	@@ -478,7 +477,6 @@
	* as an access.
	*/
	vm_page_launder(m);
	- vm_page_xunbusy(m);
	} else {
	vm_page_free(m);
	VM_OBJECT_WUNLOCK(object);
	@@ -490,6 +488,7 @@
	KASSERT(m->valid == VM_PAGE_BITS_ALL,
	("shm_dotruncate: page %p is invalid", m));
	vm_page_dirty(m);
	+ vm_page_xunbusy(m);
	vm_pager_page_unswapped(m);
	}
	}
	Index: head/sys/kern/vfs_bio.c
	===================================================================
	--- head/sys/kern/vfs_bio.c
	+++ head/sys/kern/vfs_bio.c
	@@ -2945,10 +2945,10 @@
	presid = resid > (PAGE_SIZE - poffset) ?
	(PAGE_SIZE - poffset) : resid;
	KASSERT(presid >= 0, ("brelse: extra page"));
	- while (vm_page_xbusied(m))
	- vm_page_sleep_if_xbusy(m, "mbncsh");
	+ vm_page_busy_acquire(m, VM_ALLOC_SBUSY);
	if (pmap_page_wired_mappings(m) == 0)
	vm_page_set_invalid(m, poffset, presid);
	+ vm_page_sunbusy(m);
	vm_page_release_locked(m, flags);
	resid -= presid;
	poffset = 0;
	@@ -3651,7 +3651,7 @@
	("vfs_clean_pages_dirty_buf: no buffer offset"));

	VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
	- vfs_drain_busy_pages(bp);
	+ vfs_busy_pages_acquire(bp);
	vfs_setdirty_locked_object(bp);
	for (i = 0; i < bp->b_npages; i++) {
	noff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
	@@ -3663,6 +3663,7 @@
	/* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
	foff = noff;
	}
	+ vfs_busy_pages_release(bp);
	VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
	}

	@@ -4559,28 +4560,25 @@
	}

	/*
	- * Ensure that all buffer pages are not exclusive busied. If any page is
	- * exclusive busy, drain it.
	+ * Acquire a shared busy on all pages in the buf.
	*/
	void
	-vfs_drain_busy_pages(struct buf *bp)
	+vfs_busy_pages_acquire(struct buf *bp)
	{
	- vm_page_t m;
	- int i, last_busied;
	+ int i;

	VM_OBJECT_ASSERT_WLOCKED(bp->b_bufobj->bo_object);
	- last_busied = 0;
	- for (i = 0; i < bp->b_npages; i++) {
	- m = bp->b_pages[i];
	- if (vm_page_xbusied(m)) {
	- for (; last_busied < i; last_busied++)
	- vm_page_sbusy(bp->b_pages[last_busied]);
	- while (vm_page_xbusied(m)) {
	- vm_page_sleep_if_xbusy(m, "vbpage");
	- }
	- }
	- }
	- for (i = 0; i < last_busied; i++)
	+ for (i = 0; i < bp->b_npages; i++)
	+ vm_page_busy_acquire(bp->b_pages[i], VM_ALLOC_SBUSY);
	+}
	+
	+void
	+vfs_busy_pages_release(struct buf *bp)
	+{
	+ int i;
	+
	+ VM_OBJECT_ASSERT_WLOCKED(bp->b_bufobj->bo_object);
	+ for (i = 0; i < bp->b_npages; i++)
	vm_page_sunbusy(bp->b_pages[i]);
	}

	@@ -4613,17 +4611,17 @@
	KASSERT(bp->b_offset != NOOFFSET,
	("vfs_busy_pages: no buffer offset"));
	VM_OBJECT_WLOCK(obj);
	- vfs_drain_busy_pages(bp);
	+ if ((bp->b_flags & B_CLUSTER) == 0) {
	+ vm_object_pip_add(obj, bp->b_npages);
	+ vfs_busy_pages_acquire(bp);
	+ }
	if (bp->b_bufsize != 0)
	vfs_setdirty_locked_object(bp);
	bogus = false;
	for (i = 0; i < bp->b_npages; i++) {
	m = bp->b_pages[i];
	+ vm_page_assert_sbusied(m);

	- if ((bp->b_flags & B_CLUSTER) == 0) {
	- vm_object_pip_add(obj, 1);
	- vm_page_sbusy(m);
	- }
	/*
	* When readying a buffer for a read ( i.e
	* clear_modify == 0 ), it is important to do
	Index: head/sys/kern/vfs_cluster.c
	===================================================================
	--- head/sys/kern/vfs_cluster.c
	+++ head/sys/kern/vfs_cluster.c
	@@ -418,11 +418,9 @@
	for (bn = blkno, i = 0; i < run; ++i, bn += inc) {
	if (i == 0) {
	VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
	- vfs_drain_busy_pages(tbp);
	vm_object_pip_add(tbp->b_bufobj->bo_object,
	tbp->b_npages);
	- for (k = 0; k < tbp->b_npages; k++)
	- vm_page_sbusy(tbp->b_pages[k]);
	+ vfs_busy_pages_acquire(tbp);
	VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
	} else {
	if ((bp->b_npages * PAGE_SIZE) +
	@@ -470,10 +468,9 @@
	if ((tbp->b_pages[j]->valid &
	vm_page_bits(toff, tinc)) != 0)
	break;
	- if (vm_page_xbusied(tbp->b_pages[j]))
	+ if (vm_page_trysbusy(tbp->b_pages[j]) == 0)
	break;
	vm_object_pip_add(tbp->b_bufobj->bo_object, 1);
	- vm_page_sbusy(tbp->b_pages[j]);
	off += tinc;
	tsize -= tinc;
	}
	@@ -991,11 +988,14 @@

	VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
	if (i == 0) {
	- vfs_drain_busy_pages(tbp);
	+ vfs_busy_pages_acquire(tbp);
	} else { /* if not first buffer */
	for (j = 0; j < tbp->b_npages; j += 1) {
	m = tbp->b_pages[j];
	- if (vm_page_xbusied(m)) {
	+ if (vm_page_trysbusy(m) == 0) {
	+ for (j--; j >= 0; j--)
	+ vm_page_sunbusy(
	+ tbp->b_pages[j]);
	VM_OBJECT_WUNLOCK(
	tbp->b_object);
	bqrelse(tbp);
	@@ -1003,10 +1003,10 @@
	}
	}
	}
	+ vm_object_pip_add(tbp->b_bufobj->bo_object,
	+ tbp->b_npages);
	for (j = 0; j < tbp->b_npages; j += 1) {
	m = tbp->b_pages[j];
	- vm_page_sbusy(m);
	- vm_object_pip_add(m->object, 1);
	if ((bp->b_npages == 0) \|\|
	(bp->b_pages[bp->b_npages - 1] != m)) {
	bp->b_pages[bp->b_npages] = m;
	Index: head/sys/sys/buf.h
	===================================================================
	--- head/sys/sys/buf.h
	+++ head/sys/sys/buf.h
	@@ -539,7 +539,8 @@
	void brelse(struct buf *);
	void bqrelse(struct buf *);
	int vfs_bio_awrite(struct buf *);
	-void vfs_drain_busy_pages(struct buf *bp);
	+void vfs_busy_pages_acquire(struct buf *bp);
	+void vfs_busy_pages_release(struct buf *bp);
	struct buf incore(struct bufobj , daddr_t);
	struct buf gbincore(struct bufobj , daddr_t);
	struct buf getblk(struct vnode , daddr_t, int, int, int, int);
	Index: head/sys/vm/phys_pager.c
	===================================================================
	--- head/sys/vm/phys_pager.c
	+++ head/sys/vm/phys_pager.c
	@@ -206,29 +206,13 @@
	*last = end;

	for (i = base; i <= end; i++) {
	-retry:
	- m = vm_page_lookup(object, i);
	- if (m == NULL) {
	- ahead = MIN(end - i, PHYSALLOC);
	- m = vm_page_alloc(object, i, VM_ALLOC_NORMAL \|
	- VM_ALLOC_ZERO \| VM_ALLOC_WAITFAIL \|
	- VM_ALLOC_COUNT(ahead));
	- if (m == NULL)
	- goto retry;
	- if ((m->flags & PG_ZERO) == 0)
	- pmap_zero_page(m);
	+ ahead = MIN(end - i, PHYSALLOC);
	+ m = vm_page_grab(object, i,
	+ VM_ALLOC_NORMAL \| VM_ALLOC_COUNT(ahead));
	+ if (m->valid != VM_PAGE_BITS_ALL) {
	+ vm_page_zero_invalid(m, TRUE);
	m->valid = VM_PAGE_BITS_ALL;
	- } else if (vm_page_xbusied(m)) {
	- vm_page_sleep_if_xbusy(m, "physb");
	- goto retry;
	- } else {
	- vm_page_xbusy(m);
	- if (m->valid != VM_PAGE_BITS_ALL)
	- vm_page_zero_invalid(m, TRUE);
	}
	-
	- KASSERT(m->valid == VM_PAGE_BITS_ALL,
	- ("phys_pager_populate: partially valid page %p", m));
	KASSERT(m->dirty == 0,
	("phys_pager_populate: dirty page %p", m));
	}
	Index: head/sys/vm/vm_fault.c
	===================================================================
	--- head/sys/vm/vm_fault.c
	+++ head/sys/vm/vm_fault.c
	@@ -787,7 +787,7 @@
	* around with a shared busied page except, perhaps,
	* to pmap it.
	*/
	- if (vm_page_busied(fs.m)) {
	+ if (vm_page_tryxbusy(fs.m) == 0) {
	/*
	* Reference the page before unlocking and
	* sleeping so that the page daemon is less
	@@ -819,12 +819,11 @@
	}

	/*
	- * Mark page busy for other processes, and the
	+ * The page is marked busy for other processes and the
	* pagedaemon. If it still isn't completely valid
	* (readable), jump to readrest, else break-out ( we
	* found the page ).
	*/
	- vm_page_xbusy(fs.m);
	if (fs.m->valid != VM_PAGE_BITS_ALL)
	goto readrest;
	break; /* break to PAGE HAS BEEN FOUND */
	@@ -1826,16 +1825,17 @@
	dst_m->dirty = dst_m->valid = src_m->valid;
	} else {
	dst_m = src_m;
	- if (vm_page_sleep_if_busy(dst_m, "fltupg"))
	+ if (vm_page_busy_acquire(dst_m, VM_ALLOC_WAITFAIL) == 0)
	goto again;
	- if (dst_m->pindex >= dst_object->size)
	+ if (dst_m->pindex >= dst_object->size) {
	/*
	* We are upgrading. Index can occur
	* out of bounds if the object type is
	* vnode and the file was truncated.
	*/
	+ vm_page_xunbusy(dst_m);
	break;
	- vm_page_xbusy(dst_m);
	+ }
	}
	VM_OBJECT_WUNLOCK(dst_object);

	Index: head/sys/vm/vm_object.c
	===================================================================
	--- head/sys/vm/vm_object.c
	+++ head/sys/vm/vm_object.c
	@@ -838,7 +838,7 @@
	np = TAILQ_NEXT(p, listq);
	if (p->valid == 0)
	continue;
	- if (vm_page_sleep_if_busy(p, "vpcwai")) {
	+ if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) {
	if (object->generation != curgeneration) {
	if ((flags & OBJPC_SYNC) != 0)
	goto rescan;
	@@ -848,8 +848,10 @@
	np = vm_page_find_least(object, pi);
	continue;
	}
	- if (!vm_object_page_remove_write(p, flags, &clearobjflags))
	+ if (!vm_object_page_remove_write(p, flags, &clearobjflags)) {
	+ vm_page_xunbusy(p);
	continue;
	+ }

	n = vm_object_page_collect_flush(object, p, pagerflags,
	flags, &clearobjflags, &eio);
	@@ -899,6 +901,7 @@
	int count, i, mreq, runlen;

	vm_page_lock_assert(p, MA_NOTOWNED);
	+ vm_page_assert_xbusied(p);
	VM_OBJECT_ASSERT_WLOCKED(object);

	count = 1;
	@@ -906,18 +909,22 @@

	for (tp = p; count < vm_pageout_page_count; count++) {
	tp = vm_page_next(tp);
	- if (tp == NULL \|\| vm_page_busied(tp))
	+ if (tp == NULL \|\| vm_page_tryxbusy(tp) == 0)
	break;
	- if (!vm_object_page_remove_write(tp, flags, clearobjflags))
	+ if (!vm_object_page_remove_write(tp, flags, clearobjflags)) {
	+ vm_page_xunbusy(tp);
	break;
	+ }
	}

	for (p_first = p; count < vm_pageout_page_count; count++) {
	tp = vm_page_prev(p_first);
	- if (tp == NULL \|\| vm_page_busied(tp))
	+ if (tp == NULL \|\| vm_page_tryxbusy(tp) == 0)
	break;
	- if (!vm_object_page_remove_write(tp, flags, clearobjflags))
	+ if (!vm_object_page_remove_write(tp, flags, clearobjflags)) {
	+ vm_page_xunbusy(tp);
	break;
	+ }
	p_first = tp;
	mreq++;
	}
	@@ -1158,7 +1165,7 @@
	("vm_object_madvise: page %p is fictitious", tm));
	KASSERT((tm->oflags & VPO_UNMANAGED) == 0,
	("vm_object_madvise: page %p is not managed", tm));
	- if (vm_page_busied(tm)) {
	+ if (vm_page_tryxbusy(tm) == 0) {
	if (object != tobject)
	VM_OBJECT_WUNLOCK(object);
	if (advice == MADV_WILLNEED) {
	@@ -1175,6 +1182,7 @@
	vm_page_lock(tm);
	vm_page_advise(tm, advice);
	vm_page_unlock(tm);
	+ vm_page_xunbusy(tm);
	vm_object_madvise_freespace(tobject, advice, tm->pindex, 1);
	next_pindex:
	if (tobject != object)
	@@ -1341,7 +1349,7 @@
	* We do not have to VM_PROT_NONE the page as mappings should
	* not be changed by this operation.
	*/
	- if (vm_page_busied(m)) {
	+ if (vm_page_tryxbusy(m) == 0) {
	VM_OBJECT_WUNLOCK(new_object);
	vm_page_sleep_if_busy(m, "spltwt");
	VM_OBJECT_WLOCK(new_object);
	@@ -1350,6 +1358,7 @@

	/* vm_page_rename() will dirty the page. */
	if (vm_page_rename(m, new_object, idx)) {
	+ vm_page_xunbusy(m);
	VM_OBJECT_WUNLOCK(new_object);
	VM_OBJECT_WUNLOCK(orig_object);
	vm_radix_wait();
	@@ -1357,6 +1366,8 @@
	VM_OBJECT_WLOCK(new_object);
	goto retry;
	}
	+ /* Rename released the xbusy lock. */
	+
	#if VM_NRESERVLEVEL > 0
	/*
	* If some of the reservation's allocated pages remain with
	@@ -1405,7 +1416,6 @@
	backing_object = object->backing_object;
	VM_OBJECT_ASSERT_WLOCKED(backing_object);

	- KASSERT(p == NULL \|\| vm_page_busied(p), ("unbusy page %p", p));
	KASSERT(p == NULL \|\| p->object == object \|\| p->object == backing_object,
	("invalid ownership %p %p %p", p, object, backing_object));
	if ((op & OBSC_COLLAPSE_NOWAIT) != 0)
	@@ -1510,7 +1520,7 @@
	/*
	* Check for busy page
	*/
	- if (vm_page_busied(p)) {
	+ if (vm_page_tryxbusy(p) == 0) {
	next = vm_object_collapse_scan_wait(object, p, next, op);
	continue;
	}
	@@ -1532,7 +1542,8 @@
	}

	pp = vm_page_lookup(object, new_pindex);
	- if (pp != NULL && vm_page_busied(pp)) {
	+ if (pp != NULL && vm_page_tryxbusy(pp) == 0) {
	+ vm_page_xunbusy(p);
	/*
	* The page in the parent is busy and possibly not
	* (yet) valid. Until its state is finalized by the
	@@ -1568,6 +1579,8 @@
	("freeing mapped page %p", p));
	if (vm_page_remove(p))
	vm_page_free(p);
	+ if (pp != NULL)
	+ vm_page_xunbusy(pp);
	continue;
	}

	@@ -1579,10 +1592,14 @@
	* through the rename. vm_page_rename() will dirty the page.
	*/
	if (vm_page_rename(p, object, new_pindex)) {
	+ vm_page_xunbusy(p);
	+ if (pp != NULL)
	+ vm_page_xunbusy(pp);
	next = vm_object_collapse_scan_wait(object, NULL, next,
	op);
	continue;
	}
	+ /* Rename released the xbusy lock. */

	/* Use the old pindex to free the right page. */
	if (backing_object->type == OBJT_SWAP)
	@@ -1859,7 +1876,7 @@
	* however, be invalidated if the option OBJPR_CLEANONLY is
	* not specified.
	*/
	- if (vm_page_busied(p)) {
	+ if (vm_page_tryxbusy(p) == 0) {
	vm_page_sleep_if_busy(p, "vmopar");
	goto again;
	}
	@@ -1872,6 +1889,7 @@
	p->valid = 0;
	vm_page_undirty(p);
	}
	+ vm_page_xunbusy(p);
	continue;
	}
	KASSERT((p->flags & PG_FICTITIOUS) == 0,
	@@ -1881,8 +1899,10 @@
	object->ref_count != 0 &&
	!vm_page_try_remove_write(p))
	goto wired;
	- if (p->dirty != 0)
	+ if (p->dirty != 0) {
	+ vm_page_xunbusy(p);
	continue;
	+ }
	}
	if ((options & OBJPR_NOTMAPPED) == 0 &&
	object->ref_count != 0 && !vm_page_try_remove_all(p))
	@@ -2168,7 +2188,7 @@
	tm = m;
	m = TAILQ_NEXT(m, listq);
	}
	- if (vm_page_xbusied(tm)) {
	+ if (vm_page_trysbusy(tm) == 0) {
	for (tobject = object; locked_depth >= 1;
	locked_depth--) {
	t1object = tobject->backing_object;
	@@ -2180,6 +2200,7 @@
	goto again;
	}
	vm_page_unwire(tm, queue);
	+ vm_page_sunbusy(tm);
	next_page:
	pindex++;
	}
	Index: head/sys/vm/vm_page.h
	===================================================================
	--- head/sys/vm/vm_page.h
	+++ head/sys/vm/vm_page.h
	@@ -542,6 +542,7 @@

	int vm_page_busy_acquire(vm_page_t m, int allocflags);
	void vm_page_busy_downgrade(vm_page_t m);
	+int vm_page_busy_tryupgrade(vm_page_t m);
	void vm_page_busy_sleep(vm_page_t m, const char *msg, bool nonshared);
	void vm_page_free(vm_page_t m);
	void vm_page_free_zero(vm_page_t m);
	Index: head/sys/vm/vm_page.c
	===================================================================
	--- head/sys/vm/vm_page.c
	+++ head/sys/vm/vm_page.c
	@@ -972,6 +972,32 @@
	}

	/*
	+ *
	+ * vm_page_busy_tryupgrade:
	+ *
	+ * Attempt to upgrade a single shared busy into an exclusive busy.
	+ */
	+int
	+vm_page_busy_tryupgrade(vm_page_t m)
	+{
	+ u_int x;
	+
	+ vm_page_assert_sbusied(m);
	+
	+ x = m->busy_lock;
	+ for (;;) {
	+ if (VPB_SHARERS(x) > 1)
	+ return (0);
	+ KASSERT((x & ~VPB_BIT_WAITERS) == VPB_SHARERS_WORD(1),
	+ ("vm_page_busy_tryupgrade: invalid lock state"));
	+ if (!atomic_fcmpset_acq_int(&m->busy_lock, &x,
	+ VPB_SINGLE_EXCLUSIVER \| (x & VPB_BIT_WAITERS)))
	+ continue;
	+ return (1);
	+ }
	+}
	+
	+/*
	* vm_page_sbusied:
	*
	* Return a positive value if the page is shared busied, 0 otherwise.
	@@ -2570,7 +2596,12 @@
	else if (object->memattr != VM_MEMATTR_DEFAULT)
	error = EINVAL;
	else if (vm_page_queue(m) != PQ_NONE &&
	- !vm_page_busied(m) && !vm_page_wired(m)) {
	+ vm_page_tryxbusy(m) != 0) {
	+ if (vm_page_wired(m)) {
	+ vm_page_xunbusy(m);
	+ error = EBUSY;
	+ goto unlock;
	+ }
	KASSERT(pmap_page_get_memattr(m) ==
	VM_MEMATTR_DEFAULT,
	("page %p has an unexpected memattr", m));
	@@ -2616,6 +2647,7 @@
	VM_MEMATTR_DEFAULT);
	}
	if (m_new == NULL) {
	+ vm_page_xunbusy(m);
	error = ENOMEM;
	goto unlock;
	}
	@@ -2647,7 +2679,6 @@
	m_new->valid = m->valid;
	m_new->dirty = m->dirty;
	m->flags &= ~PG_ZERO;
	- vm_page_xbusy(m);
	vm_page_dequeue(m);
	vm_page_replace_checked(m_new, object,
	m->pindex, m);
	@@ -4046,8 +4077,8 @@

	KASSERT(m->object != NULL && (m->oflags & VPO_UNMANAGED) == 0,
	("vm_page_try_blocked_op: page %p has no object", m));
	- KASSERT(!vm_page_busied(m),
	- ("vm_page_try_blocked_op: page %p is busy", m));
	+ KASSERT(vm_page_busied(m),
	+ ("vm_page_try_blocked_op: page %p is not busy", m));
	VM_OBJECT_ASSERT_LOCKED(m->object);

	old = m->ref_count;
	@@ -4163,13 +4194,18 @@
	(allocflags & VM_ALLOC_IGN_SBUSY) != 0,
	("vm_page_grab: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch"));
	pflags = allocflags &
	- ~(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITOK \| VM_ALLOC_WAITFAIL);
	+ ~(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITOK \| VM_ALLOC_WAITFAIL \|
	+ VM_ALLOC_NOBUSY);
	if ((allocflags & VM_ALLOC_NOWAIT) == 0)
	pflags \|= VM_ALLOC_WAITFAIL;
	+ if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)
	+ pflags \|= VM_ALLOC_SBUSY;
	retrylookup:
	if ((m = vm_page_lookup(object, pindex)) != NULL) {
	- sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ?
	- vm_page_xbusied(m) : vm_page_busied(m);
	+ if ((allocflags & (VM_ALLOC_IGN_SBUSY \| VM_ALLOC_SBUSY)) != 0)
	+ sleep = !vm_page_trysbusy(m);
	+ else
	+ sleep = !vm_page_tryxbusy(m);
	if (sleep) {
	if ((allocflags & VM_ALLOC_NOWAIT) != 0)
	return (NULL);
	@@ -4189,12 +4225,7 @@
	} else {
	if ((allocflags & VM_ALLOC_WIRED) != 0)
	vm_page_wire(m);
	- if ((allocflags &
	- (VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY)) == 0)
	- vm_page_xbusy(m);
	- else if ((allocflags & VM_ALLOC_SBUSY) != 0)
	- vm_page_sbusy(m);
	- return (m);
	+ goto out;
	}
	}
	if ((allocflags & VM_ALLOC_NOCREAT) != 0)
	@@ -4207,6 +4238,14 @@
	}
	if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
	pmap_zero_page(m);
	+
	+out:
	+ if ((allocflags & VM_ALLOC_NOBUSY) != 0) {
	+ if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)
	+ vm_page_sunbusy(m);
	+ else
	+ vm_page_xunbusy(m);
	+ }
	return (m);
	}

	@@ -4359,10 +4398,13 @@
	("vm_page_grab_pages: VM_ALLOC_SBUSY/IGN_SBUSY mismatch"));
	if (count == 0)
	return (0);
	- pflags = allocflags & ~(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITOK \|
	- VM_ALLOC_WAITFAIL \| VM_ALLOC_IGN_SBUSY);
	+ pflags = allocflags &
	+ ~(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITOK \| VM_ALLOC_WAITFAIL \|
	+ VM_ALLOC_NOBUSY);
	if ((allocflags & VM_ALLOC_NOWAIT) == 0)
	pflags \|= VM_ALLOC_WAITFAIL;
	+ if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)
	+ pflags \|= VM_ALLOC_SBUSY;
	i = 0;
	retrylookup:
	m = vm_radix_lookup_le(&object->rtree, pindex + i);
	@@ -4373,8 +4415,11 @@
	mpred = TAILQ_PREV(m, pglist, listq);
	for (; i < count; i++) {
	if (m != NULL) {
	- sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ?
	- vm_page_xbusied(m) : vm_page_busied(m);
	+ if ((allocflags &
	+ (VM_ALLOC_SBUSY \| VM_ALLOC_IGN_SBUSY)) != 0)
	+ sleep = !vm_page_trysbusy(m);
	+ else
	+ sleep = !vm_page_tryxbusy(m);
	if (sleep) {
	if ((allocflags & VM_ALLOC_NOWAIT) != 0)
	break;
	@@ -4392,11 +4437,6 @@
	}
	if ((allocflags & VM_ALLOC_WIRED) != 0)
	vm_page_wire(m);
	- if ((allocflags & (VM_ALLOC_NOBUSY \|
	- VM_ALLOC_SBUSY)) == 0)
	- vm_page_xbusy(m);
	- if ((allocflags & VM_ALLOC_SBUSY) != 0)
	- vm_page_sbusy(m);
	} else {
	if ((allocflags & VM_ALLOC_NOCREAT) != 0)
	break;
	@@ -4412,6 +4452,12 @@
	if ((m->flags & PG_ZERO) == 0)
	pmap_zero_page(m);
	m->valid = VM_PAGE_BITS_ALL;
	+ }
	+ if ((allocflags & VM_ALLOC_NOBUSY) != 0) {
	+ if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)
	+ vm_page_sunbusy(m);
	+ else
	+ vm_page_xunbusy(m);
	}
	ma[i] = mpred = m;
	m = vm_page_next(m);
	Index: head/sys/vm/vm_pageout.c
	===================================================================
	--- head/sys/vm/vm_pageout.c
	+++ head/sys/vm/vm_pageout.c
	@@ -334,7 +334,7 @@
	VM_OBJECT_ASSERT_WLOCKED(object);
	pindex = m->pindex;

	- vm_page_assert_unbusied(m);
	+ vm_page_assert_xbusied(m);

	mc[vm_pageout_page_count] = pb = ps = m;
	pageout_count = 1;
	@@ -360,19 +360,26 @@
	ib = 0;
	break;
	}
	- if ((p = vm_page_prev(pb)) == NULL \|\| vm_page_busied(p) \|\|
	- vm_page_wired(p)) {
	+ if ((p = vm_page_prev(pb)) == NULL \|\|
	+ vm_page_tryxbusy(p) == 0) {
	ib = 0;
	break;
	}
	+ if (vm_page_wired(p)) {
	+ ib = 0;
	+ vm_page_xunbusy(p);
	+ break;
	+ }
	vm_page_test_dirty(p);
	if (p->dirty == 0) {
	ib = 0;
	+ vm_page_xunbusy(p);
	break;
	}
	vm_page_lock(p);
	if (!vm_page_in_laundry(p) \|\| !vm_page_try_remove_write(p)) {
	vm_page_unlock(p);
	+ vm_page_xunbusy(p);
	ib = 0;
	break;
	}
	@@ -390,15 +397,22 @@
	}
	while (pageout_count < vm_pageout_page_count &&
	pindex + is < object->size) {
	- if ((p = vm_page_next(ps)) == NULL \|\| vm_page_busied(p) \|\|
	- vm_page_wired(p))
	+ if ((p = vm_page_next(ps)) == NULL \|\|
	+ vm_page_tryxbusy(p) == 0)
	break;
	+ if (vm_page_wired(p)) {
	+ vm_page_xunbusy(p);
	+ break;
	+ }
	vm_page_test_dirty(p);
	- if (p->dirty == 0)
	+ if (p->dirty == 0) {
	+ vm_page_xunbusy(p);
	break;
	+ }
	vm_page_lock(p);
	if (!vm_page_in_laundry(p) \|\| !vm_page_try_remove_write(p)) {
	vm_page_unlock(p);
	+ vm_page_xunbusy(p);
	break;
	}
	vm_page_unlock(p);
	@@ -445,8 +459,8 @@
	VM_OBJECT_ASSERT_WLOCKED(object);

	/*
	- * Initiate I/O. Mark the pages busy and verify that they're valid
	- * and read-only.
	+ * Initiate I/O. Mark the pages shared busy and verify that they're
	+ * valid and read-only.
	*
	* We do not have to fixup the clean/dirty bits here... we can
	* allow the pager to do it after the I/O completes.
	@@ -460,7 +474,7 @@
	mc[i], i, count));
	KASSERT((mc[i]->aflags & PGA_WRITEABLE) == 0,
	("vm_pageout_flush: writeable page %p", mc[i]));
	- vm_page_sbusy(mc[i]);
	+ vm_page_busy_downgrade(mc[i]);
	}
	vm_object_pip_add(object, count);

	@@ -598,6 +612,7 @@
	*/
	if (object->type == OBJT_VNODE) {
	vm_page_unlock(m);
	+ vm_page_xunbusy(m);
	vp = object->handle;
	if (vp->v_type == VREG &&
	vn_start_write(vp, &mp, V_NOWAIT) != 0) {
	@@ -648,7 +663,7 @@
	* The page may have been busied while the object and page
	* locks were released.
	*/
	- if (vm_page_busied(m)) {
	+ if (vm_page_tryxbusy(m) == 0) {
	vm_page_unlock(m);
	error = EBUSY;
	goto unlock_all;
	@@ -659,6 +674,7 @@
	* Remove all writeable mappings, failing if the page is wired.
	*/
	if (!vm_page_try_remove_write(m)) {
	+ vm_page_xunbusy(m);
	vm_page_unlock(m);
	error = EBUSY;
	goto unlock_all;
	@@ -792,7 +808,7 @@
	KASSERT(m->object == object, ("page %p does not belong to %p",
	m, object));

	- if (vm_page_busied(m))
	+ if (vm_page_tryxbusy(m) == 0)
	continue;

	/*
	@@ -804,6 +820,7 @@
	* wire count is guaranteed not to increase.
	*/
	if (__predict_false(vm_page_wired(m))) {
	+ vm_page_xunbusy(m);
	vm_page_dequeue_deferred(m);
	continue;
	}
	@@ -837,6 +854,7 @@
	}
	if (act_delta != 0) {
	if (object->ref_count != 0) {
	+ vm_page_xunbusy(m);
	VM_CNT_INC(v_reactivated);
	vm_page_activate(m);

	@@ -861,6 +879,7 @@
	launder--;
	continue;
	} else if ((object->flags & OBJ_DEAD) == 0) {
	+ vm_page_xunbusy(m);
	vm_page_requeue(m);
	continue;
	}
	@@ -876,6 +895,7 @@
	if (object->ref_count != 0) {
	vm_page_test_dirty(m);
	if (m->dirty == 0 && !vm_page_try_remove_all(m)) {
	+ vm_page_xunbusy(m);
	vm_page_dequeue_deferred(m);
	continue;
	}
	@@ -900,6 +920,7 @@
	else
	pageout_ok = true;
	if (!pageout_ok) {
	+ vm_page_xunbusy(m);
	vm_page_requeue(m);
	continue;
	}
	@@ -927,7 +948,8 @@
	}
	mtx = NULL;
	object = NULL;
	- }
	+ } else
	+ vm_page_xunbusy(m);
	}
	if (mtx != NULL) {
	mtx_unlock(mtx);
	@@ -1507,7 +1529,7 @@
	KASSERT(m->object == object, ("page %p does not belong to %p",
	m, object));

	- if (vm_page_busied(m)) {
	+ if (vm_page_tryxbusy(m) == 0) {
	/*
	* Don't mess with busy pages. Leave them at
	* the front of the queue. Most likely, they
	@@ -1529,6 +1551,7 @@
	* wire count is guaranteed not to increase.
	*/
	if (__predict_false(vm_page_wired(m))) {
	+ vm_page_xunbusy(m);
	vm_page_dequeue_deferred(m);
	continue;
	}
	@@ -1562,6 +1585,7 @@
	}
	if (act_delta != 0) {
	if (object->ref_count != 0) {
	+ vm_page_xunbusy(m);
	VM_CNT_INC(v_reactivated);
	vm_page_activate(m);

	@@ -1575,6 +1599,7 @@
	m->act_count += act_delta + ACT_ADVANCE;
	continue;
	} else if ((object->flags & OBJ_DEAD) == 0) {
	+ vm_page_xunbusy(m);
	vm_page_aflag_set(m, PGA_REQUEUE);
	goto reinsert;
	}
	@@ -1590,6 +1615,7 @@
	if (object->ref_count != 0) {
	vm_page_test_dirty(m);
	if (m->dirty == 0 && !vm_page_try_remove_all(m)) {
	+ vm_page_xunbusy(m);
	vm_page_dequeue_deferred(m);
	continue;
	}
	@@ -1615,7 +1641,10 @@
	m->queue = PQ_NONE;
	vm_page_free(m);
	page_shortage--;
	- } else if ((object->flags & OBJ_DEAD) == 0)
	+ continue;
	+ }
	+ vm_page_xunbusy(m);
	+ if ((object->flags & OBJ_DEAD) == 0)
	vm_page_launder(m);
	continue;
	reinsert:
	Index: head/sys/vm/vm_swapout.c
	===================================================================
	--- head/sys/vm/vm_swapout.c
	+++ head/sys/vm/vm_swapout.c
	@@ -208,6 +208,9 @@
	goto unlock_return;
	if (should_yield())
	goto unlock_return;
	+ if (vm_page_tryxbusy(p) == 0)
	+ continue;
	+ VM_CNT_INC(v_pdpages);

	/*
	* The page may acquire a wiring after this check.
	@@ -215,11 +218,10 @@
	* no harm done if a wiring appears while we are
	* attempting to deactivate the page.
	*/
	- if (vm_page_busied(p) \|\| vm_page_wired(p))
	+ if (vm_page_wired(p) \|\| !pmap_page_exists_quick(pmap, p)) {
	+ vm_page_xunbusy(p);
	continue;
	- VM_CNT_INC(v_pdpages);
	- if (!pmap_page_exists_quick(pmap, p))
	- continue;
	+ }
	act_delta = pmap_ts_referenced(p);
	vm_page_lock(p);
	if ((p->aflags & PGA_REFERENCED) != 0) {
	@@ -251,6 +253,7 @@
	} else if (vm_page_inactive(p))
	(void)vm_page_try_remove_all(p);
	vm_page_unlock(p);
	+ vm_page_xunbusy(p);
	}
	if ((backing_object = object->backing_object) == NULL)
	goto unlock_return;

File Metadata

Mime Type: text/plain
Expires: Fri, Jul 3, 1:49 PM (18 h, 3 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 34625613
Default Alt Text: D21548.id.diff (28 KB)

D21548.id.diffNo OneTemporaryActions

D21548.id.diffView Options

File Metadata

Event Timeline

D21548.id.diff
No OneTemporary
Actions

D21548.id.diff
View Options