Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c @@ -412,35 +412,14 @@ obj = vp->v_object; zfs_vmobject_assert_wlocked(obj); - for (;;) { - if ((pp = vm_page_lookup(obj, OFF_TO_IDX(start))) != NULL && - pp->valid) { - if (vm_page_xbusied(pp)) { - /* - * Reference the page before unlocking and - * sleeping so that the page daemon is less - * likely to reclaim it. - */ - vm_page_reference(pp); - zfs_vmobject_wunlock(obj); - vm_page_busy_sleep(pp, "zfsmwb", true); - zfs_vmobject_wlock(obj); - continue; - } - vm_page_sbusy(pp); - } else if (pp != NULL) { - ASSERT(!pp->valid); - pp = NULL; - } - - if (pp != NULL) { - ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL); - vm_object_pip_add(obj, 1); - pmap_remove_write(pp); - if (nbytes != 0) - vm_page_clear_dirty(pp, off, nbytes); - } - break; + pp = vm_page_grab(obj, OFF_TO_IDX(start), VM_ALLOC_VALID | + VM_ALLOC_SBUSY | VM_ALLOC_NORMAL | VM_ALLOC_IGN_SBUSY); + if (pp != NULL) { + ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL); + vm_object_pip_add(obj, 1); + pmap_remove_write(pp); + if (nbytes != 0) + vm_page_clear_dirty(pp, off, nbytes); } return (pp); } @@ -457,36 +436,12 @@ page_wire(vnode_t *vp, int64_t start) { vm_object_t obj; - vm_page_t pp; obj = vp->v_object; zfs_vmobject_assert_wlocked(obj); - for (;;) { - if ((pp = vm_page_lookup(obj, OFF_TO_IDX(start))) != NULL && - pp->valid) { - if (vm_page_xbusied(pp)) { - /* - * Reference the page before unlocking and - * sleeping so that the page daemon is less - * likely to reclaim it. - */ - vm_page_reference(pp); - zfs_vmobject_wunlock(obj); - vm_page_busy_sleep(pp, "zfsmwb", true); - zfs_vmobject_wlock(obj); - continue; - } - - ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL); - vm_page_lock(pp); - vm_page_wire(pp); - vm_page_unlock(pp); - } else - pp = NULL; - break; - } - return (pp); + return (vm_page_grab(obj, OFF_TO_IDX(start), VM_ALLOC_VALID | + VM_ALLOC_WIRED | VM_ALLOC_IGN_SBUSY | VM_ALLOC_NOBUSY)); } static void Index: sys/dev/drm2/ttm/ttm_bo_vm.c =================================================================== --- sys/dev/drm2/ttm/ttm_bo_vm.c +++ sys/dev/drm2/ttm/ttm_bo_vm.c @@ -233,10 +233,7 @@ } VM_OBJECT_WLOCK(vm_obj); - if (vm_page_busied(m)) { - VM_OBJECT_WUNLOCK(vm_obj); - vm_page_busy_sleep(m, "ttmpbs", false); - VM_OBJECT_WLOCK(vm_obj); + if (vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL) == 0) { ttm_mem_io_unlock(man); ttm_bo_unreserve(bo); goto retry; @@ -244,6 +241,7 @@ m1 = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); if (m1 == NULL) { if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) { + vm_page_xunbusy(m); VM_OBJECT_WUNLOCK(vm_obj); vm_wait(vm_obj); VM_OBJECT_WLOCK(vm_obj); @@ -257,7 +255,6 @@ bo, m, m1, (uintmax_t)offset)); } m->valid = VM_PAGE_BITS_ALL; - vm_page_xbusy(m); if (*mres != NULL) { KASSERT(*mres != m, ("losing %p %p", *mres, m)); vm_page_lock(*mres); @@ -381,7 +378,7 @@ m = vm_page_lookup(vm_obj, i); if (m == NULL) continue; - if (vm_page_sleep_if_busy(m, "ttm_unm")) + if (vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL) == 0) goto retry; cdev_pager_free_page(vm_obj, m); } Index: sys/dev/xen/gntdev/gntdev.c =================================================================== --- sys/dev/xen/gntdev/gntdev.c +++ sys/dev/xen/gntdev/gntdev.c @@ -606,7 +606,7 @@ m = vm_page_lookup(gmap->map->mem, i); if (m == NULL) continue; - if (vm_page_sleep_if_busy(m, "pcmdum")) + if (vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL) == 0) goto retry; cdev_pager_free_page(gmap->map->mem, m); } Index: sys/dev/xen/privcmd/privcmd.c =================================================================== --- sys/dev/xen/privcmd/privcmd.c +++ sys/dev/xen/privcmd/privcmd.c @@ -128,7 +128,7 @@ m = vm_page_lookup(map->mem, i); if (m == NULL) continue; - if (vm_page_sleep_if_busy(m, "pcmdum")) + if (vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL) == 0) goto retry; cdev_pager_free_page(map->mem, m); } Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h +++ sys/vm/vm_page.h @@ -469,6 +469,8 @@ #define VM_ALLOC_ZERO 0x0040 /* (acfgp) Allocate a prezeroed page */ #define VM_ALLOC_NOOBJ 0x0100 /* (acg) No associated object */ #define VM_ALLOC_NOBUSY 0x0200 /* (acgp) Do not excl busy the page */ +#define VM_ALLOC_VALID 0x0400 /* (gp) Only existing valid pages */ +#define VM_ALLOC_NOCREAT 0x0800 /* (gp) Don't create a page */ #define VM_ALLOC_IGN_SBUSY 0x1000 /* (gp) Ignore shared busy flag */ #define VM_ALLOC_NODUMP 0x2000 /* (ag) don't include in dump */ #define VM_ALLOC_SBUSY 0x4000 /* (acgp) Shared busy the page */ @@ -511,6 +513,7 @@ #define PS_ALL_VALID 0x2 #define PS_NONE_BUSY 0x4 +int vm_page_busy_acquire(vm_page_t m, int allocflags); void vm_page_busy_downgrade(vm_page_t m); void vm_page_busy_sleep(vm_page_t m, const char *msg, bool nonshared); void vm_page_free(vm_page_t m); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -868,6 +868,51 @@ vm_page_aflag_set(m, PGA_REFERENCED); } +/* + * vm_page_busy_acquire: + * + * Acquire the busy lock as described by VM_ALLOC_* flags. Will loop + * and drop the object lock if necessary. + */ +int +vm_page_busy_acquire(vm_page_t m, int allocflags) +{ + vm_object_t obj; + bool locked; + + /* + * The page-specific object must be cached because page + * identity can change during the sleep, causing the + * re-lock of a different object. + * It is assumed that a reference to the object is already + * held by the callers. + */ + obj = m->object; + for (;;) { + if ((allocflags & VM_ALLOC_SBUSY) == 0) { + if (vm_page_tryxbusy(m)) + return (TRUE); + } else { + if (vm_page_trysbusy(m)) + return (TRUE); + } + if ((allocflags & VM_ALLOC_NOWAIT) != 0) + return (FALSE); + if (obj != NULL) + locked = VM_OBJECT_WOWNED(obj); + else + locked = FALSE; + if (locked) + VM_OBJECT_WUNLOCK(obj); + vm_page_busy_sleep(m, "pgbacq", (allocflags & + VM_ALLOC_SBUSY) != 0); + if (locked) + VM_OBJECT_WLOCK(obj); + if ((allocflags & VM_ALLOC_WAITFAIL) != 0) + return (FALSE); + } +} + /* * vm_page_busy_downgrade: * @@ -3847,13 +3892,19 @@ * sleeping so that the page daemon is less * likely to reclaim it. */ - vm_page_aflag_set(m, PGA_REFERENCED); + if ((allocflags & VM_ALLOC_NOCREAT) != 0) + vm_page_aflag_set(m, PGA_REFERENCED); VM_OBJECT_WUNLOCK(object); vm_page_busy_sleep(m, "pgrbwt", (allocflags & VM_ALLOC_IGN_SBUSY) != 0); VM_OBJECT_WLOCK(object); + if ((allocflags & VM_ALLOC_WAITFAIL) != 0) + return (NULL); goto retrylookup; } else { + if ((allocflags & VM_ALLOC_VALID) != 0 && + m->valid != VM_PAGE_BITS_ALL) + return (NULL); if ((allocflags & VM_ALLOC_WIRED) != 0) { vm_page_lock(m); vm_page_wire(m); @@ -3862,11 +3913,13 @@ if ((allocflags & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) == 0) vm_page_xbusy(m); - if ((allocflags & VM_ALLOC_SBUSY) != 0) + else if ((allocflags & VM_ALLOC_SBUSY) != 0) vm_page_sbusy(m); return (m); } } + if ((allocflags & (VM_ALLOC_NOCREAT | VM_ALLOC_VALID)) != 0) + return (NULL); m = vm_page_alloc(object, pindex, pflags); if (m == NULL) { if ((allocflags & VM_ALLOC_NOWAIT) != 0) @@ -4045,13 +4098,16 @@ * sleeping so that the page daemon is less * likely to reclaim it. */ - vm_page_aflag_set(m, PGA_REFERENCED); + if ((allocflags & VM_ALLOC_NOCREAT) != 0) + vm_page_aflag_set(m, PGA_REFERENCED); VM_OBJECT_WUNLOCK(object); vm_page_busy_sleep(m, "grbmaw", (allocflags & VM_ALLOC_IGN_SBUSY) != 0); VM_OBJECT_WLOCK(object); goto retrylookup; } + if ((allocflags & VM_ALLOC_VALID) != 0 && !m->valid) + break; if ((allocflags & VM_ALLOC_WIRED) != 0) { vm_page_lock(m); vm_page_wire(m); @@ -4063,6 +4119,9 @@ if ((allocflags & VM_ALLOC_SBUSY) != 0) vm_page_sbusy(m); } else { + if ((allocflags & + (VM_ALLOC_VALID | VM_ALLOC_NOCREAT)) != 0) + break; m = vm_page_alloc_after(object, pindex + i, pflags | VM_ALLOC_COUNT(count - i), mpred); if (m == NULL) {