Index: sys/compat/linuxkpi/common/src/linux_page.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_page.c +++ sys/compat/linuxkpi/common/src/linux_page.c @@ -288,29 +288,11 @@ panic("GFP_NOWAIT is unimplemented"); VM_OBJECT_WLOCK(obj); - page = vm_page_grab(obj, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | - VM_ALLOC_WIRED); - if (page->valid != VM_PAGE_BITS_ALL) { - vm_page_xbusy(page); - if (vm_pager_has_page(obj, pindex, NULL, NULL)) { - rv = vm_pager_get_pages(obj, &page, 1, NULL, NULL); - if (rv != VM_PAGER_OK) { - vm_page_lock(page); - vm_page_unwire_noq(page); - vm_page_free(page); - vm_page_unlock(page); - VM_OBJECT_WUNLOCK(obj); - return (ERR_PTR(-EINVAL)); - } - MPASS(page->valid == VM_PAGE_BITS_ALL); - } else { - pmap_zero_page(page); - page->valid = VM_PAGE_BITS_ALL; - page->dirty = 0; - } - vm_page_xunbusy(page); - } + rv = vm_page_grab_valid(&page, obj, pindex, VM_ALLOC_NORMAL | + VM_ALLOC_NOBUSY | VM_ALLOC_WIRED); VM_OBJECT_WUNLOCK(obj); + if (rv != VM_PAGER_OK) + return (ERR_PTR(-EINVAL)); return (page); } Index: sys/dev/drm2/ttm/ttm_tt.c =================================================================== --- sys/dev/drm2/ttm/ttm_tt.c +++ sys/dev/drm2/ttm/ttm_tt.c @@ -288,22 +288,12 @@ VM_OBJECT_WLOCK(obj); vm_object_pip_add(obj, 1); for (i = 0; i < ttm->num_pages; ++i) { - from_page = vm_page_grab(obj, i, VM_ALLOC_NORMAL); - if (from_page->valid != VM_PAGE_BITS_ALL) { - if (vm_pager_has_page(obj, i, NULL, NULL)) { - rv = vm_pager_get_pages(obj, &from_page, 1, - NULL, NULL); - if (rv != VM_PAGER_OK) { - vm_page_lock(from_page); - vm_page_free(from_page); - vm_page_unlock(from_page); - ret = -EIO; - goto err_ret; - } - } else - vm_page_zero_invalid(from_page, TRUE); + rv = vm_page_grab_valid(&from_page, obj, i, + VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY); + if (rv != VM_PAGER_OK) { + ret = -EIO; + goto err_ret; } - vm_page_xunbusy(from_page); to_page = ttm->pages[i]; if (unlikely(to_page == NULL)) { ret = -ENOMEM; Index: sys/kern/uipc_shm.c =================================================================== --- sys/kern/uipc_shm.c +++ sys/kern/uipc_shm.c @@ -188,24 +188,13 @@ * lock to page out tobj's pages because tobj is a OBJT_SWAP * type object. */ - m = vm_page_grab(obj, idx, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY); - if (m->valid != VM_PAGE_BITS_ALL) { - vm_page_xbusy(m); - if (vm_pager_has_page(obj, idx, NULL, NULL)) { - rv = vm_pager_get_pages(obj, &m, 1, NULL, NULL); - if (rv != VM_PAGER_OK) { - printf( - "uiomove_object: vm_obj %p idx %jd valid %x pager error %d\n", - obj, idx, m->valid, rv); - vm_page_lock(m); - vm_page_free(m); - vm_page_unlock(m); - VM_OBJECT_WUNLOCK(obj); - return (EIO); - } - } else - vm_page_zero_invalid(m, TRUE); - vm_page_xunbusy(m); + rv = vm_page_grab_valid(&m, obj, idx, + VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY); + if (rv != VM_PAGER_OK) { + VM_OBJECT_WUNLOCK(obj); + printf("uiomove_object: vm_obj %p idx %jd pager error %d\n", + obj, idx, rv); + return (EIO); } vm_page_lock(m); vm_page_wire(m); Index: sys/vm/vm_glue.c =================================================================== --- sys/vm/vm_glue.c +++ sys/vm/vm_glue.c @@ -219,26 +219,11 @@ { vm_page_t m; vm_pindex_t pindex; - int rv; - VM_OBJECT_WLOCK(object); pindex = OFF_TO_IDX(offset); - m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | - VM_ALLOC_WIRED); - if (m->valid != VM_PAGE_BITS_ALL) { - vm_page_xbusy(m); - rv = vm_pager_get_pages(object, &m, 1, NULL, NULL); - if (rv != VM_PAGER_OK) { - vm_page_lock(m); - vm_page_unwire(m, PQ_NONE); - vm_page_free(m); - vm_page_unlock(m); - m = NULL; - goto out; - } - vm_page_xunbusy(m); - } -out: + VM_OBJECT_WLOCK(object); + (void) vm_page_grab_valid(&m, object, pindex, + VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED); VM_OBJECT_WUNLOCK(object); return (m); } Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -2030,16 +2030,9 @@ VM_OBJECT_ASSERT_WLOCKED(object); for (pindex = start; pindex < end; pindex++) { - m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); - if (m->valid != VM_PAGE_BITS_ALL) { - rv = vm_pager_get_pages(object, &m, 1, NULL, NULL); - if (rv != VM_PAGER_OK) { - vm_page_lock(m); - vm_page_free(m); - vm_page_unlock(m); - break; - } - } + rv = vm_page_grab_valid(&m, object, pindex, VM_ALLOC_NORMAL); + if (rv != VM_PAGER_OK) + break; /* * Keep "m" busy because a subsequent iteration may unlock * the object. Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h +++ sys/vm/vm_page.h @@ -537,6 +537,8 @@ vm_page_t vm_page_grab (vm_object_t, vm_pindex_t, int); int vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags, vm_page_t *ma, int count); +int vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, + int allocflags); void vm_page_deactivate(vm_page_t); void vm_page_deactivate_noreuse(vm_page_t); void vm_page_dequeue(vm_page_t m); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -3878,6 +3878,103 @@ return (m); } +/* + * Grab a page and make it valid, paging in if necessary. Pages missing from + * their pager are zero filled and validated. + */ +int +vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags) +{ + vm_page_t m; + bool sleep, xbusy; + int pflags; + int rv; + + KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || + (allocflags & VM_ALLOC_IGN_SBUSY) != 0, + ("vm_page_grab_valid: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch")); + KASSERT((allocflags & (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL | + VM_ALLOC_NOCREAT | VM_ALLOC_VALID | VM_ALLOC_ZERO)) == 0, + ("vm_page_grab_valid: Invalid flags 0x%X", allocflags)); + VM_OBJECT_ASSERT_WLOCKED(object); + pflags = allocflags & ~(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY); + pflags |= VM_ALLOC_WAITFAIL; + +retrylookup: + xbusy = false; + if ((m = vm_page_lookup(object, pindex)) != NULL) { + /* + * If the page is fully valid it can only become invalid + * with the object lock held. If it is not valid it can + * become valid with the busy lock held. Therefore, we + * may unnecessarily lock the exclusive busy here if we + * race with I/O completion not using the object lock. + * However, we will not end up with an invalid page and a + * shared lock. + */ + if (m->valid != VM_PAGE_BITS_ALL || + (allocflags & (VM_ALLOC_IGN_SBUSY | VM_ALLOC_SBUSY)) == 0) { + sleep = !vm_page_tryxbusy(m); + xbusy = true; + } else + sleep = !vm_page_trysbusy(m); + if (sleep) { + /* + * Reference the page before unlocking and + * sleeping so that the page daemon is less + * likely to reclaim it. + */ + vm_page_aflag_set(m, PGA_REFERENCED); + VM_OBJECT_WUNLOCK(object); + vm_page_busy_sleep(m, "pgrbwt", (allocflags & + VM_ALLOC_IGN_SBUSY) != 0); + VM_OBJECT_WLOCK(object); + goto retrylookup; + } + if ((allocflags & VM_ALLOC_WIRED) != 0) { + vm_page_lock(m); + vm_page_wire(m); + vm_page_unlock(m); + } + if (m->valid == VM_PAGE_BITS_ALL) + goto out; + } else if ((m = vm_page_alloc(object, pindex, pflags)) != NULL) { + xbusy = true; + } else { + goto retrylookup; + } + + vm_page_assert_xbusied(m); + MPASS(xbusy); + if (vm_pager_has_page(object, pindex, NULL, NULL)) { + rv = vm_pager_get_pages(object, &m, 1, NULL, NULL); + if (rv != VM_PAGER_OK) { + vm_page_lock(m); + if (allocflags & VM_ALLOC_WIRED) + vm_page_unwire_noq(m); + vm_page_free(m); + vm_page_unlock(m); + *mp = NULL; + return (rv); + } + MPASS(m->valid == VM_PAGE_BITS_ALL); + } else { + vm_page_zero_invalid(m, TRUE); + vm_page_undirty(m); /* XXX ? */ + } +out: + if ((allocflags & VM_ALLOC_NOBUSY) != 0) { + if (xbusy) + vm_page_xunbusy(m); + else + vm_page_sunbusy(m); + } + if ((allocflags & VM_ALLOC_SBUSY) != 0 && xbusy) + vm_page_busy_downgrade(m); + *mp = m; + return (VM_PAGER_OK); +} + /* * Return the specified range of pages from the given object. For each * page offset within the range, if a page already exists within the object