Index: sys/compat/linuxkpi/common/src/linux_shmemfs.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_shmemfs.c +++ sys/compat/linuxkpi/common/src/linux_shmemfs.c @@ -53,10 +53,8 @@ if ((gfp & GFP_NOWAIT) != 0) panic("GFP_NOWAIT is unimplemented"); - VM_OBJECT_WLOCK(obj); - rv = vm_page_grab_valid(&page, obj, pindex, VM_ALLOC_NORMAL | - VM_ALLOC_NOBUSY | VM_ALLOC_WIRED); - VM_OBJECT_WUNLOCK(obj); + rv = vm_page_grab_valid_unlocked(&page, obj, pindex, + VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED); if (rv != VM_PAGER_OK) return (ERR_PTR(-EINVAL)); return (page); Index: sys/kern/uipc_shm.c =================================================================== --- sys/kern/uipc_shm.c +++ sys/kern/uipc_shm.c @@ -201,8 +201,8 @@ * lock to page out tobj's pages because tobj is a OBJT_SWAP * type object. */ - rv = vm_page_grab_valid(&m, obj, idx, - VM_ALLOC_NORMAL | VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY); + rv = vm_page_grab_valid(&m, obj, idx, VM_ALLOC_NORMAL | + VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY | VM_ALLOC_ZERO); if (rv != VM_PAGER_OK) { VM_OBJECT_WUNLOCK(obj); printf("uiomove_object: vm_obj %p idx %jd pager error %d\n", Index: sys/vm/vm_glue.c =================================================================== --- sys/vm/vm_glue.c +++ sys/vm/vm_glue.c @@ -223,7 +223,7 @@ pindex = OFF_TO_IDX(offset); (void)vm_page_grab_valid_unlocked(&m, object, pindex, - VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED); + VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); return (m); } Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -2180,7 +2180,8 @@ VM_OBJECT_ASSERT_WLOCKED(object); for (pindex = start; pindex < end; pindex++) { - rv = vm_page_grab_valid(&m, object, pindex, VM_ALLOC_NORMAL); + rv = vm_page_grab_valid(&m, object, pindex, + VM_ALLOC_NORMAL | VM_ALLOC_ZERO); if (rv != VM_PAGER_OK) break; Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -4528,11 +4528,8 @@ vm_page_t ma[VM_INITIAL_PAGEIN]; int after, i, pflags, rv; - KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || - (allocflags & VM_ALLOC_IGN_SBUSY) != 0, - ("vm_page_grab_valid: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch")); - KASSERT((allocflags & - (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL | VM_ALLOC_ZERO)) == 0, + vm_page_grab_check(allocflags); + KASSERT((allocflags & (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL)) == 0, ("vm_page_grab_valid: Invalid flags 0x%X", allocflags)); VM_OBJECT_ASSERT_WLOCKED(object); /* We never want to allocate non-busy pages. */ @@ -4569,49 +4566,56 @@ } else if ((m = vm_page_alloc(object, pindex, pflags)) == NULL) { goto retrylookup; } - vm_page_assert_xbusied(m); - if (vm_pager_has_page(object, pindex, NULL, &after)) { - after = MIN(after, VM_INITIAL_PAGEIN); - after = MIN(after, allocflags >> VM_ALLOC_COUNT_SHIFT); - after = MAX(after, 1); - ma[0] = m; - for (i = 1; i < after; i++) { - if ((ma[i] = vm_page_next(ma[i - 1])) != NULL) { - if (ma[i]->valid || !vm_page_tryxbusy(ma[i])) - break; - } else { - ma[i] = vm_page_alloc(object, m->pindex + i, - VM_ALLOC_NORMAL); - if (ma[i] == NULL) - break; - } - } - after = i; - vm_object_pip_add(object, after); - VM_OBJECT_WUNLOCK(object); - rv = vm_pager_get_pages(object, ma, after, NULL, NULL); - VM_OBJECT_WLOCK(object); - vm_object_pip_wakeupn(object, after); - /* Pager may have replaced a page. */ - m = ma[0]; - if (rv != VM_PAGER_OK) { - for (i = 0; i < after; i++) { - if (!vm_page_wired(ma[i])) - vm_page_free(ma[i]); - else - vm_page_xunbusy(ma[i]); - } + + if (!vm_pager_has_page(object, pindex, NULL, &after)) { + if ((allocflags & VM_ALLOC_ZERO) == 0) { + if (!vm_page_wired(m)) + vm_page_free(m); + else + vm_page_xunbusy(m); *mp = NULL; - return (rv); + return (VM_PAGER_FAIL); } - for (i = 1; i < after; i++) - vm_page_readahead_finish(ma[i]); - MPASS(vm_page_all_valid(m)); - } else { - /* Unlike other grab functions zero is implied. */ - vm_page_zero_invalid(m, TRUE); + goto out; } + + after = MIN(after, VM_INITIAL_PAGEIN); + after = MIN(after, allocflags >> VM_ALLOC_COUNT_SHIFT); + after = MAX(after, 1); + ma[0] = m; + for (i = 1; i < after; i++) { + if ((ma[i] = vm_page_next(ma[i - 1])) != NULL) { + if (ma[i]->valid || !vm_page_tryxbusy(ma[i])) + break; + } else { + ma[i] = vm_page_alloc(object, m->pindex + i, + VM_ALLOC_NORMAL); + if (ma[i] == NULL) + break; + } + } + after = i; + vm_object_pip_add(object, after); + VM_OBJECT_WUNLOCK(object); + rv = vm_pager_get_pages(object, ma, after, NULL, NULL); + VM_OBJECT_WLOCK(object); + vm_object_pip_wakeupn(object, after); + /* Pager may have replaced a page. */ + m = ma[0]; + if (rv != VM_PAGER_OK) { + for (i = 0; i < after; i++) { + if (!vm_page_wired(ma[i])) + vm_page_free(ma[i]); + else + vm_page_xunbusy(ma[i]); + } + *mp = NULL; + return (rv); + } + for (i = 1; i < after; i++) + vm_page_readahead_finish(ma[i]); + MPASS(vm_page_all_valid(m)); out: vm_page_grab_release(m, allocflags); *mp = m; @@ -4630,12 +4634,8 @@ int flags; int error; - KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 || - (allocflags & VM_ALLOC_IGN_SBUSY) != 0, - ("vm_page_grab_valid_unlocked: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY " - "mismatch")); - KASSERT((allocflags & - (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL | VM_ALLOC_ZERO)) == 0, + vm_page_grab_check(allocflags); + KASSERT((allocflags & (VM_ALLOC_NOWAIT | VM_ALLOC_WAITFAIL)) == 0, ("vm_page_grab_valid_unlocked: Invalid flags 0x%X", allocflags)); /*