Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -1423,13 +1423,41 @@ #define OBSC_COLLAPSE_NOWAIT 0x0002 #define OBSC_COLLAPSE_WAIT 0x0004 +static vm_page_t +vm_object_backing_scan_wait(vm_object_t object, vm_page_t p, vm_page_t next, + int op) +{ + vm_object_t backing_object; + + VM_OBJECT_ASSERT_WLOCKED(object); + backing_object = object->backing_object; + VM_OBJECT_ASSERT_WLOCKED(backing_object); + + KASSERT(p == NULL || vm_page_busied(p), ("unbusy page %p", p)); + KASSERT(p == NULL || p->object == object || p->object == backing_object, + ("invalid ownership %p %p %p", p, object, backing_object)); + if ((op & OBSC_COLLAPSE_NOWAIT) != 0) + return (next); + if (p != NULL) + vm_page_lock(p); + VM_OBJECT_WUNLOCK(object); + VM_OBJECT_WUNLOCK(backing_object); + if (p == NULL) + VM_WAIT; + else + vm_page_busy_sleep(p, "vmocol"); + VM_OBJECT_WLOCK(object); + VM_OBJECT_WLOCK(backing_object); + return (TAILQ_FIRST(&backing_object->memq)); +} + static int vm_object_backing_scan(vm_object_t object, int op) { - int r = 1; - vm_page_t p; + vm_page_t next, p, pp; vm_object_t backing_object; - vm_pindex_t backing_offset_index; + vm_pindex_t new_pindex, backing_offset_index; + int r; VM_OBJECT_ASSERT_WLOCKED(object); VM_OBJECT_ASSERT_WLOCKED(object->backing_object); @@ -1458,16 +1486,17 @@ vm_object_set_flag(backing_object, OBJ_DEAD); } + r = 1; + /* * Our scan */ p = TAILQ_FIRST(&backing_object->memq); while (p) { - vm_page_t next = TAILQ_NEXT(p, listq); - vm_pindex_t new_pindex = p->pindex - backing_offset_index; + next = TAILQ_NEXT(p, listq); + new_pindex = p->pindex - backing_offset_index; if (op & OBSC_TEST_ALL_SHADOWED) { - vm_page_t pp; /* * Ignore pages outside the parent object's range @@ -1509,31 +1538,10 @@ * Check for busy page */ if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) { - vm_page_t pp; - - if (op & OBSC_COLLAPSE_NOWAIT) { - if (!p->valid || vm_page_busied(p)) { - p = next; - continue; - } - } else if (op & OBSC_COLLAPSE_WAIT) { - if (vm_page_busied(p)) { - VM_OBJECT_WUNLOCK(object); - vm_page_lock(p); - VM_OBJECT_WUNLOCK(backing_object); - vm_page_busy_sleep(p, "vmocol"); - VM_OBJECT_WLOCK(object); - VM_OBJECT_WLOCK(backing_object); - /* - * If we slept, anything could have - * happened. Since the object is - * marked dead, the backing offset - * should not have changed so we - * just restart our scan. - */ - p = TAILQ_FIRST(&backing_object->memq); - continue; - } + if (vm_page_busied(p)) { + p = vm_object_backing_scan_wait(object, p, + next, op); + continue; } KASSERT( @@ -1566,43 +1574,41 @@ } pp = vm_page_lookup(object, new_pindex); - if ( - (op & OBSC_COLLAPSE_NOWAIT) != 0 && - (pp != NULL && pp->valid == 0) - ) { - if (backing_object->type == OBJT_SWAP) - swap_pager_freespace(backing_object, - p->pindex, 1); - + if (pp != NULL && vm_page_busied(pp)) { /* - * The page in the parent is not (yet) valid. - * We don't know anything about the state of - * the original page. It might be mapped, - * so we must avoid the next if here. + * The page in the parent is busy and + * possibly not (yet) valid. We don't + * know anything about the state of + * the original page. It might be + * mapped, so we must avoid the next + * if here. * - * This is due to a race in vm_fault() where - * we must unbusy the original (backing_obj) - * page before we can (re)lock the parent. - * Hence we can get here. + * This is due to a race in vm_fault() + * where we must unbusy the original + * (backing_obj) page before we can + * (re)lock the parent. Hence we can + * get here. */ - p = next; + p = vm_object_backing_scan_wait(object, pp, + next, op); continue; } - if ( - pp != NULL || - vm_pager_has_page(object, new_pindex, NULL, NULL) - ) { - if (backing_object->type == OBJT_SWAP) - swap_pager_freespace(backing_object, - p->pindex, 1); + KASSERT(pp == NULL || pp->valid != 0, + ("unbusy invalid page %p", pp)); + + if (pp != NULL || vm_pager_has_page(object, + new_pindex, NULL, NULL)) { /* - * page already exists in parent OR swap exists - * for this location in the parent. Destroy - * the original page from the backing object. - * - * Leave the parent's page alone + * Page already exists in parent OR + * swap exists for this location in + * the parent. Destroy the original + * page from the backing object. + * Leave the parent's page alone. */ + if (backing_object->type == OBJT_SWAP) + swap_pager_freespace(backing_object, + p->pindex, 1); vm_page_lock(p); KASSERT(!pmap_page_is_mapped(p), ("freeing mapped page %p", p)); @@ -1624,16 +1630,8 @@ * vm_page_rename() will handle dirty and cache. */ if (vm_page_rename(p, object, new_pindex)) { - if (op & OBSC_COLLAPSE_NOWAIT) { - p = next; - continue; - } - VM_OBJECT_WUNLOCK(backing_object); - VM_OBJECT_WUNLOCK(object); - VM_WAIT; - VM_OBJECT_WLOCK(object); - VM_OBJECT_WLOCK(backing_object); - p = TAILQ_FIRST(&backing_object->memq); + p = vm_object_backing_scan_wait(object, NULL, + next, op); continue; }