Index: sys/fs/nfsclient/nfs_clvnops.c =================================================================== --- sys/fs/nfsclient/nfs_clvnops.c +++ sys/fs/nfsclient/nfs_clvnops.c @@ -646,7 +646,7 @@ */ if (vp->v_writecount <= -1) { if ((obj = vp->v_object) != NULL && - (obj->flags & OBJ_MIGHTBEDIRTY) != 0) { + vm_object_mightbedirty(obj)) { VM_OBJECT_WLOCK(obj); vm_object_page_clean(obj, 0, 0, OBJPC_SYNC); VM_OBJECT_WUNLOCK(obj); Index: sys/fs/nfsserver/nfs_nfsdport.c =================================================================== --- sys/fs/nfsserver/nfs_nfsdport.c +++ sys/fs/nfsserver/nfs_nfsdport.c @@ -1498,8 +1498,7 @@ /* * Give up and do the whole thing */ - if (vp->v_object && - (vp->v_object->flags & OBJ_MIGHTBEDIRTY)) { + if (vp->v_object && vm_object_mightbedirty(vp->v_object)) { VM_OBJECT_WLOCK(vp->v_object); vm_object_page_clean(vp->v_object, 0, 0, OBJPC_SYNC); VM_OBJECT_WUNLOCK(vp->v_object); @@ -1529,8 +1528,7 @@ } lblkno = off / iosize; - if (vp->v_object && - (vp->v_object->flags & OBJ_MIGHTBEDIRTY)) { + if (vp->v_object && vm_object_mightbedirty(vp->v_object)) { VM_OBJECT_WLOCK(vp->v_object); vm_object_page_clean(vp->v_object, off, off + cnt, OBJPC_SYNC); Index: sys/fs/tmpfs/tmpfs_subr.c =================================================================== --- sys/fs/tmpfs/tmpfs_subr.c +++ sys/fs/tmpfs/tmpfs_subr.c @@ -1477,10 +1477,10 @@ KASSERT((obj->flags & (OBJ_TMPFS_NODE | OBJ_TMPFS)) == (OBJ_TMPFS_NODE | OBJ_TMPFS), ("non-tmpfs obj")); /* unlocked read */ - if ((obj->flags & OBJ_TMPFS_DIRTY) != 0) { + if (obj->generation != obj->cleangeneration) { VM_OBJECT_WLOCK(obj); - if ((obj->flags & OBJ_TMPFS_DIRTY) != 0) { - obj->flags &= ~OBJ_TMPFS_DIRTY; + if (obj->generation != obj->cleangeneration) { + obj->cleangeneration = obj->generation; node = VP_TO_TMPFS_NODE(vp); node->tn_status |= TMPFS_NODE_MODIFIED | TMPFS_NODE_CHANGED; Index: sys/fs/tmpfs/tmpfs_vfsops.c =================================================================== --- sys/fs/tmpfs/tmpfs_vfsops.c +++ sys/fs/tmpfs/tmpfs_vfsops.c @@ -172,7 +172,7 @@ * For non-lazy case, we must flush all pending * metadata changes now. */ - if (!lazy || (obj->flags & OBJ_TMPFS_DIRTY) != 0) { + if (!lazy || obj->generation != obj->cleangeneration) { if (vget(vp, LK_EXCLUSIVE | LK_INTERLOCK, curthread) != 0) continue; Index: sys/fs/tmpfs/tmpfs_vnops.c =================================================================== --- sys/fs/tmpfs/tmpfs_vnops.c +++ sys/fs/tmpfs/tmpfs_vnops.c @@ -1323,7 +1323,7 @@ goto need; if (vp->v_type == VREG) { obj = vp->v_object; - if ((obj->flags & OBJ_TMPFS_DIRTY) != 0) + if (obj->generation != obj->cleangeneration) goto need; } return (0); Index: sys/kern/vfs_subr.c =================================================================== --- sys/kern/vfs_subr.c +++ sys/kern/vfs_subr.c @@ -3342,7 +3342,7 @@ * pending I/O and dirty pages in the object. */ if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 && - (obj->flags & OBJ_MIGHTBEDIRTY) != 0) { + vm_object_mightbedirty(obj)) { VM_OBJECT_WLOCK(obj); vm_object_page_clean(obj, 0, 0, 0); VM_OBJECT_WUNLOCK(obj); @@ -4400,7 +4400,7 @@ MNT_VNODE_FOREACH_ACTIVE(vp, mp, mvp) { obj = vp->v_object; - if (obj != NULL && (obj->flags & OBJ_MIGHTBEDIRTY) != 0 && + if (obj != NULL && vm_object_mightbedirty(obj) && (flags == MNT_WAIT || VOP_ISLOCKED(vp) == 0)) { if (!vget(vp, LK_EXCLUSIVE | LK_RETRY | LK_INTERLOCK, @@ -4690,7 +4690,7 @@ MPASS(mtx_owned(VI_MTX(vp))); need = 0; if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 && - (obj->flags & OBJ_MIGHTBEDIRTY) != 0) + vm_object_mightbedirty(obj)) need = 1; return (need); } Index: sys/ufs/ffs/ffs_rawread.c =================================================================== --- sys/ufs/ffs/ffs_rawread.c +++ sys/ufs/ffs/ffs_rawread.c @@ -109,7 +109,7 @@ if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0 || ((obj = vp->v_object) != NULL && - (obj->flags & OBJ_MIGHTBEDIRTY) != 0)) { + vm_object_mightbedirty(obj))) { VI_UNLOCK(vp); BO_UNLOCK(bo); @@ -140,7 +140,7 @@ } /* Attempt to msync mmap() regions to clean dirty mmap */ if ((obj = vp->v_object) != NULL && - (obj->flags & OBJ_MIGHTBEDIRTY) != 0) { + vm_object_mightbedirty(obj)) { VI_UNLOCK(vp); VM_OBJECT_WLOCK(obj); vm_object_page_clean(obj, 0, 0, OBJPC_SYNC); Index: sys/vm/vm_fault.c =================================================================== --- sys/vm/vm_fault.c +++ sys/vm/vm_fault.c @@ -208,7 +208,7 @@ static void vm_fault_dirty(vm_map_entry_t entry, vm_page_t m, vm_prot_t prot, - vm_prot_t fault_type, int fault_flags, bool set_wd) + vm_prot_t fault_type, int fault_flags, bool excl) { bool need_dirty; @@ -224,11 +224,11 @@ (fault_flags & VM_FAULT_WIRE) == 0) || (fault_flags & VM_FAULT_DIRTY) != 0; - if (set_wd) - vm_object_set_writeable_dirty(m->object); - else + vm_object_set_writeable_dirty(m->object); + + if (!excl) /* - * If two callers of vm_fault_dirty() with set_wd == + * If two callers of vm_fault_dirty() with excl == * FALSE, one for the map entry with MAP_ENTRY_NOSYNC * flag set, other with flag clear, race, it is * possible for the no-NOSYNC thread to see m->dirty @@ -265,7 +265,7 @@ */ if (need_dirty) vm_page_dirty(m); - if (!set_wd) + if (!excl) vm_page_unlock(m); else if (need_dirty) vm_pager_page_unswapped(m); @@ -711,29 +711,17 @@ /* * Try to avoid lock contention on the top-level object through * special-case handling of some types of page faults, specifically, - * those that are both (1) mapping an existing page from the top- - * level object and (2) not having to mark that object as containing - * dirty pages. Under these conditions, a read lock on the top-level - * object suffices, allowing multiple page faults of a similar type to - * run in parallel on the same top-level object. + * those that are mapping an existing page from the top-level object. + * Under this condition, a read lock on the object suffices, allowing + * multiple page faults of a similar type to run in parallel. */ if (fs.vp == NULL /* avoid locked vnode leak */ && - (fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0 && - /* avoid calling vm_object_set_writeable_dirty() */ - ((prot & VM_PROT_WRITE) == 0 || - (fs.first_object->type != OBJT_VNODE && - (fs.first_object->flags & OBJ_TMPFS_NODE) == 0) || - (fs.first_object->flags & OBJ_MIGHTBEDIRTY) != 0)) { + (fault_flags & (VM_FAULT_WIRE | VM_FAULT_DIRTY)) == 0) { VM_OBJECT_RLOCK(fs.first_object); - if ((prot & VM_PROT_WRITE) == 0 || - (fs.first_object->type != OBJT_VNODE && - (fs.first_object->flags & OBJ_TMPFS_NODE) == 0) || - (fs.first_object->flags & OBJ_MIGHTBEDIRTY) != 0) { - rv = vm_fault_soft_fast(&fs, vaddr, prot, fault_type, - fault_flags, wired, m_hold); - if (rv == KERN_SUCCESS) - return (rv); - } + rv = vm_fault_soft_fast(&fs, vaddr, prot, fault_type, + fault_flags, wired, m_hold); + if (rv == KERN_SUCCESS) + return (rv); if (!VM_OBJECT_TRYUPGRADE(fs.first_object)) { VM_OBJECT_RUNLOCK(fs.first_object); VM_OBJECT_WLOCK(fs.first_object); Index: sys/vm/vm_object.h =================================================================== --- sys/vm/vm_object.h +++ sys/vm/vm_object.h @@ -105,7 +105,8 @@ struct vm_radix rtree; /* root of the resident page radix trie*/ vm_pindex_t size; /* Object size */ struct domainset_ref domain; /* NUMA policy. */ - int generation; /* generation ID */ + volatile int generation; /* generation ID */ + int cleangeneration; /* Generation at clean time */ volatile u_int ref_count; /* How many refs?? */ int shadow_count; /* how many objects that this is a shadow for */ vm_memattr_t memattr; /* default memory attribute for pages */ @@ -187,9 +188,7 @@ #define OBJ_NOSPLIT 0x0010 /* dont split this object */ #define OBJ_UMTXDEAD 0x0020 /* umtx pshared was terminated */ #define OBJ_PG_DTOR 0x0080 /* dont reset object, leave that for dtor */ -#define OBJ_MIGHTBEDIRTY 0x0100 /* object might be dirty, only for vnode */ #define OBJ_TMPFS_NODE 0x0200 /* object belongs to tmpfs VREG node */ -#define OBJ_TMPFS_DIRTY 0x0400 /* dirty tmpfs obj */ #define OBJ_COLORED 0x1000 /* pg_color is defined */ #define OBJ_ONEMAPPING 0x2000 /* One USE (a single, non-forked) mapping flag */ #define OBJ_TMPFS 0x8000 /* has tmpfs vnode allocated */ @@ -308,6 +307,14 @@ return (false); } +static __inline bool +vm_object_mightbedirty(vm_object_t object) +{ + + return (object->type == OBJT_VNODE && + object->generation != object->cleangeneration); +} + void vm_object_clear_flag(vm_object_t object, u_short bits); void vm_object_pip_add(vm_object_t object, short i); void vm_object_pip_wakeup(vm_object_t object); Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -112,10 +112,10 @@ "Use old (insecure) msync behavior"); static int vm_object_page_collect_flush(vm_object_t object, vm_page_t p, - int pagerflags, int flags, boolean_t *clearobjflags, + int pagerflags, int flags, boolean_t *allclean, boolean_t *eio); static boolean_t vm_object_page_remove_write(vm_page_t p, int flags, - boolean_t *clearobjflags); + boolean_t *allclean); static void vm_object_qcollapse(vm_object_t object); static void vm_object_vndeallocate(vm_object_t object); @@ -282,6 +282,7 @@ object->size = size; object->domain.dr_policy = NULL; object->generation = 1; + object->cleangeneration = 1; refcount_init(&object->ref_count, 1); object->memattr = VM_MEMATTR_DEFAULT; object->cred = NULL; @@ -769,7 +770,7 @@ * page should be flushed, and FALSE otherwise. */ static boolean_t -vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *clearobjflags) +vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *allclean) { vm_page_assert_busied(p); @@ -780,7 +781,7 @@ * cleared in this case so we do not have to set them. */ if ((flags & OBJPC_NOSYNC) != 0 && (p->aflags & PGA_NOSYNC) != 0) { - *clearobjflags = FALSE; + *allclean = FALSE; return (FALSE); } else { pmap_remove_write(p); @@ -813,16 +814,11 @@ vm_page_t np, p; vm_pindex_t pi, tend, tstart; int curgeneration, n, pagerflags; - boolean_t clearobjflags, eio, res; + boolean_t eio, res, allclean; VM_OBJECT_ASSERT_WLOCKED(object); - /* - * The OBJ_MIGHTBEDIRTY flag is only set for OBJT_VNODE - * objects. The check below prevents the function from - * operating on non-vnode objects. - */ - if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 || + if (object->type != OBJT_VNODE || !vm_object_mightbedirty(object) || object->resident_page_count == 0) return (TRUE); @@ -832,7 +828,7 @@ tstart = OFF_TO_IDX(start); tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK); - clearobjflags = tstart == 0 && tend >= object->size; + allclean = tstart == 0 && tend >= object->size; res = TRUE; rescan: @@ -846,32 +842,26 @@ if (vm_page_none_valid(p)) continue; if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) { - if (object->generation != curgeneration) { - if ((flags & OBJPC_SYNC) != 0) - goto rescan; - else - clearobjflags = FALSE; - } + if (object->generation != curgeneration && + (flags & OBJPC_SYNC) != 0) + goto rescan; np = vm_page_find_least(object, pi); continue; } - if (!vm_object_page_remove_write(p, flags, &clearobjflags)) { + if (!vm_object_page_remove_write(p, flags, &allclean)) { vm_page_xunbusy(p); continue; } n = vm_object_page_collect_flush(object, p, pagerflags, - flags, &clearobjflags, &eio); + flags, &allclean, &eio); if (eio) { res = FALSE; - clearobjflags = FALSE; - } - if (object->generation != curgeneration) { - if ((flags & OBJPC_SYNC) != 0) - goto rescan; - else - clearobjflags = FALSE; + allclean = FALSE; } + if (object->generation != curgeneration && + (flags & OBJPC_SYNC) != 0) + goto rescan; /* * If the VOP_PUTPAGES() did a truncated write, so @@ -887,7 +877,7 @@ */ if (n == 0) { n = 1; - clearobjflags = FALSE; + allclean = FALSE; } np = vm_page_find_least(object, pi + n); } @@ -895,14 +885,14 @@ VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0); #endif - if (clearobjflags) - vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY); + if (allclean) + object->cleangeneration = curgeneration; return (res); } static int vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags, - int flags, boolean_t *clearobjflags, boolean_t *eio) + int flags, boolean_t *allclean, boolean_t *eio) { vm_page_t ma[vm_pageout_page_count], p_first, tp; int count, i, mreq, runlen; @@ -918,7 +908,7 @@ tp = vm_page_next(tp); if (tp == NULL || vm_page_tryxbusy(tp) == 0) break; - if (!vm_object_page_remove_write(tp, flags, clearobjflags)) { + if (!vm_object_page_remove_write(tp, flags, allclean)) { vm_page_xunbusy(tp); break; } @@ -928,7 +918,7 @@ tp = vm_page_prev(p_first); if (tp == NULL || vm_page_tryxbusy(tp) == 0) break; - if (!vm_object_page_remove_write(tp, flags, clearobjflags)) { + if (!vm_object_page_remove_write(tp, flags, allclean)) { vm_page_xunbusy(tp); break; } @@ -993,7 +983,7 @@ * I/O. */ if (object->type == OBJT_VNODE && - (object->flags & OBJ_MIGHTBEDIRTY) != 0 && + vm_object_mightbedirty(object) != 0 && ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) { VM_OBJECT_WUNLOCK(object); (void) vn_start_write(vp, &mp, V_WAIT); @@ -2130,18 +2120,13 @@ vm_object_set_writeable_dirty(vm_object_t object) { - VM_OBJECT_ASSERT_WLOCKED(object); - if (object->type != OBJT_VNODE) { - if ((object->flags & OBJ_TMPFS_NODE) != 0) { - KASSERT(object->type == OBJT_SWAP, ("non-swap tmpfs")); - vm_object_set_flag(object, OBJ_TMPFS_DIRTY); - } - return; - } - object->generation++; - if ((object->flags & OBJ_MIGHTBEDIRTY) != 0) + VM_OBJECT_ASSERT_LOCKED(object); + + /* Only set for vnodes & tmpfs */ + if (object->type != OBJT_VNODE && + (object->flags & OBJ_TMPFS_NODE) == 0) return; - vm_object_set_flag(object, OBJ_MIGHTBEDIRTY); + atomic_add_int(&object->generation, 1); } /* Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -1547,7 +1547,7 @@ /* * Since we are inserting a new and possibly dirty page, - * update the object's OBJ_MIGHTBEDIRTY flag. + * update the object's generation count. */ if (pmap_page_is_write_mapped(m)) vm_object_set_writeable_dirty(object); @@ -1717,7 +1717,8 @@ /* * The object's resident_page_count does not change because we have - * swapped one page for another, but OBJ_MIGHTBEDIRTY. + * swapped one page for another, but the generation count should + * change if the page is dirty. */ if (pmap_page_is_write_mapped(mnew)) vm_object_set_writeable_dirty(object);