Index: sys/dev/ksyms/ksyms.c =================================================================== --- sys/dev/ksyms/ksyms.c +++ sys/dev/ksyms/ksyms.c @@ -444,7 +444,6 @@ object = vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(round_page(elfsz))); - vm_object_set_flag(object, OBJ_NOSPLIT); sc->sc_obj = object; sc->sc_objsz = elfsz; Index: sys/fs/tmpfs/tmpfs_subr.c =================================================================== --- sys/fs/tmpfs/tmpfs_subr.c +++ sys/fs/tmpfs/tmpfs_subr.c @@ -273,8 +273,7 @@ NULL /* XXXKIB - tmpfs needs swap reservation */); VM_OBJECT_WLOCK(obj); /* OBJ_TMPFS is set together with the setting of vp->v_object */ - vm_object_set_flag(obj, OBJ_NOSPLIT | OBJ_TMPFS_NODE); - vm_object_clear_flag(obj, OBJ_ONEMAPPING); + vm_object_set_flag(obj, OBJ_TMPFS_NODE); VM_OBJECT_WUNLOCK(obj); break; Index: sys/kern/sysv_shm.c =================================================================== --- sys/kern/sysv_shm.c +++ sys/kern/sysv_shm.c @@ -751,11 +751,6 @@ #endif return (ENOMEM); } - shm_object->pg_color = 0; - VM_OBJECT_WLOCK(shm_object); - vm_object_clear_flag(shm_object, OBJ_ONEMAPPING); - vm_object_set_flag(shm_object, OBJ_COLORED | OBJ_NOSPLIT); - VM_OBJECT_WUNLOCK(shm_object); shmseg->object = shm_object; shmseg->u.shm_perm.cuid = shmseg->u.shm_perm.uid = cred->cr_uid; Index: sys/kern/uipc_shm.c =================================================================== --- sys/kern/uipc_shm.c +++ sys/kern/uipc_shm.c @@ -573,11 +573,6 @@ shmfd->shm_object = vm_pager_allocate(OBJT_SWAP, NULL, shmfd->shm_size, VM_PROT_DEFAULT, 0, ucred); KASSERT(shmfd->shm_object != NULL, ("shm_create: vm_pager_allocate")); - shmfd->shm_object->pg_color = 0; - VM_OBJECT_WLOCK(shmfd->shm_object); - vm_object_clear_flag(shmfd->shm_object, OBJ_ONEMAPPING); - vm_object_set_flag(shmfd->shm_object, OBJ_COLORED | OBJ_NOSPLIT); - VM_OBJECT_WUNLOCK(shmfd->shm_object); vfs_timestamp(&shmfd->shm_birthtime); shmfd->shm_atime = shmfd->shm_mtime = shmfd->shm_ctime = shmfd->shm_birthtime; Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -3002,7 +3002,7 @@ { VM_OBJECT_WLOCK(object); - KASSERT((object->flags & OBJ_NOSPLIT) != 0, + KASSERT((object->flags & OBJ_ANON) == 0, ("Splittable object with writecount")); object->un_pager.swp.writemappings += (vm_ooffset_t)end - start; VM_OBJECT_WUNLOCK(object); @@ -3014,7 +3014,7 @@ { VM_OBJECT_WLOCK(object); - KASSERT((object->flags & OBJ_NOSPLIT) != 0, + KASSERT((object->flags & OBJ_ANON) == 0, ("Splittable object with writecount")); object->un_pager.swp.writemappings -= (vm_ooffset_t)end - start; VM_OBJECT_WUNLOCK(object); Index: sys/vm/vm_fault.c =================================================================== --- sys/vm/vm_fault.c +++ sys/vm/vm_fault.c @@ -1263,8 +1263,7 @@ /* * No other ways to look the object up */ - ((fs.object->type == OBJT_DEFAULT) || - (fs.object->type == OBJT_SWAP)) && + ((fs.object->flags & OBJ_ANON) != 0) && (is_first_object_locked = VM_OBJECT_TRYWLOCK(fs.first_object)) && /* * We don't chase down the shadow chain @@ -1765,7 +1764,7 @@ * Create the top-level object for the destination entry. (Doesn't * actually shadow anything - we copy the pages directly.) */ - dst_object = vm_object_allocate(OBJT_DEFAULT, + dst_object = vm_object_allocate_anon( atop(dst_entry->end - dst_entry->start)); #if VM_NRESERVLEVEL > 0 dst_object->flags |= OBJ_COLORED; Index: sys/vm/vm_map.c =================================================================== --- sys/vm/vm_map.c +++ sys/vm/vm_map.c @@ -1473,10 +1473,12 @@ * reference counting is insufficient to recognize * aliases with precision.) */ - VM_OBJECT_WLOCK(object); - if (object->ref_count > 1 || object->shadow_count != 0) - vm_object_clear_flag(object, OBJ_ONEMAPPING); - VM_OBJECT_WUNLOCK(object); + if ((object->flags & OBJ_ANON) != 0) { + VM_OBJECT_WLOCK(object); + if (object->ref_count > 1 || object->shadow_count != 0) + vm_object_clear_flag(object, OBJ_ONEMAPPING); + VM_OBJECT_WUNLOCK(object); + } } else if ((prev_entry->eflags & ~MAP_ENTRY_USER_WIRED) == protoeflags && (cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP | @@ -2069,8 +2071,7 @@ ("map entry %p has backing object", entry)); KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0, ("map entry %p is a submap", entry)); - object = vm_object_allocate(OBJT_DEFAULT, - atop(entry->end - entry->start)); + object = vm_object_allocate_anon(atop(entry->end - entry->start)); entry->object.vm_object = object; entry->offset = 0; if (entry->cred != NULL) { @@ -3450,6 +3451,9 @@ if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 && (object != NULL)) { + if ((object->flags & OBJ_ANON) == 0 && + object != kernel_object) + goto out; KASSERT(entry->cred == NULL || object->cred == NULL || (entry->eflags & MAP_ENTRY_NEEDS_COPY), ("OVERCOMMIT vm_map_entry_delete: both cred %p", entry)); @@ -3457,8 +3461,8 @@ offidxstart = OFF_TO_IDX(entry->offset); offidxend = offidxstart + count; VM_OBJECT_WLOCK(object); - if (object->ref_count != 1 && ((object->flags & (OBJ_NOSPLIT | - OBJ_ONEMAPPING)) == OBJ_ONEMAPPING || + if (object->ref_count != 1 && + ((object->flags & OBJ_ONEMAPPING) != 0 || object == kernel_object)) { vm_object_collapse(object); @@ -3490,6 +3494,7 @@ VM_OBJECT_WUNLOCK(object); } else entry->object.vm_object = NULL; +out: if (map->system_map) vm_map_entry_deallocate(entry, TRUE); else { @@ -3708,11 +3713,9 @@ VM_OBJECT_WLOCK(src_object); charged = ENTRY_CHARGED(src_entry); if (src_object->handle == NULL && - (src_object->type == OBJT_DEFAULT || - src_object->type == OBJT_SWAP)) { + (src_object->flags & OBJ_ANON) != 0) { vm_object_collapse(src_object); - if ((src_object->flags & (OBJ_NOSPLIT | - OBJ_ONEMAPPING)) == OBJ_ONEMAPPING) { + if ((src_object->flags & OBJ_ONEMAPPING) != 0) { vm_object_split(src_entry); src_object = src_entry->object.vm_object; @@ -4643,8 +4646,7 @@ !map->system_map) { if (vm_map_lock_upgrade(map)) goto RetryLookup; - entry->object.vm_object = vm_object_allocate(OBJT_DEFAULT, - atop(size)); + entry->object.vm_object = vm_object_allocate_anon(atop(size)); entry->offset = 0; if (entry->cred != NULL) { VM_OBJECT_WLOCK(entry->object.vm_object); Index: sys/vm/vm_meter.c =================================================================== --- sys/vm/vm_meter.c +++ sys/vm/vm_meter.c @@ -258,7 +258,7 @@ continue; } if (object->ref_count == 1 && - (object->flags & OBJ_NOSPLIT) != 0) { + (object->flags & OBJ_ANON) == 0) { /* * Also skip otherwise unreferenced swap * objects backing tmpfs vnodes, and POSIX or Index: sys/vm/vm_object.h =================================================================== --- sys/vm/vm_object.h +++ sys/vm/vm_object.h @@ -185,7 +185,7 @@ #define OBJ_UNMANAGED 0x0002 /* (c) contains unmanaged pages */ #define OBJ_POPULATE 0x0004 /* pager implements populate() */ #define OBJ_DEAD 0x0008 /* dead objects (during rundown) */ -#define OBJ_NOSPLIT 0x0010 /* dont split this object */ +#define OBJ_ANON 0x0010 /* object backs anonymous memory */ #define OBJ_UMTXDEAD 0x0020 /* umtx pshared was terminated */ #define OBJ_SIZEVNLOCK 0x0040 /* lock vnode to check obj size */ #define OBJ_PG_DTOR 0x0080 /* dont reset object, leave that for dtor */ @@ -340,6 +340,7 @@ extern int umtx_shm_vnobj_persistent; vm_object_t vm_object_allocate (objtype_t, vm_pindex_t); +vm_object_t vm_object_allocate_anon(vm_pindex_t); boolean_t vm_object_coalesce(vm_object_t, vm_ooffset_t, vm_size_t, vm_size_t, boolean_t); void vm_object_collapse (vm_object_t); Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -261,7 +261,7 @@ panic("_vm_object_allocate: can't create OBJT_DEAD"); case OBJT_DEFAULT: case OBJT_SWAP: - object->flags = OBJ_ONEMAPPING; + object->flags = OBJ_COLORED; break; case OBJT_DEVICE: case OBJT_SG: @@ -283,6 +283,7 @@ object->domain.dr_policy = NULL; object->generation = 1; object->cleangeneration = 1; + object->pg_color = 0; refcount_init(&object->ref_count, 1); object->memattr = VM_MEMATTR_DEFAULT; object->cred = NULL; @@ -433,6 +434,25 @@ return (object); } +/* + * vm_object_allocate_anon: + * + * Returns a new default object of the given size and marked as + * anonymous memory for special split/collapse handling. Color + * to be initialized by the caller. + */ +vm_object_t +vm_object_allocate_anon(vm_pindex_t size) +{ + vm_object_t object; + + object = vm_object_allocate(OBJT_DEFAULT, size); + object->flags |= OBJ_ANON | OBJ_ONEMAPPING; + object->flags &= ~OBJ_COLORED; + + return (object); +} + /* * vm_object_reference: @@ -522,7 +542,10 @@ * being 0 or 1. These cases require a write lock on the * object. */ - released = refcount_release_if_gt(&object->ref_count, 2); + if ((object->flags & OBJ_ANON) == 0) + released = refcount_release_if_gt(&object->ref_count, 1); + else + released = refcount_release_if_gt(&object->ref_count, 2); VM_OBJECT_RUNLOCK(object); if (released) return; @@ -538,14 +561,11 @@ } else if (object->ref_count == 1) { if (object->shadow_count == 0 && object->handle == NULL && - (object->type == OBJT_DEFAULT || - (object->type == OBJT_SWAP && - (object->flags & OBJ_TMPFS_NODE) == 0))) { + (object->flags & OBJ_ANON) != 0) { vm_object_set_flag(object, OBJ_ONEMAPPING); } else if ((object->shadow_count == 1) && (object->handle == NULL) && - (object->type == OBJT_DEFAULT || - object->type == OBJT_SWAP)) { + (object->flags & OBJ_ANON) != 0) { vm_object_t robject; robject = LIST_FIRST(&object->shadow_head); @@ -576,10 +596,9 @@ * be deallocated by the thread that is * deallocating its shadow. */ - if ((robject->flags & OBJ_DEAD) == 0 && - (robject->handle == NULL) && - (robject->type == OBJT_DEFAULT || - robject->type == OBJT_SWAP)) { + if ((robject->flags & + (OBJ_DEAD | OBJ_ANON)) == OBJ_ANON && + robject->handle == NULL) { refcount_acquire(&robject->ref_count); retry: @@ -1049,8 +1068,8 @@ return (false); if (advice != MADV_FREE) return (true); - return ((object->type == OBJT_DEFAULT || object->type == OBJT_SWAP) && - (object->flags & OBJ_ONEMAPPING) != 0); + return ((object->flags & (OBJ_ONEMAPPING | OBJ_ANON)) == + (OBJ_ONEMAPPING | OBJ_ANON)); } static void @@ -1211,23 +1230,20 @@ /* * Don't create the new object if the old object isn't shared. + * + * If we hold the only reference we can guarantee that it won't + * increase while we have the map locked. Otherwise the race is + * harmless and we will end up with an extra shadow object that + * will be collapsed later. */ - if (source != NULL) { - VM_OBJECT_RLOCK(source); - if (source->ref_count == 1 && - source->handle == NULL && - (source->type == OBJT_DEFAULT || - source->type == OBJT_SWAP)) { - VM_OBJECT_RUNLOCK(source); - return; - } - VM_OBJECT_RUNLOCK(source); - } + if (source != NULL && source->ref_count == 1 && + source->handle == NULL && (source->flags & OBJ_ANON) != 0) + return; /* * Allocate a new object with the given length. */ - result = vm_object_allocate(OBJT_DEFAULT, atop(length)); + result = vm_object_allocate_anon(atop(length)); /* * The new object shadows the source object, adding a reference to it. @@ -1282,7 +1298,7 @@ vm_size_t size; orig_object = entry->object.vm_object; - if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP) + if ((orig_object->flags & OBJ_ANON) == 0) return; if (orig_object->ref_count <= 1) return; @@ -1295,7 +1311,7 @@ * If swap_pager_copy() is later called, it will convert new_object * into a swap object. */ - new_object = vm_object_allocate(OBJT_DEFAULT, size); + new_object = vm_object_allocate_anon(size); /* * At this point, the new object is still private, so the order in @@ -1443,8 +1459,7 @@ backing_object = object->backing_object; - if (backing_object->type != OBJT_DEFAULT && - backing_object->type != OBJT_SWAP) + if ((backing_object->flags & OBJ_ANON) == 0) return (false); pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset); @@ -1668,15 +1683,13 @@ * we check the backing object first, because it is most likely * not collapsable. */ + if ((backing_object->flags & OBJ_ANON) == 0) + break; VM_OBJECT_WLOCK(backing_object); if (backing_object->handle != NULL || - (backing_object->type != OBJT_DEFAULT && - backing_object->type != OBJT_SWAP) || - (backing_object->flags & (OBJ_DEAD | OBJ_NOSPLIT)) != 0 || + (backing_object->flags & OBJ_DEAD) != 0 || object->handle != NULL || - (object->type != OBJT_DEFAULT && - object->type != OBJT_SWAP) || - (object->flags & OBJ_DEAD)) { + (object->flags & OBJ_DEAD) != 0) { VM_OBJECT_WUNLOCK(backing_object); break; } @@ -2027,14 +2040,10 @@ if (prev_object == NULL) return (TRUE); - VM_OBJECT_WLOCK(prev_object); - if ((prev_object->type != OBJT_DEFAULT && - prev_object->type != OBJT_SWAP) || - (prev_object->flags & OBJ_NOSPLIT) != 0) { - VM_OBJECT_WUNLOCK(prev_object); + if ((prev_object->flags & OBJ_ANON) == 0) return (FALSE); - } + VM_OBJECT_WLOCK(prev_object); /* * Try to collapse the object first */ Index: sys/vm/vm_reserv.c =================================================================== --- sys/vm/vm_reserv.c +++ sys/vm/vm_reserv.c @@ -717,20 +717,15 @@ /* * Would the last new reservation extend past the end of the object? + * + * If the object is unlikely to grow don't allocate a reservation for + * the tail. */ - if (first + maxpages > object->size) { - /* - * Don't allocate the last new reservation if the object is a - * vnode or backed by another object that is a vnode. - */ - if (object->type == OBJT_VNODE || - (object->backing_object != NULL && - object->backing_object->type == OBJT_VNODE)) { - if (maxpages == VM_LEVEL_0_NPAGES) - return (NULL); - allocpages = minpages; - } - /* Speculate that the object may grow. */ + if ((object->flags & OBJ_ANON) == 0 && + first + maxpages > object->size) { + if (maxpages == VM_LEVEL_0_NPAGES) + return (NULL); + allocpages = minpages; } /* @@ -876,19 +871,14 @@ vm_reserv_object_unlock(object); /* - * Would a new reservation extend past the end of the object? + * Would the last new reservation extend past the end of the object? + * + * If the object is unlikely to grow don't allocate a reservation for + * the tail. */ - if (first + VM_LEVEL_0_NPAGES > object->size) { - /* - * Don't allocate a new reservation if the object is a vnode or - * backed by another object that is a vnode. - */ - if (object->type == OBJT_VNODE || - (object->backing_object != NULL && - object->backing_object->type == OBJT_VNODE)) - return (NULL); - /* Speculate that the object may grow. */ - } + if ((object->flags & OBJ_ANON) == 0 && + first + VM_LEVEL_0_NPAGES > object->size) + return (NULL); /* * Allocate and populate the new reservation.