Index: sys/vm/vm_map.c
===================================================================
--- sys/vm/vm_map.c
+++ sys/vm/vm_map.c
@@ -2361,9 +2361,7 @@
 
 	if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL)
 		return;
-	VM_OBJECT_RLOCK(object);
 	if (object->type == OBJT_DEVICE || object->type == OBJT_SG) {
-		VM_OBJECT_RUNLOCK(object);
 		VM_OBJECT_WLOCK(object);
 		if (object->type == OBJT_DEVICE || object->type == OBJT_SG) {
 			pmap_object_init_pt(map->pmap, addr, object, pindex,
@@ -2372,7 +2370,8 @@
 			return;
 		}
 		VM_OBJECT_LOCK_DOWNGRADE(object);
-	}
+	} else
+		VM_OBJECT_RLOCK(object);
 
 	psize = atop(size);
 	if (psize + pindex > object->size) {
@@ -2539,6 +2538,8 @@
 			continue;
 		}
 
+		if ((obj->flags & OBJ_ANON) == 0)
+			continue;
 		VM_OBJECT_WLOCK(obj);
 		if (obj->type != OBJT_DEFAULT && obj->type != OBJT_SWAP) {
 			VM_OBJECT_WUNLOCK(obj);
Index: sys/vm/vm_mmap.c
===================================================================
--- sys/vm/vm_mmap.c
+++ sys/vm/vm_mmap.c
@@ -1325,12 +1325,14 @@
 	} else {
 		KASSERT(obj->type == OBJT_DEFAULT || obj->type == OBJT_SWAP,
 		    ("wrong object type"));
-		VM_OBJECT_WLOCK(obj);
-		vm_object_reference_locked(obj);
+		vm_object_reference(obj);
 #if VM_NRESERVLEVEL > 0
-		vm_object_color(obj, 0);
+		if ((obj->flags & OBJ_COLORED) == 0) {
+			VM_OBJECT_WLOCK(obj);
+			vm_object_color(obj, 0);
+			VM_OBJECT_WUNLOCK(obj);
+		}
 #endif
-		VM_OBJECT_WUNLOCK(obj);
 	}
 	*objp = obj;
 	*flagsp = flags;
Index: sys/vm/vm_object.c
===================================================================
--- sys/vm/vm_object.c
+++ sys/vm/vm_object.c
@@ -528,7 +528,7 @@
 	if (!umtx_shm_vnobj_persistent)
 		umtx_shm_object_terminated(object);
 
-	VM_OBJECT_WUNLOCK(object);
+	VM_OBJECT_RUNLOCK(object);
 	/* vrele may need the vnode lock. */
 	vrele(vp);
 }
@@ -548,7 +548,7 @@
 vm_object_deallocate(vm_object_t object)
 {
 	vm_object_t robject, temp;
-	bool last, released;
+	bool released;
 
 	while (object != NULL) {
 		/*
@@ -565,18 +565,23 @@
 		if (released)
 			return;
 
+		if (object->type == OBJT_VNODE) {
+			VM_OBJECT_RLOCK(object);
+			if (object->type == OBJT_VNODE) {
+				if (refcount_release(&object->ref_count))
+					vm_object_vndeallocate(object);
+				else
+					VM_OBJECT_RUNLOCK(object);
+				return;
+			}
+			VM_OBJECT_RUNLOCK(object);
+		}
+
 		VM_OBJECT_WLOCK(object);
 		KASSERT(object->ref_count != 0,
 			("vm_object_deallocate: object deallocated too many times: %d", object->type));
 
-		last = refcount_release(&object->ref_count);
-		if (object->type == OBJT_VNODE) {
-			if (last)
-				vm_object_vndeallocate(object);
-			else
-				VM_OBJECT_WUNLOCK(object);
-			return;
-		}
+		refcount_release(&object->ref_count);
 		if (object->ref_count > 1) {
 			VM_OBJECT_WUNLOCK(object);
 			return;