diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -731,9 +731,9 @@
 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,
 	    ("vm_object_backing_remove: Removing collapsing object."));
 
+	refcount_release(&backing_object->shadow_count);
 	if ((object->flags & OBJ_SHADOWLIST) != 0) {
 		LIST_REMOVE(object, shadow_list);
-		backing_object->shadow_count--;
 		object->flags &= ~OBJ_SHADOWLIST;
 	}
 	object->backing_object = NULL;
@@ -746,13 +746,15 @@
 
 	VM_OBJECT_ASSERT_WLOCKED(object);
 
+	backing_object = object->backing_object;
 	if ((object->flags & OBJ_SHADOWLIST) != 0) {
-		backing_object = object->backing_object;
 		VM_OBJECT_WLOCK(backing_object);
 		vm_object_backing_remove_locked(object);
 		VM_OBJECT_WUNLOCK(backing_object);
-	} else
+	} else {
 		object->backing_object = NULL;
+		refcount_release(&backing_object->shadow_count);
+	}
 }
 
 static void
@@ -761,11 +763,11 @@
 
 	VM_OBJECT_ASSERT_WLOCKED(object);
 
+	refcount_acquire(&backing_object->shadow_count);
 	if ((backing_object->flags & OBJ_ANON) != 0) {
 		VM_OBJECT_ASSERT_WLOCKED(backing_object);
 		LIST_INSERT_HEAD(&backing_object->shadow_head, object,
 		    shadow_list);
-		backing_object->shadow_count++;
 		object->flags |= OBJ_SHADOWLIST;
 	}
 	object->backing_object = backing_object;
@@ -781,8 +783,10 @@
 		VM_OBJECT_WLOCK(backing_object);
 		vm_object_backing_insert_locked(object, backing_object);
 		VM_OBJECT_WUNLOCK(backing_object);
-	} else
+	} else {
 		object->backing_object = backing_object;
+		refcount_acquire(&backing_object->shadow_count);
+	}
 }
 
 /*