diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -4485,6 +4485,7 @@
 umtx_shm_create_reg(struct thread *td, const struct umtx_key *key,
     struct umtx_shm_reg **res)
 {
+	struct shmfd *shm;
 	struct umtx_shm_reg *reg, *reg1;
 	struct ucred *cred;
 	int error;
@@ -4504,9 +4505,14 @@
 	cred = td->td_ucred;
 	if (!chgumtxcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_UMTXP)))
 		return (ENOMEM);
+	shm = shm_alloc(td->td_ucred, O_RDWR, false);
+	if (shm == NULL) {
+		chgumtxcnt(cred->cr_ruidinfo, -1, 0);
+		return (ENOMEM);
+	}
 	reg = uma_zalloc(umtx_shm_reg_zone, M_WAITOK | M_ZERO);
 	bcopy(key, &reg->ushm_key, sizeof(*key));
-	reg->ushm_obj = shm_alloc(td->td_ucred, O_RDWR, false);
+	reg->ushm_obj = shm;
 	reg->ushm_cred = crhold(cred);
 	error = shm_dotruncate(reg->ushm_obj, PAGE_SIZE);
 	if (error != 0) {
diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c
--- a/sys/kern/uipc_shm.c
+++ b/sys/kern/uipc_shm.c
@@ -952,6 +952,13 @@
 	} else {
 		obj = vm_pager_allocate(shmfd_pager_type, NULL,
 		    shmfd->shm_size, VM_PROT_DEFAULT, 0, ucred);
+		if (obj != NULL) {
+			/*
+			 * swap reservation limits can cause object allocation
+			 * to fail.
+			 */
+			return (NULL);
+		}
 		obj->un_pager.swp.swp_priv = shmfd;
 	}
 	KASSERT(obj != NULL, ("shm_create: vm_pager_allocate"));
@@ -1246,6 +1253,11 @@
 			return (EINVAL);
 		}
 		shmfd = shm_alloc(td->td_ucred, cmode, largepage);
+		if (shmfd == NULL) {
+			fdclose(td, fp, fd);
+			fdrop(fp, td);
+			return (ENOMEM);
+		}
 		shmfd->shm_seals = initial_seals;
 		shmfd->shm_flags = shmflags;
 	} else {
@@ -1262,14 +1274,18 @@
 #endif
 					shmfd = shm_alloc(td->td_ucred, cmode,
 					    largepage);
-					shmfd->shm_seals = initial_seals;
-					shmfd->shm_flags = shmflags;
-					shm_insert(path, fnv, shmfd);
+					if (shmfd == NULL) {
+						error = ENOMEM;
+					} else {
+						shmfd->shm_seals =
+						    initial_seals;
+						shmfd->shm_flags = shmflags;
+						shm_insert(path, fnv, shmfd);
+					}
 #ifdef MAC
 				}
 #endif
 			} else {
-				free(path, M_SHMFD);
 				error = ENOENT;
 			}
 		} else {
@@ -1285,12 +1301,6 @@
 			 */
 			initial_seals &= ~shmfd->shm_seals;
 
-			/*
-			 * Object already exists, obtain a new
-			 * reference if requested and permitted.
-			 */
-			free(path, M_SHMFD);
-
 			/*
 			 * initial_seals can't set additional seals if we've
 			 * already been set F_SEAL_SEAL.  If F_SEAL_SEAL is set,
@@ -1349,7 +1359,8 @@
 		}
 		sx_xunlock(&shm_dict_lock);
 
-		if (error) {
+		free(path, M_SHMFD);
+		if (error != 0) {
 			fdclose(td, fp, fd);
 			fdrop(fp, td);
 			return (error);