Index: share/man/man9/casuword.9
===================================================================
--- share/man/man9/casuword.9
+++ share/man/man9/casuword.9
@@ -1,4 +1,4 @@
-.\" Copyright (c) 2014 The FreeBSD Foundation
+.\" Copyright (c) 2014, 2019 The FreeBSD Foundation
 .\" All rights reserved.
 .\"
 .\" Part of this documentation was written by
@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 21, 2014
+.Dd April 19, 2019
 .Dt CASU 9
 .Os
 .Sh NAME
@@ -41,13 +41,31 @@
 .In sys/types.h
 .In sys/systm.h
 .Ft int
-.Fn casueword "volatile u_long *base" "u_long oldval" "u_long *oldvalp" "u_long newval"
+.Fo casueword
+.Fa "volatile u_long *base"
+.Fa "u_long oldval"
+.Fa "u_long *oldvalp"
+.Fa "u_long newval"
+.Fc
 .Ft int
-.Fn casueword32 "volatile uint32_t *base" "uint32_t oldval" "uint32_t *oldvalp" "uint32_t newval"
+.Fo casueword32
+.Fa "volatile uint32_t *base"
+.Fa "uint32_t oldval"
+.Fa "uint32_t *oldvalp"
+.Fa "uint32_t newval"
+.Fc
 .Ft u_long
-.Fn casuword "volatile u_long *base" "u_long oldval" "u_long newval"
+.Fo casuword
+.Fa "volatile u_long *base"
+.Fa "u_long oldval"
+.Fa "u_long newval"
+.Fc
 .Ft uint32_t
-.Fn casuword32 "volatile uint32_t *base" "uint32_t oldval" "uint32_t newval"
+.Fo casuword32
+.Fa "volatile uint32_t *base"
+.Fa "uint32_t oldval"
+.Fa "uint32_t newval"
+.Fc
 .Sh DESCRIPTION
 The
 .Nm
@@ -88,7 +106,9 @@
 .Fn casueword
 and
 .Fn casueword32
-functions return 0 on success and -1 on failure.
+functions return 0 on success, -1 on failure to access memory,
+and 1 when comparison or store failed.
+The store can fail on load-linked/store-conditional architectures.
 .Sh SEE ALSO
 .Xr atomic 9 ,
 .Xr fetch 9 ,
Index: sys/amd64/amd64/support.S
===================================================================
--- sys/amd64/amd64/support.S
+++ sys/amd64/amd64/support.S
@@ -811,6 +811,7 @@
 	lock
 #endif
 	cmpxchgl %ecx,(%rdi)			/* new = %ecx */
+	setne	%cl
 
 	/*
 	 * The old value is in %eax.  If the store succeeded it will be the
@@ -828,6 +829,7 @@
 	 */
 	movl	%esi,(%rdx)			/* oldp = %rdx */
 	POP_FRAME_POINTER
+	movzbl	%cl, %eax
 	ret
 END(casueword32_nosmap)
 
@@ -847,6 +849,7 @@
 #endif
 	cmpxchgl %ecx,(%rdi)			/* new = %ecx */
 	clac
+	setne	%cl
 
 	/*
 	 * The old value is in %eax.  If the store succeeded it will be the
@@ -864,6 +867,7 @@
 	 */
 	movl	%esi,(%rdx)			/* oldp = %rdx */
 	POP_FRAME_POINTER
+	movzbl	%cl, %eax
 	ret
 END(casueword32_smap)
 
@@ -886,6 +890,7 @@
 	lock
 #endif
 	cmpxchgq %rcx,(%rdi)			/* new = %rcx */
+	setne	%cl
 
 	/*
 	 * The old value is in %rax.  If the store succeeded it will be the
@@ -897,6 +902,7 @@
 	movq	%rax,PCB_ONFAULT(%r8)
 	movq	%rsi,(%rdx)
 	POP_FRAME_POINTER
+	movzbl	%cl, %eax
 	ret
 END(casueword_nosmap)
 
@@ -916,6 +922,7 @@
 #endif
 	cmpxchgq %rcx,(%rdi)			/* new = %rcx */
 	clac
+	setne	%cl
 
 	/*
 	 * The old value is in %rax.  If the store succeeded it will be the
@@ -927,6 +934,7 @@
 	movq	%rax,PCB_ONFAULT(%r8)
 	movq	%rsi,(%rdx)
 	POP_FRAME_POINTER
+	movzbl	%cl, %eax
 	ret
 END(casueword_smap)
 
Index: sys/arm/arm/fusu.S
===================================================================
--- sys/arm/arm/fusu.S
+++ sys/arm/arm/fusu.S
@@ -63,7 +63,7 @@
 	ldr	r4, =(VM_MAXUSER_ADDRESS-3)
 	cmp	r0, r4
 	mvncs	r0, #0
-	bcs	2f
+	bcs	1f
 
 	GET_PCB(r6)
 	ldr	r6, [r6]
@@ -78,12 +78,10 @@
 	str	r4, [r6, #PCB_ONFAULT]
 
 #if __ARM_ARCH >= 6
-1:
+	mov	r5, #1
 	ldrex	r4, [r0]
 	cmp	r4, r1
 	strexeq	r5, r3, [r0]
-	cmpeq	r5, #1
-	beq	1b
 #else
 	ldrt	r4, [r0]
 	cmp	r4, r1
@@ -92,7 +90,10 @@
 	str	r4, [r2]
 	mov	r0, #0
 	str	r0, [r6, #PCB_ONFAULT]
-2:
+#if __ARM_ARCH >= 6
+	mov	r0, r5
+#endif
+1:
 	ldmfd	sp!, {r4, r5, r6}
 	RET
 EEND(casueword32)
Index: sys/arm64/arm64/support.S
===================================================================
--- sys/arm64/arm64/support.S
+++ sys/arm64/arm64/support.S
@@ -63,11 +63,10 @@
 	cmp	w4, w1			/* Compare */
 	b.ne	2f			/* Not equal, exit */
 	stxr	w5, w3, [x0]		/* Store the new data */
-	cbnz	w5, 1b			/* Retry on failure */
 2:	EXIT_USER_ACCESS(w6)
 	SET_FAULT_HANDLER(xzr, x5)	/* Reset the fault handler */
 	str	w4, [x2]		/* Store the read data */
-	mov	x0, #0			/* Success */
+	mov	w0, w5			/* Result same as store status */
 	ret				/* Return */
 END(casueword32)
 
@@ -85,11 +84,10 @@
 	cmp	x4, x1			/* Compare */
 	b.ne	2f			/* Not equal, exit */
 	stxr	w5, x3, [x0]		/* Store the new data */
-	cbnz	w5, 1b			/* Retry on failure */
 2:	EXIT_USER_ACCESS(w6)
 	SET_FAULT_HANDLER(xzr, x5)	/* Reset the fault handler */
 	str	x4, [x2]		/* Store the read data */
-	mov	x0, #0			/* Success */
+	mov	w0, w5			/* Result same as store status */
 	ret				/* Return */
 END(casueword)
 
Index: sys/i386/i386/copyout.c
===================================================================
--- sys/i386/i386/copyout.c
+++ sys/i386/i386/copyout.c
@@ -428,6 +428,7 @@
 struct casueword_arg0 {
 	uint32_t oldval;
 	uint32_t newval;
+	int res;
 };
 
 static void
@@ -436,7 +437,8 @@
 	struct casueword_arg0 *ca;
 
 	ca = arg;
-	atomic_fcmpset_int((u_int *)kva, &ca->oldval, ca->newval);
+	ca->res = 1 - atomic_fcmpset_int((u_int *)kva, &ca->oldval,
+	    ca->newval);
 }
 
 int
@@ -452,7 +454,7 @@
 	    casueword_slow0, &ca);
 	if (res == 0) {
 		*oldvalp = ca.oldval;
-		return (0);
+		return (ca.res);
 	}
 	return (-1);
 }
@@ -469,7 +471,7 @@
 	    casueword_slow0, &ca);
 	if (res == 0) {
 		*oldvalp = ca.oldval;
-		return (0);
+		return (ca.res);
 	}
 	return (-1);
 }
Index: sys/kern/kern_umtx.c
===================================================================
--- sys/kern/kern_umtx.c
+++ sys/kern/kern_umtx.c
@@ -691,7 +691,7 @@
 }
 
 static int
-umtxq_check_susp(struct thread *td)
+umtxq_check_susp(struct thread *td, bool sleep)
 {
 	struct proc *p;
 	int error;
@@ -710,7 +710,7 @@
 		if (p->p_flag & P_SINGLE_EXIT)
 			error = EINTR;
 		else
-			error = ERESTART;
+			error = sleep ? thread_suspend_check(0) : ERESTART;
 	}
 	PROC_UNLOCK(p);
 	return (error);
@@ -1049,9 +1049,12 @@
 				    id | UMUTEX_CONTESTED);
 				if (rv == -1)
 					return (EFAULT);
-				if (owner == UMUTEX_RB_OWNERDEAD)
+				if (rv == 0) {
+					MPASS(owner == UMUTEX_RB_OWNERDEAD);
 					return (EOWNERDEAD); /* success */
-				rv = umtxq_check_susp(td);
+				}
+				MPASS(rv == 1);
+				rv = umtxq_check_susp(td, false);
 				if (rv != 0)
 					return (rv);
 				continue;
@@ -1070,13 +1073,16 @@
 				return (EFAULT);
 
 			/* The acquire succeeded. */
-			if (owner == UMUTEX_UNOWNED)
+			if (rv == 0) {
+				MPASS(owner == UMUTEX_UNOWNED);
 				return (0);
+			}
 
 			/*
 			 * If no one owns it but it is contested try
 			 * to acquire it.
 			 */
+			MPASS(rv == 1);
 			if (owner == UMUTEX_CONTESTED) {
 				rv = casueword32(&m->m_owner,
 				    UMUTEX_CONTESTED, &owner,
@@ -1084,13 +1090,15 @@
 				/* The address was invalid. */
 				if (rv == -1)
 					return (EFAULT);
-
-				if (owner == UMUTEX_CONTESTED)
+				if (rv == 0) {
+					MPASS(owner == UMUTEX_CONTESTED);
 					return (0);
-
-				rv = umtxq_check_susp(td);
-				if (rv != 0)
-					return (rv);
+				}
+				if (rv == 1) {
+					rv = umtxq_check_susp(td, false);
+					if (rv != 0)
+						return (rv);
+				}
 
 				/*
 				 * If this failed the lock has
@@ -1098,6 +1106,11 @@
 				 */
 				continue;
 			}
+
+			/* rv == 1 but not contested, likely store failure */
+			rv = umtxq_check_susp(td, false);
+			if (rv != 0)
+				return (rv);
 		}
 
 		if (mode == _UMUTEX_TRY)
@@ -1128,14 +1141,21 @@
 		rv = casueword32(&m->m_owner, owner, &old,
 		    owner | UMUTEX_CONTESTED);
 
-		/* The address was invalid. */
-		if (rv == -1) {
+		/* The address was invalid or casueword failed to store. */
+		if (rv == -1 || rv == 1) {
 			umtxq_lock(&uq->uq_key);
 			umtxq_remove(uq);
 			umtxq_unbusy(&uq->uq_key);
 			umtxq_unlock(&uq->uq_key);
 			umtx_key_release(&uq->uq_key);
-			return (EFAULT);
+			if (rv == -1)
+				return (EFAULT);
+			if (rv == 1) {
+				rv = umtxq_check_susp(td, true);
+				if (rv != 0)
+					return (rv);
+			}
+			continue;
 		}
 
 		/*
@@ -1145,15 +1165,15 @@
 		 */
 		umtxq_lock(&uq->uq_key);
 		umtxq_unbusy(&uq->uq_key);
-		if (old == owner)
-			error = umtxq_sleep(uq, "umtxn", timeout == NULL ?
-			    NULL : &timo);
+		MPASS(old == owner);
+		error = umtxq_sleep(uq, "umtxn", timeout == NULL ?
+		    NULL : &timo);
 		umtxq_remove(uq);
 		umtxq_unlock(&uq->uq_key);
 		umtx_key_release(&uq->uq_key);
 
 		if (error == 0)
-			error = umtxq_check_susp(td);
+			error = umtxq_check_susp(td, false);
 	}
 
 	return (0);
@@ -1170,6 +1190,8 @@
 	int error, count;
 
 	id = td->td_tid;
+
+again:
 	/*
 	 * Make sure we own this mtx.
 	 */
@@ -1185,9 +1207,14 @@
 		error = casueword32(&m->m_owner, owner, &old, newlock);
 		if (error == -1)
 			return (EFAULT);
-		if (old == owner)
-			return (0);
-		owner = old;
+		if (error == 1) {
+			error = umtxq_check_susp(td, true);
+			if (error != 0)
+				return (error);
+			goto again;
+		}
+		MPASS(old == owner);
+		return (0);
 	}
 
 	/* We should only ever be in here for contested locks */
@@ -1215,8 +1242,14 @@
 	umtx_key_release(&key);
 	if (error == -1)
 		return (EFAULT);
-	if (old != owner)
-		return (EINVAL);
+	if (error == 1) {
+		if (old != owner)
+			return (EINVAL);
+		error = umtxq_check_susp(td, true);
+		if (error != 0)
+			return (error);
+		goto again;
+	}
 	return (0);
 }
 
@@ -1233,6 +1266,7 @@
 	int error;
 	int count;
 
+again:
 	error = fueword32(&m->m_owner, &owner);
 	if (error == -1)
 		return (EFAULT);
@@ -1259,14 +1293,26 @@
 	    owner != UMUTEX_RB_NOTRECOV) {
 		error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
 		    UMUTEX_UNOWNED);
-		if (error == -1)
+		if (error == -1) {
 			error = EFAULT;
+		} else if (error == 1) {
+			umtxq_unbusy(&key);
+			umtxq_unlock(&key);
+			umtx_key_release(&key);
+			error = umtxq_check_susp(td, true);
+			if (error != 0)
+				return (error);
+			goto again;
+		}
 	}
 
 	umtxq_lock(&key);
-	if (error == 0 && count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 ||
-	    owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV))
+	if (error == 0 && count != 0) {
+		MPASS((owner & ~UMUTEX_CONTESTED) == 0 ||
+		    owner == UMUTEX_RB_OWNERDEAD ||
+		    owner == UMUTEX_RB_NOTRECOV);
 		umtxq_signal(&key, 1);
+	}
 	umtxq_unbusy(&key);
 	umtxq_unlock(&key);
 	umtx_key_release(&key);
@@ -1330,12 +1376,12 @@
 				error = EFAULT;
 				break;
 			}
-			if (old == owner)
+			if (error == 0) {
+				MPASS(old == owner);
 				break;
+			}
 			owner = old;
-			error = umtxq_check_susp(td);
-			if (error != 0)
-				break;
+			error = umtxq_check_susp(td, true);
 		}
 	} else if (count == 1) {
 		error = fueword32(&m->m_owner, &owner);
@@ -1349,12 +1395,12 @@
 				error = EFAULT;
 				break;
 			}
-			if (old == owner)
+			if (error == 0) {
+				MPASS(old == owner);
 				break;
+			}
 			owner = old;
-			error = umtxq_check_susp(td);
-			if (error != 0)
-				break;
+			error = umtxq_check_susp(td, true);
 		}
 	}
 	umtxq_lock(&key);
@@ -1842,13 +1888,17 @@
 			error = EFAULT;
 			break;
 		}
-
 		/* The acquire succeeded. */
-		if (owner == UMUTEX_UNOWNED) {
+		if (rv == 0) {
+			MPASS(owner == UMUTEX_UNOWNED);
 			error = 0;
 			break;
 		}
 
+		error = umtxq_check_susp(td, true);
+		if (error != 0)
+			break;
+
 		if (owner == UMUTEX_RB_NOTRECOV) {
 			error = ENOTRECOVERABLE;
 			break;
@@ -1864,36 +1914,37 @@
 				error = EFAULT;
 				break;
 			}
-
-			if (owner == old_owner) {
-				umtxq_lock(&uq->uq_key);
-				umtxq_busy(&uq->uq_key);
-				error = umtx_pi_claim(pi, td);
-				umtxq_unbusy(&uq->uq_key);
-				umtxq_unlock(&uq->uq_key);
-				if (error != 0) {
-					/*
-					 * Since we're going to return an
-					 * error, restore the m_owner to its
-					 * previous, unowned state to avoid
-					 * compounding the problem.
-					 */
-					(void)casuword32(&m->m_owner,
-					    id | UMUTEX_CONTESTED,
-					    old_owner);
-				}
-				if (error == 0 &&
-				    old_owner == UMUTEX_RB_OWNERDEAD)
-					error = EOWNERDEAD;
-				break;
+			if (rv == 1) {
+				error = umtxq_check_susp(td, true);
+				if (error != 0)
+					return (error);
+				/*
+				 * If this failed the lock could
+				 * changed, restart.
+				 */
+				continue;
 			}
 
-			error = umtxq_check_susp(td);
-			if (error != 0)
-				break;
-
-			/* If this failed the lock has changed, restart. */
-			continue;
+			MPASS(rv == 0);
+			MPASS(owner == old_owner);
+			umtxq_lock(&uq->uq_key);
+			umtxq_busy(&uq->uq_key);
+			error = umtx_pi_claim(pi, td);
+			umtxq_unbusy(&uq->uq_key);
+			umtxq_unlock(&uq->uq_key);
+			if (error != 0) {
+				/*
+				 * Since we're going to return an
+				 * error, restore the m_owner to its
+				 * previous, unowned state to avoid
+				 * compounding the problem.
+				 */
+				(void)casuword32(&m->m_owner,
+				    id | UMUTEX_CONTESTED, old_owner);
+			}
+			if (error == 0 && old_owner == UMUTEX_RB_OWNERDEAD)
+				error = EOWNERDEAD;
+			break;
 		}
 
 		if ((owner & ~UMUTEX_CONTESTED) == id) {
@@ -1932,27 +1983,32 @@
 			error = EFAULT;
 			break;
 		}
-
-		umtxq_lock(&uq->uq_key);
-		/*
-		 * We set the contested bit, sleep. Otherwise the lock changed
-		 * and we need to retry or we lost a race to the thread
-		 * unlocking the umtx.  Note that the UMUTEX_RB_OWNERDEAD
-		 * value for owner is impossible there.
-		 */
-		if (old == owner) {
-			error = umtxq_sleep_pi(uq, pi,
-			    owner & ~UMUTEX_CONTESTED,
-			    "umtxpi", timeout == NULL ? NULL : &timo,
-			    (flags & USYNC_PROCESS_SHARED) != 0);
+		if (rv == 1) {
+			umtxq_unbusy_unlocked(&uq->uq_key);
+			error = umtxq_check_susp(td, true);
 			if (error != 0)
-				continue;
-		} else {
-			umtxq_unbusy(&uq->uq_key);
-			umtxq_unlock(&uq->uq_key);
+				break;
+
+			/*
+			 * The lock changed and we need to retry or we
+			 * lost a race to the thread unlocking the
+			 * umtx.  Note that the UMUTEX_RB_OWNERDEAD
+			 * value for owner is impossible there.
+			 */
+			continue;
 		}
 
-		error = umtxq_check_susp(td);
+		umtxq_lock(&uq->uq_key);
+
+		/* We set the contested bit, sleep. */
+		MPASS(old == owner);
+		error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
+		    "umtxpi", timeout == NULL ? NULL : &timo,
+		    (flags & USYNC_PROCESS_SHARED) != 0);
+		if (error != 0)
+			continue;
+
+		error = umtxq_check_susp(td, true);
 		if (error != 0)
 			break;
 	}
@@ -1978,6 +2034,8 @@
 	int count, error, pri;
 
 	id = td->td_tid;
+
+usrloop:
 	/*
 	 * Make sure we own this mtx.
 	 */
@@ -1995,6 +2053,12 @@
 		error = casueword32(&m->m_owner, owner, &old, new_owner);
 		if (error == -1)
 			return (EFAULT);
+		if (error == 1) {
+			error = umtxq_check_susp(td, true);
+			if (error != 0)
+				return (error);
+			goto usrloop;
+		}
 		if (old == owner)
 			return (0);
 		owner = old;
@@ -2074,15 +2138,20 @@
 
 	if (count > 1)
 		new_owner |= UMUTEX_CONTESTED;
+again:
 	error = casueword32(&m->m_owner, owner, &old, new_owner);
-
+	if (error == 1) {
+		error = umtxq_check_susp(td, true);
+		if (error == 0)
+			goto again;
+	}
 	umtxq_unbusy_unlocked(&key);
 	umtx_key_release(&key);
 	if (error == -1)
 		return (EFAULT);
-	if (old != owner)
+	if (error == 0 && old != owner)
 		return (EINVAL);
-	return (0);
+	return (error);
 }
 
 /*
@@ -2148,26 +2217,31 @@
 		if (rv == -1) {
 			error = EFAULT;
 			break;
-		}
-
-		if (owner == UMUTEX_CONTESTED) {
-			error = 0;
-			break;
-		} else if (owner == UMUTEX_RB_OWNERDEAD) {
-			rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD,
-			    &owner, id | UMUTEX_CONTESTED);
-			if (rv == -1) {
-				error = EFAULT;
+		} else 	if (rv == 0) {
+			if (owner == UMUTEX_CONTESTED) {
+				error = 0;
 				break;
-			}
-			if (owner == UMUTEX_RB_OWNERDEAD) {
-				error = EOWNERDEAD; /* success */
+			} else if (owner == UMUTEX_RB_OWNERDEAD) {
+				rv = casueword32(&m->m_owner,
+				    UMUTEX_RB_OWNERDEAD,
+				    &owner, id | UMUTEX_CONTESTED);
+				if (rv == -1) {
+					error = EFAULT;
+					break;
+				} else if (rv == 0) {
+					MPASS(owner == UMUTEX_RB_OWNERDEAD);
+					error = EOWNERDEAD; /* success */
+					break;
+				} else if (rv == 1) {
+					/* XXXKIB */
+				}
+				error = 0;
+			} else if (owner == UMUTEX_RB_NOTRECOV) {
+				error = ENOTRECOVERABLE;
 				break;
 			}
-			error = 0;
-		} else if (owner == UMUTEX_RB_NOTRECOV) {
-			error = ENOTRECOVERABLE;
-			break;
+		} else /* rv == 1 */ {
+			/* XXXKIB */
 		}
 
 		if (try != 0) {
@@ -2668,11 +2742,12 @@
 				umtx_key_release(&uq->uq_key);
 				return (EFAULT);
 			}
-			if (oldstate == state) {
+			if (rv == 0) {
+				MPASS(oldstate == state);
 				umtx_key_release(&uq->uq_key);
 				return (0);
 			}
-			error = umtxq_check_susp(td);
+			error = umtxq_check_susp(td, true);
 			if (error != 0)
 				break;
 			state = oldstate;
@@ -2703,10 +2778,12 @@
 				error = EFAULT;
 				break;
 			}
-			if (oldstate == state)
+			if (rv == 0) {
+				MPASS(oldstate == state);
 				goto sleep;
+			}
 			state = oldstate;
-			error = umtxq_check_susp(td);
+			error = umtxq_check_susp(td, true);
 			if (error != 0)
 				break;
 		}
@@ -2718,7 +2795,7 @@
 		/* state is changed while setting flags, restart */
 		if (!(state & wrflags)) {
 			umtxq_unbusy_unlocked(&uq->uq_key);
-			error = umtxq_check_susp(td);
+			error = umtxq_check_susp(td, true);
 			if (error != 0)
 				break;
 			continue;
@@ -2781,10 +2858,12 @@
 					error = EFAULT;
 					break;
 				}
-				if (oldstate == state)
+				if (rv == 0) {
+					MPASS(oldstate == state);
 					break;
+				}
 				state = oldstate;
-				error1 = umtxq_check_susp(td);
+				error1 = umtxq_check_susp(td, true);
 				if (error1 != 0) {
 					if (error == 0)
 						error = error1;
@@ -2840,22 +2919,25 @@
 				umtx_key_release(&uq->uq_key);
 				return (EFAULT);
 			}
-			if (oldstate == state) {
+			if (rv == 0) {
+				MPASS(oldstate == state);
 				umtx_key_release(&uq->uq_key);
 				return (0);
 			}
 			state = oldstate;
-			error = umtxq_check_susp(td);
+			error = umtxq_check_susp(td, true);
 			if (error != 0)
 				break;
 		}
 
 		if (error) {
-			if (!(state & (URWLOCK_WRITE_OWNER|URWLOCK_WRITE_WAITERS)) &&
+			if ((state & (URWLOCK_WRITE_OWNER |
+			    URWLOCK_WRITE_WAITERS)) == 0 &&
 			    blocked_readers != 0) {
 				umtxq_lock(&uq->uq_key);
 				umtxq_busy(&uq->uq_key);
-				umtxq_signal_queue(&uq->uq_key, INT_MAX, UMTX_SHARED_QUEUE);
+				umtxq_signal_queue(&uq->uq_key, INT_MAX,
+				    UMTX_SHARED_QUEUE);
 				umtxq_unbusy(&uq->uq_key);
 				umtxq_unlock(&uq->uq_key);
 			}
@@ -2885,10 +2967,12 @@
 				error = EFAULT;
 				break;
 			}
-			if (oldstate == state)
+			if (rv == 0) {
+				MPASS(oldstate == state);
 				goto sleep;
+			}
 			state = oldstate;
-			error = umtxq_check_susp(td);
+			error = umtxq_check_susp(td, true);
 			if (error != 0)
 				break;
 		}
@@ -2900,7 +2984,7 @@
 		if ((state & URWLOCK_WRITE_OWNER) == 0 &&
 		    URWLOCK_READER_COUNT(state) == 0) {
 			umtxq_unbusy_unlocked(&uq->uq_key);
-			error = umtxq_check_susp(td);
+			error = umtxq_check_susp(td, false);
 			if (error != 0)
 				break;
 			continue;
@@ -2958,10 +3042,12 @@
 					error = EFAULT;
 					break;
 				}
-				if (oldstate == state)
+				if (rv == 0) {
+					MPASS(oldstate == state);
 					break;
+				}
 				state = oldstate;
-				error1 = umtxq_check_susp(td);
+				error1 = umtxq_check_susp(td, true);
 				/*
 				 * We are leaving the URWLOCK_WRITE_WAITERS
 				 * behind, but this should not harm the
@@ -3021,13 +3107,13 @@
 				error = EFAULT;
 				goto out;
 			}
-			if (oldstate != state) {
+			if (rv == 1) {
 				state = oldstate;
 				if (!(oldstate & URWLOCK_WRITE_OWNER)) {
 					error = EPERM;
 					goto out;
 				}
-				error = umtxq_check_susp(td);
+				error = umtxq_check_susp(td, true);
 				if (error != 0)
 					goto out;
 			} else
@@ -3041,13 +3127,13 @@
 				error = EFAULT;
 				goto out;
 			}
-			if (oldstate != state) {
+			if (rv == 1) {
 				state = oldstate;
 				if (URWLOCK_READER_COUNT(oldstate) == 0) {
 					error = EPERM;
 					goto out;
 				}
-				error = umtxq_check_susp(td);
+				error = umtxq_check_susp(td, true);
 				if (error != 0)
 					goto out;
 			} else
@@ -3097,7 +3183,7 @@
 	struct abs_timeout timo;
 	struct umtx_q *uq;
 	uint32_t flags, count, count1;
-	int error, rv;
+	int error, rv, rv1;
 
 	uq = td->td_umtxq;
 	error = fueword32(&sem->_flags, &flags);
@@ -3114,15 +3200,24 @@
 	umtxq_busy(&uq->uq_key);
 	umtxq_insert(uq);
 	umtxq_unlock(&uq->uq_key);
+again:
 	rv = casueword32(&sem->_has_waiters, 0, &count1, 1);
 	if (rv == 0)
-		rv = fueword32(&sem->_count, &count);
-	if (rv == -1 || count != 0) {
+		rv1 = fueword32(&sem->_count, &count);
+	if (rv == -1 || (rv == 0 && (rv1 == -1 || count != 0)) || rv == 1) {
 		umtxq_lock(&uq->uq_key);
 		umtxq_unbusy(&uq->uq_key);
 		umtxq_remove(uq);
 		umtxq_unlock(&uq->uq_key);
 		umtx_key_release(&uq->uq_key);
+		if (rv == 1) {
+			rv = umtxq_check_susp(td, true);
+			if (rv == 0)
+				goto again;
+			return (rv);
+		}
+		if (rv == 0)
+			rv = rv1;
 		return (rv == -1 ? EFAULT : 0);
 	}
 	umtxq_lock(&uq->uq_key);
@@ -3201,6 +3296,7 @@
 	if (timeout != NULL)
 		abs_timeout_init2(&timo, timeout);
 
+again:
 	umtxq_lock(&uq->uq_key);
 	umtxq_busy(&uq->uq_key);
 	umtxq_insert(uq);
@@ -3226,16 +3322,19 @@
 		if (count == USEM_HAS_WAITERS)
 			break;
 		rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS);
-		if (rv == -1) {
-			umtxq_lock(&uq->uq_key);
-			umtxq_unbusy(&uq->uq_key);
-			umtxq_remove(uq);
-			umtxq_unlock(&uq->uq_key);
-			umtx_key_release(&uq->uq_key);
-			return (EFAULT);
-		}
-		if (count == 0)
+		if (rv == 0)
 			break;
+		umtxq_lock(&uq->uq_key);
+		umtxq_unbusy(&uq->uq_key);
+		umtxq_remove(uq);
+		umtxq_unlock(&uq->uq_key);
+		umtx_key_release(&uq->uq_key);
+		if (rv == -1)
+			return (EFAULT);
+		rv = umtxq_check_susp(td, true);
+		if (rv != 0)
+			return (rv);
+		goto again;
 	}
 	umtxq_lock(&uq->uq_key);
 	umtxq_unbusy(&uq->uq_key);
@@ -3288,11 +3387,20 @@
 		if (cnt == 1) {
 			umtxq_unlock(&key);
 			rv = fueword32(&sem->_count, &count);
-			while (rv != -1 && count & USEM_HAS_WAITERS)
+			while (rv != -1 && count & USEM_HAS_WAITERS) {
 				rv = casueword32(&sem->_count, count, &count,
 				    count & ~USEM_HAS_WAITERS);
+				if (rv == 1) {
+					rv = umtxq_check_susp(td, true);
+					if (rv != 0)
+						break;
+				}
+			}
 			if (rv == -1)
 				error = EFAULT;
+			else if (rv > 0) {
+				error = rv;
+			}
 			umtxq_lock(&key);
 		}
 
Index: sys/mips/mips/support.S
===================================================================
--- sys/mips/mips/support.S
+++ sys/mips/mips/support.S
@@ -336,19 +336,15 @@
 	GET_CPU_PCPU(v1)
 	PTR_L	v1, PC_CURPCB(v1)
 	PTR_S	v0, U_PCB_ONFAULT(v1)
-1:
+
+	li	v0, 1
 	move	t0, a3
 	ll	t1, 0(a0)
-	bne	a1, t1, 2f
+	bne	a1, t1, 1f
 	nop
 	sc	t0, 0(a0)		# store word
-	beqz	t0, 1b
-	nop
-	j	3f
-	li	v0, 0
-2:
-	li	v0, -1
-3:
+	xori	v0, t0, 1
+1:
 	PTR_S	zero, U_PCB_ONFAULT(v1)
 	jr	ra
 	sw	t1, 0(a2)		# unconditionally store old word
@@ -363,19 +359,15 @@
 	GET_CPU_PCPU(v1)
 	PTR_L	v1, PC_CURPCB(v1)
 	PTR_S	v0, U_PCB_ONFAULT(v1)
-1:
+
+	li	v0, 1
 	move	t0, a3
 	lld	t1, 0(a0)
-	bne	a1, t1, 2f
+	bne	a1, t1, 1f
 	nop
 	scd	t0, 0(a0)		# store double word
-	beqz	t0, 1b
-	nop
-	j	3f
-	li	v0, 0
-2:
-	li	v0, -1
-3:
+	xori	v0, t0, 1
+1:
 	PTR_S	zero, U_PCB_ONFAULT(v1)
 	jr	ra
 	sd	t1, 0(a2)		# unconditionally store old word
Index: sys/powerpc/powerpc/copyinout.c
===================================================================
--- sys/powerpc/powerpc/copyinout.c
+++ sys/powerpc/powerpc/copyinout.c
@@ -433,6 +433,7 @@
 	pmap_t pm;
 	jmp_buf		env;
 	uint32_t *p, val;
+	int res;
 
 	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
@@ -449,24 +450,26 @@
 		return (-1);
 	}
 
+	res = 0;
 	__asm __volatile (
-		"1:\tlwarx %0, 0, %2\n\t"	/* load old value */
-		"cmplw %3, %0\n\t"		/* compare */
-		"bne 2f\n\t"			/* exit if not equal */
-		"stwcx. %4, 0, %2\n\t"      	/* attempt to store */
-		"bne- 1b\n\t"			/* spin if failed */
-		"b 3f\n\t"			/* we've succeeded */
+		"lwarx %0, 0, %3\n\t"		/* load old value */
+		"cmplw %4, %0\n\t"		/* compare */
+		"bne 1f\n\t"			/* exit if not equal */
+		"stwcx. %5, 0, %3\n\t"      	/* attempt to store */
+		"bne- 1f\n\t"			/* if failed */
+		"b 2f\n\t"			/* we've succeeded */
+		"1:\n\t"
+		"stwcx. %0, 0, %4\n\t"       	/* clear reservation (74xx) */
+		"li %2, 1\n\t"
 		"2:\n\t"
-		"stwcx. %0, 0, %2\n\t"       	/* clear reservation (74xx) */
-		"3:\n\t"
-		: "=&r" (val), "=m" (*p)
+		: "=&r" (val), "=m" (*p), "=&r" (res)
 		: "r" (p), "r" (old), "r" (new), "m" (*p)
 		: "cr0", "memory");
 
 	td->td_pcb->pcb_onfault = NULL;
 
 	*oldvalp = val;
-	return (0);
+	return (res);
 }
 
 #ifndef __powerpc64__
@@ -485,6 +488,7 @@
 	pmap_t pm;
 	jmp_buf		env;
 	u_long *p, val;
+	int res;
 
 	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
@@ -501,23 +505,25 @@
 		return (-1);
 	}
 
+	res = 0;
 	__asm __volatile (
-		"1:\tldarx %0, 0, %2\n\t"	/* load old value */
-		"cmpld %3, %0\n\t"		/* compare */
-		"bne 2f\n\t"			/* exit if not equal */
-		"stdcx. %4, 0, %2\n\t"      	/* attempt to store */
-		"bne- 1b\n\t"			/* spin if failed */
-		"b 3f\n\t"			/* we've succeeded */
+		"ldarx %0, 0, %3\n\t"		/* load old value */
+		"cmpld %4, %0\n\t"		/* compare */
+		"bne 1f\n\t"			/* exit if not equal */
+		"stdcx. %5, 0, %3\n\t"      	/* attempt to store */
+		"bne- 1f\n\t"			/* if failed */
+		"b 2f\n\t"			/* we've succeeded */
+		"1:\n\t"
+		"stdcx. %0, 0, %3\n\t"       	/* clear reservation (74xx) */
+		"li %2, 1\n\t"
 		"2:\n\t"
-		"stdcx. %0, 0, %2\n\t"       	/* clear reservation (74xx) */
-		"3:\n\t"
-		: "=&r" (val), "=m" (*p)
+		: "=&r" (val), "=m" (*p), "=&r" (res)
 		: "r" (p), "r" (old), "r" (new), "m" (*p)
 		: "cr0", "memory");
 
 	td->td_pcb->pcb_onfault = NULL;
 
 	*oldvalp = val;
-	return (0);
+	return (res);
 }
 #endif
Index: sys/riscv/riscv/support.S
===================================================================
--- sys/riscv/riscv/support.S
+++ sys/riscv/riscv/support.S
@@ -60,14 +60,15 @@
 	la	a6, fsu_fault		/* Load the fault handler */
 	SET_FAULT_HANDLER(a6, a4)	/* And set it */
 	ENTER_USER_ACCESS(a4)
-1:	lr.w	a4, 0(a0)		/* Load-exclusive the data */
-	bne	a4, a1, 2f		/* If not equal then exit */
+	lr.w	a4, 0(a0)		/* Load-exclusive the data */
+	bne	a4, a1, 1f		/* If not equal then exit */
 	sc.w	a5, a3, 0(a0)		/* Store the new data */
-	bnez	a5, 1b			/* Retry on failure */
-2:	EXIT_USER_ACCESS(a5)
+	beq	x0, a5, 1f		/* Success */
+	li	a5, 1			/* Normalize failure result */
+1:	EXIT_USER_ACCESS(a5)
 	SET_FAULT_HANDLER(x0, a5)	/* Reset the fault handler */
 	sw	a4, 0(a2)		/* Store the read data */
-	li	a0, 0			/* Success */
+	mv	a0, a5			/* Success indicator */
 	ret				/* Return */
 END(casueword32)
 
@@ -80,14 +81,15 @@
 	la	a6, fsu_fault		/* Load the fault handler */
 	SET_FAULT_HANDLER(a6, a4)	/* And set it */
 	ENTER_USER_ACCESS(a4)
-1:	lr.d	a4, 0(a0)		/* Load-exclusive the data */
-	bne	a4, a1, 2f		/* If not equal then exit */
+	lr.d	a4, 0(a0)		/* Load-exclusive the data */
+	bne	a4, a1, 1f		/* If not equal then exit */
 	sc.d	a5, a3, 0(a0)		/* Store the new data */
-	bnez	a5, 1b			/* Retry on failure */
-2:	EXIT_USER_ACCESS(a5)
+	beq	x0, a5, 1f		/* Success */
+	li	a5, 1			/* Normalize failure result */
+1:	EXIT_USER_ACCESS(a5)
 	SET_FAULT_HANDLER(x0, a5)	/* Reset the fault handler */
 	sd	a4, 0(a2)		/* Store the read data */
-	li	a0, 0			/* Success */
+	mv	a0, a5			/* Success indicator */
 	ret				/* Return */
 END(casueword)
 
Index: sys/sparc64/sparc64/support.S
===================================================================
--- sys/sparc64/sparc64/support.S
+++ sys/sparc64/sparc64/support.S
@@ -403,28 +403,27 @@
 	.set	susword, suword16
 	.set	suword, suword64
 
-	.globl	casuword32, casuword, fuptr, suptr
-	.set	casuword, casuword64
+	.globl	casuword32_int, casuword64_int, fuptr, suptr
 	.set	fuptr, fuword64
 	.set	suptr, suword64
 
 /*
  * int32_t casuword32(volatile int32_t *p, int32_t e, int32_t s)
  */
-ENTRY(casuword32)
+ENTRY(casuword32_int)
 	casa	[%o0] ASI_AIUP, %o1, %o2
 	retl
 	 mov	%o2, %o0
-END(casuword32)
+END(casuword32_int)
 
 /*
  * int64_t casuword64(volatile int64_t *p, int64_t e, int64_t s)
  */
-ENTRY(casuword64)
+ENTRY(casuword64_int)
 	casxa	[%o0] ASI_AIUP, %o1, %o2
 	retl
 	 mov	%o2, %o0
-END(casuword64)
+END(casuword64_int)
 
 /*
  * int fuword8(const void *base)
Index: sys/sparc64/sparc64/vm_machdep.c
===================================================================
--- sys/sparc64/sparc64/vm_machdep.c
+++ sys/sparc64/sparc64/vm_machdep.c
@@ -462,3 +462,28 @@
 	pmap_qremove(sf->kva, 1);
 	return (1);
 }
+
+uint32_t casuword32_int(volatile uint32_t *base, uint32_t oldval,
+    uint32_t newval);
+uint32_t
+casuword32(volatile uint32_t *base, uint32_t oldval, uint32_t newval)
+{
+	uint32_t ret;
+
+	ret = casuword32_int(base, oldval, newval);
+	if (ret != -1)
+		ret = ret != oldval;
+	return (ret);
+}
+
+u_long casuword64_int(volatile u_long *p, u_long oldval, u_long newval);
+u_long
+casuword(volatile u_long *p, u_long oldval, u_long newval)
+{
+	u_long ret;
+
+	ret = casuword64_int(p, oldval, newval);
+	if (ret != -1L)
+		ret = ret != oldval;
+	return (ret);
+}