D20772.diff
No OneTemporary
Actions

Size

31 KB

Referenced Files

None

Subscribers

None

D20772.diff
View Options

	Index: head/share/man/man9/casuword.9
	===================================================================
	--- head/share/man/man9/casuword.9
	+++ head/share/man/man9/casuword.9
	@@ -1,4 +1,4 @@
	-.\" Copyright (c) 2014 The FreeBSD Foundation
	+.\" Copyright (c) 2014, 2019 The FreeBSD Foundation
	.\" All rights reserved.
	.\"
	.\" Part of this documentation was written by
	@@ -28,7 +28,7 @@
	.\"
	.\" $FreeBSD$
	.\"
	-.Dd October 21, 2014
	+.Dd April 19, 2019
	.Dt CASU 9
	.Os
	.Sh NAME
	@@ -106,7 +106,9 @@
	.Fn casueword
	and
	.Fn casueword32
	-functions return 0 on success and -1 on failure.
	+functions return 0 on success, -1 on failure to access memory,
	+and 1 when comparison or store failed.
	+The store can fail on load-linked/store-conditional architectures.
	.Sh SEE ALSO
	.Xr atomic 9 ,
	.Xr fetch 9 ,
	Index: head/sys/amd64/amd64/support.S
	===================================================================
	--- head/sys/amd64/amd64/support.S
	+++ head/sys/amd64/amd64/support.S
	@@ -811,6 +811,7 @@
	lock
	#endif
	cmpxchgl %ecx,(%rdi) /* new = %ecx */
	+ setne %cl

	/*
	* The old value is in %eax. If the store succeeded it will be the
	@@ -828,6 +829,7 @@
	*/
	movl %esi,(%rdx) /* oldp = %rdx */
	POP_FRAME_POINTER
	+ movzbl %cl, %eax
	ret
	END(casueword32_nosmap)

	@@ -847,6 +849,7 @@
	#endif
	cmpxchgl %ecx,(%rdi) /* new = %ecx */
	clac
	+ setne %cl

	/*
	* The old value is in %eax. If the store succeeded it will be the
	@@ -864,6 +867,7 @@
	*/
	movl %esi,(%rdx) /* oldp = %rdx */
	POP_FRAME_POINTER
	+ movzbl %cl, %eax
	ret
	END(casueword32_smap)

	@@ -886,6 +890,7 @@
	lock
	#endif
	cmpxchgq %rcx,(%rdi) /* new = %rcx */
	+ setne %cl

	/*
	* The old value is in %rax. If the store succeeded it will be the
	@@ -897,6 +902,7 @@
	movq %rax,PCB_ONFAULT(%r8)
	movq %rsi,(%rdx)
	POP_FRAME_POINTER
	+ movzbl %cl, %eax
	ret
	END(casueword_nosmap)

	@@ -916,6 +922,7 @@
	#endif
	cmpxchgq %rcx,(%rdi) /* new = %rcx */
	clac
	+ setne %cl

	/*
	* The old value is in %rax. If the store succeeded it will be the
	@@ -927,6 +934,7 @@
	movq %rax,PCB_ONFAULT(%r8)
	movq %rsi,(%rdx)
	POP_FRAME_POINTER
	+ movzbl %cl, %eax
	ret
	END(casueword_smap)

	Index: head/sys/arm/arm/fusu.S
	===================================================================
	--- head/sys/arm/arm/fusu.S
	+++ head/sys/arm/arm/fusu.S
	@@ -63,7 +63,7 @@
	ldr r4, =(VM_MAXUSER_ADDRESS-3)
	cmp r0, r4
	mvncs r0, #0
	- bcs 2f
	+ bcs 1f

	GET_PCB(r6)
	ldr r6, [r6]
	@@ -78,12 +78,10 @@
	str r4, [r6, #PCB_ONFAULT]

	#if __ARM_ARCH >= 6
	-1:
	+ mov r5, #1
	ldrex r4, [r0]
	cmp r4, r1
	strexeq r5, r3, [r0]
	- cmpeq r5, #1
	- beq 1b
	#else
	ldrt r4, [r0]
	cmp r4, r1
	@@ -92,7 +90,10 @@
	str r4, [r2]
	mov r0, #0
	str r0, [r6, #PCB_ONFAULT]
	-2:
	+#if __ARM_ARCH >= 6
	+ mov r0, r5
	+#endif
	+1:
	ldmfd sp!, {r4, r5, r6}
	RET
	EEND(casueword32)
	Index: head/sys/arm64/arm64/support.S
	===================================================================
	--- head/sys/arm64/arm64/support.S
	+++ head/sys/arm64/arm64/support.S
	@@ -57,17 +57,17 @@
	cmp x0, x4
	b.cs fsu_fault_nopcb
	adr x6, fsu_fault /* Load the fault handler */
	+ mov w5, #1
	SET_FAULT_HANDLER(x6, x4) /* And set it */
	ENTER_USER_ACCESS(w6, x4)
	1: ldxr w4, [x0] /* Load-exclusive the data */
	cmp w4, w1 /* Compare */
	b.ne 2f /* Not equal, exit */
	stxr w5, w3, [x0] /* Store the new data */
	- cbnz w5, 1b /* Retry on failure */
	2: EXIT_USER_ACCESS(w6)
	- SET_FAULT_HANDLER(xzr, x5) /* Reset the fault handler */
	+ SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */
	str w4, [x2] /* Store the read data */
	- mov x0, #0 /* Success */
	+ mov w0, w5 /* Result same as store status */
	ret /* Return */
	END(casueword32)

	@@ -79,17 +79,17 @@
	cmp x0, x4
	b.cs fsu_fault_nopcb
	adr x6, fsu_fault /* Load the fault handler */
	+ mov w5, #1
	SET_FAULT_HANDLER(x6, x4) /* And set it */
	ENTER_USER_ACCESS(w6, x4)
	1: ldxr x4, [x0] /* Load-exclusive the data */
	cmp x4, x1 /* Compare */
	b.ne 2f /* Not equal, exit */
	stxr w5, x3, [x0] /* Store the new data */
	- cbnz w5, 1b /* Retry on failure */
	2: EXIT_USER_ACCESS(w6)
	- SET_FAULT_HANDLER(xzr, x5) /* Reset the fault handler */
	+ SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */
	str x4, [x2] /* Store the read data */
	- mov x0, #0 /* Success */
	+ mov w0, w5 /* Result same as store status */
	ret /* Return */
	END(casueword)

	Index: head/sys/i386/i386/copyout.c
	===================================================================
	--- head/sys/i386/i386/copyout.c
	+++ head/sys/i386/i386/copyout.c
	@@ -428,6 +428,7 @@
	struct casueword_arg0 {
	uint32_t oldval;
	uint32_t newval;
	+ int res;
	};

	static void
	@@ -436,7 +437,8 @@
	struct casueword_arg0 *ca;

	ca = arg;
	- atomic_fcmpset_int((u_int *)kva, &ca->oldval, ca->newval);
	+ ca->res = 1 - atomic_fcmpset_int((u_int *)kva, &ca->oldval,
	+ ca->newval);
	}

	int
	@@ -452,7 +454,7 @@
	casueword_slow0, &ca);
	if (res == 0) {
	*oldvalp = ca.oldval;
	- return (0);
	+ return (ca.res);
	}
	return (-1);
	}
	@@ -469,7 +471,7 @@
	casueword_slow0, &ca);
	if (res == 0) {
	*oldvalp = ca.oldval;
	- return (0);
	+ return (ca.res);
	}
	return (-1);
	}
	Index: head/sys/kern/kern_umtx.c
	===================================================================
	--- head/sys/kern/kern_umtx.c
	+++ head/sys/kern/kern_umtx.c
	@@ -690,8 +690,26 @@
	return (0);
	}

	+/*
	+ * Check for possible stops and suspensions while executing a umtx
	+ * locking operation.
	+ *
	+ * The sleep argument controls whether the function can handle a stop
	+ * request itself or it should return ERESTART and the request is
	+ * proceed at the kernel/user boundary in ast.
	+ *
	+ * Typically, when retrying due to casueword(9) failure (rv == 1), we
	+ * should handle the stop requests there, with exception of cases when
	+ * the thread busied the umtx key, or when functions return
	+ * immediately if umtxq_check_susp() returned non-zero. On the other
	+ * hand, retrying the whole lock operation, we better not stop there
	+ * but delegate the handling to ast.
	+ *
	+ * If the request is for thread termination P_SINGLE_EXIT, we cannot
	+ * handle it at all, and simply return EINTR.
	+ */
	static int
	-umtxq_check_susp(struct thread *td)
	+umtxq_check_susp(struct thread *td, bool sleep)
	{
	struct proc *p;
	int error;
	@@ -710,7 +728,7 @@
	if (p->p_flag & P_SINGLE_EXIT)
	error = EINTR;
	else
	- error = ERESTART;
	+ error = sleep ? thread_suspend_check(0) : ERESTART;
	}
	PROC_UNLOCK(p);
	return (error);
	@@ -1049,9 +1067,12 @@
	id \| UMUTEX_CONTESTED);
	if (rv == -1)
	return (EFAULT);
	- if (owner == UMUTEX_RB_OWNERDEAD)
	+ if (rv == 0) {
	+ MPASS(owner == UMUTEX_RB_OWNERDEAD);
	return (EOWNERDEAD); /* success */
	- rv = umtxq_check_susp(td);
	+ }
	+ MPASS(rv == 1);
	+ rv = umtxq_check_susp(td, false);
	if (rv != 0)
	return (rv);
	continue;
	@@ -1070,13 +1091,16 @@
	return (EFAULT);

	/* The acquire succeeded. */
	- if (owner == UMUTEX_UNOWNED)
	+ if (rv == 0) {
	+ MPASS(owner == UMUTEX_UNOWNED);
	return (0);
	+ }

	/*
	* If no one owns it but it is contested try
	* to acquire it.
	*/
	+ MPASS(rv == 1);
	if (owner == UMUTEX_CONTESTED) {
	rv = casueword32(&m->m_owner,
	UMUTEX_CONTESTED, &owner,
	@@ -1084,20 +1108,27 @@
	/* The address was invalid. */
	if (rv == -1)
	return (EFAULT);
	-
	- if (owner == UMUTEX_CONTESTED)
	+ if (rv == 0) {
	+ MPASS(owner == UMUTEX_CONTESTED);
	return (0);
	+ }
	+ if (rv == 1) {
	+ rv = umtxq_check_susp(td, false);
	+ if (rv != 0)
	+ return (rv);
	+ }

	- rv = umtxq_check_susp(td);
	- if (rv != 0)
	- return (rv);
	-
	/*
	* If this failed the lock has
	* changed, restart.
	*/
	continue;
	}
	+
	+ /* rv == 1 but not contested, likely store failure */
	+ rv = umtxq_check_susp(td, false);
	+ if (rv != 0)
	+ return (rv);
	}

	if (mode == _UMUTEX_TRY)
	@@ -1128,14 +1159,21 @@
	rv = casueword32(&m->m_owner, owner, &old,
	owner \| UMUTEX_CONTESTED);

	- /* The address was invalid. */
	- if (rv == -1) {
	+ /* The address was invalid or casueword failed to store. */
	+ if (rv == -1 \|\| rv == 1) {
	umtxq_lock(&uq->uq_key);
	umtxq_remove(uq);
	umtxq_unbusy(&uq->uq_key);
	umtxq_unlock(&uq->uq_key);
	umtx_key_release(&uq->uq_key);
	- return (EFAULT);
	+ if (rv == -1)
	+ return (EFAULT);
	+ if (rv == 1) {
	+ rv = umtxq_check_susp(td, false);
	+ if (rv != 0)
	+ return (rv);
	+ }
	+ continue;
	}

	/*
	@@ -1145,15 +1183,15 @@
	*/
	umtxq_lock(&uq->uq_key);
	umtxq_unbusy(&uq->uq_key);
	- if (old == owner)
	- error = umtxq_sleep(uq, "umtxn", timeout == NULL ?
	- NULL : &timo);
	+ MPASS(old == owner);
	+ error = umtxq_sleep(uq, "umtxn", timeout == NULL ?
	+ NULL : &timo);
	umtxq_remove(uq);
	umtxq_unlock(&uq->uq_key);
	umtx_key_release(&uq->uq_key);

	if (error == 0)
	- error = umtxq_check_susp(td);
	+ error = umtxq_check_susp(td, false);
	}

	return (0);
	@@ -1170,6 +1208,8 @@
	int error, count;

	id = td->td_tid;
	+
	+again:
	/*
	* Make sure we own this mtx.
	*/
	@@ -1185,9 +1225,14 @@
	error = casueword32(&m->m_owner, owner, &old, newlock);
	if (error == -1)
	return (EFAULT);
	- if (old == owner)
	- return (0);
	- owner = old;
	+ if (error == 1) {
	+ error = umtxq_check_susp(td, false);
	+ if (error != 0)
	+ return (error);
	+ goto again;
	+ }
	+ MPASS(old == owner);
	+ return (0);
	}

	/* We should only ever be in here for contested locks */
	@@ -1215,8 +1260,14 @@
	umtx_key_release(&key);
	if (error == -1)
	return (EFAULT);
	- if (old != owner)
	- return (EINVAL);
	+ if (error == 1) {
	+ if (old != owner)
	+ return (EINVAL);
	+ error = umtxq_check_susp(td, false);
	+ if (error != 0)
	+ return (error);
	+ goto again;
	+ }
	return (0);
	}

	@@ -1233,6 +1284,7 @@
	int error;
	int count;

	+again:
	error = fueword32(&m->m_owner, &owner);
	if (error == -1)
	return (EFAULT);
	@@ -1259,14 +1311,27 @@
	owner != UMUTEX_RB_NOTRECOV) {
	error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
	UMUTEX_UNOWNED);
	- if (error == -1)
	+ if (error == -1) {
	error = EFAULT;
	+ } else if (error == 1) {
	+ umtxq_lock(&key);
	+ umtxq_unbusy(&key);
	+ umtxq_unlock(&key);
	+ umtx_key_release(&key);
	+ error = umtxq_check_susp(td, false);
	+ if (error != 0)
	+ return (error);
	+ goto again;
	+ }
	}

	umtxq_lock(&key);
	- if (error == 0 && count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 \|\|
	- owner == UMUTEX_RB_OWNERDEAD \|\| owner == UMUTEX_RB_NOTRECOV))
	+ if (error == 0 && count != 0) {
	+ MPASS((owner & ~UMUTEX_CONTESTED) == 0 \|\|
	+ owner == UMUTEX_RB_OWNERDEAD \|\|
	+ owner == UMUTEX_RB_NOTRECOV);
	umtxq_signal(&key, 1);
	+ }
	umtxq_unbusy(&key);
	umtxq_unlock(&key);
	umtx_key_release(&key);
	@@ -1314,49 +1379,32 @@
	umtxq_busy(&key);
	count = umtxq_count(&key);
	umtxq_unlock(&key);
	+
	+ error = fueword32(&m->m_owner, &owner);
	+ if (error == -1)
	+ error = EFAULT;
	+
	/*
	- * Only repair contention bit if there is a waiter, this means the mutex
	- * is still being referenced by userland code, otherwise don't update
	- * any memory.
	+ * Only repair contention bit if there is a waiter, this means
	+ * the mutex is still being referenced by userland code,
	+ * otherwise don't update any memory.
	*/
	- if (count > 1) {
	- error = fueword32(&m->m_owner, &owner);
	- if (error == -1)
	+ while (error == 0 && (owner & UMUTEX_CONTESTED) == 0 &&
	+ (count > 1 \|\| (count == 1 && (owner & ~UMUTEX_CONTESTED) != 0))) {
	+ error = casueword32(&m->m_owner, owner, &old,
	+ owner \| UMUTEX_CONTESTED);
	+ if (error == -1) {
	error = EFAULT;
	- while (error == 0 && (owner & UMUTEX_CONTESTED) == 0) {
	- error = casueword32(&m->m_owner, owner, &old,
	- owner \| UMUTEX_CONTESTED);
	- if (error == -1) {
	- error = EFAULT;
	- break;
	- }
	- if (old == owner)
	- break;
	- owner = old;
	- error = umtxq_check_susp(td);
	- if (error != 0)
	- break;
	+ break;
	}
	- } else if (count == 1) {
	- error = fueword32(&m->m_owner, &owner);
	- if (error == -1)
	- error = EFAULT;
	- while (error == 0 && (owner & ~UMUTEX_CONTESTED) != 0 &&
	- (owner & UMUTEX_CONTESTED) == 0) {
	- error = casueword32(&m->m_owner, owner, &old,
	- owner \| UMUTEX_CONTESTED);
	- if (error == -1) {
	- error = EFAULT;
	- break;
	- }
	- if (old == owner)
	- break;
	- owner = old;
	- error = umtxq_check_susp(td);
	- if (error != 0)
	- break;
	+ if (error == 0) {
	+ MPASS(old == owner);
	+ break;
	}
	+ owner = old;
	+ error = umtxq_check_susp(td, false);
	}
	+
	umtxq_lock(&key);
	if (error == EFAULT) {
	umtxq_signal(&key, INT_MAX);
	@@ -1842,9 +1890,9 @@
	error = EFAULT;
	break;
	}
	-
	/* The acquire succeeded. */
	- if (owner == UMUTEX_UNOWNED) {
	+ if (rv == 0) {
	+ MPASS(owner == UMUTEX_UNOWNED);
	error = 0;
	break;
	}
	@@ -1854,6 +1902,16 @@
	break;
	}

	+ /*
	+ * Avoid overwriting a possible error from sleep due
	+ * to the pending signal with suspension check result.
	+ */
	+ if (error == 0) {
	+ error = umtxq_check_susp(td, true);
	+ if (error != 0)
	+ break;
	+ }
	+
	/* If no one owns it but it is contested try to acquire it. */
	if (owner == UMUTEX_CONTESTED \|\| owner == UMUTEX_RB_OWNERDEAD) {
	old_owner = owner;
	@@ -1864,36 +1922,40 @@
	error = EFAULT;
	break;
	}
	-
	- if (owner == old_owner) {
	- umtxq_lock(&uq->uq_key);
	- umtxq_busy(&uq->uq_key);
	- error = umtx_pi_claim(pi, td);
	- umtxq_unbusy(&uq->uq_key);
	- umtxq_unlock(&uq->uq_key);
	- if (error != 0) {
	- /*
	- * Since we're going to return an
	- * error, restore the m_owner to its
	- * previous, unowned state to avoid
	- * compounding the problem.
	- */
	- (void)casuword32(&m->m_owner,
	- id \| UMUTEX_CONTESTED,
	- old_owner);
	+ if (rv == 1) {
	+ if (error == 0) {
	+ error = umtxq_check_susp(td, true);
	+ if (error != 0)
	+ return (error);
	}
	- if (error == 0 &&
	- old_owner == UMUTEX_RB_OWNERDEAD)
	- error = EOWNERDEAD;
	- break;
	+
	+ /*
	+ * If this failed the lock could
	+ * changed, restart.
	+ */
	+ continue;
	}

	- error = umtxq_check_susp(td);
	- if (error != 0)
	- break;
	-
	- /* If this failed the lock has changed, restart. */
	- continue;
	+ MPASS(rv == 0);
	+ MPASS(owner == old_owner);
	+ umtxq_lock(&uq->uq_key);
	+ umtxq_busy(&uq->uq_key);
	+ error = umtx_pi_claim(pi, td);
	+ umtxq_unbusy(&uq->uq_key);
	+ umtxq_unlock(&uq->uq_key);
	+ if (error != 0) {
	+ /*
	+ * Since we're going to return an
	+ * error, restore the m_owner to its
	+ * previous, unowned state to avoid
	+ * compounding the problem.
	+ */
	+ (void)casuword32(&m->m_owner,
	+ id \| UMUTEX_CONTESTED, old_owner);
	+ }
	+ if (error == 0 && old_owner == UMUTEX_RB_OWNERDEAD)
	+ error = EOWNERDEAD;
	+ break;
	}

	if ((owner & ~UMUTEX_CONTESTED) == id) {
	@@ -1932,28 +1994,33 @@
	error = EFAULT;
	break;
	}
	-
	- umtxq_lock(&uq->uq_key);
	- /*
	- * We set the contested bit, sleep. Otherwise the lock changed
	- * and we need to retry or we lost a race to the thread
	- * unlocking the umtx. Note that the UMUTEX_RB_OWNERDEAD
	- * value for owner is impossible there.
	- */
	- if (old == owner) {
	- error = umtxq_sleep_pi(uq, pi,
	- owner & ~UMUTEX_CONTESTED,
	- "umtxpi", timeout == NULL ? NULL : &timo,
	- (flags & USYNC_PROCESS_SHARED) != 0);
	+ if (rv == 1) {
	+ umtxq_unbusy_unlocked(&uq->uq_key);
	+ error = umtxq_check_susp(td, true);
	if (error != 0)
	- continue;
	- } else {
	- umtxq_unbusy(&uq->uq_key);
	- umtxq_unlock(&uq->uq_key);
	+ break;
	+
	+ /*
	+ * The lock changed and we need to retry or we
	+ * lost a race to the thread unlocking the
	+ * umtx. Note that the UMUTEX_RB_OWNERDEAD
	+ * value for owner is impossible there.
	+ */
	+ continue;
	}

	- error = umtxq_check_susp(td);
	+ umtxq_lock(&uq->uq_key);
	+
	+ /* We set the contested bit, sleep. */
	+ MPASS(old == owner);
	+ error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
	+ "umtxpi", timeout == NULL ? NULL : &timo,
	+ (flags & USYNC_PROCESS_SHARED) != 0);
	if (error != 0)
	+ continue;
	+
	+ error = umtxq_check_susp(td, false);
	+ if (error != 0)
	break;
	}

	@@ -1978,6 +2045,8 @@
	int count, error, pri;

	id = td->td_tid;
	+
	+usrloop:
	/*
	* Make sure we own this mtx.
	*/
	@@ -1995,6 +2064,12 @@
	error = casueword32(&m->m_owner, owner, &old, new_owner);
	if (error == -1)
	return (EFAULT);
	+ if (error == 1) {
	+ error = umtxq_check_susp(td, true);
	+ if (error != 0)
	+ return (error);
	+ goto usrloop;
	+ }
	if (old == owner)
	return (0);
	owner = old;
	@@ -2074,15 +2149,20 @@

	if (count > 1)
	new_owner \|= UMUTEX_CONTESTED;
	+again:
	error = casueword32(&m->m_owner, owner, &old, new_owner);
	-
	+ if (error == 1) {
	+ error = umtxq_check_susp(td, false);
	+ if (error == 0)
	+ goto again;
	+ }
	umtxq_unbusy_unlocked(&key);
	umtx_key_release(&key);
	if (error == -1)
	return (EFAULT);
	- if (old != owner)
	+ if (error == 0 && old != owner)
	return (EINVAL);
	- return (0);
	+ return (error);
	}

	/*
	@@ -2149,31 +2229,49 @@
	error = EFAULT;
	break;
	}
	-
	- if (owner == UMUTEX_CONTESTED) {
	+ if (rv == 0) {
	+ MPASS(owner == UMUTEX_CONTESTED);
	error = 0;
	break;
	- } else if (owner == UMUTEX_RB_OWNERDEAD) {
	+ }
	+ /* rv == 1 */
	+ if (owner == UMUTEX_RB_OWNERDEAD) {
	rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD,
	&owner, id \| UMUTEX_CONTESTED);
	if (rv == -1) {
	error = EFAULT;
	break;
	}
	- if (owner == UMUTEX_RB_OWNERDEAD) {
	+ if (rv == 0) {
	+ MPASS(owner == UMUTEX_RB_OWNERDEAD);
	error = EOWNERDEAD; /* success */
	break;
	}
	- error = 0;
	+
	+ /*
	+ * rv == 1, only check for suspension if we
	+ * did not already catched a signal. If we
	+ * get an error from the check, the same
	+ * condition is checked by the umtxq_sleep()
	+ * call below, so we should obliterate the
	+ * error to not skip the last loop iteration.
	+ */
	+ if (error == 0) {
	+ error = umtxq_check_susp(td, false);
	+ if (error == 0) {
	+ if (try != 0)
	+ error = EBUSY;
	+ else
	+ continue;
	+ }
	+ error = 0;
	+ }
	} else if (owner == UMUTEX_RB_NOTRECOV) {
	error = ENOTRECOVERABLE;
	- break;
	}

	- if (try != 0) {
	+ if (try != 0)
	error = EBUSY;
	- break;
	- }

	/*
	* If we caught a signal, we have retried and now
	@@ -2668,11 +2766,12 @@
	umtx_key_release(&uq->uq_key);
	return (EFAULT);
	}
	- if (oldstate == state) {
	+ if (rv == 0) {
	+ MPASS(oldstate == state);
	umtx_key_release(&uq->uq_key);
	return (0);
	}
	- error = umtxq_check_susp(td);
	+ error = umtxq_check_susp(td, true);
	if (error != 0)
	break;
	state = oldstate;
	@@ -2703,10 +2802,12 @@
	error = EFAULT;
	break;
	}
	- if (oldstate == state)
	+ if (rv == 0) {
	+ MPASS(oldstate == state);
	goto sleep;
	+ }
	state = oldstate;
	- error = umtxq_check_susp(td);
	+ error = umtxq_check_susp(td, false);
	if (error != 0)
	break;
	}
	@@ -2718,7 +2819,7 @@
	/* state is changed while setting flags, restart */
	if (!(state & wrflags)) {
	umtxq_unbusy_unlocked(&uq->uq_key);
	- error = umtxq_check_susp(td);
	+ error = umtxq_check_susp(td, true);
	if (error != 0)
	break;
	continue;
	@@ -2781,10 +2882,12 @@
	error = EFAULT;
	break;
	}
	- if (oldstate == state)
	+ if (rv == 0) {
	+ MPASS(oldstate == state);
	break;
	+ }
	state = oldstate;
	- error1 = umtxq_check_susp(td);
	+ error1 = umtxq_check_susp(td, false);
	if (error1 != 0) {
	if (error == 0)
	error = error1;
	@@ -2840,22 +2943,25 @@
	umtx_key_release(&uq->uq_key);
	return (EFAULT);
	}
	- if (oldstate == state) {
	+ if (rv == 0) {
	+ MPASS(oldstate == state);
	umtx_key_release(&uq->uq_key);
	return (0);
	}
	state = oldstate;
	- error = umtxq_check_susp(td);
	+ error = umtxq_check_susp(td, true);
	if (error != 0)
	break;
	}

	if (error) {
	- if (!(state & (URWLOCK_WRITE_OWNER\|URWLOCK_WRITE_WAITERS)) &&
	+ if ((state & (URWLOCK_WRITE_OWNER \|
	+ URWLOCK_WRITE_WAITERS)) == 0 &&
	blocked_readers != 0) {
	umtxq_lock(&uq->uq_key);
	umtxq_busy(&uq->uq_key);
	- umtxq_signal_queue(&uq->uq_key, INT_MAX, UMTX_SHARED_QUEUE);
	+ umtxq_signal_queue(&uq->uq_key, INT_MAX,
	+ UMTX_SHARED_QUEUE);
	umtxq_unbusy(&uq->uq_key);
	umtxq_unlock(&uq->uq_key);
	}
	@@ -2885,10 +2991,12 @@
	error = EFAULT;
	break;
	}
	- if (oldstate == state)
	+ if (rv == 0) {
	+ MPASS(oldstate == state);
	goto sleep;
	+ }
	state = oldstate;
	- error = umtxq_check_susp(td);
	+ error = umtxq_check_susp(td, false);
	if (error != 0)
	break;
	}
	@@ -2900,7 +3008,7 @@
	if ((state & URWLOCK_WRITE_OWNER) == 0 &&
	URWLOCK_READER_COUNT(state) == 0) {
	umtxq_unbusy_unlocked(&uq->uq_key);
	- error = umtxq_check_susp(td);
	+ error = umtxq_check_susp(td, false);
	if (error != 0)
	break;
	continue;
	@@ -2958,10 +3066,12 @@
	error = EFAULT;
	break;
	}
	- if (oldstate == state)
	+ if (rv == 0) {
	+ MPASS(oldstate == state);
	break;
	+ }
	state = oldstate;
	- error1 = umtxq_check_susp(td);
	+ error1 = umtxq_check_susp(td, false);
	/*
	* We are leaving the URWLOCK_WRITE_WAITERS
	* behind, but this should not harm the
	@@ -3021,13 +3131,13 @@
	error = EFAULT;
	goto out;
	}
	- if (oldstate != state) {
	+ if (rv == 1) {
	state = oldstate;
	if (!(oldstate & URWLOCK_WRITE_OWNER)) {
	error = EPERM;
	goto out;
	}
	- error = umtxq_check_susp(td);
	+ error = umtxq_check_susp(td, true);
	if (error != 0)
	goto out;
	} else
	@@ -3041,13 +3151,13 @@
	error = EFAULT;
	goto out;
	}
	- if (oldstate != state) {
	+ if (rv == 1) {
	state = oldstate;
	if (URWLOCK_READER_COUNT(oldstate) == 0) {
	error = EPERM;
	goto out;
	}
	- error = umtxq_check_susp(td);
	+ error = umtxq_check_susp(td, true);
	if (error != 0)
	goto out;
	} else
	@@ -3097,7 +3207,7 @@
	struct abs_timeout timo;
	struct umtx_q *uq;
	uint32_t flags, count, count1;
	- int error, rv;
	+ int error, rv, rv1;

	uq = td->td_umtxq;
	error = fueword32(&sem->_flags, &flags);
	@@ -3110,20 +3220,30 @@
	if (timeout != NULL)
	abs_timeout_init2(&timo, timeout);

	+again:
	umtxq_lock(&uq->uq_key);
	umtxq_busy(&uq->uq_key);
	umtxq_insert(uq);
	umtxq_unlock(&uq->uq_key);
	rv = casueword32(&sem->_has_waiters, 0, &count1, 1);
	if (rv == 0)
	- rv = fueword32(&sem->_count, &count);
	- if (rv == -1 \|\| count != 0) {
	+ rv1 = fueword32(&sem->_count, &count);
	+ if (rv == -1 \|\| (rv == 0 && (rv1 == -1 \|\| count != 0)) \|\| rv == 1) {
	umtxq_lock(&uq->uq_key);
	umtxq_unbusy(&uq->uq_key);
	umtxq_remove(uq);
	umtxq_unlock(&uq->uq_key);
	- umtx_key_release(&uq->uq_key);
	- return (rv == -1 ? EFAULT : 0);
	+ if (rv == 1) {
	+ rv = umtxq_check_susp(td, true);
	+ if (rv == 0)
	+ goto again;
	+ error = rv;
	+ goto out;
	+ }
	+ if (rv == 0)
	+ rv = rv1;
	+ error = rv == -1 ? EFAULT : 0;
	+ goto out;
	}
	umtxq_lock(&uq->uq_key);
	umtxq_unbusy(&uq->uq_key);
	@@ -3140,6 +3260,7 @@
	error = EINTR;
	}
	umtxq_unlock(&uq->uq_key);
	+out:
	umtx_key_release(&uq->uq_key);
	return (error);
	}
	@@ -3201,6 +3322,7 @@
	if (timeout != NULL)
	abs_timeout_init2(&timo, timeout);

	+again:
	umtxq_lock(&uq->uq_key);
	umtxq_busy(&uq->uq_key);
	umtxq_insert(uq);
	@@ -3226,16 +3348,19 @@
	if (count == USEM_HAS_WAITERS)
	break;
	rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS);
	- if (rv == -1) {
	- umtxq_lock(&uq->uq_key);
	- umtxq_unbusy(&uq->uq_key);
	- umtxq_remove(uq);
	- umtxq_unlock(&uq->uq_key);
	- umtx_key_release(&uq->uq_key);
	- return (EFAULT);
	- }
	- if (count == 0)
	+ if (rv == 0)
	break;
	+ umtxq_lock(&uq->uq_key);
	+ umtxq_unbusy(&uq->uq_key);
	+ umtxq_remove(uq);
	+ umtxq_unlock(&uq->uq_key);
	+ umtx_key_release(&uq->uq_key);
	+ if (rv == -1)
	+ return (EFAULT);
	+ rv = umtxq_check_susp(td, true);
	+ if (rv != 0)
	+ return (rv);
	+ goto again;
	}
	umtxq_lock(&uq->uq_key);
	umtxq_unbusy(&uq->uq_key);
	@@ -3288,11 +3413,20 @@
	if (cnt == 1) {
	umtxq_unlock(&key);
	rv = fueword32(&sem->_count, &count);
	- while (rv != -1 && count & USEM_HAS_WAITERS)
	+ while (rv != -1 && count & USEM_HAS_WAITERS) {
	rv = casueword32(&sem->_count, count, &count,
	count & ~USEM_HAS_WAITERS);
	+ if (rv == 1) {
	+ rv = umtxq_check_susp(td, true);
	+ if (rv != 0)
	+ break;
	+ }
	+ }
	if (rv == -1)
	error = EFAULT;
	+ else if (rv > 0) {
	+ error = rv;
	+ }
	umtxq_lock(&key);
	}

	Index: head/sys/mips/mips/support.S
	===================================================================
	--- head/sys/mips/mips/support.S
	+++ head/sys/mips/mips/support.S
	@@ -336,19 +336,15 @@
	GET_CPU_PCPU(v1)
	PTR_L v1, PC_CURPCB(v1)
	PTR_S v0, U_PCB_ONFAULT(v1)
	-1:
	+
	+ li v0, 1
	move t0, a3
	ll t1, 0(a0)
	- bne a1, t1, 2f
	+ bne a1, t1, 1f
	nop
	sc t0, 0(a0) # store word
	- beqz t0, 1b
	- nop
	- j 3f
	- li v0, 0
	-2:
	- li v0, -1
	-3:
	+ xori v0, t0, 1
	+1:
	PTR_S zero, U_PCB_ONFAULT(v1)
	jr ra
	sw t1, 0(a2) # unconditionally store old word
	@@ -363,19 +359,15 @@
	GET_CPU_PCPU(v1)
	PTR_L v1, PC_CURPCB(v1)
	PTR_S v0, U_PCB_ONFAULT(v1)
	-1:
	+
	+ li v0, 1
	move t0, a3
	lld t1, 0(a0)
	- bne a1, t1, 2f
	+ bne a1, t1, 1f
	nop
	scd t0, 0(a0) # store double word
	- beqz t0, 1b
	- nop
	- j 3f
	- li v0, 0
	-2:
	- li v0, -1
	-3:
	+ xori v0, t0, 1
	+1:
	PTR_S zero, U_PCB_ONFAULT(v1)
	jr ra
	sd t1, 0(a2) # unconditionally store old word
	Index: head/sys/powerpc/powerpc/copyinout.c
	===================================================================
	--- head/sys/powerpc/powerpc/copyinout.c
	+++ head/sys/powerpc/powerpc/copyinout.c
	@@ -433,6 +433,7 @@
	pmap_t pm;
	jmp_buf env;
	uint32_t *p, val;
	+ int res;

	td = curthread;
	pm = &td->td_proc->p_vmspace->vm_pmap;
	@@ -449,24 +450,26 @@
	return (-1);
	}

	+ res = 0;
	__asm __volatile (
	- "1:\tlwarx %0, 0, %2\n\t" /* load old value */
	- "cmplw %3, %0\n\t" /* compare */
	- "bne 2f\n\t" /* exit if not equal */
	- "stwcx. %4, 0, %2\n\t" /* attempt to store */
	- "bne- 1b\n\t" /* spin if failed */
	- "b 3f\n\t" /* we've succeeded */
	+ "lwarx %0, 0, %3\n\t" /* load old value */
	+ "cmplw %4, %0\n\t" /* compare */
	+ "bne 1f\n\t" /* exit if not equal */
	+ "stwcx. %5, 0, %3\n\t" /* attempt to store */
	+ "bne- 1f\n\t" /* if failed */
	+ "b 2f\n\t" /* we've succeeded */
	+ "1:\n\t"
	+ "stwcx. %0, 0, %4\n\t" /* clear reservation (74xx) */
	+ "li %2, 1\n\t"
	"2:\n\t"
	- "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
	- "3:\n\t"
	- : "=&r" (val), "=m" (*p)
	+ : "=&r" (val), "=m" (*p), "=&r" (res)
	: "r" (p), "r" (old), "r" (new), "m" (*p)
	: "cr0", "memory");

	td->td_pcb->pcb_onfault = NULL;

	*oldvalp = val;
	- return (0);
	+ return (res);
	}

	#ifndef __powerpc64__
	@@ -485,6 +488,7 @@
	pmap_t pm;
	jmp_buf env;
	u_long *p, val;
	+ int res;

	td = curthread;
	pm = &td->td_proc->p_vmspace->vm_pmap;
	@@ -501,23 +505,25 @@
	return (-1);
	}

	+ res = 0;
	__asm __volatile (
	- "1:\tldarx %0, 0, %2\n\t" /* load old value */
	- "cmpld %3, %0\n\t" /* compare */
	- "bne 2f\n\t" /* exit if not equal */
	- "stdcx. %4, 0, %2\n\t" /* attempt to store */
	- "bne- 1b\n\t" /* spin if failed */
	- "b 3f\n\t" /* we've succeeded */
	+ "ldarx %0, 0, %3\n\t" /* load old value */
	+ "cmpld %4, %0\n\t" /* compare */
	+ "bne 1f\n\t" /* exit if not equal */
	+ "stdcx. %5, 0, %3\n\t" /* attempt to store */
	+ "bne- 1f\n\t" /* if failed */
	+ "b 2f\n\t" /* we've succeeded */
	+ "1:\n\t"
	+ "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
	+ "li %2, 1\n\t"
	"2:\n\t"
	- "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
	- "3:\n\t"
	- : "=&r" (val), "=m" (*p)
	+ : "=&r" (val), "=m" (*p), "=&r" (res)
	: "r" (p), "r" (old), "r" (new), "m" (*p)
	: "cr0", "memory");

	td->td_pcb->pcb_onfault = NULL;

	*oldvalp = val;
	- return (0);
	+ return (res);
	}
	#endif
	Index: head/sys/riscv/riscv/support.S
	===================================================================
	--- head/sys/riscv/riscv/support.S
	+++ head/sys/riscv/riscv/support.S
	@@ -60,14 +60,15 @@
	la a6, fsu_fault /* Load the fault handler */
	SET_FAULT_HANDLER(a6, a4) /* And set it */
	ENTER_USER_ACCESS(a4)
	-1: lr.w a4, 0(a0) /* Load-exclusive the data */
	- bne a4, a1, 2f /* If not equal then exit */
	+ lr.w a4, 0(a0) /* Load-exclusive the data */
	+ bne a4, a1, 1f /* If not equal then exit */
	sc.w a5, a3, 0(a0) /* Store the new data */
	- bnez a5, 1b /* Retry on failure */
	-2: EXIT_USER_ACCESS(a5)
	- SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */
	+ beqz a5, 2f /* Success */
	+1: li a5, 1 /* Normalize failure result */
	+2: EXIT_USER_ACCESS(a6)
	+ SET_FAULT_HANDLER(x0, a6) /* Reset the fault handler */
	sw a4, 0(a2) /* Store the read data */
	- li a0, 0 /* Success */
	+ mv a0, a5 /* Success indicator */
	ret /* Return */
	END(casueword32)

	@@ -80,14 +81,15 @@
	la a6, fsu_fault /* Load the fault handler */
	SET_FAULT_HANDLER(a6, a4) /* And set it */
	ENTER_USER_ACCESS(a4)
	-1: lr.d a4, 0(a0) /* Load-exclusive the data */
	- bne a4, a1, 2f /* If not equal then exit */
	+ lr.d a4, 0(a0) /* Load-exclusive the data */
	+ bne a4, a1, 1f /* If not equal then exit */
	sc.d a5, a3, 0(a0) /* Store the new data */
	- bnez a5, 1b /* Retry on failure */
	-2: EXIT_USER_ACCESS(a5)
	- SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */
	+ beqz a5, 2f /* Success */
	+1: li a5, 1 /* Normalize failure result */
	+2: EXIT_USER_ACCESS(a6)
	+ SET_FAULT_HANDLER(x0, a6) /* Reset the fault handler */
	sd a4, 0(a2) /* Store the read data */
	- li a0, 0 /* Success */
	+ mv a0, a5 /* Success indicator */
	ret /* Return */
	END(casueword)

	Index: head/sys/sparc64/sparc64/support.S
	===================================================================
	--- head/sys/sparc64/sparc64/support.S
	+++ head/sys/sparc64/sparc64/support.S
	@@ -403,28 +403,27 @@
	.set susword, suword16
	.set suword, suword64

	- .globl casuword32, casuword, fuptr, suptr
	- .set casuword, casuword64
	+ .globl casuword32_int, casuword64_int, fuptr, suptr
	.set fuptr, fuword64
	.set suptr, suword64

	/*
	* int32_t casuword32(volatile int32_t *p, int32_t e, int32_t s)
	*/
	-ENTRY(casuword32)
	+ENTRY(casuword32_int)
	casa [%o0] ASI_AIUP, %o1, %o2
	retl
	mov %o2, %o0
	-END(casuword32)
	+END(casuword32_int)

	/*
	* int64_t casuword64(volatile int64_t *p, int64_t e, int64_t s)
	*/
	-ENTRY(casuword64)
	+ENTRY(casuword64_int)
	casxa [%o0] ASI_AIUP, %o1, %o2
	retl
	mov %o2, %o0
	-END(casuword64)
	+END(casuword64_int)

	/*
	* int fuword8(const void *base)
	Index: head/sys/sparc64/sparc64/vm_machdep.c
	===================================================================
	--- head/sys/sparc64/sparc64/vm_machdep.c
	+++ head/sys/sparc64/sparc64/vm_machdep.c
	@@ -462,3 +462,28 @@
	pmap_qremove(sf->kva, 1);
	return (1);
	}
	+
	+uint32_t casuword32_int(volatile uint32_t *base, uint32_t oldval,
	+ uint32_t newval);
	+uint32_t
	+casuword32(volatile uint32_t *base, uint32_t oldval, uint32_t newval)
	+{
	+ uint32_t ret;
	+
	+ ret = casuword32_int(base, oldval, newval);
	+ if (ret != -1)
	+ ret = ret != oldval;
	+ return (ret);
	+}
	+
	+u_long casuword64_int(volatile u_long *p, u_long oldval, u_long newval);
	+u_long
	+casuword(volatile u_long *p, u_long oldval, u_long newval)
	+{
	+ u_long ret;
	+
	+ ret = casuword64_int(p, oldval, newval);
	+ if (ret != -1L)
	+ ret = ret != oldval;
	+ return (ret);
	+}

File Metadata

Mime Type: text/plain
Expires: Wed, Nov 19, 3:13 PM (3 h, 6 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 25646165
Default Alt Text: D20772.diff (31 KB)

D20772.diffNo OneTemporaryActions

D20772.diffView Options

File Metadata

Event Timeline

D20772.diff
No OneTemporary
Actions

D20772.diff
View Options