Page MenuHomeFreeBSD
Files F140308222

D20772.id59112.diff
No OneTemporary
Actions

D20772.id59112.diff
View Options

Index: share/man/man9/casuword.9
===================================================================
--- share/man/man9/casuword.9
+++ share/man/man9/casuword.9
@@ -1,4 +1,4 @@
-.\" Copyright (c) 2014 The FreeBSD Foundation
+.\" Copyright (c) 2014, 2019 The FreeBSD Foundation
.\" All rights reserved.
.\"
.\" Part of this documentation was written by
@@ -28,7 +28,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd October 21, 2014
+.Dd April 19, 2019
.Dt CASU 9
.Os
.Sh NAME
@@ -41,13 +41,31 @@
.In sys/types.h
.In sys/systm.h
.Ft int
-.Fn casueword "volatile u_long *base" "u_long oldval" "u_long *oldvalp" "u_long newval"
+.Fo casueword
+.Fa "volatile u_long *base"
+.Fa "u_long oldval"
+.Fa "u_long *oldvalp"
+.Fa "u_long newval"
+.Fc
.Ft int
-.Fn casueword32 "volatile uint32_t *base" "uint32_t oldval" "uint32_t *oldvalp" "uint32_t newval"
+.Fo casueword32
+.Fa "volatile uint32_t *base"
+.Fa "uint32_t oldval"
+.Fa "uint32_t *oldvalp"
+.Fa "uint32_t newval"
+.Fc
.Ft u_long
-.Fn casuword "volatile u_long *base" "u_long oldval" "u_long newval"
+.Fo casuword
+.Fa "volatile u_long *base"
+.Fa "u_long oldval"
+.Fa "u_long newval"
+.Fc
.Ft uint32_t
-.Fn casuword32 "volatile uint32_t *base" "uint32_t oldval" "uint32_t newval"
+.Fo casuword32
+.Fa "volatile uint32_t *base"
+.Fa "uint32_t oldval"
+.Fa "uint32_t newval"
+.Fc
.Sh DESCRIPTION
The
.Nm
@@ -88,7 +106,9 @@
.Fn casueword
and
.Fn casueword32
-functions return 0 on success and -1 on failure.
+functions return 0 on success, -1 on failure to access memory,
+and 1 when comparison or store failed.
+The store can fail on load-linked/store-conditional architectures.
.Sh SEE ALSO
.Xr atomic 9 ,
.Xr fetch 9 ,
Index: sys/amd64/amd64/support.S
===================================================================
--- sys/amd64/amd64/support.S
+++ sys/amd64/amd64/support.S
@@ -811,6 +811,7 @@
lock
#endif
cmpxchgl %ecx,(%rdi) /* new = %ecx */
+ setne %cl
/*
* The old value is in %eax. If the store succeeded it will be the
@@ -828,6 +829,7 @@
*/
movl %esi,(%rdx) /* oldp = %rdx */
POP_FRAME_POINTER
+ movzbl %cl, %eax
ret
END(casueword32_nosmap)
@@ -847,6 +849,7 @@
#endif
cmpxchgl %ecx,(%rdi) /* new = %ecx */
clac
+ setne %cl
/*
* The old value is in %eax. If the store succeeded it will be the
@@ -864,6 +867,7 @@
*/
movl %esi,(%rdx) /* oldp = %rdx */
POP_FRAME_POINTER
+ movzbl %cl, %eax
ret
END(casueword32_smap)
@@ -886,6 +890,7 @@
lock
#endif
cmpxchgq %rcx,(%rdi) /* new = %rcx */
+ setne %cl
/*
* The old value is in %rax. If the store succeeded it will be the
@@ -897,6 +902,7 @@
movq %rax,PCB_ONFAULT(%r8)
movq %rsi,(%rdx)
POP_FRAME_POINTER
+ movzbl %cl, %eax
ret
END(casueword_nosmap)
@@ -916,6 +922,7 @@
#endif
cmpxchgq %rcx,(%rdi) /* new = %rcx */
clac
+ setne %cl
/*
* The old value is in %rax. If the store succeeded it will be the
@@ -927,6 +934,7 @@
movq %rax,PCB_ONFAULT(%r8)
movq %rsi,(%rdx)
POP_FRAME_POINTER
+ movzbl %cl, %eax
ret
END(casueword_smap)
Index: sys/arm/arm/fusu.S
===================================================================
--- sys/arm/arm/fusu.S
+++ sys/arm/arm/fusu.S
@@ -63,7 +63,7 @@
ldr r4, =(VM_MAXUSER_ADDRESS-3)
cmp r0, r4
mvncs r0, #0
- bcs 2f
+ bcs 1f
GET_PCB(r6)
ldr r6, [r6]
@@ -78,12 +78,10 @@
str r4, [r6, #PCB_ONFAULT]
#if __ARM_ARCH >= 6
-1:
+ mov r5, #1
ldrex r4, [r0]
cmp r4, r1
strexeq r5, r3, [r0]
- cmpeq r5, #1
- beq 1b
#else
ldrt r4, [r0]
cmp r4, r1
@@ -92,7 +90,10 @@
str r4, [r2]
mov r0, #0
str r0, [r6, #PCB_ONFAULT]
-2:
+#if __ARM_ARCH >= 6
+ mov r0, r5
+#endif
+1:
ldmfd sp!, {r4, r5, r6}
RET
EEND(casueword32)
Index: sys/arm64/arm64/support.S
===================================================================
--- sys/arm64/arm64/support.S
+++ sys/arm64/arm64/support.S
@@ -63,11 +63,10 @@
cmp w4, w1 /* Compare */
b.ne 2f /* Not equal, exit */
stxr w5, w3, [x0] /* Store the new data */
- cbnz w5, 1b /* Retry on failure */
2: EXIT_USER_ACCESS(w6)
SET_FAULT_HANDLER(xzr, x5) /* Reset the fault handler */
str w4, [x2] /* Store the read data */
- mov x0, #0 /* Success */
+ mov w0, w5 /* Result same as store status */
ret /* Return */
END(casueword32)
@@ -85,11 +84,10 @@
cmp x4, x1 /* Compare */
b.ne 2f /* Not equal, exit */
stxr w5, x3, [x0] /* Store the new data */
- cbnz w5, 1b /* Retry on failure */
2: EXIT_USER_ACCESS(w6)
SET_FAULT_HANDLER(xzr, x5) /* Reset the fault handler */
str x4, [x2] /* Store the read data */
- mov x0, #0 /* Success */
+ mov w0, w5 /* Result same as store status */
ret /* Return */
END(casueword)
Index: sys/i386/i386/copyout.c
===================================================================
--- sys/i386/i386/copyout.c
+++ sys/i386/i386/copyout.c
@@ -428,6 +428,7 @@
struct casueword_arg0 {
uint32_t oldval;
uint32_t newval;
+ int res;
};
static void
@@ -436,7 +437,7 @@
struct casueword_arg0 *ca;
ca = arg;
- atomic_fcmpset_int((u_int *)kva, &ca->oldval, ca->newval);
+ ca->res = atomic_fcmpset_int((u_int *)kva, &ca->oldval, ca->newval);
}
int
@@ -452,7 +453,7 @@
casueword_slow0, &ca);
if (res == 0) {
*oldvalp = ca.oldval;
- return (0);
+ return (ca.res);
}
return (-1);
}
@@ -469,7 +470,7 @@
casueword_slow0, &ca);
if (res == 0) {
*oldvalp = ca.oldval;
- return (0);
+ return (ca.res);
}
return (-1);
}
Index: sys/kern/kern_umtx.c
===================================================================
--- sys/kern/kern_umtx.c
+++ sys/kern/kern_umtx.c
@@ -691,7 +691,7 @@
}
static int
-umtxq_check_susp(struct thread *td)
+umtxq_check_susp(struct thread *td, bool sleep)
{
struct proc *p;
int error;
@@ -710,7 +710,7 @@
if (p->p_flag & P_SINGLE_EXIT)
error = EINTR;
else
- error = ERESTART;
+ error = sleep ? thread_suspend_check(0) : ERESTART;
}
PROC_UNLOCK(p);
return (error);
@@ -1026,6 +1026,7 @@
* Care must be exercised when dealing with umtx structure. It
* can fault on any access.
*/
+again:
for (;;) {
rv = fueword32(&m->m_owner, &owner);
if (rv == -1)
@@ -1049,9 +1050,12 @@
id | UMUTEX_CONTESTED);
if (rv == -1)
return (EFAULT);
- if (owner == UMUTEX_RB_OWNERDEAD)
+ if (rv == 0) {
+ MPASS(owner == UMUTEX_RB_OWNERDEAD);
return (EOWNERDEAD); /* success */
- rv = umtxq_check_susp(td);
+ }
+ MPASS(rv == 1);
+ rv = umtxq_check_susp(td, true);
if (rv != 0)
return (rv);
continue;
@@ -1059,7 +1063,6 @@
if (owner == UMUTEX_RB_NOTRECOV)
return (ENOTRECOVERABLE);
-
/*
* Try the uncontested case. This should be
* done in userland.
@@ -1071,13 +1074,16 @@
return (EFAULT);
/* The acquire succeeded. */
- if (owner == UMUTEX_UNOWNED)
+ if (rv == 0) {
+ MPASS(owner == UMUTEX_UNOWNED);
return (0);
+ }
/*
* If no one owns it but it is contested try
* to acquire it.
*/
+ MPASS(rv == 1);
if (owner == UMUTEX_CONTESTED) {
rv = casueword32(&m->m_owner,
UMUTEX_CONTESTED, &owner,
@@ -1085,13 +1091,15 @@
/* The address was invalid. */
if (rv == -1)
return (EFAULT);
-
- if (owner == UMUTEX_CONTESTED)
+ if (rv == 0) {
+ MPASS(owner == UMUTEX_CONTESTED);
return (0);
-
- rv = umtxq_check_susp(td);
- if (rv != 0)
- return (rv);
+ }
+ if (rv == 1) {
+ rv = umtxq_check_susp(td, false);
+ if (rv != 0)
+ return (rv);
+ }
/*
* If this failed the lock has
@@ -1099,6 +1107,11 @@
*/
continue;
}
+
+ /* rv == 1 but not contested, likely store failure */
+ rv = umtxq_check_susp(td, false);
+ if (rv != 0)
+ return (rv);
}
if (mode == _UMUTEX_TRY)
@@ -1129,14 +1142,21 @@
rv = casueword32(&m->m_owner, owner, &old,
owner | UMUTEX_CONTESTED);
- /* The address was invalid. */
- if (rv == -1) {
+ /* The address was invalid or casueword failed to store. */
+ if (rv == -1 || rv == 1) {
umtxq_lock(&uq->uq_key);
umtxq_remove(uq);
umtxq_unbusy(&uq->uq_key);
umtxq_unlock(&uq->uq_key);
umtx_key_release(&uq->uq_key);
- return (EFAULT);
+ if (rv == -1)
+ return (EFAULT);
+ if (rv == 1) {
+ rv = umtxq_check_susp(td, true);
+ if (rv != 0)
+ return (rv);
+ }
+ goto again;
}
/*
@@ -1154,7 +1174,7 @@
umtx_key_release(&uq->uq_key);
if (error == 0)
- error = umtxq_check_susp(td);
+ error = umtxq_check_susp(td, false);
}
return (0);
@@ -1171,6 +1191,8 @@
int error, count;
id = td->td_tid;
+
+again:
/*
* Make sure we own this mtx.
*/
@@ -1186,6 +1208,12 @@
error = casueword32(&m->m_owner, owner, &old, newlock);
if (error == -1)
return (EFAULT);
+ if (error == 1) {
+ error = umtxq_check_susp(td, true);
+ if (error != 0)
+ return (error);
+ goto again;
+ }
if (old == owner)
return (0);
owner = old;
@@ -1216,6 +1244,12 @@
umtx_key_release(&key);
if (error == -1)
return (EFAULT);
+ if (error == 1) {
+ error = umtxq_check_susp(td, true);
+ if (error != 0)
+ return (error);
+ goto again;
+ }
if (old != owner)
return (EINVAL);
return (0);
@@ -1234,6 +1268,7 @@
int error;
int count;
+again:
error = fueword32(&m->m_owner, &owner);
if (error == -1)
return (EFAULT);
@@ -1260,8 +1295,17 @@
owner != UMUTEX_RB_NOTRECOV) {
error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
UMUTEX_UNOWNED);
- if (error == -1)
+ if (error == -1) {
error = EFAULT;
+ } else if (error == 1) {
+ umtxq_unbusy(&key);
+ umtxq_unlock(&key);
+ umtx_key_release(&key);
+ error = umtxq_check_susp(td, true);
+ if (error != 0)
+ return (error);
+ goto again;
+ }
}
umtxq_lock(&key);
@@ -1331,12 +1375,10 @@
error = EFAULT;
break;
}
- if (old == owner)
+ if (error == 0 && old == owner)
break;
owner = old;
- error = umtxq_check_susp(td);
- if (error != 0)
- break;
+ error = umtxq_check_susp(td, true);
}
} else if (count == 1) {
error = fueword32(&m->m_owner, &owner);
@@ -1350,12 +1392,12 @@
error = EFAULT;
break;
}
- if (old == owner)
+ if (error == 0) {
+ MPASS(old == owner);
break;
+ }
owner = old;
- error = umtxq_check_susp(td);
- if (error != 0)
- break;
+ error = umtxq_check_susp(td, true);
}
}
umtxq_lock(&key);
@@ -1843,6 +1885,11 @@
error = EFAULT;
break;
}
+ if (rv == 1) {
+ error = umtxq_check_susp(td, true);
+ if (error != 0)
+ break;
+ }
/* The acquire succeeded. */
if (owner == UMUTEX_UNOWNED) {
@@ -1865,8 +1912,15 @@
error = EFAULT;
break;
}
+ if (rv == 1) {
+ error = umtxq_check_susp(td, true);
+ if (error != 0)
+ return (error);
+ continue;
+ }
- if (owner == old_owner) {
+ if (rv == 0) {
+ MPASS(owner == old_owner);
umtxq_lock(&uq->uq_key);
umtxq_busy(&uq->uq_key);
error = umtx_pi_claim(pi, td);
@@ -1889,7 +1943,7 @@
break;
}
- error = umtxq_check_susp(td);
+ error = umtxq_check_susp(td, false);
if (error != 0)
break;
@@ -1933,6 +1987,13 @@
error = EFAULT;
break;
}
+ if (rv == 1) {
+ umtxq_unbusy_unlocked(&uq->uq_key);
+ error = umtxq_check_susp(td, true);
+ if (error != 0)
+ break;
+ continue;
+ }
umtxq_lock(&uq->uq_key);
/*
@@ -1953,7 +2014,7 @@
umtxq_unlock(&uq->uq_key);
}
- error = umtxq_check_susp(td);
+ error = umtxq_check_susp(td, true);
if (error != 0)
break;
}
@@ -1979,6 +2040,8 @@
int count, error, pri;
id = td->td_tid;
+
+usrloop:
/*
* Make sure we own this mtx.
*/
@@ -1996,6 +2059,12 @@
error = casueword32(&m->m_owner, owner, &old, new_owner);
if (error == -1)
return (EFAULT);
+ if (error == 1) {
+ error = umtxq_check_susp(td, true);
+ if (error != 0)
+ return (error);
+ goto usrloop;
+ }
if (old == owner)
return (0);
owner = old;
@@ -2075,15 +2144,20 @@
if (count > 1)
new_owner |= UMUTEX_CONTESTED;
+again:
error = casueword32(&m->m_owner, owner, &old, new_owner);
-
+ if (error == 1) {
+ error = umtxq_check_susp(td, true);
+ if (error == 0)
+ goto again;
+ }
umtxq_unbusy_unlocked(&key);
umtx_key_release(&key);
if (error == -1)
return (EFAULT);
- if (old != owner)
+ if (error == 0 && old != owner)
return (EINVAL);
- return (0);
+ return (error);
}
/*
@@ -2149,26 +2223,31 @@
if (rv == -1) {
error = EFAULT;
break;
- }
-
- if (owner == UMUTEX_CONTESTED) {
- error = 0;
- break;
- } else if (owner == UMUTEX_RB_OWNERDEAD) {
- rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD,
- &owner, id | UMUTEX_CONTESTED);
- if (rv == -1) {
- error = EFAULT;
+ } else if (rv == 0) {
+ if (owner == UMUTEX_CONTESTED) {
+ error = 0;
break;
- }
- if (owner == UMUTEX_RB_OWNERDEAD) {
- error = EOWNERDEAD; /* success */
+ } else if (owner == UMUTEX_RB_OWNERDEAD) {
+ rv = casueword32(&m->m_owner,
+ UMUTEX_RB_OWNERDEAD,
+ &owner, id | UMUTEX_CONTESTED);
+ if (rv == -1) {
+ error = EFAULT;
+ break;
+ } else if (rv == 0) {
+ MPASS(owner == UMUTEX_RB_OWNERDEAD);
+ error = EOWNERDEAD; /* success */
+ break;
+ } else if (rv == 1) {
+ /* XXXKIB */
+ }
+ error = 0;
+ } else if (owner == UMUTEX_RB_NOTRECOV) {
+ error = ENOTRECOVERABLE;
break;
}
- error = 0;
- } else if (owner == UMUTEX_RB_NOTRECOV) {
- error = ENOTRECOVERABLE;
- break;
+ } else /* rv == 1 */ {
+ /* XXXKIB */
}
if (try != 0) {
@@ -2624,7 +2703,8 @@
}
static int
-do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx_time *timeout)
+do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag,
+ struct _umtx_time *timeout)
{
struct abs_timeout timo;
struct umtx_q *uq;
@@ -2657,7 +2737,8 @@
/* try to lock it */
while (!(state & wrflags)) {
- if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) {
+ if (__predict_false(URWLOCK_READER_COUNT(state) ==
+ URWLOCK_MAX_READERS)) {
umtx_key_release(&uq->uq_key);
return (EAGAIN);
}
@@ -2667,11 +2748,12 @@
umtx_key_release(&uq->uq_key);
return (EFAULT);
}
- if (oldstate == state) {
+ if (rv == 0) {
+ MPASS(oldstate == state);
umtx_key_release(&uq->uq_key);
return (0);
}
- error = umtxq_check_susp(td);
+ error = umtxq_check_susp(td, true);
if (error != 0)
break;
state = oldstate;
@@ -2702,10 +2784,12 @@
error = EFAULT;
break;
}
- if (oldstate == state)
+ if (rv == 0) {
+ MPASS(oldstate == state);
goto sleep;
+ }
state = oldstate;
- error = umtxq_check_susp(td);
+ error = umtxq_check_susp(td, true);
if (error != 0)
break;
}
@@ -2717,14 +2801,17 @@
/* state is changed while setting flags, restart */
if (!(state & wrflags)) {
umtxq_unbusy_unlocked(&uq->uq_key);
- error = umtxq_check_susp(td);
+ error = umtxq_check_susp(td, true);
if (error != 0)
break;
continue;
}
sleep:
- /* contention bit is set, before sleeping, increase read waiter count */
+ /*
+ * Contention bit is set, before sleeping, increase
+ * read waiter count.
+ */
rv = fueword32(&rwlock->rw_blocked_readers,
&blocked_readers);
if (rv == -1) {
@@ -2777,10 +2864,12 @@
error = EFAULT;
break;
}
- if (oldstate == state)
+ if (rv == 0) {
+ MPASS(oldstate == state);
break;
+ }
state = oldstate;
- error1 = umtxq_check_susp(td);
+ error1 = umtxq_check_susp(td, true);
if (error1 != 0) {
if (error == 0)
error = error1;
@@ -2828,29 +2917,33 @@
umtx_key_release(&uq->uq_key);
return (EFAULT);
}
- while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
+ while ((state & URWLOCK_WRITE_OWNER) == 0 &&
+ URWLOCK_READER_COUNT(state) == 0) {
rv = casueword32(&rwlock->rw_state, state,
&oldstate, state | URWLOCK_WRITE_OWNER);
if (rv == -1) {
umtx_key_release(&uq->uq_key);
return (EFAULT);
}
- if (oldstate == state) {
+ if (rv == 0) {
+ MPASS(oldstate == state);
umtx_key_release(&uq->uq_key);
return (0);
}
state = oldstate;
- error = umtxq_check_susp(td);
+ error = umtxq_check_susp(td, true);
if (error != 0)
break;
}
if (error) {
- if (!(state & (URWLOCK_WRITE_OWNER|URWLOCK_WRITE_WAITERS)) &&
+ if ((state & (URWLOCK_WRITE_OWNER |
+ URWLOCK_WRITE_WAITERS)) == 0 &&
blocked_readers != 0) {
umtxq_lock(&uq->uq_key);
umtxq_busy(&uq->uq_key);
- umtxq_signal_queue(&uq->uq_key, INT_MAX, UMTX_SHARED_QUEUE);
+ umtxq_signal_queue(&uq->uq_key, INT_MAX,
+ UMTX_SHARED_QUEUE);
umtxq_unbusy(&uq->uq_key);
umtxq_unlock(&uq->uq_key);
}
@@ -2864,8 +2957,8 @@
umtxq_unlock(&uq->uq_key);
/*
- * re-read the state, in case it changed between the try-lock above
- * and the check below
+ * Re-read the state, in case it changed between the
+ * try-lock above and the check below.
*/
rv = fueword32(&rwlock->rw_state, &state);
if (rv == -1)
@@ -2880,10 +2973,12 @@
error = EFAULT;
break;
}
- if (oldstate == state)
+ if (rv == 0) {
+ MPASS(oldstate == state);
goto sleep;
+ }
state = oldstate;
- error = umtxq_check_susp(td);
+ error = umtxq_check_susp(td, true);
if (error != 0)
break;
}
@@ -2892,9 +2987,10 @@
break;
}
- if (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
+ if ((state & URWLOCK_WRITE_OWNER) == 0 &&
+ URWLOCK_READER_COUNT(state) == 0) {
umtxq_unbusy_unlocked(&uq->uq_key);
- error = umtxq_check_susp(td);
+ error = umtxq_check_susp(td, false);
if (error != 0)
break;
continue;
@@ -2907,9 +3003,10 @@
error = EFAULT;
break;
}
- suword32(&rwlock->rw_blocked_writers, blocked_writers+1);
+ suword32(&rwlock->rw_blocked_writers, blocked_writers + 1);
- while ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) {
+ while ((state & URWLOCK_WRITE_OWNER) ||
+ URWLOCK_READER_COUNT(state) != 0) {
umtxq_lock(&uq->uq_key);
umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
umtxq_unbusy(&uq->uq_key);
@@ -2951,10 +3048,12 @@
error = EFAULT;
break;
}
- if (oldstate == state)
+ if (rv == 0) {
+ MPASS(oldstate == state);
break;
+ }
state = oldstate;
- error1 = umtxq_check_susp(td);
+ error1 = umtxq_check_susp(td, true);
/*
* We are leaving the URWLOCK_WRITE_WAITERS
* behind, but this should not harm the
@@ -3014,13 +3113,13 @@
error = EFAULT;
goto out;
}
- if (oldstate != state) {
+ if (rv == 1 || oldstate != state) {
state = oldstate;
if (!(oldstate & URWLOCK_WRITE_OWNER)) {
error = EPERM;
goto out;
}
- error = umtxq_check_susp(td);
+ error = umtxq_check_susp(td, true);
if (error != 0)
goto out;
} else
@@ -3034,13 +3133,13 @@
error = EFAULT;
goto out;
}
- if (oldstate != state) {
+ if (rv == 1 || oldstate != state) {
state = oldstate;
if (URWLOCK_READER_COUNT(oldstate) == 0) {
error = EPERM;
goto out;
}
- error = umtxq_check_susp(td);
+ error = umtxq_check_susp(td, true);
if (error != 0)
goto out;
} else
@@ -3090,7 +3189,7 @@
struct abs_timeout timo;
struct umtx_q *uq;
uint32_t flags, count, count1;
- int error, rv;
+ int error, rv, rv1;
uq = td->td_umtxq;
error = fueword32(&sem->_flags, &flags);
@@ -3107,15 +3206,24 @@
umtxq_busy(&uq->uq_key);
umtxq_insert(uq);
umtxq_unlock(&uq->uq_key);
+again:
rv = casueword32(&sem->_has_waiters, 0, &count1, 1);
if (rv == 0)
- rv = fueword32(&sem->_count, &count);
- if (rv == -1 || count != 0) {
+ rv1 = fueword32(&sem->_count, &count);
+ if (rv == -1 || (rv == 0 && (rv1 == -1 || count != 0)) || rv == 1) {
umtxq_lock(&uq->uq_key);
umtxq_unbusy(&uq->uq_key);
umtxq_remove(uq);
umtxq_unlock(&uq->uq_key);
umtx_key_release(&uq->uq_key);
+ if (rv == 1) {
+ rv = umtxq_check_susp(td, true);
+ if (rv == 0)
+ goto again;
+ return (rv);
+ }
+ if (rv == 0)
+ rv = rv1;
return (rv == -1 ? EFAULT : 0);
}
umtxq_lock(&uq->uq_key);
@@ -3194,6 +3302,7 @@
if (timeout != NULL)
abs_timeout_init2(&timo, timeout);
+again:
umtxq_lock(&uq->uq_key);
umtxq_busy(&uq->uq_key);
umtxq_insert(uq);
@@ -3219,13 +3328,18 @@
if (count == USEM_HAS_WAITERS)
break;
rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS);
- if (rv == -1) {
+ if (rv == -1 || rv == 1) {
umtxq_lock(&uq->uq_key);
umtxq_unbusy(&uq->uq_key);
umtxq_remove(uq);
umtxq_unlock(&uq->uq_key);
umtx_key_release(&uq->uq_key);
- return (EFAULT);
+ if (rv == -1)
+ return (EFAULT);
+ rv = umtxq_check_susp(td, true);
+ if (rv != 0)
+ return (rv);
+ goto again;
}
if (count == 0)
break;
@@ -3281,11 +3395,19 @@
if (cnt == 1) {
umtxq_unlock(&key);
rv = fueword32(&sem->_count, &count);
- while (rv != -1 && count & USEM_HAS_WAITERS)
+ while (rv != -1 && count & USEM_HAS_WAITERS) {
rv = casueword32(&sem->_count, count, &count,
count & ~USEM_HAS_WAITERS);
+ if (rv == 1) {
+ rv = umtxq_check_susp(td, true);
+ if (rv != 0)
+ break;
+ }
+ }
if (rv == -1)
error = EFAULT;
+ if (rv > 0)
+ error = rv;
umtxq_lock(&key);
}
Index: sys/mips/mips/support.S
===================================================================
--- sys/mips/mips/support.S
+++ sys/mips/mips/support.S
@@ -336,19 +336,15 @@
GET_CPU_PCPU(v1)
PTR_L v1, PC_CURPCB(v1)
PTR_S v0, U_PCB_ONFAULT(v1)
-1:
+
+ li v0, 1
move t0, a3
ll t1, 0(a0)
- bne a1, t1, 2f
+ bne a1, t1, 1f
nop
sc t0, 0(a0) # store word
- beqz t0, 1b
- nop
- j 3f
- li v0, 0
-2:
- li v0, -1
-3:
+ xori v0, t0, 1
+1:
PTR_S zero, U_PCB_ONFAULT(v1)
jr ra
sw t1, 0(a2) # unconditionally store old word
@@ -363,19 +359,15 @@
GET_CPU_PCPU(v1)
PTR_L v1, PC_CURPCB(v1)
PTR_S v0, U_PCB_ONFAULT(v1)
-1:
+
+ li v0, 1
move t0, a3
lld t1, 0(a0)
- bne a1, t1, 2f
+ bne a1, t1, 1f
nop
scd t0, 0(a0) # store double word
- beqz t0, 1b
- nop
- j 3f
- li v0, 0
-2:
- li v0, -1
-3:
+ xori v0, t0, 1
+1:
PTR_S zero, U_PCB_ONFAULT(v1)
jr ra
sd t1, 0(a2) # unconditionally store old word
Index: sys/powerpc/powerpc/copyinout.c
===================================================================
--- sys/powerpc/powerpc/copyinout.c
+++ sys/powerpc/powerpc/copyinout.c
@@ -433,6 +433,7 @@
pmap_t pm;
jmp_buf env;
uint32_t *p, val;
+ int res;
td = curthread;
pm = &td->td_proc->p_vmspace->vm_pmap;
@@ -449,24 +450,26 @@
return (-1);
}
+ res = 0;
__asm __volatile (
- "1:\tlwarx %0, 0, %2\n\t" /* load old value */
- "cmplw %3, %0\n\t" /* compare */
- "bne 2f\n\t" /* exit if not equal */
- "stwcx. %4, 0, %2\n\t" /* attempt to store */
- "bne- 1b\n\t" /* spin if failed */
- "b 3f\n\t" /* we've succeeded */
+ "lwarx %0, 0, %3\n\t" /* load old value */
+ "cmplw %4, %0\n\t" /* compare */
+ "bne 1f\n\t" /* exit if not equal */
+ "stwcx. %5, 0, %3\n\t" /* attempt to store */
+ "bne- 1f\n\t" /* if failed */
+ "b 2f\n\t" /* we've succeeded */
+ "1:\n\t"
+ "stwcx. %0, 0, %4\n\t" /* clear reservation (74xx) */
+ "li %2, 1\n\t"
"2:\n\t"
- "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
- "3:\n\t"
- : "=&r" (val), "=m" (*p)
+ : "=&r" (val), "=m" (*p), "=&r" (res)
: "r" (p), "r" (old), "r" (new), "m" (*p)
: "cr0", "memory");
td->td_pcb->pcb_onfault = NULL;
*oldvalp = val;
- return (0);
+ return (res);
}
#ifndef __powerpc64__
@@ -485,6 +488,7 @@
pmap_t pm;
jmp_buf env;
u_long *p, val;
+ int res;
td = curthread;
pm = &td->td_proc->p_vmspace->vm_pmap;
@@ -501,23 +505,25 @@
return (-1);
}
+ res = 0;
__asm __volatile (
- "1:\tldarx %0, 0, %2\n\t" /* load old value */
- "cmpld %3, %0\n\t" /* compare */
- "bne 2f\n\t" /* exit if not equal */
- "stdcx. %4, 0, %2\n\t" /* attempt to store */
- "bne- 1b\n\t" /* spin if failed */
- "b 3f\n\t" /* we've succeeded */
+ "ldarx %0, 0, %3\n\t" /* load old value */
+ "cmpld %4, %0\n\t" /* compare */
+ "bne 1f\n\t" /* exit if not equal */
+ "stdcx. %5, 0, %3\n\t" /* attempt to store */
+ "bne- 1f\n\t" /* if failed */
+ "b 2f\n\t" /* we've succeeded */
+ "1:\n\t"
+ "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
+ "li %2, 1\n\t"
"2:\n\t"
- "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
- "3:\n\t"
- : "=&r" (val), "=m" (*p)
+ : "=&r" (val), "=m" (*p), "=&r" (res)
: "r" (p), "r" (old), "r" (new), "m" (*p)
: "cr0", "memory");
td->td_pcb->pcb_onfault = NULL;
*oldvalp = val;
- return (0);
+ return (res);
}
#endif
Index: sys/riscv/riscv/support.S
===================================================================
--- sys/riscv/riscv/support.S
+++ sys/riscv/riscv/support.S
@@ -60,14 +60,15 @@
la a6, fsu_fault /* Load the fault handler */
SET_FAULT_HANDLER(a6, a4) /* And set it */
ENTER_USER_ACCESS(a4)
-1: lr.w a4, 0(a0) /* Load-exclusive the data */
- bne a4, a1, 2f /* If not equal then exit */
+ lr.w a4, 0(a0) /* Load-exclusive the data */
+ bne a4, a1, 1f /* If not equal then exit */
sc.w a5, a3, 0(a0) /* Store the new data */
- bnez a5, 1b /* Retry on failure */
-2: EXIT_USER_ACCESS(a5)
+ beq x0, a5, 1f /* Success */
+ li a5, 1 /* Normalize failure result */
+1: EXIT_USER_ACCESS(a5)
SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */
sw a4, 0(a2) /* Store the read data */
- li a0, 0 /* Success */
+ mv a0, a5 /* Success indicator */
ret /* Return */
END(casueword32)
@@ -80,14 +81,15 @@
la a6, fsu_fault /* Load the fault handler */
SET_FAULT_HANDLER(a6, a4) /* And set it */
ENTER_USER_ACCESS(a4)
-1: lr.d a4, 0(a0) /* Load-exclusive the data */
- bne a4, a1, 2f /* If not equal then exit */
+ lr.d a4, 0(a0) /* Load-exclusive the data */
+ bne a4, a1, 1f /* If not equal then exit */
sc.d a5, a3, 0(a0) /* Store the new data */
- bnez a5, 1b /* Retry on failure */
-2: EXIT_USER_ACCESS(a5)
+ beq x0, a5, 1f /* Success */
+ li a5, 1 /* Normalize failure result */
+1: EXIT_USER_ACCESS(a5)
SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */
sd a4, 0(a2) /* Store the read data */
- li a0, 0 /* Success */
+ mv a0, a5 /* Success indicator */
ret /* Return */
END(casueword)
Index: sys/sparc64/sparc64/support.S
===================================================================
--- sys/sparc64/sparc64/support.S
+++ sys/sparc64/sparc64/support.S
@@ -403,28 +403,27 @@
.set susword, suword16
.set suword, suword64
- .globl casuword32, casuword, fuptr, suptr
- .set casuword, casuword64
+ .globl casuword32_int, casuword64_int, fuptr, suptr
.set fuptr, fuword64
.set suptr, suword64
/*
* int32_t casuword32(volatile int32_t *p, int32_t e, int32_t s)
*/
-ENTRY(casuword32)
+ENTRY(casuword32_int)
casa [%o0] ASI_AIUP, %o1, %o2
retl
mov %o2, %o0
-END(casuword32)
+END(casuword32_int)
/*
* int64_t casuword64(volatile int64_t *p, int64_t e, int64_t s)
*/
-ENTRY(casuword64)
+ENTRY(casuword64_int)
casxa [%o0] ASI_AIUP, %o1, %o2
retl
mov %o2, %o0
-END(casuword64)
+END(casuword64_int)
/*
* int fuword8(const void *base)
Index: sys/sparc64/sparc64/vm_machdep.c
===================================================================
--- sys/sparc64/sparc64/vm_machdep.c
+++ sys/sparc64/sparc64/vm_machdep.c
@@ -462,3 +462,28 @@
pmap_qremove(sf->kva, 1);
return (1);
}
+
+uint32_t casuword32_int(volatile uint32_t *base, uint32_t oldval,
+ uint32_t newval);
+uint32_t
+casuword32(volatile uint32_t *base, uint32_t oldval, uint32_t newval)
+{
+ uint32_t ret;
+
+ ret = casuword32_int(base, oldval, newval);
+ if (ret != -1)
+ ret = ret != oldval;
+ return (ret);
+}
+
+u_long casuword64_int(volatile u_long *p, u_long oldval, u_long newval);
+u_long
+casuword(volatile u_long *p, u_long oldval, u_long newval)
+{
+ u_long ret;
+
+ ret = casuword64_int(p, oldval, newval);
+ if (ret != -1L)
+ ret = ret != oldval;
+ return (ret);
+}

File Metadata

Mime Type
text/plain
Expires
Tue, Dec 23, 1:48 PM (2 h, 48 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
27182436
Default Alt Text
D20772.id59112.diff (27 KB)

Event Timeline