Index: head/share/man/man9/casuword.9 =================================================================== --- head/share/man/man9/casuword.9 +++ head/share/man/man9/casuword.9 @@ -1,4 +1,4 @@ -.\" Copyright (c) 2014 The FreeBSD Foundation +.\" Copyright (c) 2014, 2019 The FreeBSD Foundation .\" All rights reserved. .\" .\" Part of this documentation was written by @@ -28,7 +28,7 @@ .\" .\" $FreeBSD$ .\" -.Dd October 21, 2014 +.Dd April 19, 2019 .Dt CASU 9 .Os .Sh NAME @@ -106,7 +106,9 @@ .Fn casueword and .Fn casueword32 -functions return 0 on success and -1 on failure. +functions return 0 on success, -1 on failure to access memory, +and 1 when comparison or store failed. +The store can fail on load-linked/store-conditional architectures. .Sh SEE ALSO .Xr atomic 9 , .Xr fetch 9 , Index: head/sys/amd64/amd64/support.S =================================================================== --- head/sys/amd64/amd64/support.S +++ head/sys/amd64/amd64/support.S @@ -811,6 +811,7 @@ lock #endif cmpxchgl %ecx,(%rdi) /* new = %ecx */ + setne %cl /* * The old value is in %eax. If the store succeeded it will be the @@ -828,6 +829,7 @@ */ movl %esi,(%rdx) /* oldp = %rdx */ POP_FRAME_POINTER + movzbl %cl, %eax ret END(casueword32_nosmap) @@ -847,6 +849,7 @@ #endif cmpxchgl %ecx,(%rdi) /* new = %ecx */ clac + setne %cl /* * The old value is in %eax. If the store succeeded it will be the @@ -864,6 +867,7 @@ */ movl %esi,(%rdx) /* oldp = %rdx */ POP_FRAME_POINTER + movzbl %cl, %eax ret END(casueword32_smap) @@ -886,6 +890,7 @@ lock #endif cmpxchgq %rcx,(%rdi) /* new = %rcx */ + setne %cl /* * The old value is in %rax. If the store succeeded it will be the @@ -897,6 +902,7 @@ movq %rax,PCB_ONFAULT(%r8) movq %rsi,(%rdx) POP_FRAME_POINTER + movzbl %cl, %eax ret END(casueword_nosmap) @@ -916,6 +922,7 @@ #endif cmpxchgq %rcx,(%rdi) /* new = %rcx */ clac + setne %cl /* * The old value is in %rax. If the store succeeded it will be the @@ -927,6 +934,7 @@ movq %rax,PCB_ONFAULT(%r8) movq %rsi,(%rdx) POP_FRAME_POINTER + movzbl %cl, %eax ret END(casueword_smap) Index: head/sys/arm/arm/fusu.S =================================================================== --- head/sys/arm/arm/fusu.S +++ head/sys/arm/arm/fusu.S @@ -63,7 +63,7 @@ ldr r4, =(VM_MAXUSER_ADDRESS-3) cmp r0, r4 mvncs r0, #0 - bcs 2f + bcs 1f GET_PCB(r6) ldr r6, [r6] @@ -78,12 +78,10 @@ str r4, [r6, #PCB_ONFAULT] #if __ARM_ARCH >= 6 -1: + mov r5, #1 ldrex r4, [r0] cmp r4, r1 strexeq r5, r3, [r0] - cmpeq r5, #1 - beq 1b #else ldrt r4, [r0] cmp r4, r1 @@ -92,7 +90,10 @@ str r4, [r2] mov r0, #0 str r0, [r6, #PCB_ONFAULT] -2: +#if __ARM_ARCH >= 6 + mov r0, r5 +#endif +1: ldmfd sp!, {r4, r5, r6} RET EEND(casueword32) Index: head/sys/arm64/arm64/support.S =================================================================== --- head/sys/arm64/arm64/support.S +++ head/sys/arm64/arm64/support.S @@ -57,17 +57,17 @@ cmp x0, x4 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ + mov w5, #1 SET_FAULT_HANDLER(x6, x4) /* And set it */ ENTER_USER_ACCESS(w6, x4) 1: ldxr w4, [x0] /* Load-exclusive the data */ cmp w4, w1 /* Compare */ b.ne 2f /* Not equal, exit */ stxr w5, w3, [x0] /* Store the new data */ - cbnz w5, 1b /* Retry on failure */ 2: EXIT_USER_ACCESS(w6) - SET_FAULT_HANDLER(xzr, x5) /* Reset the fault handler */ + SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ str w4, [x2] /* Store the read data */ - mov x0, #0 /* Success */ + mov w0, w5 /* Result same as store status */ ret /* Return */ END(casueword32) @@ -79,17 +79,17 @@ cmp x0, x4 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ + mov w5, #1 SET_FAULT_HANDLER(x6, x4) /* And set it */ ENTER_USER_ACCESS(w6, x4) 1: ldxr x4, [x0] /* Load-exclusive the data */ cmp x4, x1 /* Compare */ b.ne 2f /* Not equal, exit */ stxr w5, x3, [x0] /* Store the new data */ - cbnz w5, 1b /* Retry on failure */ 2: EXIT_USER_ACCESS(w6) - SET_FAULT_HANDLER(xzr, x5) /* Reset the fault handler */ + SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ str x4, [x2] /* Store the read data */ - mov x0, #0 /* Success */ + mov w0, w5 /* Result same as store status */ ret /* Return */ END(casueword) Index: head/sys/i386/i386/copyout.c =================================================================== --- head/sys/i386/i386/copyout.c +++ head/sys/i386/i386/copyout.c @@ -428,6 +428,7 @@ struct casueword_arg0 { uint32_t oldval; uint32_t newval; + int res; }; static void @@ -436,7 +437,8 @@ struct casueword_arg0 *ca; ca = arg; - atomic_fcmpset_int((u_int *)kva, &ca->oldval, ca->newval); + ca->res = 1 - atomic_fcmpset_int((u_int *)kva, &ca->oldval, + ca->newval); } int @@ -452,7 +454,7 @@ casueword_slow0, &ca); if (res == 0) { *oldvalp = ca.oldval; - return (0); + return (ca.res); } return (-1); } @@ -469,7 +471,7 @@ casueword_slow0, &ca); if (res == 0) { *oldvalp = ca.oldval; - return (0); + return (ca.res); } return (-1); } Index: head/sys/kern/kern_umtx.c =================================================================== --- head/sys/kern/kern_umtx.c +++ head/sys/kern/kern_umtx.c @@ -690,8 +690,26 @@ return (0); } +/* + * Check for possible stops and suspensions while executing a umtx + * locking operation. + * + * The sleep argument controls whether the function can handle a stop + * request itself or it should return ERESTART and the request is + * proceed at the kernel/user boundary in ast. + * + * Typically, when retrying due to casueword(9) failure (rv == 1), we + * should handle the stop requests there, with exception of cases when + * the thread busied the umtx key, or when functions return + * immediately if umtxq_check_susp() returned non-zero. On the other + * hand, retrying the whole lock operation, we better not stop there + * but delegate the handling to ast. + * + * If the request is for thread termination P_SINGLE_EXIT, we cannot + * handle it at all, and simply return EINTR. + */ static int -umtxq_check_susp(struct thread *td) +umtxq_check_susp(struct thread *td, bool sleep) { struct proc *p; int error; @@ -710,7 +728,7 @@ if (p->p_flag & P_SINGLE_EXIT) error = EINTR; else - error = ERESTART; + error = sleep ? thread_suspend_check(0) : ERESTART; } PROC_UNLOCK(p); return (error); @@ -1049,9 +1067,12 @@ id | UMUTEX_CONTESTED); if (rv == -1) return (EFAULT); - if (owner == UMUTEX_RB_OWNERDEAD) + if (rv == 0) { + MPASS(owner == UMUTEX_RB_OWNERDEAD); return (EOWNERDEAD); /* success */ - rv = umtxq_check_susp(td); + } + MPASS(rv == 1); + rv = umtxq_check_susp(td, false); if (rv != 0) return (rv); continue; @@ -1070,13 +1091,16 @@ return (EFAULT); /* The acquire succeeded. */ - if (owner == UMUTEX_UNOWNED) + if (rv == 0) { + MPASS(owner == UMUTEX_UNOWNED); return (0); + } /* * If no one owns it but it is contested try * to acquire it. */ + MPASS(rv == 1); if (owner == UMUTEX_CONTESTED) { rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, @@ -1084,20 +1108,27 @@ /* The address was invalid. */ if (rv == -1) return (EFAULT); - - if (owner == UMUTEX_CONTESTED) + if (rv == 0) { + MPASS(owner == UMUTEX_CONTESTED); return (0); + } + if (rv == 1) { + rv = umtxq_check_susp(td, false); + if (rv != 0) + return (rv); + } - rv = umtxq_check_susp(td); - if (rv != 0) - return (rv); - /* * If this failed the lock has * changed, restart. */ continue; } + + /* rv == 1 but not contested, likely store failure */ + rv = umtxq_check_susp(td, false); + if (rv != 0) + return (rv); } if (mode == _UMUTEX_TRY) @@ -1128,14 +1159,21 @@ rv = casueword32(&m->m_owner, owner, &old, owner | UMUTEX_CONTESTED); - /* The address was invalid. */ - if (rv == -1) { + /* The address was invalid or casueword failed to store. */ + if (rv == -1 || rv == 1) { umtxq_lock(&uq->uq_key); umtxq_remove(uq); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); - return (EFAULT); + if (rv == -1) + return (EFAULT); + if (rv == 1) { + rv = umtxq_check_susp(td, false); + if (rv != 0) + return (rv); + } + continue; } /* @@ -1145,15 +1183,15 @@ */ umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); - if (old == owner) - error = umtxq_sleep(uq, "umtxn", timeout == NULL ? - NULL : &timo); + MPASS(old == owner); + error = umtxq_sleep(uq, "umtxn", timeout == NULL ? + NULL : &timo); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); if (error == 0) - error = umtxq_check_susp(td); + error = umtxq_check_susp(td, false); } return (0); @@ -1170,6 +1208,8 @@ int error, count; id = td->td_tid; + +again: /* * Make sure we own this mtx. */ @@ -1185,9 +1225,14 @@ error = casueword32(&m->m_owner, owner, &old, newlock); if (error == -1) return (EFAULT); - if (old == owner) - return (0); - owner = old; + if (error == 1) { + error = umtxq_check_susp(td, false); + if (error != 0) + return (error); + goto again; + } + MPASS(old == owner); + return (0); } /* We should only ever be in here for contested locks */ @@ -1215,8 +1260,14 @@ umtx_key_release(&key); if (error == -1) return (EFAULT); - if (old != owner) - return (EINVAL); + if (error == 1) { + if (old != owner) + return (EINVAL); + error = umtxq_check_susp(td, false); + if (error != 0) + return (error); + goto again; + } return (0); } @@ -1233,6 +1284,7 @@ int error; int count; +again: error = fueword32(&m->m_owner, &owner); if (error == -1) return (EFAULT); @@ -1259,14 +1311,27 @@ owner != UMUTEX_RB_NOTRECOV) { error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, UMUTEX_UNOWNED); - if (error == -1) + if (error == -1) { error = EFAULT; + } else if (error == 1) { + umtxq_lock(&key); + umtxq_unbusy(&key); + umtxq_unlock(&key); + umtx_key_release(&key); + error = umtxq_check_susp(td, false); + if (error != 0) + return (error); + goto again; + } } umtxq_lock(&key); - if (error == 0 && count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 || - owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV)) + if (error == 0 && count != 0) { + MPASS((owner & ~UMUTEX_CONTESTED) == 0 || + owner == UMUTEX_RB_OWNERDEAD || + owner == UMUTEX_RB_NOTRECOV); umtxq_signal(&key, 1); + } umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); @@ -1314,49 +1379,32 @@ umtxq_busy(&key); count = umtxq_count(&key); umtxq_unlock(&key); + + error = fueword32(&m->m_owner, &owner); + if (error == -1) + error = EFAULT; + /* - * Only repair contention bit if there is a waiter, this means the mutex - * is still being referenced by userland code, otherwise don't update - * any memory. + * Only repair contention bit if there is a waiter, this means + * the mutex is still being referenced by userland code, + * otherwise don't update any memory. */ - if (count > 1) { - error = fueword32(&m->m_owner, &owner); - if (error == -1) + while (error == 0 && (owner & UMUTEX_CONTESTED) == 0 && + (count > 1 || (count == 1 && (owner & ~UMUTEX_CONTESTED) != 0))) { + error = casueword32(&m->m_owner, owner, &old, + owner | UMUTEX_CONTESTED); + if (error == -1) { error = EFAULT; - while (error == 0 && (owner & UMUTEX_CONTESTED) == 0) { - error = casueword32(&m->m_owner, owner, &old, - owner | UMUTEX_CONTESTED); - if (error == -1) { - error = EFAULT; - break; - } - if (old == owner) - break; - owner = old; - error = umtxq_check_susp(td); - if (error != 0) - break; + break; } - } else if (count == 1) { - error = fueword32(&m->m_owner, &owner); - if (error == -1) - error = EFAULT; - while (error == 0 && (owner & ~UMUTEX_CONTESTED) != 0 && - (owner & UMUTEX_CONTESTED) == 0) { - error = casueword32(&m->m_owner, owner, &old, - owner | UMUTEX_CONTESTED); - if (error == -1) { - error = EFAULT; - break; - } - if (old == owner) - break; - owner = old; - error = umtxq_check_susp(td); - if (error != 0) - break; + if (error == 0) { + MPASS(old == owner); + break; } + owner = old; + error = umtxq_check_susp(td, false); } + umtxq_lock(&key); if (error == EFAULT) { umtxq_signal(&key, INT_MAX); @@ -1842,9 +1890,9 @@ error = EFAULT; break; } - /* The acquire succeeded. */ - if (owner == UMUTEX_UNOWNED) { + if (rv == 0) { + MPASS(owner == UMUTEX_UNOWNED); error = 0; break; } @@ -1854,6 +1902,16 @@ break; } + /* + * Avoid overwriting a possible error from sleep due + * to the pending signal with suspension check result. + */ + if (error == 0) { + error = umtxq_check_susp(td, true); + if (error != 0) + break; + } + /* If no one owns it but it is contested try to acquire it. */ if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) { old_owner = owner; @@ -1864,36 +1922,40 @@ error = EFAULT; break; } - - if (owner == old_owner) { - umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); - error = umtx_pi_claim(pi, td); - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); - if (error != 0) { - /* - * Since we're going to return an - * error, restore the m_owner to its - * previous, unowned state to avoid - * compounding the problem. - */ - (void)casuword32(&m->m_owner, - id | UMUTEX_CONTESTED, - old_owner); + if (rv == 1) { + if (error == 0) { + error = umtxq_check_susp(td, true); + if (error != 0) + return (error); } - if (error == 0 && - old_owner == UMUTEX_RB_OWNERDEAD) - error = EOWNERDEAD; - break; + + /* + * If this failed the lock could + * changed, restart. + */ + continue; } - error = umtxq_check_susp(td); - if (error != 0) - break; - - /* If this failed the lock has changed, restart. */ - continue; + MPASS(rv == 0); + MPASS(owner == old_owner); + umtxq_lock(&uq->uq_key); + umtxq_busy(&uq->uq_key); + error = umtx_pi_claim(pi, td); + umtxq_unbusy(&uq->uq_key); + umtxq_unlock(&uq->uq_key); + if (error != 0) { + /* + * Since we're going to return an + * error, restore the m_owner to its + * previous, unowned state to avoid + * compounding the problem. + */ + (void)casuword32(&m->m_owner, + id | UMUTEX_CONTESTED, old_owner); + } + if (error == 0 && old_owner == UMUTEX_RB_OWNERDEAD) + error = EOWNERDEAD; + break; } if ((owner & ~UMUTEX_CONTESTED) == id) { @@ -1932,28 +1994,33 @@ error = EFAULT; break; } - - umtxq_lock(&uq->uq_key); - /* - * We set the contested bit, sleep. Otherwise the lock changed - * and we need to retry or we lost a race to the thread - * unlocking the umtx. Note that the UMUTEX_RB_OWNERDEAD - * value for owner is impossible there. - */ - if (old == owner) { - error = umtxq_sleep_pi(uq, pi, - owner & ~UMUTEX_CONTESTED, - "umtxpi", timeout == NULL ? NULL : &timo, - (flags & USYNC_PROCESS_SHARED) != 0); + if (rv == 1) { + umtxq_unbusy_unlocked(&uq->uq_key); + error = umtxq_check_susp(td, true); if (error != 0) - continue; - } else { - umtxq_unbusy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + break; + + /* + * The lock changed and we need to retry or we + * lost a race to the thread unlocking the + * umtx. Note that the UMUTEX_RB_OWNERDEAD + * value for owner is impossible there. + */ + continue; } - error = umtxq_check_susp(td); + umtxq_lock(&uq->uq_key); + + /* We set the contested bit, sleep. */ + MPASS(old == owner); + error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, + "umtxpi", timeout == NULL ? NULL : &timo, + (flags & USYNC_PROCESS_SHARED) != 0); if (error != 0) + continue; + + error = umtxq_check_susp(td, false); + if (error != 0) break; } @@ -1978,6 +2045,8 @@ int count, error, pri; id = td->td_tid; + +usrloop: /* * Make sure we own this mtx. */ @@ -1995,6 +2064,12 @@ error = casueword32(&m->m_owner, owner, &old, new_owner); if (error == -1) return (EFAULT); + if (error == 1) { + error = umtxq_check_susp(td, true); + if (error != 0) + return (error); + goto usrloop; + } if (old == owner) return (0); owner = old; @@ -2074,15 +2149,20 @@ if (count > 1) new_owner |= UMUTEX_CONTESTED; +again: error = casueword32(&m->m_owner, owner, &old, new_owner); - + if (error == 1) { + error = umtxq_check_susp(td, false); + if (error == 0) + goto again; + } umtxq_unbusy_unlocked(&key); umtx_key_release(&key); if (error == -1) return (EFAULT); - if (old != owner) + if (error == 0 && old != owner) return (EINVAL); - return (0); + return (error); } /* @@ -2149,31 +2229,49 @@ error = EFAULT; break; } - - if (owner == UMUTEX_CONTESTED) { + if (rv == 0) { + MPASS(owner == UMUTEX_CONTESTED); error = 0; break; - } else if (owner == UMUTEX_RB_OWNERDEAD) { + } + /* rv == 1 */ + if (owner == UMUTEX_RB_OWNERDEAD) { rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD, &owner, id | UMUTEX_CONTESTED); if (rv == -1) { error = EFAULT; break; } - if (owner == UMUTEX_RB_OWNERDEAD) { + if (rv == 0) { + MPASS(owner == UMUTEX_RB_OWNERDEAD); error = EOWNERDEAD; /* success */ break; } - error = 0; + + /* + * rv == 1, only check for suspension if we + * did not already catched a signal. If we + * get an error from the check, the same + * condition is checked by the umtxq_sleep() + * call below, so we should obliterate the + * error to not skip the last loop iteration. + */ + if (error == 0) { + error = umtxq_check_susp(td, false); + if (error == 0) { + if (try != 0) + error = EBUSY; + else + continue; + } + error = 0; + } } else if (owner == UMUTEX_RB_NOTRECOV) { error = ENOTRECOVERABLE; - break; } - if (try != 0) { + if (try != 0) error = EBUSY; - break; - } /* * If we caught a signal, we have retried and now @@ -2668,11 +2766,12 @@ umtx_key_release(&uq->uq_key); return (EFAULT); } - if (oldstate == state) { + if (rv == 0) { + MPASS(oldstate == state); umtx_key_release(&uq->uq_key); return (0); } - error = umtxq_check_susp(td); + error = umtxq_check_susp(td, true); if (error != 0) break; state = oldstate; @@ -2703,10 +2802,12 @@ error = EFAULT; break; } - if (oldstate == state) + if (rv == 0) { + MPASS(oldstate == state); goto sleep; + } state = oldstate; - error = umtxq_check_susp(td); + error = umtxq_check_susp(td, false); if (error != 0) break; } @@ -2718,7 +2819,7 @@ /* state is changed while setting flags, restart */ if (!(state & wrflags)) { umtxq_unbusy_unlocked(&uq->uq_key); - error = umtxq_check_susp(td); + error = umtxq_check_susp(td, true); if (error != 0) break; continue; @@ -2781,10 +2882,12 @@ error = EFAULT; break; } - if (oldstate == state) + if (rv == 0) { + MPASS(oldstate == state); break; + } state = oldstate; - error1 = umtxq_check_susp(td); + error1 = umtxq_check_susp(td, false); if (error1 != 0) { if (error == 0) error = error1; @@ -2840,22 +2943,25 @@ umtx_key_release(&uq->uq_key); return (EFAULT); } - if (oldstate == state) { + if (rv == 0) { + MPASS(oldstate == state); umtx_key_release(&uq->uq_key); return (0); } state = oldstate; - error = umtxq_check_susp(td); + error = umtxq_check_susp(td, true); if (error != 0) break; } if (error) { - if (!(state & (URWLOCK_WRITE_OWNER|URWLOCK_WRITE_WAITERS)) && + if ((state & (URWLOCK_WRITE_OWNER | + URWLOCK_WRITE_WAITERS)) == 0 && blocked_readers != 0) { umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); - umtxq_signal_queue(&uq->uq_key, INT_MAX, UMTX_SHARED_QUEUE); + umtxq_signal_queue(&uq->uq_key, INT_MAX, + UMTX_SHARED_QUEUE); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); } @@ -2885,10 +2991,12 @@ error = EFAULT; break; } - if (oldstate == state) + if (rv == 0) { + MPASS(oldstate == state); goto sleep; + } state = oldstate; - error = umtxq_check_susp(td); + error = umtxq_check_susp(td, false); if (error != 0) break; } @@ -2900,7 +3008,7 @@ if ((state & URWLOCK_WRITE_OWNER) == 0 && URWLOCK_READER_COUNT(state) == 0) { umtxq_unbusy_unlocked(&uq->uq_key); - error = umtxq_check_susp(td); + error = umtxq_check_susp(td, false); if (error != 0) break; continue; @@ -2958,10 +3066,12 @@ error = EFAULT; break; } - if (oldstate == state) + if (rv == 0) { + MPASS(oldstate == state); break; + } state = oldstate; - error1 = umtxq_check_susp(td); + error1 = umtxq_check_susp(td, false); /* * We are leaving the URWLOCK_WRITE_WAITERS * behind, but this should not harm the @@ -3021,13 +3131,13 @@ error = EFAULT; goto out; } - if (oldstate != state) { + if (rv == 1) { state = oldstate; if (!(oldstate & URWLOCK_WRITE_OWNER)) { error = EPERM; goto out; } - error = umtxq_check_susp(td); + error = umtxq_check_susp(td, true); if (error != 0) goto out; } else @@ -3041,13 +3151,13 @@ error = EFAULT; goto out; } - if (oldstate != state) { + if (rv == 1) { state = oldstate; if (URWLOCK_READER_COUNT(oldstate) == 0) { error = EPERM; goto out; } - error = umtxq_check_susp(td); + error = umtxq_check_susp(td, true); if (error != 0) goto out; } else @@ -3097,7 +3207,7 @@ struct abs_timeout timo; struct umtx_q *uq; uint32_t flags, count, count1; - int error, rv; + int error, rv, rv1; uq = td->td_umtxq; error = fueword32(&sem->_flags, &flags); @@ -3110,20 +3220,30 @@ if (timeout != NULL) abs_timeout_init2(&timo, timeout); +again: umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); rv = casueword32(&sem->_has_waiters, 0, &count1, 1); if (rv == 0) - rv = fueword32(&sem->_count, &count); - if (rv == -1 || count != 0) { + rv1 = fueword32(&sem->_count, &count); + if (rv == -1 || (rv == 0 && (rv1 == -1 || count != 0)) || rv == 1) { umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); umtxq_remove(uq); umtxq_unlock(&uq->uq_key); - umtx_key_release(&uq->uq_key); - return (rv == -1 ? EFAULT : 0); + if (rv == 1) { + rv = umtxq_check_susp(td, true); + if (rv == 0) + goto again; + error = rv; + goto out; + } + if (rv == 0) + rv = rv1; + error = rv == -1 ? EFAULT : 0; + goto out; } umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); @@ -3140,6 +3260,7 @@ error = EINTR; } umtxq_unlock(&uq->uq_key); +out: umtx_key_release(&uq->uq_key); return (error); } @@ -3201,6 +3322,7 @@ if (timeout != NULL) abs_timeout_init2(&timo, timeout); +again: umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); umtxq_insert(uq); @@ -3226,16 +3348,19 @@ if (count == USEM_HAS_WAITERS) break; rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS); - if (rv == -1) { - umtxq_lock(&uq->uq_key); - umtxq_unbusy(&uq->uq_key); - umtxq_remove(uq); - umtxq_unlock(&uq->uq_key); - umtx_key_release(&uq->uq_key); - return (EFAULT); - } - if (count == 0) + if (rv == 0) break; + umtxq_lock(&uq->uq_key); + umtxq_unbusy(&uq->uq_key); + umtxq_remove(uq); + umtxq_unlock(&uq->uq_key); + umtx_key_release(&uq->uq_key); + if (rv == -1) + return (EFAULT); + rv = umtxq_check_susp(td, true); + if (rv != 0) + return (rv); + goto again; } umtxq_lock(&uq->uq_key); umtxq_unbusy(&uq->uq_key); @@ -3288,11 +3413,20 @@ if (cnt == 1) { umtxq_unlock(&key); rv = fueword32(&sem->_count, &count); - while (rv != -1 && count & USEM_HAS_WAITERS) + while (rv != -1 && count & USEM_HAS_WAITERS) { rv = casueword32(&sem->_count, count, &count, count & ~USEM_HAS_WAITERS); + if (rv == 1) { + rv = umtxq_check_susp(td, true); + if (rv != 0) + break; + } + } if (rv == -1) error = EFAULT; + else if (rv > 0) { + error = rv; + } umtxq_lock(&key); } Index: head/sys/mips/mips/support.S =================================================================== --- head/sys/mips/mips/support.S +++ head/sys/mips/mips/support.S @@ -336,19 +336,15 @@ GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) -1: + + li v0, 1 move t0, a3 ll t1, 0(a0) - bne a1, t1, 2f + bne a1, t1, 1f nop sc t0, 0(a0) # store word - beqz t0, 1b - nop - j 3f - li v0, 0 -2: - li v0, -1 -3: + xori v0, t0, 1 +1: PTR_S zero, U_PCB_ONFAULT(v1) jr ra sw t1, 0(a2) # unconditionally store old word @@ -363,19 +359,15 @@ GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) -1: + + li v0, 1 move t0, a3 lld t1, 0(a0) - bne a1, t1, 2f + bne a1, t1, 1f nop scd t0, 0(a0) # store double word - beqz t0, 1b - nop - j 3f - li v0, 0 -2: - li v0, -1 -3: + xori v0, t0, 1 +1: PTR_S zero, U_PCB_ONFAULT(v1) jr ra sd t1, 0(a2) # unconditionally store old word Index: head/sys/powerpc/powerpc/copyinout.c =================================================================== --- head/sys/powerpc/powerpc/copyinout.c +++ head/sys/powerpc/powerpc/copyinout.c @@ -433,6 +433,7 @@ pmap_t pm; jmp_buf env; uint32_t *p, val; + int res; td = curthread; pm = &td->td_proc->p_vmspace->vm_pmap; @@ -449,24 +450,26 @@ return (-1); } + res = 0; __asm __volatile ( - "1:\tlwarx %0, 0, %2\n\t" /* load old value */ - "cmplw %3, %0\n\t" /* compare */ - "bne 2f\n\t" /* exit if not equal */ - "stwcx. %4, 0, %2\n\t" /* attempt to store */ - "bne- 1b\n\t" /* spin if failed */ - "b 3f\n\t" /* we've succeeded */ + "lwarx %0, 0, %3\n\t" /* load old value */ + "cmplw %4, %0\n\t" /* compare */ + "bne 1f\n\t" /* exit if not equal */ + "stwcx. %5, 0, %3\n\t" /* attempt to store */ + "bne- 1f\n\t" /* if failed */ + "b 2f\n\t" /* we've succeeded */ + "1:\n\t" + "stwcx. %0, 0, %4\n\t" /* clear reservation (74xx) */ + "li %2, 1\n\t" "2:\n\t" - "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ - "3:\n\t" - : "=&r" (val), "=m" (*p) + : "=&r" (val), "=m" (*p), "=&r" (res) : "r" (p), "r" (old), "r" (new), "m" (*p) : "cr0", "memory"); td->td_pcb->pcb_onfault = NULL; *oldvalp = val; - return (0); + return (res); } #ifndef __powerpc64__ @@ -485,6 +488,7 @@ pmap_t pm; jmp_buf env; u_long *p, val; + int res; td = curthread; pm = &td->td_proc->p_vmspace->vm_pmap; @@ -501,23 +505,25 @@ return (-1); } + res = 0; __asm __volatile ( - "1:\tldarx %0, 0, %2\n\t" /* load old value */ - "cmpld %3, %0\n\t" /* compare */ - "bne 2f\n\t" /* exit if not equal */ - "stdcx. %4, 0, %2\n\t" /* attempt to store */ - "bne- 1b\n\t" /* spin if failed */ - "b 3f\n\t" /* we've succeeded */ + "ldarx %0, 0, %3\n\t" /* load old value */ + "cmpld %4, %0\n\t" /* compare */ + "bne 1f\n\t" /* exit if not equal */ + "stdcx. %5, 0, %3\n\t" /* attempt to store */ + "bne- 1f\n\t" /* if failed */ + "b 2f\n\t" /* we've succeeded */ + "1:\n\t" + "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ + "li %2, 1\n\t" "2:\n\t" - "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ - "3:\n\t" - : "=&r" (val), "=m" (*p) + : "=&r" (val), "=m" (*p), "=&r" (res) : "r" (p), "r" (old), "r" (new), "m" (*p) : "cr0", "memory"); td->td_pcb->pcb_onfault = NULL; *oldvalp = val; - return (0); + return (res); } #endif Index: head/sys/riscv/riscv/support.S =================================================================== --- head/sys/riscv/riscv/support.S +++ head/sys/riscv/riscv/support.S @@ -60,14 +60,15 @@ la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a4) /* And set it */ ENTER_USER_ACCESS(a4) -1: lr.w a4, 0(a0) /* Load-exclusive the data */ - bne a4, a1, 2f /* If not equal then exit */ + lr.w a4, 0(a0) /* Load-exclusive the data */ + bne a4, a1, 1f /* If not equal then exit */ sc.w a5, a3, 0(a0) /* Store the new data */ - bnez a5, 1b /* Retry on failure */ -2: EXIT_USER_ACCESS(a5) - SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */ + beqz a5, 2f /* Success */ +1: li a5, 1 /* Normalize failure result */ +2: EXIT_USER_ACCESS(a6) + SET_FAULT_HANDLER(x0, a6) /* Reset the fault handler */ sw a4, 0(a2) /* Store the read data */ - li a0, 0 /* Success */ + mv a0, a5 /* Success indicator */ ret /* Return */ END(casueword32) @@ -80,14 +81,15 @@ la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a4) /* And set it */ ENTER_USER_ACCESS(a4) -1: lr.d a4, 0(a0) /* Load-exclusive the data */ - bne a4, a1, 2f /* If not equal then exit */ + lr.d a4, 0(a0) /* Load-exclusive the data */ + bne a4, a1, 1f /* If not equal then exit */ sc.d a5, a3, 0(a0) /* Store the new data */ - bnez a5, 1b /* Retry on failure */ -2: EXIT_USER_ACCESS(a5) - SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */ + beqz a5, 2f /* Success */ +1: li a5, 1 /* Normalize failure result */ +2: EXIT_USER_ACCESS(a6) + SET_FAULT_HANDLER(x0, a6) /* Reset the fault handler */ sd a4, 0(a2) /* Store the read data */ - li a0, 0 /* Success */ + mv a0, a5 /* Success indicator */ ret /* Return */ END(casueword) Index: head/sys/sparc64/sparc64/support.S =================================================================== --- head/sys/sparc64/sparc64/support.S +++ head/sys/sparc64/sparc64/support.S @@ -403,28 +403,27 @@ .set susword, suword16 .set suword, suword64 - .globl casuword32, casuword, fuptr, suptr - .set casuword, casuword64 + .globl casuword32_int, casuword64_int, fuptr, suptr .set fuptr, fuword64 .set suptr, suword64 /* * int32_t casuword32(volatile int32_t *p, int32_t e, int32_t s) */ -ENTRY(casuword32) +ENTRY(casuword32_int) casa [%o0] ASI_AIUP, %o1, %o2 retl mov %o2, %o0 -END(casuword32) +END(casuword32_int) /* * int64_t casuword64(volatile int64_t *p, int64_t e, int64_t s) */ -ENTRY(casuword64) +ENTRY(casuword64_int) casxa [%o0] ASI_AIUP, %o1, %o2 retl mov %o2, %o0 -END(casuword64) +END(casuword64_int) /* * int fuword8(const void *base) Index: head/sys/sparc64/sparc64/vm_machdep.c =================================================================== --- head/sys/sparc64/sparc64/vm_machdep.c +++ head/sys/sparc64/sparc64/vm_machdep.c @@ -462,3 +462,28 @@ pmap_qremove(sf->kva, 1); return (1); } + +uint32_t casuword32_int(volatile uint32_t *base, uint32_t oldval, + uint32_t newval); +uint32_t +casuword32(volatile uint32_t *base, uint32_t oldval, uint32_t newval) +{ + uint32_t ret; + + ret = casuword32_int(base, oldval, newval); + if (ret != -1) + ret = ret != oldval; + return (ret); +} + +u_long casuword64_int(volatile u_long *p, u_long oldval, u_long newval); +u_long +casuword(volatile u_long *p, u_long oldval, u_long newval) +{ + u_long ret; + + ret = casuword64_int(p, oldval, newval); + if (ret != -1L) + ret = ret != oldval; + return (ret); +}