Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_umtx.c
Show First 20 Lines • Show All 685 Lines • ▼ Show 20 Lines | umtxq_count_pi(struct umtx_key *key, struct umtx_q **first) | ||||
if (uh != NULL) { | if (uh != NULL) { | ||||
*first = TAILQ_FIRST(&uh->head); | *first = TAILQ_FIRST(&uh->head); | ||||
return (uh->length); | return (uh->length); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Check for possible stops and suspensions while executing a umtx | * Check for possible stops and suspensions while executing a casueword | ||||
* locking operation. | * operation. | ||||
* | * | ||||
* The sleep argument controls whether the function can handle a stop | * The sleep argument controls whether the function can handle a stop | ||||
* request itself or it should return ERESTART and the request is | * request itself or it should return ERESTART and the request is | ||||
* proceed at the kernel/user boundary in ast. | * proceed at the kernel/user boundary in ast. | ||||
* | * | ||||
* Typically, when retrying due to casueword(9) failure (rv == 1), we | * Typically, when retrying due to casueword(9) failure (rv == 1), we | ||||
* should handle the stop requests there, with exception of cases when | * should handle the stop requests there, with exception of cases when | ||||
* the thread busied the umtx key, or when functions return | * the thread owns a kernel resource, for instance busied the umtx | ||||
* immediately if umtxq_check_susp() returned non-zero. On the other | * key, or when functions return immediately if casueword_check_susp() | ||||
* hand, retrying the whole lock operation, we better not stop there | * returned non-zero. On the other hand, retrying the whole lock | ||||
* but delegate the handling to ast. | * operation, we better not stop there but delegate the handling to | ||||
* ast. | |||||
* | * | ||||
* If the request is for thread termination P_SINGLE_EXIT, we cannot | * If the request is for thread termination P_SINGLE_EXIT, we cannot | ||||
* handle it at all, and simply return EINTR. | * handle it at all, and simply return EINTR. | ||||
*/ | */ | ||||
static int | int | ||||
umtxq_check_susp(struct thread *td, bool sleep) | casueword_check_susp(struct thread *td, bool sleep) | ||||
{ | { | ||||
struct proc *p; | struct proc *p; | ||||
int error; | int error; | ||||
/* | /* | ||||
* The check for TDF_NEEDSUSPCHK is racy, but it is enough to | * The check for TDF_NEEDSUSPCHK is racy, but it is enough to | ||||
* eventually break the lockstep loop. | * eventually break the lockstep loop. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 344 Lines • ▼ Show 20 Lines | if (mode == _UMUTEX_WAIT) { | ||||
id | UMUTEX_CONTESTED); | id | UMUTEX_CONTESTED); | ||||
if (rv == -1) | if (rv == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (rv == 0) { | if (rv == 0) { | ||||
MPASS(owner == UMUTEX_RB_OWNERDEAD); | MPASS(owner == UMUTEX_RB_OWNERDEAD); | ||||
return (EOWNERDEAD); /* success */ | return (EOWNERDEAD); /* success */ | ||||
} | } | ||||
MPASS(rv == 1); | MPASS(rv == 1); | ||||
rv = umtxq_check_susp(td, false); | rv = casueword_check_susp(td, false); | ||||
if (rv != 0) | if (rv != 0) | ||||
return (rv); | return (rv); | ||||
continue; | continue; | ||||
} | } | ||||
if (owner == UMUTEX_RB_NOTRECOV) | if (owner == UMUTEX_RB_NOTRECOV) | ||||
return (ENOTRECOVERABLE); | return (ENOTRECOVERABLE); | ||||
/* | /* | ||||
Show All 24 Lines | if (mode == _UMUTEX_WAIT) { | ||||
/* The address was invalid. */ | /* The address was invalid. */ | ||||
if (rv == -1) | if (rv == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (rv == 0) { | if (rv == 0) { | ||||
MPASS(owner == UMUTEX_CONTESTED); | MPASS(owner == UMUTEX_CONTESTED); | ||||
return (0); | return (0); | ||||
} | } | ||||
if (rv == 1) { | if (rv == 1) { | ||||
rv = umtxq_check_susp(td, false); | rv = casueword_check_susp(td, false); | ||||
if (rv != 0) | if (rv != 0) | ||||
return (rv); | return (rv); | ||||
} | } | ||||
/* | /* | ||||
* If this failed the lock has | * If this failed the lock has | ||||
* changed, restart. | * changed, restart. | ||||
*/ | */ | ||||
continue; | continue; | ||||
} | } | ||||
/* rv == 1 but not contested, likely store failure */ | /* rv == 1 but not contested, likely store failure */ | ||||
rv = umtxq_check_susp(td, false); | rv = casueword_check_susp(td, false); | ||||
if (rv != 0) | if (rv != 0) | ||||
return (rv); | return (rv); | ||||
} | } | ||||
if (mode == _UMUTEX_TRY) | if (mode == _UMUTEX_TRY) | ||||
return (EBUSY); | return (EBUSY); | ||||
/* | /* | ||||
Show All 26 Lines | if (rv == -1 || rv == 1) { | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_remove(uq); | umtxq_remove(uq); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
if (rv == -1) | if (rv == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (rv == 1) { | if (rv == 1) { | ||||
rv = umtxq_check_susp(td, false); | rv = casueword_check_susp(td, false); | ||||
if (rv != 0) | if (rv != 0) | ||||
return (rv); | return (rv); | ||||
} | } | ||||
continue; | continue; | ||||
} | } | ||||
/* | /* | ||||
* We set the contested bit, sleep. Otherwise the lock changed | * We set the contested bit, sleep. Otherwise the lock changed | ||||
* and we need to retry or we lost a race to the thread | * and we need to retry or we lost a race to the thread | ||||
* unlocking the umtx. | * unlocking the umtx. | ||||
*/ | */ | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
MPASS(old == owner); | MPASS(old == owner); | ||||
error = umtxq_sleep(uq, "umtxn", timeout == NULL ? | error = umtxq_sleep(uq, "umtxn", timeout == NULL ? | ||||
NULL : &timo); | NULL : &timo); | ||||
umtxq_remove(uq); | umtxq_remove(uq); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
if (error == 0) | if (error == 0) | ||||
error = umtxq_check_susp(td, false); | error = casueword_check_susp(td, false); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. | * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. | ||||
*/ | */ | ||||
Show All 18 Lines | if ((owner & ~UMUTEX_CONTESTED) != id) | ||||
return (EPERM); | return (EPERM); | ||||
newlock = umtx_unlock_val(flags, rb); | newlock = umtx_unlock_val(flags, rb); | ||||
if ((owner & UMUTEX_CONTESTED) == 0) { | if ((owner & UMUTEX_CONTESTED) == 0) { | ||||
error = casueword32(&m->m_owner, owner, &old, newlock); | error = casueword32(&m->m_owner, owner, &old, newlock); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (error == 1) { | if (error == 1) { | ||||
error = umtxq_check_susp(td, false); | error = casueword_check_susp(td, false); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
goto again; | goto again; | ||||
} | } | ||||
MPASS(old == owner); | MPASS(old == owner); | ||||
return (0); | return (0); | ||||
} | } | ||||
Show All 20 Lines | again: | ||||
umtxq_unbusy(&key); | umtxq_unbusy(&key); | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (error == 1) { | if (error == 1) { | ||||
if (old != owner) | if (old != owner) | ||||
return (EINVAL); | return (EINVAL); | ||||
error = umtxq_check_susp(td, false); | error = casueword_check_susp(td, false); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
goto again; | goto again; | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
Show All 38 Lines | error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, | ||||
UMUTEX_UNOWNED); | UMUTEX_UNOWNED); | ||||
if (error == -1) { | if (error == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
} else if (error == 1) { | } else if (error == 1) { | ||||
umtxq_lock(&key); | umtxq_lock(&key); | ||||
umtxq_unbusy(&key); | umtxq_unbusy(&key); | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
error = umtxq_check_susp(td, false); | error = casueword_check_susp(td, false); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
goto again; | goto again; | ||||
} | } | ||||
} | } | ||||
umtxq_lock(&key); | umtxq_lock(&key); | ||||
if (error == 0 && count != 0) { | if (error == 0 && count != 0) { | ||||
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | if (error == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (error == 0) { | if (error == 0) { | ||||
MPASS(old == owner); | MPASS(old == owner); | ||||
break; | break; | ||||
} | } | ||||
owner = old; | owner = old; | ||||
error = umtxq_check_susp(td, false); | error = casueword_check_susp(td, false); | ||||
} | } | ||||
umtxq_lock(&key); | umtxq_lock(&key); | ||||
if (error == EFAULT) { | if (error == EFAULT) { | ||||
umtxq_signal(&key, INT_MAX); | umtxq_signal(&key, INT_MAX); | ||||
} else if (count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 || | } else if (count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 || | ||||
owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV)) | owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV)) | ||||
umtxq_signal(&key, 1); | umtxq_signal(&key, 1); | ||||
▲ Show 20 Lines • Show All 488 Lines • ▼ Show 20 Lines | if (owner == UMUTEX_RB_NOTRECOV) { | ||||
break; | break; | ||||
} | } | ||||
/* | /* | ||||
* Avoid overwriting a possible error from sleep due | * Avoid overwriting a possible error from sleep due | ||||
* to the pending signal with suspension check result. | * to the pending signal with suspension check result. | ||||
*/ | */ | ||||
if (error == 0) { | if (error == 0) { | ||||
error = umtxq_check_susp(td, true); | error = casueword_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
} | } | ||||
/* If no one owns it but it is contested try to acquire it. */ | /* If no one owns it but it is contested try to acquire it. */ | ||||
if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) { | if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) { | ||||
old_owner = owner; | old_owner = owner; | ||||
rv = casueword32(&m->m_owner, owner, &owner, | rv = casueword32(&m->m_owner, owner, &owner, | ||||
id | UMUTEX_CONTESTED); | id | UMUTEX_CONTESTED); | ||||
/* The address was invalid. */ | /* The address was invalid. */ | ||||
if (rv == -1) { | if (rv == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (rv == 1) { | if (rv == 1) { | ||||
if (error == 0) { | if (error == 0) { | ||||
error = umtxq_check_susp(td, true); | error = casueword_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
} | } | ||||
/* | /* | ||||
* If this failed the lock could | * If this failed the lock could | ||||
* changed, restart. | * changed, restart. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines | for (;;) { | ||||
/* The address was invalid. */ | /* The address was invalid. */ | ||||
if (rv == -1) { | if (rv == -1) { | ||||
umtxq_unbusy_unlocked(&uq->uq_key); | umtxq_unbusy_unlocked(&uq->uq_key); | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (rv == 1) { | if (rv == 1) { | ||||
umtxq_unbusy_unlocked(&uq->uq_key); | umtxq_unbusy_unlocked(&uq->uq_key); | ||||
error = umtxq_check_susp(td, true); | error = casueword_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
/* | /* | ||||
* The lock changed and we need to retry or we | * The lock changed and we need to retry or we | ||||
* lost a race to the thread unlocking the | * lost a race to the thread unlocking the | ||||
* umtx. Note that the UMUTEX_RB_OWNERDEAD | * umtx. Note that the UMUTEX_RB_OWNERDEAD | ||||
* value for owner is impossible there. | * value for owner is impossible there. | ||||
*/ | */ | ||||
continue; | continue; | ||||
} | } | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
/* We set the contested bit, sleep. */ | /* We set the contested bit, sleep. */ | ||||
MPASS(old == owner); | MPASS(old == owner); | ||||
error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, | error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, | ||||
"umtxpi", timeout == NULL ? NULL : &timo, | "umtxpi", timeout == NULL ? NULL : &timo, | ||||
(flags & USYNC_PROCESS_SHARED) != 0); | (flags & USYNC_PROCESS_SHARED) != 0); | ||||
if (error != 0) | if (error != 0) | ||||
continue; | continue; | ||||
error = umtxq_check_susp(td, false); | error = casueword_check_susp(td, false); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
} | } | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtx_pi_unref(pi); | umtx_pi_unref(pi); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
Show All 29 Lines | usrloop: | ||||
new_owner = umtx_unlock_val(flags, rb); | new_owner = umtx_unlock_val(flags, rb); | ||||
/* This should be done in userland */ | /* This should be done in userland */ | ||||
if ((owner & UMUTEX_CONTESTED) == 0) { | if ((owner & UMUTEX_CONTESTED) == 0) { | ||||
error = casueword32(&m->m_owner, owner, &old, new_owner); | error = casueword32(&m->m_owner, owner, &old, new_owner); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (error == 1) { | if (error == 1) { | ||||
error = umtxq_check_susp(td, true); | error = casueword_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
goto usrloop; | goto usrloop; | ||||
} | } | ||||
if (old == owner) | if (old == owner) | ||||
return (0); | return (0); | ||||
owner = old; | owner = old; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | usrloop: | ||||
* Otherwise, it must be marked as contested. | * Otherwise, it must be marked as contested. | ||||
*/ | */ | ||||
if (count > 1) | if (count > 1) | ||||
new_owner |= UMUTEX_CONTESTED; | new_owner |= UMUTEX_CONTESTED; | ||||
again: | again: | ||||
error = casueword32(&m->m_owner, owner, &old, new_owner); | error = casueword32(&m->m_owner, owner, &old, new_owner); | ||||
if (error == 1) { | if (error == 1) { | ||||
error = umtxq_check_susp(td, false); | error = casueword_check_susp(td, false); | ||||
if (error == 0) | if (error == 0) | ||||
goto again; | goto again; | ||||
} | } | ||||
umtxq_unbusy_unlocked(&key); | umtxq_unbusy_unlocked(&key); | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (error == 0 && old != owner) | if (error == 0 && old != owner) | ||||
▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines | if (owner == UMUTEX_RB_OWNERDEAD) { | ||||
* rv == 1, only check for suspension if we | * rv == 1, only check for suspension if we | ||||
* did not already catched a signal. If we | * did not already catched a signal. If we | ||||
* get an error from the check, the same | * get an error from the check, the same | ||||
* condition is checked by the umtxq_sleep() | * condition is checked by the umtxq_sleep() | ||||
* call below, so we should obliterate the | * call below, so we should obliterate the | ||||
* error to not skip the last loop iteration. | * error to not skip the last loop iteration. | ||||
*/ | */ | ||||
if (error == 0) { | if (error == 0) { | ||||
error = umtxq_check_susp(td, false); | error = casueword_check_susp(td, false); | ||||
if (error == 0) { | if (error == 0) { | ||||
if (try != 0) | if (try != 0) | ||||
error = EBUSY; | error = EBUSY; | ||||
else | else | ||||
continue; | continue; | ||||
} | } | ||||
error = 0; | error = 0; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 498 Lines • ▼ Show 20 Lines | while (!(state & wrflags)) { | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (EFAULT); | return (EFAULT); | ||||
} | } | ||||
if (rv == 0) { | if (rv == 0) { | ||||
MPASS(oldstate == state); | MPASS(oldstate == state); | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (0); | return (0); | ||||
} | } | ||||
error = umtxq_check_susp(td, true); | error = casueword_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
state = oldstate; | state = oldstate; | ||||
} | } | ||||
if (error) | if (error) | ||||
break; | break; | ||||
Show All 19 Lines | while (error == 0 && (state & wrflags) && | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (rv == 0) { | if (rv == 0) { | ||||
MPASS(oldstate == state); | MPASS(oldstate == state); | ||||
goto sleep; | goto sleep; | ||||
} | } | ||||
state = oldstate; | state = oldstate; | ||||
error = umtxq_check_susp(td, false); | error = casueword_check_susp(td, false); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
} | } | ||||
if (error != 0) { | if (error != 0) { | ||||
umtxq_unbusy_unlocked(&uq->uq_key); | umtxq_unbusy_unlocked(&uq->uq_key); | ||||
break; | break; | ||||
} | } | ||||
/* state is changed while setting flags, restart */ | /* state is changed while setting flags, restart */ | ||||
if (!(state & wrflags)) { | if (!(state & wrflags)) { | ||||
umtxq_unbusy_unlocked(&uq->uq_key); | umtxq_unbusy_unlocked(&uq->uq_key); | ||||
error = umtxq_check_susp(td, true); | error = casueword_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
continue; | continue; | ||||
} | } | ||||
sleep: | sleep: | ||||
/* | /* | ||||
* Contention bit is set, before sleeping, increase | * Contention bit is set, before sleeping, increase | ||||
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | if (blocked_readers == 1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (rv == 0) { | if (rv == 0) { | ||||
MPASS(oldstate == state); | MPASS(oldstate == state); | ||||
break; | break; | ||||
} | } | ||||
state = oldstate; | state = oldstate; | ||||
error1 = umtxq_check_susp(td, false); | error1 = casueword_check_susp(td, false); | ||||
if (error1 != 0) { | if (error1 != 0) { | ||||
if (error == 0) | if (error == 0) | ||||
error = error1; | error = error1; | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | while ((state & URWLOCK_WRITE_OWNER) == 0 && | ||||
return (EFAULT); | return (EFAULT); | ||||
} | } | ||||
if (rv == 0) { | if (rv == 0) { | ||||
MPASS(oldstate == state); | MPASS(oldstate == state); | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (0); | return (0); | ||||
} | } | ||||
state = oldstate; | state = oldstate; | ||||
error = umtxq_check_susp(td, true); | error = casueword_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
} | } | ||||
if (error) { | if (error) { | ||||
if ((state & (URWLOCK_WRITE_OWNER | | if ((state & (URWLOCK_WRITE_OWNER | | ||||
URWLOCK_WRITE_WAITERS)) == 0 && | URWLOCK_WRITE_WAITERS)) == 0 && | ||||
blocked_readers != 0) { | blocked_readers != 0) { | ||||
Show All 30 Lines | while (error == 0 && ((state & URWLOCK_WRITE_OWNER) || | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (rv == 0) { | if (rv == 0) { | ||||
MPASS(oldstate == state); | MPASS(oldstate == state); | ||||
goto sleep; | goto sleep; | ||||
} | } | ||||
state = oldstate; | state = oldstate; | ||||
error = umtxq_check_susp(td, false); | error = casueword_check_susp(td, false); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
} | } | ||||
if (error != 0) { | if (error != 0) { | ||||
umtxq_unbusy_unlocked(&uq->uq_key); | umtxq_unbusy_unlocked(&uq->uq_key); | ||||
break; | break; | ||||
} | } | ||||
if ((state & URWLOCK_WRITE_OWNER) == 0 && | if ((state & URWLOCK_WRITE_OWNER) == 0 && | ||||
URWLOCK_READER_COUNT(state) == 0) { | URWLOCK_READER_COUNT(state) == 0) { | ||||
umtxq_unbusy_unlocked(&uq->uq_key); | umtxq_unbusy_unlocked(&uq->uq_key); | ||||
error = umtxq_check_susp(td, false); | error = casueword_check_susp(td, false); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
continue; | continue; | ||||
} | } | ||||
sleep: | sleep: | ||||
rv = fueword32(&rwlock->rw_blocked_writers, | rv = fueword32(&rwlock->rw_blocked_writers, | ||||
&blocked_writers); | &blocked_writers); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | if (blocked_writers == 1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (rv == 0) { | if (rv == 0) { | ||||
MPASS(oldstate == state); | MPASS(oldstate == state); | ||||
break; | break; | ||||
} | } | ||||
state = oldstate; | state = oldstate; | ||||
error1 = umtxq_check_susp(td, false); | error1 = casueword_check_susp(td, false); | ||||
/* | /* | ||||
* We are leaving the URWLOCK_WRITE_WAITERS | * We are leaving the URWLOCK_WRITE_WAITERS | ||||
* behind, but this should not harm the | * behind, but this should not harm the | ||||
* correctness. | * correctness. | ||||
*/ | */ | ||||
if (error1 != 0) { | if (error1 != 0) { | ||||
if (error == 0) | if (error == 0) | ||||
error = error1; | error = error1; | ||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | for (;;) { | ||||
goto out; | goto out; | ||||
} | } | ||||
if (rv == 1) { | if (rv == 1) { | ||||
state = oldstate; | state = oldstate; | ||||
if (!(oldstate & URWLOCK_WRITE_OWNER)) { | if (!(oldstate & URWLOCK_WRITE_OWNER)) { | ||||
error = EPERM; | error = EPERM; | ||||
goto out; | goto out; | ||||
} | } | ||||
error = umtxq_check_susp(td, true); | error = casueword_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
goto out; | goto out; | ||||
} else | } else | ||||
break; | break; | ||||
} | } | ||||
} else if (URWLOCK_READER_COUNT(state) != 0) { | } else if (URWLOCK_READER_COUNT(state) != 0) { | ||||
for (;;) { | for (;;) { | ||||
rv = casueword32(&rwlock->rw_state, state, | rv = casueword32(&rwlock->rw_state, state, | ||||
&oldstate, state - 1); | &oldstate, state - 1); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
goto out; | goto out; | ||||
} | } | ||||
if (rv == 1) { | if (rv == 1) { | ||||
state = oldstate; | state = oldstate; | ||||
if (URWLOCK_READER_COUNT(oldstate) == 0) { | if (URWLOCK_READER_COUNT(oldstate) == 0) { | ||||
error = EPERM; | error = EPERM; | ||||
goto out; | goto out; | ||||
} | } | ||||
error = umtxq_check_susp(td, true); | error = casueword_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
goto out; | goto out; | ||||
} else | } else | ||||
break; | break; | ||||
} | } | ||||
} else { | } else { | ||||
error = EPERM; | error = EPERM; | ||||
goto out; | goto out; | ||||
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines | if (rv == 0) | ||||
rv1 = fueword32(&sem->_count, &count); | rv1 = fueword32(&sem->_count, &count); | ||||
if (rv == -1 || (rv == 0 && (rv1 == -1 || count != 0)) || | if (rv == -1 || (rv == 0 && (rv1 == -1 || count != 0)) || | ||||
(rv == 1 && count1 == 0)) { | (rv == 1 && count1 == 0)) { | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
umtxq_remove(uq); | umtxq_remove(uq); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
if (rv == 1) { | if (rv == 1) { | ||||
rv = umtxq_check_susp(td, true); | rv = casueword_check_susp(td, true); | ||||
if (rv == 0) | if (rv == 0) | ||||
goto again; | goto again; | ||||
error = rv; | error = rv; | ||||
goto out; | goto out; | ||||
} | } | ||||
if (rv == 0) | if (rv == 0) | ||||
rv = rv1; | rv = rv1; | ||||
error = rv == -1 ? EFAULT : 0; | error = rv == -1 ? EFAULT : 0; | ||||
▲ Show 20 Lines • Show All 105 Lines • ▼ Show 20 Lines | if (rv == 0) | ||||
break; | break; | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
umtxq_remove(uq); | umtxq_remove(uq); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
if (rv == -1) | if (rv == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
rv = umtxq_check_susp(td, true); | rv = casueword_check_susp(td, true); | ||||
if (rv != 0) | if (rv != 0) | ||||
return (rv); | return (rv); | ||||
goto again; | goto again; | ||||
} | } | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo); | error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo); | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | if (cnt > 0) { | ||||
*/ | */ | ||||
if (cnt == 1) { | if (cnt == 1) { | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
rv = fueword32(&sem->_count, &count); | rv = fueword32(&sem->_count, &count); | ||||
while (rv != -1 && count & USEM_HAS_WAITERS) { | while (rv != -1 && count & USEM_HAS_WAITERS) { | ||||
rv = casueword32(&sem->_count, count, &count, | rv = casueword32(&sem->_count, count, &count, | ||||
count & ~USEM_HAS_WAITERS); | count & ~USEM_HAS_WAITERS); | ||||
if (rv == 1) { | if (rv == 1) { | ||||
rv = umtxq_check_susp(td, true); | rv = casueword_check_susp(td, true); | ||||
if (rv != 0) | if (rv != 0) | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
if (rv == -1) | if (rv == -1) | ||||
error = EFAULT; | error = EFAULT; | ||||
else if (rv > 0) { | else if (rv > 0) { | ||||
error = rv; | error = rv; | ||||
▲ Show 20 Lines • Show All 1,285 Lines • Show Last 20 Lines |