Changeset View
Standalone View
sys/kern/kern_umtx.c
Show First 20 Lines • Show All 684 Lines • ▼ Show 20 Lines | umtxq_count_pi(struct umtx_key *key, struct umtx_q **first) | ||||
uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE); | uh = umtxq_queue_lookup(key, UMTX_SHARED_QUEUE); | ||||
if (uh != NULL) { | if (uh != NULL) { | ||||
*first = TAILQ_FIRST(&uh->head); | *first = TAILQ_FIRST(&uh->head); | ||||
return (uh->length); | return (uh->length); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | |||||
* Check for possible stops and suspensions while executing a umtx | |||||
* locking operation. | |||||
markj: s/operations/operation/ | |||||
* | |||||
* The sleep argument controls whether the function can handle a stop | |||||
* request itself or it should return ERESTART and the request is | |||||
* proceed at the kernel/user boundary in ast. | |||||
* | |||||
* Typically, when retrying due to casueword(9) failure (rv == 1), we | |||||
* should handle the stop requests there, with exception of cases when | |||||
* the thread busied the umtx key, or when functions return | |||||
* immediately if umtxq_check_susp() returned non-zero. On the other | |||||
* hand, retrying the whole lock operation, we better not stop there | |||||
* but delegate the handling to ast. | |||||
* | |||||
* If the request is for thread termination P_SINGLE_EXIT, we cannot | |||||
* handle it at all, and simply return EINTR. | |||||
*/ | |||||
static int | static int | ||||
umtxq_check_susp(struct thread *td) | umtxq_check_susp(struct thread *td, bool sleep) | ||||
{ | { | ||||
struct proc *p; | struct proc *p; | ||||
int error; | int error; | ||||
/* | /* | ||||
* The check for TDF_NEEDSUSPCHK is racy, but it is enough to | * The check for TDF_NEEDSUSPCHK is racy, but it is enough to | ||||
* eventually break the lockstep loop. | * eventually break the lockstep loop. | ||||
*/ | */ | ||||
if ((td->td_flags & TDF_NEEDSUSPCHK) == 0) | if ((td->td_flags & TDF_NEEDSUSPCHK) == 0) | ||||
return (0); | return (0); | ||||
error = 0; | error = 0; | ||||
p = td->td_proc; | p = td->td_proc; | ||||
PROC_LOCK(p); | PROC_LOCK(p); | ||||
if (P_SHOULDSTOP(p) || | if (P_SHOULDSTOP(p) || | ||||
((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) { | ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) { | ||||
if (p->p_flag & P_SINGLE_EXIT) | if (p->p_flag & P_SINGLE_EXIT) | ||||
error = EINTR; | error = EINTR; | ||||
else | else | ||||
error = ERESTART; | error = sleep ? thread_suspend_check(0) : ERESTART; | ||||
} | } | ||||
PROC_UNLOCK(p); | PROC_UNLOCK(p); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Wake up threads waiting on an userland object. | * Wake up threads waiting on an userland object. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 322 Lines • ▼ Show 20 Lines | if (mode == _UMUTEX_WAIT) { | ||||
* by the common userspace code. | * by the common userspace code. | ||||
*/ | */ | ||||
if (owner == UMUTEX_RB_OWNERDEAD) { | if (owner == UMUTEX_RB_OWNERDEAD) { | ||||
rv = casueword32(&m->m_owner, | rv = casueword32(&m->m_owner, | ||||
UMUTEX_RB_OWNERDEAD, &owner, | UMUTEX_RB_OWNERDEAD, &owner, | ||||
id | UMUTEX_CONTESTED); | id | UMUTEX_CONTESTED); | ||||
if (rv == -1) | if (rv == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (owner == UMUTEX_RB_OWNERDEAD) | if (rv == 0) { | ||||
MPASS(owner == UMUTEX_RB_OWNERDEAD); | |||||
return (EOWNERDEAD); /* success */ | return (EOWNERDEAD); /* success */ | ||||
rv = umtxq_check_susp(td); | } | ||||
MPASS(rv == 1); | |||||
Done Inline ActionsHow can we have rv != 1 at this point? markj: How can we have rv != 1 at this point? | |||||
rv = umtxq_check_susp(td, false); | |||||
if (rv != 0) | if (rv != 0) | ||||
return (rv); | return (rv); | ||||
continue; | continue; | ||||
} | } | ||||
if (owner == UMUTEX_RB_NOTRECOV) | if (owner == UMUTEX_RB_NOTRECOV) | ||||
return (ENOTRECOVERABLE); | return (ENOTRECOVERABLE); | ||||
/* | /* | ||||
* Try the uncontested case. This should be | * Try the uncontested case. This should be | ||||
* done in userland. | * done in userland. | ||||
*/ | */ | ||||
rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, | rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, | ||||
&owner, id); | &owner, id); | ||||
/* The address was invalid. */ | /* The address was invalid. */ | ||||
if (rv == -1) | if (rv == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
/* The acquire succeeded. */ | /* The acquire succeeded. */ | ||||
if (owner == UMUTEX_UNOWNED) | if (rv == 0) { | ||||
MPASS(owner == UMUTEX_UNOWNED); | |||||
return (0); | return (0); | ||||
} | |||||
/* | /* | ||||
* If no one owns it but it is contested try | * If no one owns it but it is contested try | ||||
* to acquire it. | * to acquire it. | ||||
*/ | */ | ||||
MPASS(rv == 1); | |||||
if (owner == UMUTEX_CONTESTED) { | if (owner == UMUTEX_CONTESTED) { | ||||
Done Inline ActionsI think this block is dead code now. Previously, casueword32 would return 0 even if the comparison failed. Now it returns 1 in that case, but here we know that rv == 0, so it must be that owner == UMUTEX_UNOWNED. Am I missing something? markj: I think this block is dead code now. Previously, casueword32 would return 0 even if the… | |||||
Done Inline ActionsSure, this is the bug. The block must not be dead, I moved the checks for rv == 1 further. We must handle UMUTEX_CONTESTED there. kib: Sure, this is the bug. The block must not be dead, I moved the checks for rv == 1 further. We… | |||||
rv = casueword32(&m->m_owner, | rv = casueword32(&m->m_owner, | ||||
UMUTEX_CONTESTED, &owner, | UMUTEX_CONTESTED, &owner, | ||||
id | UMUTEX_CONTESTED); | id | UMUTEX_CONTESTED); | ||||
/* The address was invalid. */ | /* The address was invalid. */ | ||||
if (rv == -1) | if (rv == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (rv == 0) { | |||||
if (owner == UMUTEX_CONTESTED) | MPASS(owner == UMUTEX_CONTESTED); | ||||
return (0); | return (0); | ||||
} | |||||
rv = umtxq_check_susp(td); | if (rv == 1) { | ||||
rv = umtxq_check_susp(td, false); | |||||
if (rv != 0) | if (rv != 0) | ||||
return (rv); | return (rv); | ||||
} | |||||
/* | /* | ||||
* If this failed the lock has | * If this failed the lock has | ||||
* changed, restart. | * changed, restart. | ||||
*/ | */ | ||||
continue; | continue; | ||||
} | } | ||||
/* rv == 1 but not contested, likely store failure */ | |||||
rv = umtxq_check_susp(td, false); | |||||
if (rv != 0) | |||||
return (rv); | |||||
} | } | ||||
if (mode == _UMUTEX_TRY) | if (mode == _UMUTEX_TRY) | ||||
return (EBUSY); | return (EBUSY); | ||||
/* | /* | ||||
* If we caught a signal, we have retried and now | * If we caught a signal, we have retried and now | ||||
* exit immediately. | * exit immediately. | ||||
Show All 14 Lines | for (;;) { | ||||
* Set the contested bit so that a release in user space | * Set the contested bit so that a release in user space | ||||
* knows to use the system call for unlock. If this fails | * knows to use the system call for unlock. If this fails | ||||
* either some one else has acquired the lock or it has been | * either some one else has acquired the lock or it has been | ||||
* released. | * released. | ||||
*/ | */ | ||||
rv = casueword32(&m->m_owner, owner, &old, | rv = casueword32(&m->m_owner, owner, &old, | ||||
owner | UMUTEX_CONTESTED); | owner | UMUTEX_CONTESTED); | ||||
/* The address was invalid. */ | /* The address was invalid or casueword failed to store. */ | ||||
if (rv == -1) { | if (rv == -1 || rv == 1) { | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_remove(uq); | umtxq_remove(uq); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
if (rv == -1) | |||||
return (EFAULT); | return (EFAULT); | ||||
if (rv == 1) { | |||||
rv = umtxq_check_susp(td, false); | |||||
if (rv != 0) | |||||
return (rv); | |||||
} | } | ||||
continue; | |||||
Done Inline ActionsWhy not just continue? markj: Why not just `continue`? | |||||
} | |||||
/* | /* | ||||
* We set the contested bit, sleep. Otherwise the lock changed | * We set the contested bit, sleep. Otherwise the lock changed | ||||
* and we need to retry or we lost a race to the thread | * and we need to retry or we lost a race to the thread | ||||
* unlocking the umtx. | * unlocking the umtx. | ||||
*/ | */ | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
if (old == owner) | MPASS(old == owner); | ||||
Done Inline ActionsThis condition is always true. markj: This condition is always true. | |||||
error = umtxq_sleep(uq, "umtxn", timeout == NULL ? | error = umtxq_sleep(uq, "umtxn", timeout == NULL ? | ||||
NULL : &timo); | NULL : &timo); | ||||
umtxq_remove(uq); | umtxq_remove(uq); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
if (error == 0) | if (error == 0) | ||||
error = umtxq_check_susp(td); | error = umtxq_check_susp(td, false); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. | * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. | ||||
*/ | */ | ||||
static int | static int | ||||
do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags, bool rb) | do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags, bool rb) | ||||
{ | { | ||||
struct umtx_key key; | struct umtx_key key; | ||||
uint32_t owner, old, id, newlock; | uint32_t owner, old, id, newlock; | ||||
int error, count; | int error, count; | ||||
id = td->td_tid; | id = td->td_tid; | ||||
again: | |||||
/* | /* | ||||
* Make sure we own this mtx. | * Make sure we own this mtx. | ||||
*/ | */ | ||||
error = fueword32(&m->m_owner, &owner); | error = fueword32(&m->m_owner, &owner); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if ((owner & ~UMUTEX_CONTESTED) != id) | if ((owner & ~UMUTEX_CONTESTED) != id) | ||||
return (EPERM); | return (EPERM); | ||||
newlock = umtx_unlock_val(flags, rb); | newlock = umtx_unlock_val(flags, rb); | ||||
if ((owner & UMUTEX_CONTESTED) == 0) { | if ((owner & UMUTEX_CONTESTED) == 0) { | ||||
error = casueword32(&m->m_owner, owner, &old, newlock); | error = casueword32(&m->m_owner, owner, &old, newlock); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (old == owner) | if (error == 1) { | ||||
Done Inline ActionsIt is guaranteed that old == owner here. markj: It is guaranteed that old == owner here. | |||||
error = umtxq_check_susp(td, false); | |||||
if (error != 0) | |||||
return (error); | |||||
goto again; | |||||
} | |||||
MPASS(old == owner); | |||||
return (0); | return (0); | ||||
owner = old; | |||||
} | } | ||||
/* We should only ever be in here for contested locks */ | /* We should only ever be in here for contested locks */ | ||||
if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), | if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), | ||||
&key)) != 0) | &key)) != 0) | ||||
return (error); | return (error); | ||||
umtxq_lock(&key); | umtxq_lock(&key); | ||||
Show All 11 Lines | again: | ||||
error = casueword32(&m->m_owner, owner, &old, newlock); | error = casueword32(&m->m_owner, owner, &old, newlock); | ||||
umtxq_lock(&key); | umtxq_lock(&key); | ||||
umtxq_signal(&key, 1); | umtxq_signal(&key, 1); | ||||
umtxq_unbusy(&key); | umtxq_unbusy(&key); | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (error == 1) { | |||||
if (old != owner) | if (old != owner) | ||||
return (EINVAL); | return (EINVAL); | ||||
error = umtxq_check_susp(td, false); | |||||
if (error != 0) | |||||
return (error); | |||||
goto again; | |||||
} | |||||
Done Inline ActionsIt is guaranteed that old == owner here. markj: It is guaranteed that old == owner here. | |||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Check if the mutex is available and wake up a waiter, | * Check if the mutex is available and wake up a waiter, | ||||
* only for simple mutex. | * only for simple mutex. | ||||
*/ | */ | ||||
static int | static int | ||||
do_wake_umutex(struct thread *td, struct umutex *m) | do_wake_umutex(struct thread *td, struct umutex *m) | ||||
{ | { | ||||
struct umtx_key key; | struct umtx_key key; | ||||
uint32_t owner; | uint32_t owner; | ||||
uint32_t flags; | uint32_t flags; | ||||
int error; | int error; | ||||
int count; | int count; | ||||
again: | |||||
error = fueword32(&m->m_owner, &owner); | error = fueword32(&m->m_owner, &owner); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if ((owner & ~UMUTEX_CONTESTED) != 0 && owner != UMUTEX_RB_OWNERDEAD && | if ((owner & ~UMUTEX_CONTESTED) != 0 && owner != UMUTEX_RB_OWNERDEAD && | ||||
owner != UMUTEX_RB_NOTRECOV) | owner != UMUTEX_RB_NOTRECOV) | ||||
return (0); | return (0); | ||||
Show All 10 Lines | again: | ||||
umtxq_busy(&key); | umtxq_busy(&key); | ||||
count = umtxq_count(&key); | count = umtxq_count(&key); | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
if (count <= 1 && owner != UMUTEX_RB_OWNERDEAD && | if (count <= 1 && owner != UMUTEX_RB_OWNERDEAD && | ||||
owner != UMUTEX_RB_NOTRECOV) { | owner != UMUTEX_RB_NOTRECOV) { | ||||
error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, | error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, | ||||
UMUTEX_UNOWNED); | UMUTEX_UNOWNED); | ||||
if (error == -1) | if (error == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
} else if (error == 1) { | |||||
umtxq_lock(&key); | |||||
umtxq_unbusy(&key); | |||||
umtxq_unlock(&key); | |||||
umtx_key_release(&key); | |||||
error = umtxq_check_susp(td, false); | |||||
if (error != 0) | |||||
return (error); | |||||
goto again; | |||||
} | } | ||||
} | |||||
umtxq_lock(&key); | umtxq_lock(&key); | ||||
if (error == 0 && count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 || | if (error == 0 && count != 0) { | ||||
owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV)) | MPASS((owner & ~UMUTEX_CONTESTED) == 0 || | ||||
owner == UMUTEX_RB_OWNERDEAD || | |||||
owner == UMUTEX_RB_NOTRECOV); | |||||
Done Inline ActionsIf error == 0 then we must have owner == UMUTEX_CONTESTED, so (owner & ~UMUTEX_CONTESTED) == 0 is always true. markj: If error == 0 then we must have owner == UMUTEX_CONTESTED, so (owner & ~UMUTEX_CONTESTED) == 0… | |||||
Done Inline Actionserror == 0 case can happen when count > 1, in which case we did not executed casueword32() at all. And then either owner == UMUTEX_CONTESTED or this is a dying robust mutex. I think it would be cleaner just to move the part after second '&&' into MPASS(). kib: error == 0 case can happen when count > 1, in which case we did not executed casueword32() at… | |||||
umtxq_signal(&key, 1); | umtxq_signal(&key, 1); | ||||
} | |||||
umtxq_unbusy(&key); | umtxq_unbusy(&key); | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Check if the mutex has waiters and tries to fix contention bit. | * Check if the mutex has waiters and tries to fix contention bit. | ||||
Show All 31 Lines | do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags) | ||||
if ((error = umtx_key_get(m, type, GET_SHARE(flags), &key)) != 0) | if ((error = umtx_key_get(m, type, GET_SHARE(flags), &key)) != 0) | ||||
return (error); | return (error); | ||||
owner = 0; | owner = 0; | ||||
umtxq_lock(&key); | umtxq_lock(&key); | ||||
umtxq_busy(&key); | umtxq_busy(&key); | ||||
count = umtxq_count(&key); | count = umtxq_count(&key); | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
/* | |||||
* Only repair contention bit if there is a waiter, this means the mutex | |||||
* is still being referenced by userland code, otherwise don't update | |||||
* any memory. | |||||
*/ | |||||
if (count > 1) { | |||||
error = fueword32(&m->m_owner, &owner); | error = fueword32(&m->m_owner, &owner); | ||||
if (error == -1) | if (error == -1) | ||||
error = EFAULT; | error = EFAULT; | ||||
while (error == 0 && (owner & UMUTEX_CONTESTED) == 0) { | |||||
/* | |||||
* Only repair contention bit if there is a waiter, this means | |||||
* the mutex is still being referenced by userland code, | |||||
* otherwise don't update any memory. | |||||
*/ | |||||
while (error == 0 && (owner & UMUTEX_CONTESTED) == 0 && | |||||
(count > 1 || (count == 1 && (owner & ~UMUTEX_CONTESTED) != 0))) { | |||||
error = casueword32(&m->m_owner, owner, &old, | error = casueword32(&m->m_owner, owner, &old, | ||||
owner | UMUTEX_CONTESTED); | owner | UMUTEX_CONTESTED); | ||||
if (error == -1) { | if (error == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
Done Inline ActionsIf error == 0 we know that old == owner. markj: If error == 0 we know that old == owner. | |||||
} | } | ||||
if (old == owner) | if (error == 0) { | ||||
MPASS(old == owner); | |||||
break; | break; | ||||
owner = old; | |||||
error = umtxq_check_susp(td); | |||||
if (error != 0) | |||||
break; | |||||
} | } | ||||
} else if (count == 1) { | |||||
error = fueword32(&m->m_owner, &owner); | |||||
if (error == -1) | |||||
error = EFAULT; | |||||
while (error == 0 && (owner & ~UMUTEX_CONTESTED) != 0 && | |||||
(owner & UMUTEX_CONTESTED) == 0) { | |||||
error = casueword32(&m->m_owner, owner, &old, | |||||
owner | UMUTEX_CONTESTED); | |||||
if (error == -1) { | |||||
error = EFAULT; | |||||
break; | |||||
} | |||||
if (old == owner) | |||||
break; | |||||
owner = old; | owner = old; | ||||
error = umtxq_check_susp(td); | error = umtxq_check_susp(td, false); | ||||
if (error != 0) | |||||
break; | |||||
} | } | ||||
} | |||||
umtxq_lock(&key); | umtxq_lock(&key); | ||||
if (error == EFAULT) { | if (error == EFAULT) { | ||||
umtxq_signal(&key, INT_MAX); | umtxq_signal(&key, INT_MAX); | ||||
} else if (count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 || | } else if (count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 || | ||||
owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV)) | owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV)) | ||||
umtxq_signal(&key, 1); | umtxq_signal(&key, 1); | ||||
umtxq_unbusy(&key); | umtxq_unbusy(&key); | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
▲ Show 20 Lines • Show All 469 Lines • ▼ Show 20 Lines | for (;;) { | ||||
* Try the uncontested case. This should be done in userland. | * Try the uncontested case. This should be done in userland. | ||||
*/ | */ | ||||
rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id); | rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id); | ||||
/* The address was invalid. */ | /* The address was invalid. */ | ||||
if (rv == -1) { | if (rv == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
/* The acquire succeeded. */ | /* The acquire succeeded. */ | ||||
if (owner == UMUTEX_UNOWNED) { | if (rv == 0) { | ||||
MPASS(owner == UMUTEX_UNOWNED); | |||||
Done Inline ActionsWhat if rv == 1? markj: What if rv == 1? | |||||
error = 0; | error = 0; | ||||
break; | break; | ||||
} | } | ||||
if (owner == UMUTEX_RB_NOTRECOV) { | if (owner == UMUTEX_RB_NOTRECOV) { | ||||
error = ENOTRECOVERABLE; | error = ENOTRECOVERABLE; | ||||
break; | break; | ||||
} | } | ||||
/* | |||||
* Avoid overwriting a possible error from sleep due | |||||
* to the pending singal with suspension check result. | |||||
markjUnsubmitted Not Done Inline ActionsTypo: signal markj: Typo: signal | |||||
*/ | |||||
if (error == 0) { | |||||
error = umtxq_check_susp(td, true); | |||||
if (error != 0) | |||||
break; | |||||
} | |||||
/* If no one owns it but it is contested try to acquire it. */ | /* If no one owns it but it is contested try to acquire it. */ | ||||
if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) { | if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) { | ||||
old_owner = owner; | old_owner = owner; | ||||
rv = casueword32(&m->m_owner, owner, &owner, | rv = casueword32(&m->m_owner, owner, &owner, | ||||
id | UMUTEX_CONTESTED); | id | UMUTEX_CONTESTED); | ||||
/* The address was invalid. */ | /* The address was invalid. */ | ||||
if (rv == -1) { | if (rv == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (rv == 1) { | |||||
if (error == 0) { | |||||
error = umtxq_check_susp(td, true); | |||||
if (error != 0) | |||||
return (error); | |||||
} | |||||
if (owner == old_owner) { | /* | ||||
Done Inline ActionsThis code is unreachable. markj: This code is unreachable. | |||||
* If this failed the lock could | |||||
* changed, restart. | |||||
*/ | |||||
continue; | |||||
} | |||||
MPASS(rv == 0); | |||||
MPASS(owner == old_owner); | |||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_busy(&uq->uq_key); | umtxq_busy(&uq->uq_key); | ||||
error = umtx_pi_claim(pi, td); | error = umtx_pi_claim(pi, td); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
if (error != 0) { | if (error != 0) { | ||||
/* | /* | ||||
* Since we're going to return an | * Since we're going to return an | ||||
* error, restore the m_owner to its | * error, restore the m_owner to its | ||||
* previous, unowned state to avoid | * previous, unowned state to avoid | ||||
* compounding the problem. | * compounding the problem. | ||||
*/ | */ | ||||
(void)casuword32(&m->m_owner, | (void)casuword32(&m->m_owner, | ||||
id | UMUTEX_CONTESTED, | id | UMUTEX_CONTESTED, old_owner); | ||||
old_owner); | |||||
} | } | ||||
if (error == 0 && | if (error == 0 && old_owner == UMUTEX_RB_OWNERDEAD) | ||||
old_owner == UMUTEX_RB_OWNERDEAD) | |||||
error = EOWNERDEAD; | error = EOWNERDEAD; | ||||
break; | break; | ||||
} | } | ||||
error = umtxq_check_susp(td); | |||||
if (error != 0) | |||||
break; | |||||
/* If this failed the lock has changed, restart. */ | |||||
continue; | |||||
} | |||||
if ((owner & ~UMUTEX_CONTESTED) == id) { | if ((owner & ~UMUTEX_CONTESTED) == id) { | ||||
error = EDEADLK; | error = EDEADLK; | ||||
break; | break; | ||||
} | } | ||||
if (try != 0) { | if (try != 0) { | ||||
error = EBUSY; | error = EBUSY; | ||||
break; | break; | ||||
Show All 19 Lines | for (;;) { | ||||
rv = casueword32(&m->m_owner, owner, &old, owner | | rv = casueword32(&m->m_owner, owner, &old, owner | | ||||
UMUTEX_CONTESTED); | UMUTEX_CONTESTED); | ||||
/* The address was invalid. */ | /* The address was invalid. */ | ||||
if (rv == -1) { | if (rv == -1) { | ||||
umtxq_unbusy_unlocked(&uq->uq_key); | umtxq_unbusy_unlocked(&uq->uq_key); | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (rv == 1) { | |||||
Done Inline ActionsThis condition must be true. markj: This condition must be true. | |||||
umtxq_unbusy_unlocked(&uq->uq_key); | |||||
error = umtxq_check_susp(td, true); | |||||
if (error != 0) | |||||
break; | |||||
umtxq_lock(&uq->uq_key); | |||||
/* | /* | ||||
* We set the contested bit, sleep. Otherwise the lock changed | * The lock changed and we need to retry or we | ||||
* and we need to retry or we lost a race to the thread | * lost a race to the thread unlocking the | ||||
* unlocking the umtx. Note that the UMUTEX_RB_OWNERDEAD | * umtx. Note that the UMUTEX_RB_OWNERDEAD | ||||
* value for owner is impossible there. | * value for owner is impossible there. | ||||
*/ | */ | ||||
if (old == owner) { | continue; | ||||
error = umtxq_sleep_pi(uq, pi, | } | ||||
owner & ~UMUTEX_CONTESTED, | |||||
umtxq_lock(&uq->uq_key); | |||||
/* We set the contested bit, sleep. */ | |||||
MPASS(old == owner); | |||||
error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, | |||||
"umtxpi", timeout == NULL ? NULL : &timo, | "umtxpi", timeout == NULL ? NULL : &timo, | ||||
(flags & USYNC_PROCESS_SHARED) != 0); | (flags & USYNC_PROCESS_SHARED) != 0); | ||||
if (error != 0) | if (error != 0) | ||||
continue; | continue; | ||||
} else { | |||||
umtxq_unbusy(&uq->uq_key); | |||||
umtxq_unlock(&uq->uq_key); | |||||
} | |||||
error = umtxq_check_susp(td); | error = umtxq_check_susp(td, false); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
} | } | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtx_pi_unref(pi); | umtx_pi_unref(pi); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
Show All 9 Lines | |||||
{ | { | ||||
struct umtx_key key; | struct umtx_key key; | ||||
struct umtx_q *uq_first, *uq_first2, *uq_me; | struct umtx_q *uq_first, *uq_first2, *uq_me; | ||||
struct umtx_pi *pi, *pi2; | struct umtx_pi *pi, *pi2; | ||||
uint32_t id, new_owner, old, owner; | uint32_t id, new_owner, old, owner; | ||||
int count, error, pri; | int count, error, pri; | ||||
id = td->td_tid; | id = td->td_tid; | ||||
usrloop: | |||||
/* | /* | ||||
* Make sure we own this mtx. | * Make sure we own this mtx. | ||||
*/ | */ | ||||
error = fueword32(&m->m_owner, &owner); | error = fueword32(&m->m_owner, &owner); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if ((owner & ~UMUTEX_CONTESTED) != id) | if ((owner & ~UMUTEX_CONTESTED) != id) | ||||
return (EPERM); | return (EPERM); | ||||
new_owner = umtx_unlock_val(flags, rb); | new_owner = umtx_unlock_val(flags, rb); | ||||
/* This should be done in userland */ | /* This should be done in userland */ | ||||
if ((owner & UMUTEX_CONTESTED) == 0) { | if ((owner & UMUTEX_CONTESTED) == 0) { | ||||
error = casueword32(&m->m_owner, owner, &old, new_owner); | error = casueword32(&m->m_owner, owner, &old, new_owner); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (error == 1) { | |||||
error = umtxq_check_susp(td, true); | |||||
if (error != 0) | |||||
return (error); | |||||
goto usrloop; | |||||
} | |||||
if (old == owner) | if (old == owner) | ||||
return (0); | return (0); | ||||
owner = old; | owner = old; | ||||
} | } | ||||
/* We should only ever be in here for contested locks */ | /* We should only ever be in here for contested locks */ | ||||
if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? | if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? | ||||
TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), | TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), | ||||
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | usrloop: | ||||
/* | /* | ||||
* When unlocking the umtx, it must be marked as unowned if | * When unlocking the umtx, it must be marked as unowned if | ||||
* there is zero or one thread only waiting for it. | * there is zero or one thread only waiting for it. | ||||
* Otherwise, it must be marked as contested. | * Otherwise, it must be marked as contested. | ||||
*/ | */ | ||||
if (count > 1) | if (count > 1) | ||||
new_owner |= UMUTEX_CONTESTED; | new_owner |= UMUTEX_CONTESTED; | ||||
again: | |||||
error = casueword32(&m->m_owner, owner, &old, new_owner); | error = casueword32(&m->m_owner, owner, &old, new_owner); | ||||
if (error == 1) { | |||||
error = umtxq_check_susp(td, false); | |||||
Done Inline ActionsThe key is busied here, so we cannot sleep. markj: The key is busied here, so we cannot sleep. | |||||
if (error == 0) | |||||
goto again; | |||||
} | |||||
umtxq_unbusy_unlocked(&key); | umtxq_unbusy_unlocked(&key); | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (old != owner) | if (error == 0 && old != owner) | ||||
return (EINVAL); | return (EINVAL); | ||||
return (0); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Lock a PP mutex. | * Lock a PP mutex. | ||||
*/ | */ | ||||
static int | static int | ||||
do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, | do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, | ||||
struct _umtx_time *timeout, int try) | struct _umtx_time *timeout, int try) | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | for (;;) { | ||||
rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, | rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, | ||||
id | UMUTEX_CONTESTED); | id | UMUTEX_CONTESTED); | ||||
/* The address was invalid. */ | /* The address was invalid. */ | ||||
if (rv == -1) { | if (rv == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (rv == 0) { | |||||
if (owner == UMUTEX_CONTESTED) { | MPASS(owner == UMUTEX_CONTESTED); | ||||
error = 0; | error = 0; | ||||
break; | break; | ||||
} else if (owner == UMUTEX_RB_OWNERDEAD) { | } | ||||
/* rv == 1 */ | |||||
if (owner == UMUTEX_RB_OWNERDEAD) { | |||||
rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD, | rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD, | ||||
&owner, id | UMUTEX_CONTESTED); | &owner, id | UMUTEX_CONTESTED); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
Done Inline ActionsWhitespace looks wrong here. markj: Whitespace looks wrong here. | |||||
break; | break; | ||||
Done Inline ActionsIf rv == 0 we must have owner == UMUTEX_CONTESTED. markj: If rv == 0 we must have owner == UMUTEX_CONTESTED. | |||||
Done Inline ActionsI re-wrote the logic for rv == 1 completely. kib: I re-wrote the logic for rv == 1 completely. | |||||
} | } | ||||
if (owner == UMUTEX_RB_OWNERDEAD) { | if (rv == 0) { | ||||
MPASS(owner == UMUTEX_RB_OWNERDEAD); | |||||
error = EOWNERDEAD; /* success */ | error = EOWNERDEAD; /* success */ | ||||
break; | break; | ||||
} | } | ||||
/* | |||||
* rv == 1, only check for suspension if we | |||||
* did not already catched a signal. If we | |||||
* get an error from the check, the same | |||||
* condition is checked by the umtxq_sleep() | |||||
* call below, so we should obliterate the | |||||
* error to not skip the last loop iteration. | |||||
*/ | |||||
if (error == 0) { | |||||
error = umtxq_check_susp(td, false); | |||||
if (error == 0) { | |||||
if (try != 0) | |||||
error = EBUSY; | |||||
else | |||||
continue; | |||||
} | |||||
error = 0; | error = 0; | ||||
} | |||||
} else if (owner == UMUTEX_RB_NOTRECOV) { | } else if (owner == UMUTEX_RB_NOTRECOV) { | ||||
error = ENOTRECOVERABLE; | error = ENOTRECOVERABLE; | ||||
break; | |||||
} | } | ||||
if (try != 0) { | if (try != 0) | ||||
error = EBUSY; | error = EBUSY; | ||||
break; | |||||
} | |||||
/* | /* | ||||
* If we caught a signal, we have retried and now | * If we caught a signal, we have retried and now | ||||
* exit immediately. | * exit immediately. | ||||
*/ | */ | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
▲ Show 20 Lines • Show All 434 Lines • ▼ Show 20 Lines | do_cv_broadcast(struct thread *td, struct ucond *cv) | ||||
umtxq_unbusy_unlocked(&key); | umtxq_unbusy_unlocked(&key); | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, | do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, | ||||
struct _umtx_time *timeout) | struct _umtx_time *timeout) | ||||
Done Inline Actionsunrelated style change should probably be a separate commit (also the only conflict against CheriBSD) brooks: unrelated style change should probably be a separate commit (also the only conflict against… | |||||
{ | { | ||||
struct abs_timeout timo; | struct abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
uint32_t flags, wrflags; | uint32_t flags, wrflags; | ||||
int32_t state, oldstate; | int32_t state, oldstate; | ||||
int32_t blocked_readers; | int32_t blocked_readers; | ||||
int error, error1, rv; | int error, error1, rv; | ||||
Show All 27 Lines | while (!(state & wrflags)) { | ||||
return (EAGAIN); | return (EAGAIN); | ||||
} | } | ||||
rv = casueword32(&rwlock->rw_state, state, | rv = casueword32(&rwlock->rw_state, state, | ||||
&oldstate, state + 1); | &oldstate, state + 1); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (EFAULT); | return (EFAULT); | ||||
} | } | ||||
if (oldstate == state) { | if (rv == 0) { | ||||
MPASS(oldstate == state); | |||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (0); | return (0); | ||||
} | } | ||||
error = umtxq_check_susp(td); | error = umtxq_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
state = oldstate; | state = oldstate; | ||||
} | } | ||||
if (error) | if (error) | ||||
break; | break; | ||||
Show All 14 Lines | for (;;) { | ||||
while (error == 0 && (state & wrflags) && | while (error == 0 && (state & wrflags) && | ||||
!(state & URWLOCK_READ_WAITERS)) { | !(state & URWLOCK_READ_WAITERS)) { | ||||
rv = casueword32(&rwlock->rw_state, state, | rv = casueword32(&rwlock->rw_state, state, | ||||
&oldstate, state | URWLOCK_READ_WAITERS); | &oldstate, state | URWLOCK_READ_WAITERS); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (oldstate == state) | if (rv == 0) { | ||||
MPASS(oldstate == state); | |||||
goto sleep; | goto sleep; | ||||
} | |||||
state = oldstate; | state = oldstate; | ||||
error = umtxq_check_susp(td); | error = umtxq_check_susp(td, false); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
} | } | ||||
if (error != 0) { | if (error != 0) { | ||||
umtxq_unbusy_unlocked(&uq->uq_key); | umtxq_unbusy_unlocked(&uq->uq_key); | ||||
break; | break; | ||||
} | } | ||||
/* state is changed while setting flags, restart */ | /* state is changed while setting flags, restart */ | ||||
if (!(state & wrflags)) { | if (!(state & wrflags)) { | ||||
umtxq_unbusy_unlocked(&uq->uq_key); | umtxq_unbusy_unlocked(&uq->uq_key); | ||||
error = umtxq_check_susp(td); | error = umtxq_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
continue; | continue; | ||||
} | } | ||||
sleep: | sleep: | ||||
/* | /* | ||||
* Contention bit is set, before sleeping, increase | * Contention bit is set, before sleeping, increase | ||||
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | if (blocked_readers == 1) { | ||||
} | } | ||||
for (;;) { | for (;;) { | ||||
rv = casueword32(&rwlock->rw_state, state, | rv = casueword32(&rwlock->rw_state, state, | ||||
&oldstate, state & ~URWLOCK_READ_WAITERS); | &oldstate, state & ~URWLOCK_READ_WAITERS); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (oldstate == state) | if (rv == 0) { | ||||
MPASS(oldstate == state); | |||||
break; | break; | ||||
} | |||||
state = oldstate; | state = oldstate; | ||||
error1 = umtxq_check_susp(td); | error1 = umtxq_check_susp(td, false); | ||||
if (error1 != 0) { | if (error1 != 0) { | ||||
if (error == 0) | if (error == 0) | ||||
error = error1; | error = error1; | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
Show All 32 Lines | do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeout) | ||||
blocked_readers = 0; | blocked_readers = 0; | ||||
for (;;) { | for (;;) { | ||||
rv = fueword32(&rwlock->rw_state, &state); | rv = fueword32(&rwlock->rw_state, &state); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (EFAULT); | return (EFAULT); | ||||
} | } | ||||
while ((state & URWLOCK_WRITE_OWNER) == 0 && | while ((state & URWLOCK_WRITE_OWNER) == 0 && | ||||
URWLOCK_READER_COUNT(state) == 0) { | URWLOCK_READER_COUNT(state) == 0) { | ||||
Done Inline ActionsThe line wraps could be committed separately. markj: The line wraps could be committed separately. | |||||
rv = casueword32(&rwlock->rw_state, state, | rv = casueword32(&rwlock->rw_state, state, | ||||
&oldstate, state | URWLOCK_WRITE_OWNER); | &oldstate, state | URWLOCK_WRITE_OWNER); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (EFAULT); | return (EFAULT); | ||||
} | } | ||||
if (oldstate == state) { | if (rv == 0) { | ||||
MPASS(oldstate == state); | |||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (0); | return (0); | ||||
} | } | ||||
state = oldstate; | state = oldstate; | ||||
error = umtxq_check_susp(td); | error = umtxq_check_susp(td, true); | ||||
Not Done Inline ActionsIt seems inconsistent that we sleep here. markj: It seems inconsistent that we sleep here. | |||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
} | } | ||||
if (error) { | if (error) { | ||||
if (!(state & (URWLOCK_WRITE_OWNER|URWLOCK_WRITE_WAITERS)) && | if ((state & (URWLOCK_WRITE_OWNER | | ||||
URWLOCK_WRITE_WAITERS)) == 0 && | |||||
blocked_readers != 0) { | blocked_readers != 0) { | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_busy(&uq->uq_key); | umtxq_busy(&uq->uq_key); | ||||
umtxq_signal_queue(&uq->uq_key, INT_MAX, UMTX_SHARED_QUEUE); | umtxq_signal_queue(&uq->uq_key, INT_MAX, | ||||
UMTX_SHARED_QUEUE); | |||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
} | } | ||||
break; | break; | ||||
} | } | ||||
/* grab monitor lock */ | /* grab monitor lock */ | ||||
Show All 13 Lines | while (error == 0 && ((state & URWLOCK_WRITE_OWNER) || | ||||
URWLOCK_READER_COUNT(state) != 0) && | URWLOCK_READER_COUNT(state) != 0) && | ||||
(state & URWLOCK_WRITE_WAITERS) == 0) { | (state & URWLOCK_WRITE_WAITERS) == 0) { | ||||
rv = casueword32(&rwlock->rw_state, state, | rv = casueword32(&rwlock->rw_state, state, | ||||
&oldstate, state | URWLOCK_WRITE_WAITERS); | &oldstate, state | URWLOCK_WRITE_WAITERS); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (oldstate == state) | if (rv == 0) { | ||||
MPASS(oldstate == state); | |||||
goto sleep; | goto sleep; | ||||
} | |||||
state = oldstate; | state = oldstate; | ||||
error = umtxq_check_susp(td); | error = umtxq_check_susp(td, false); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
} | } | ||||
if (error != 0) { | if (error != 0) { | ||||
umtxq_unbusy_unlocked(&uq->uq_key); | umtxq_unbusy_unlocked(&uq->uq_key); | ||||
break; | break; | ||||
} | } | ||||
if ((state & URWLOCK_WRITE_OWNER) == 0 && | if ((state & URWLOCK_WRITE_OWNER) == 0 && | ||||
URWLOCK_READER_COUNT(state) == 0) { | URWLOCK_READER_COUNT(state) == 0) { | ||||
umtxq_unbusy_unlocked(&uq->uq_key); | umtxq_unbusy_unlocked(&uq->uq_key); | ||||
error = umtxq_check_susp(td); | error = umtxq_check_susp(td, false); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
continue; | continue; | ||||
} | } | ||||
sleep: | sleep: | ||||
rv = fueword32(&rwlock->rw_blocked_writers, | rv = fueword32(&rwlock->rw_blocked_writers, | ||||
&blocked_writers); | &blocked_writers); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | if (blocked_writers == 1) { | ||||
} | } | ||||
for (;;) { | for (;;) { | ||||
rv = casueword32(&rwlock->rw_state, state, | rv = casueword32(&rwlock->rw_state, state, | ||||
&oldstate, state & ~URWLOCK_WRITE_WAITERS); | &oldstate, state & ~URWLOCK_WRITE_WAITERS); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
break; | break; | ||||
} | } | ||||
if (oldstate == state) | if (rv == 0) { | ||||
MPASS(oldstate == state); | |||||
break; | break; | ||||
} | |||||
state = oldstate; | state = oldstate; | ||||
error1 = umtxq_check_susp(td); | error1 = umtxq_check_susp(td, false); | ||||
/* | /* | ||||
* We are leaving the URWLOCK_WRITE_WAITERS | * We are leaving the URWLOCK_WRITE_WAITERS | ||||
* behind, but this should not harm the | * behind, but this should not harm the | ||||
* correctness. | * correctness. | ||||
*/ | */ | ||||
if (error1 != 0) { | if (error1 != 0) { | ||||
if (error == 0) | if (error == 0) | ||||
error = error1; | error = error1; | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | do_rw_unlock(struct thread *td, struct urwlock *rwlock) | ||||
if (state & URWLOCK_WRITE_OWNER) { | if (state & URWLOCK_WRITE_OWNER) { | ||||
for (;;) { | for (;;) { | ||||
rv = casueword32(&rwlock->rw_state, state, | rv = casueword32(&rwlock->rw_state, state, | ||||
&oldstate, state & ~URWLOCK_WRITE_OWNER); | &oldstate, state & ~URWLOCK_WRITE_OWNER); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
goto out; | goto out; | ||||
} | } | ||||
if (oldstate != state) { | if (rv == 1) { | ||||
Done Inline ActionsIf oldstate != state and rv != -1, then rv == 1. markj: If oldstate != state and rv != -1, then rv == 1. | |||||
state = oldstate; | state = oldstate; | ||||
if (!(oldstate & URWLOCK_WRITE_OWNER)) { | if (!(oldstate & URWLOCK_WRITE_OWNER)) { | ||||
error = EPERM; | error = EPERM; | ||||
goto out; | goto out; | ||||
} | } | ||||
error = umtxq_check_susp(td); | error = umtxq_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
goto out; | goto out; | ||||
} else | } else | ||||
break; | break; | ||||
} | } | ||||
} else if (URWLOCK_READER_COUNT(state) != 0) { | } else if (URWLOCK_READER_COUNT(state) != 0) { | ||||
for (;;) { | for (;;) { | ||||
rv = casueword32(&rwlock->rw_state, state, | rv = casueword32(&rwlock->rw_state, state, | ||||
&oldstate, state - 1); | &oldstate, state - 1); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
error = EFAULT; | error = EFAULT; | ||||
goto out; | goto out; | ||||
} | } | ||||
if (oldstate != state) { | if (rv == 1) { | ||||
Done Inline ActionsSame here. markj: Same here. | |||||
state = oldstate; | state = oldstate; | ||||
if (URWLOCK_READER_COUNT(oldstate) == 0) { | if (URWLOCK_READER_COUNT(oldstate) == 0) { | ||||
error = EPERM; | error = EPERM; | ||||
goto out; | goto out; | ||||
} | } | ||||
error = umtxq_check_susp(td); | error = umtxq_check_susp(td, true); | ||||
if (error != 0) | if (error != 0) | ||||
goto out; | goto out; | ||||
} else | } else | ||||
break; | break; | ||||
} | } | ||||
} else { | } else { | ||||
error = EPERM; | error = EPERM; | ||||
goto out; | goto out; | ||||
Show All 33 Lines | |||||
#if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) | #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) | ||||
static int | static int | ||||
do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout) | do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout) | ||||
{ | { | ||||
struct abs_timeout timo; | struct abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
uint32_t flags, count, count1; | uint32_t flags, count, count1; | ||||
int error, rv; | int error, rv, rv1; | ||||
uq = td->td_umtxq; | uq = td->td_umtxq; | ||||
error = fueword32(&sem->_flags, &flags); | error = fueword32(&sem->_flags, &flags); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); | error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
if (timeout != NULL) | if (timeout != NULL) | ||||
abs_timeout_init2(&timo, timeout); | abs_timeout_init2(&timo, timeout); | ||||
again: | |||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_busy(&uq->uq_key); | umtxq_busy(&uq->uq_key); | ||||
umtxq_insert(uq); | umtxq_insert(uq); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
rv = casueword32(&sem->_has_waiters, 0, &count1, 1); | rv = casueword32(&sem->_has_waiters, 0, &count1, 1); | ||||
if (rv == 0) | if (rv == 0) | ||||
rv = fueword32(&sem->_count, &count); | rv1 = fueword32(&sem->_count, &count); | ||||
if (rv == -1 || count != 0) { | if (rv == -1 || (rv == 0 && (rv1 == -1 || count != 0)) || rv == 1) { | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
umtxq_remove(uq); | umtxq_remove(uq); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
umtx_key_release(&uq->uq_key); | if (rv == 1) { | ||||
return (rv == -1 ? EFAULT : 0); | rv = umtxq_check_susp(td, true); | ||||
if (rv == 0) | |||||
goto again; | |||||
error = rv; | |||||
goto out; | |||||
} | } | ||||
if (rv == 0) | |||||
rv = rv1; | |||||
error = rv == -1 ? EFAULT : 0; | |||||
goto out; | |||||
} | |||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo); | error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo); | ||||
if ((uq->uq_flags & UQF_UMTXQ) == 0) | if ((uq->uq_flags & UQF_UMTXQ) == 0) | ||||
error = 0; | error = 0; | ||||
else { | else { | ||||
umtxq_remove(uq); | umtxq_remove(uq); | ||||
/* A relative timeout cannot be restarted. */ | /* A relative timeout cannot be restarted. */ | ||||
if (error == ERESTART && timeout != NULL && | if (error == ERESTART && timeout != NULL && | ||||
(timeout->_flags & UMTX_ABSTIME) == 0) | (timeout->_flags & UMTX_ABSTIME) == 0) | ||||
error = EINTR; | error = EINTR; | ||||
} | } | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
out: | |||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Signal a userland semaphore. | * Signal a userland semaphore. | ||||
*/ | */ | ||||
static int | static int | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout) | ||||
flags = fuword32(&sem->_flags); | flags = fuword32(&sem->_flags); | ||||
error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); | error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
if (timeout != NULL) | if (timeout != NULL) | ||||
abs_timeout_init2(&timo, timeout); | abs_timeout_init2(&timo, timeout); | ||||
again: | |||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_busy(&uq->uq_key); | umtxq_busy(&uq->uq_key); | ||||
umtxq_insert(uq); | umtxq_insert(uq); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
rv = fueword32(&sem->_count, &count); | rv = fueword32(&sem->_count, &count); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
Show All 9 Lines | if (USEM_COUNT(count) != 0) { | ||||
umtxq_remove(uq); | umtxq_remove(uq); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (0); | return (0); | ||||
} | } | ||||
if (count == USEM_HAS_WAITERS) | if (count == USEM_HAS_WAITERS) | ||||
break; | break; | ||||
rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS); | rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS); | ||||
if (rv == -1) { | if (rv == 0) | ||||
Done Inline ActionsThis test is redundant. markj: This test is redundant. | |||||
break; | |||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
umtxq_remove(uq); | umtxq_remove(uq); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
if (rv == -1) | |||||
return (EFAULT); | return (EFAULT); | ||||
rv = umtxq_check_susp(td, true); | |||||
if (rv != 0) | |||||
return (rv); | |||||
goto again; | |||||
} | } | ||||
if (count == 0) | |||||
break; | |||||
} | |||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_unbusy(&uq->uq_key); | umtxq_unbusy(&uq->uq_key); | ||||
error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo); | error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo); | ||||
if ((uq->uq_flags & UQF_UMTXQ) == 0) | if ((uq->uq_flags & UQF_UMTXQ) == 0) | ||||
error = 0; | error = 0; | ||||
else { | else { | ||||
Show All 35 Lines | do_sem2_wake(struct thread *td, struct _usem2 *sem) | ||||
if (cnt > 0) { | if (cnt > 0) { | ||||
/* | /* | ||||
* If this was the last sleeping thread, clear the waiters | * If this was the last sleeping thread, clear the waiters | ||||
* flag in _count. | * flag in _count. | ||||
*/ | */ | ||||
if (cnt == 1) { | if (cnt == 1) { | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
rv = fueword32(&sem->_count, &count); | rv = fueword32(&sem->_count, &count); | ||||
while (rv != -1 && count & USEM_HAS_WAITERS) | while (rv != -1 && count & USEM_HAS_WAITERS) { | ||||
rv = casueword32(&sem->_count, count, &count, | rv = casueword32(&sem->_count, count, &count, | ||||
count & ~USEM_HAS_WAITERS); | count & ~USEM_HAS_WAITERS); | ||||
if (rv == 1) { | |||||
rv = umtxq_check_susp(td, true); | |||||
if (rv != 0) | |||||
break; | |||||
} | |||||
} | |||||
if (rv == -1) | if (rv == -1) | ||||
error = EFAULT; | error = EFAULT; | ||||
else if (rv > 0) { | |||||
Done Inline ActionsThis should be an else if. markj: This should be an else if. | |||||
error = rv; | |||||
} | |||||
umtxq_lock(&key); | umtxq_lock(&key); | ||||
} | } | ||||
umtxq_signal(&key, 1); | umtxq_signal(&key, 1); | ||||
} | } | ||||
umtxq_unbusy(&key); | umtxq_unbusy(&key); | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
▲ Show 20 Lines • Show All 1,276 Lines • Show Last 20 Lines |
s/operations/operation/