Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_umtx.c
Show First 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | |||||
#define _UMUTEX_WAIT 2 | #define _UMUTEX_WAIT 2 | ||||
#ifdef UMTX_PROFILING | #ifdef UMTX_PROFILING | ||||
#define UPROF_PERC_BIGGER(w, f, sw, sf) \ | #define UPROF_PERC_BIGGER(w, f, sw, sf) \ | ||||
(((w) > (sw)) || ((w) == (sw) && (f) > (sf))) | (((w) > (sw)) || ((w) == (sw) && (f) > (sf))) | ||||
#endif | #endif | ||||
#define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) | #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) | ||||
#ifdef INVARIANTS | |||||
kib: arg should be key and not uc | |||||
#define UMTXQ_LOCKED_ASSERT_BUSY(key) do { \ | |||||
kibUnsubmitted Not Done Inline ActionsUMTXQ_ASSERT_LOCKED_BUSY kib: UMTXQ_ASSERT_LOCKED_BUSY | |||||
struct umtxq_chain *uc; \ | |||||
\ | |||||
uc = umtxq_getchain(key); \ | |||||
mtx_assert(&uc->uc_lock, MA_OWNED); \ | |||||
KASSERT(uc->uc_busy != 0, ("umtx chain is not busy")); \ | |||||
Not Done Inline ActionsThis is better, but please do #ifdef INVARIANTS #define UMTXQ_ASSERT_LOCKED_BUSY(key) <your definition> #else #define UMTXQ_ASSERT_LOCKED_BUSY(key) do {} while (0) #endif So that uc is not calculated for !INVARIANTS case. Also note a suggested name change kib: This is better, but please do
```
#ifdef INVARIANTS
#define UMTXQ_ASSERT_LOCKED_BUSY(key) <your… | |||||
} while (0) | |||||
#else | |||||
#define UMTXQ_LOCKED_ASSERT_BUSY(key) do {} while (0) | |||||
#endif | |||||
/* | /* | ||||
* Don't propagate time-sharing priority, there is a security reason, | * Don't propagate time-sharing priority, there is a security reason, | ||||
* a user can simply introduce PI-mutex, let thread A lock the mutex, | * a user can simply introduce PI-mutex, let thread A lock the mutex, | ||||
* and let another thread B block on the mutex, because B is | * and let another thread B block on the mutex, because B is | ||||
* sleeping, its priority will be boosted, this causes A's priority to | * sleeping, its priority will be boosted, this causes A's priority to | ||||
* be boosted via priority propagating too and will never be lowered even | * be boosted via priority propagating too and will never be lowered even | ||||
* if it is using 100%CPU, this is unfair to other processes. | * if it is using 100%CPU, this is unfair to other processes. | ||||
▲ Show 20 Lines • Show All 1,609 Lines • ▼ Show 20 Lines | umtx_pi_insert(struct umtx_pi *pi) | ||||
struct umtxq_chain *uc; | struct umtxq_chain *uc; | ||||
uc = umtxq_getchain(&pi->pi_key); | uc = umtxq_getchain(&pi->pi_key); | ||||
UMTXQ_LOCKED_ASSERT(uc); | UMTXQ_LOCKED_ASSERT(uc); | ||||
TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink); | TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink); | ||||
} | } | ||||
/* | /* | ||||
* Drop a PI mutex and wakeup a top waiter. | |||||
*/ | |||||
int | |||||
umtx_pi_drop(struct thread *td, struct umtx_key *key, bool rb, int *count) | |||||
{ | |||||
struct umtx_q *uq_first, *uq_first2, *uq_me; | |||||
struct umtx_pi *pi, *pi2; | |||||
int pri; | |||||
Not Done Inline ActionsIf exposing such function as the generic KPI, we should assert there that key is locked and busy. For that, you need to add corresponding primitives. kib: If exposing such function as the generic KPI, we should assert there that key is locked and… | |||||
UMTXQ_LOCKED_ASSERT_BUSY(key); | |||||
*count = umtxq_count_pi(key, &uq_first); | |||||
if (uq_first != NULL) { | |||||
mtx_lock(&umtx_lock); | |||||
pi = uq_first->uq_pi_blocked; | |||||
Not Done Inline ActionsNo, pass the key to UMTXQ_LOCKED_ASSERT_BUSY(), so that the INVARIANTS block above is not needed. kib: No, pass the key to UMTXQ_LOCKED_ASSERT_BUSY(), so that the INVARIANTS block above is not… | |||||
KASSERT(pi != NULL, ("pi == NULL?")); | |||||
if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) { | |||||
Not Done Inline ActionsShouldn't #endif go before this line? Otherwise uc is not defined. It might be better to define some helper e.g. UMTXQ_ASSERT_LOCKED_BUSY(key) and expand it into block as you write for INVARIANTS, otherwise keep it empty. kib: Shouldn't #endif go before this line? Otherwise uc is not defined.
It might be better to… | |||||
mtx_unlock(&umtx_lock); | |||||
/* userland messed the mutex */ | |||||
return (EPERM); | |||||
} | |||||
uq_me = td->td_umtxq; | |||||
if (pi->pi_owner == td) | |||||
umtx_pi_disown(pi); | |||||
/* get highest priority thread which is still sleeping. */ | |||||
uq_first = TAILQ_FIRST(&pi->pi_blocked); | |||||
while (uq_first != NULL && | |||||
(uq_first->uq_flags & UQF_UMTXQ) == 0) { | |||||
uq_first = TAILQ_NEXT(uq_first, uq_lockq); | |||||
} | |||||
pri = PRI_MAX; | |||||
TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { | |||||
uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); | |||||
if (uq_first2 != NULL) { | |||||
if (pri > UPRI(uq_first2->uq_thread)) | |||||
pri = UPRI(uq_first2->uq_thread); | |||||
} | |||||
} | |||||
thread_lock(td); | |||||
sched_lend_user_prio(td, pri); | |||||
thread_unlock(td); | |||||
mtx_unlock(&umtx_lock); | |||||
if (uq_first) | |||||
umtxq_signal_thread(uq_first); | |||||
} else { | |||||
pi = umtx_pi_lookup(key); | |||||
/* | |||||
* A umtx_pi can exist if a signal or timeout removed the | |||||
* last waiter from the umtxq, but there is still | |||||
* a thread in do_lock_pi() holding the umtx_pi. | |||||
*/ | |||||
if (pi != NULL) { | |||||
/* | |||||
* The umtx_pi can be unowned, such as when a thread | |||||
* has just entered do_lock_pi(), allocated the | |||||
* umtx_pi, and unlocked the umtxq. | |||||
* If the current thread owns it, it must disown it. | |||||
*/ | |||||
mtx_lock(&umtx_lock); | |||||
if (pi->pi_owner == td) | |||||
umtx_pi_disown(pi); | |||||
mtx_unlock(&umtx_lock); | |||||
} | |||||
} | |||||
return (0); | |||||
} | |||||
/* | |||||
* Lock a PI mutex. | * Lock a PI mutex. | ||||
*/ | */ | ||||
static int | static int | ||||
do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, | do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, | ||||
struct _umtx_time *timeout, int try) | struct _umtx_time *timeout, int try) | ||||
{ | { | ||||
struct umtx_abs_timeout timo; | struct umtx_abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
▲ Show 20 Lines • Show All 193 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Unlock a PI mutex. | * Unlock a PI mutex. | ||||
*/ | */ | ||||
static int | static int | ||||
do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb) | do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb) | ||||
{ | { | ||||
struct umtx_key key; | struct umtx_key key; | ||||
struct umtx_q *uq_first, *uq_first2, *uq_me; | |||||
struct umtx_pi *pi, *pi2; | |||||
uint32_t id, new_owner, old, owner; | uint32_t id, new_owner, old, owner; | ||||
int count, error, pri; | int count, error; | ||||
id = td->td_tid; | id = td->td_tid; | ||||
usrloop: | usrloop: | ||||
/* | /* | ||||
* Make sure we own this mtx. | * Make sure we own this mtx. | ||||
*/ | */ | ||||
error = fueword32(&m->m_owner, &owner); | error = fueword32(&m->m_owner, &owner); | ||||
Show All 24 Lines | usrloop: | ||||
/* We should only ever be in here for contested locks */ | /* We should only ever be in here for contested locks */ | ||||
if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? | if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? | ||||
TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), | TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), | ||||
&key)) != 0) | &key)) != 0) | ||||
return (error); | return (error); | ||||
umtxq_lock(&key); | umtxq_lock(&key); | ||||
umtxq_busy(&key); | umtxq_busy(&key); | ||||
count = umtxq_count_pi(&key, &uq_first); | error = umtx_pi_drop(td, &key, rb, &count); | ||||
if (uq_first != NULL) { | if (error != 0) { | ||||
mtx_lock(&umtx_lock); | |||||
pi = uq_first->uq_pi_blocked; | |||||
KASSERT(pi != NULL, ("pi == NULL?")); | |||||
if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) { | |||||
mtx_unlock(&umtx_lock); | |||||
umtxq_unbusy(&key); | umtxq_unbusy(&key); | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
/* userland messed the mutex */ | /* userland messed the mutex */ | ||||
return (EPERM); | return (error); | ||||
} | |||||
uq_me = td->td_umtxq; | |||||
if (pi->pi_owner == td) | |||||
umtx_pi_disown(pi); | |||||
/* get highest priority thread which is still sleeping. */ | |||||
uq_first = TAILQ_FIRST(&pi->pi_blocked); | |||||
while (uq_first != NULL && | |||||
(uq_first->uq_flags & UQF_UMTXQ) == 0) { | |||||
uq_first = TAILQ_NEXT(uq_first, uq_lockq); | |||||
} | |||||
pri = PRI_MAX; | |||||
TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { | |||||
uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); | |||||
if (uq_first2 != NULL) { | |||||
if (pri > UPRI(uq_first2->uq_thread)) | |||||
pri = UPRI(uq_first2->uq_thread); | |||||
} | |||||
} | |||||
thread_lock(td); | |||||
sched_lend_user_prio(td, pri); | |||||
thread_unlock(td); | |||||
mtx_unlock(&umtx_lock); | |||||
if (uq_first) | |||||
umtxq_signal_thread(uq_first); | |||||
} else { | |||||
pi = umtx_pi_lookup(&key); | |||||
/* | |||||
* A umtx_pi can exist if a signal or timeout removed the | |||||
* last waiter from the umtxq, but there is still | |||||
* a thread in do_lock_pi() holding the umtx_pi. | |||||
*/ | |||||
if (pi != NULL) { | |||||
/* | |||||
* The umtx_pi can be unowned, such as when a thread | |||||
* has just entered do_lock_pi(), allocated the | |||||
* umtx_pi, and unlocked the umtxq. | |||||
* If the current thread owns it, it must disown it. | |||||
*/ | |||||
mtx_lock(&umtx_lock); | |||||
if (pi->pi_owner == td) | |||||
umtx_pi_disown(pi); | |||||
mtx_unlock(&umtx_lock); | |||||
} | |||||
} | } | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
/* | /* | ||||
* When unlocking the umtx, it must be marked as unowned if | * When unlocking the umtx, it must be marked as unowned if | ||||
* there is zero or one thread only waiting for it. | * there is zero or one thread only waiting for it. | ||||
* Otherwise, it must be marked as contested. | * Otherwise, it must be marked as contested. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 2,589 Lines • Show Last 20 Lines |
arg should be key and not uc