Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_umtx.c
Show First 20 Lines • Show All 83 Lines • ▼ Show 20 Lines | |||||
#define _UMUTEX_WAIT 2 | #define _UMUTEX_WAIT 2 | ||||
#ifdef UMTX_PROFILING | #ifdef UMTX_PROFILING | ||||
#define UPROF_PERC_BIGGER(w, f, sw, sf) \ | #define UPROF_PERC_BIGGER(w, f, sw, sf) \ | ||||
(((w) > (sw)) || ((w) == (sw) && (f) > (sf))) | (((w) > (sw)) || ((w) == (sw) && (f) > (sf))) | ||||
#endif | #endif | ||||
#define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) | #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) | ||||
kib: arg should be key and not uc | |||||
/* | /* | ||||
Not Done Inline ActionsUMTXQ_ASSERT_LOCKED_BUSY kib: UMTXQ_ASSERT_LOCKED_BUSY | |||||
* Don't propagate time-sharing priority, there is a security reason, | * Don't propagate time-sharing priority, there is a security reason, | ||||
* a user can simply introduce PI-mutex, let thread A lock the mutex, | * a user can simply introduce PI-mutex, let thread A lock the mutex, | ||||
* and let another thread B block on the mutex, because B is | * and let another thread B block on the mutex, because B is | ||||
* sleeping, its priority will be boosted, this causes A's priority to | * sleeping, its priority will be boosted, this causes A's priority to | ||||
* be boosted via priority propagating too and will never be lowered even | * be boosted via priority propagating too and will never be lowered even | ||||
Not Done Inline ActionsThis is better, but please do #ifdef INVARIANTS #define UMTXQ_ASSERT_LOCKED_BUSY(key) <your definition> #else #define UMTXQ_ASSERT_LOCKED_BUSY(key) do {} while (0) #endif So that uc is not calculated for !INVARIANTS case. Also note a suggested name change kib: This is better, but please do
```
#ifdef INVARIANTS
#define UMTXQ_ASSERT_LOCKED_BUSY(key) <your… | |||||
* if it is using 100%CPU, this is unfair to other processes. | * if it is using 100%CPU, this is unfair to other processes. | ||||
*/ | */ | ||||
#define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\ | #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\ | ||||
(td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\ | (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\ | ||||
PRI_MAX_TIMESHARE : (td)->td_user_pri) | PRI_MAX_TIMESHARE : (td)->td_user_pri) | ||||
#define GOLDEN_RATIO_PRIME 2654404609U | #define GOLDEN_RATIO_PRIME 2654404609U | ||||
▲ Show 20 Lines • Show All 1,597 Lines • ▼ Show 20 Lines | umtx_pi_insert(struct umtx_pi *pi) | ||||
struct umtxq_chain *uc; | struct umtxq_chain *uc; | ||||
uc = umtxq_getchain(&pi->pi_key); | uc = umtxq_getchain(&pi->pi_key); | ||||
UMTXQ_LOCKED_ASSERT(uc); | UMTXQ_LOCKED_ASSERT(uc); | ||||
TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink); | TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink); | ||||
} | } | ||||
/* | /* | ||||
* Drop a PI mutex and wakeup a top waiter. | |||||
*/ | |||||
int | |||||
umtx_pi_drop(struct thread *td, struct umtx_key *key, bool rb, int *count) | |||||
{ | |||||
struct umtx_q *uq_first, *uq_first2, *uq_me; | |||||
struct umtx_pi *pi, *pi2; | |||||
int pri; | |||||
kibUnsubmitted Not Done Inline ActionsIf exposing such function as the generic KPI, we should assert there that key is locked and busy. For that, you need to add corresponding primitives. kib: If exposing such function as the generic KPI, we should assert there that key is locked and… | |||||
*count = umtxq_count_pi(key, &uq_first); | |||||
if (uq_first != NULL) { | |||||
mtx_lock(&umtx_lock); | |||||
pi = uq_first->uq_pi_blocked; | |||||
KASSERT(pi != NULL, ("pi == NULL?")); | |||||
Not Done Inline ActionsNo, pass the key to UMTXQ_LOCKED_ASSERT_BUSY(), so that the INVARIANTS block above is not needed. kib: No, pass the key to UMTXQ_LOCKED_ASSERT_BUSY(), so that the INVARIANTS block above is not… | |||||
if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) { | |||||
mtx_unlock(&umtx_lock); | |||||
Not Done Inline ActionsShouldn't #endif go before this line? Otherwise uc is not defined. It might be better to define some helper e.g. UMTXQ_ASSERT_LOCKED_BUSY(key) and expand it into block as you write for INVARIANTS, otherwise keep it empty. kib: Shouldn't #endif go before this line? Otherwise uc is not defined.
It might be better to… | |||||
/* userland messed the mutex */ | |||||
return (EPERM); | |||||
} | |||||
uq_me = td->td_umtxq; | |||||
if (pi->pi_owner == td) | |||||
umtx_pi_disown(pi); | |||||
/* get highest priority thread which is still sleeping. */ | |||||
uq_first = TAILQ_FIRST(&pi->pi_blocked); | |||||
while (uq_first != NULL && | |||||
(uq_first->uq_flags & UQF_UMTXQ) == 0) { | |||||
uq_first = TAILQ_NEXT(uq_first, uq_lockq); | |||||
} | |||||
pri = PRI_MAX; | |||||
TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { | |||||
uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); | |||||
if (uq_first2 != NULL) { | |||||
if (pri > UPRI(uq_first2->uq_thread)) | |||||
pri = UPRI(uq_first2->uq_thread); | |||||
} | |||||
} | |||||
thread_lock(td); | |||||
sched_lend_user_prio(td, pri); | |||||
thread_unlock(td); | |||||
mtx_unlock(&umtx_lock); | |||||
if (uq_first) | |||||
umtxq_signal_thread(uq_first); | |||||
} else { | |||||
pi = umtx_pi_lookup(key); | |||||
/* | |||||
* A umtx_pi can exist if a signal or timeout removed the | |||||
* last waiter from the umtxq, but there is still | |||||
* a thread in do_lock_pi() holding the umtx_pi. | |||||
*/ | |||||
if (pi != NULL) { | |||||
/* | |||||
* The umtx_pi can be unowned, such as when a thread | |||||
* has just entered do_lock_pi(), allocated the | |||||
* umtx_pi, and unlocked the umtxq. | |||||
* If the current thread owns it, it must disown it. | |||||
*/ | |||||
mtx_lock(&umtx_lock); | |||||
if (pi->pi_owner == td) | |||||
umtx_pi_disown(pi); | |||||
mtx_unlock(&umtx_lock); | |||||
} | |||||
} | |||||
return (0); | |||||
} | |||||
/* | |||||
* Lock a PI mutex. | * Lock a PI mutex. | ||||
*/ | */ | ||||
static int | static int | ||||
do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, | do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, | ||||
struct _umtx_time *timeout, int try) | struct _umtx_time *timeout, int try) | ||||
{ | { | ||||
struct umtx_abs_timeout timo; | struct umtx_abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
▲ Show 20 Lines • Show All 193 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Unlock a PI mutex. | * Unlock a PI mutex. | ||||
*/ | */ | ||||
static int | static int | ||||
do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb) | do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb) | ||||
{ | { | ||||
struct umtx_key key; | struct umtx_key key; | ||||
struct umtx_q *uq_first, *uq_first2, *uq_me; | |||||
struct umtx_pi *pi, *pi2; | |||||
uint32_t id, new_owner, old, owner; | uint32_t id, new_owner, old, owner; | ||||
int count, error, pri; | int count, error; | ||||
id = td->td_tid; | id = td->td_tid; | ||||
usrloop: | usrloop: | ||||
/* | /* | ||||
* Make sure we own this mtx. | * Make sure we own this mtx. | ||||
*/ | */ | ||||
error = fueword32(&m->m_owner, &owner); | error = fueword32(&m->m_owner, &owner); | ||||
Show All 24 Lines | usrloop: | ||||
/* We should only ever be in here for contested locks */ | /* We should only ever be in here for contested locks */ | ||||
if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? | if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? | ||||
TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), | TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), | ||||
&key)) != 0) | &key)) != 0) | ||||
return (error); | return (error); | ||||
umtxq_lock(&key); | umtxq_lock(&key); | ||||
umtxq_busy(&key); | umtxq_busy(&key); | ||||
count = umtxq_count_pi(&key, &uq_first); | error = umtx_pi_drop(td, &key, rb, &count); | ||||
if (uq_first != NULL) { | if (error != 0) { | ||||
mtx_lock(&umtx_lock); | |||||
pi = uq_first->uq_pi_blocked; | |||||
KASSERT(pi != NULL, ("pi == NULL?")); | |||||
if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) { | |||||
mtx_unlock(&umtx_lock); | |||||
umtxq_unbusy(&key); | umtxq_unbusy(&key); | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
/* userland messed the mutex */ | /* userland messed the mutex */ | ||||
return (EPERM); | return (error); | ||||
} | |||||
uq_me = td->td_umtxq; | |||||
if (pi->pi_owner == td) | |||||
umtx_pi_disown(pi); | |||||
/* get highest priority thread which is still sleeping. */ | |||||
uq_first = TAILQ_FIRST(&pi->pi_blocked); | |||||
while (uq_first != NULL && | |||||
(uq_first->uq_flags & UQF_UMTXQ) == 0) { | |||||
uq_first = TAILQ_NEXT(uq_first, uq_lockq); | |||||
} | |||||
pri = PRI_MAX; | |||||
TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { | |||||
uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); | |||||
if (uq_first2 != NULL) { | |||||
if (pri > UPRI(uq_first2->uq_thread)) | |||||
pri = UPRI(uq_first2->uq_thread); | |||||
} | |||||
} | |||||
thread_lock(td); | |||||
sched_lend_user_prio(td, pri); | |||||
thread_unlock(td); | |||||
mtx_unlock(&umtx_lock); | |||||
if (uq_first) | |||||
umtxq_signal_thread(uq_first); | |||||
} else { | |||||
pi = umtx_pi_lookup(&key); | |||||
/* | |||||
* A umtx_pi can exist if a signal or timeout removed the | |||||
* last waiter from the umtxq, but there is still | |||||
* a thread in do_lock_pi() holding the umtx_pi. | |||||
*/ | |||||
if (pi != NULL) { | |||||
/* | |||||
* The umtx_pi can be unowned, such as when a thread | |||||
* has just entered do_lock_pi(), allocated the | |||||
* umtx_pi, and unlocked the umtxq. | |||||
* If the current thread owns it, it must disown it. | |||||
*/ | |||||
mtx_lock(&umtx_lock); | |||||
if (pi->pi_owner == td) | |||||
umtx_pi_disown(pi); | |||||
mtx_unlock(&umtx_lock); | |||||
} | |||||
} | } | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
/* | /* | ||||
* When unlocking the umtx, it must be marked as unowned if | * When unlocking the umtx, it must be marked as unowned if | ||||
* there is zero or one thread only waiting for it. | * there is zero or one thread only waiting for it. | ||||
* Otherwise, it must be marked as contested. | * Otherwise, it must be marked as contested. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 2,589 Lines • Show Last 20 Lines |
arg should be key and not uc