Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_umtx.c
Show First 20 Lines • Show All 207 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
#define UMTX_SHIFTS (__WORD_BIT - 9) | #define UMTX_SHIFTS (__WORD_BIT - 9) | ||||
#define GET_SHARE(flags) \ | #define GET_SHARE(flags) \ | ||||
(((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE) | (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE) | ||||
#define BUSY_SPINS 200 | #define BUSY_SPINS 200 | ||||
struct abs_timeout { | |||||
int clockid; | |||||
bool is_abs_real; /* TIMER_ABSTIME && CLOCK_REALTIME* */ | |||||
struct timespec cur; | |||||
struct timespec end; | |||||
}; | |||||
struct umtx_copyops { | struct umtx_copyops { | ||||
int (*copyin_timeout)(const void *uaddr, struct timespec *tsp); | int (*copyin_timeout)(const void *uaddr, struct timespec *tsp); | ||||
int (*copyin_umtx_time)(const void *uaddr, size_t size, | int (*copyin_umtx_time)(const void *uaddr, size_t size, | ||||
struct _umtx_time *tp); | struct _umtx_time *tp); | ||||
int (*copyin_robust_lists)(const void *uaddr, size_t size, | int (*copyin_robust_lists)(const void *uaddr, size_t size, | ||||
struct umtx_robust_lists_params *rbp); | struct umtx_robust_lists_params *rbp); | ||||
int (*copyout_timeout)(void *uaddr, size_t size, | int (*copyout_timeout)(void *uaddr, size_t size, | ||||
struct timespec *tsp); | struct timespec *tsp); | ||||
Show All 31 Lines | |||||
#ifdef UMTX_PROFILING | #ifdef UMTX_PROFILING | ||||
static long max_length; | static long max_length; | ||||
SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length"); | SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length"); | ||||
static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | ||||
"umtx chain stats"); | "umtx chain stats"); | ||||
#endif | #endif | ||||
static void abs_timeout_update(struct abs_timeout *timo); | static inline void umtx_abs_timeout_init2(struct umtx_abs_timeout *timo, | ||||
const struct _umtx_time *umtxtime); | |||||
static int umtx_abs_timeout_gethz(struct umtx_abs_timeout *timo); | |||||
static inline void umtx_abs_timeout_update(struct umtx_abs_timeout *timo); | |||||
kib: I am not sure, but 'inline' is probably more important in declaration. | |||||
static void umtx_shm_init(void); | static void umtx_shm_init(void); | ||||
static void umtxq_sysinit(void *); | static void umtxq_sysinit(void *); | ||||
static void umtxq_hash(struct umtx_key *key); | static void umtxq_hash(struct umtx_key *key); | ||||
static struct umtxq_chain *umtxq_getchain(struct umtx_key *key); | static struct umtxq_chain *umtxq_getchain(struct umtx_key *key); | ||||
static void umtxq_unlock(struct umtx_key *key); | static void umtxq_unlock(struct umtx_key *key); | ||||
static void umtxq_busy(struct umtx_key *key); | static void umtxq_busy(struct umtx_key *key); | ||||
static void umtxq_unbusy(struct umtx_key *key); | static void umtxq_unbusy(struct umtx_key *key); | ||||
static void umtxq_insert_queue(struct umtx_q *uq, int q); | static void umtxq_insert_queue(struct umtx_q *uq, int q); | ||||
static void umtxq_remove_queue(struct umtx_q *uq, int q); | static void umtxq_remove_queue(struct umtx_q *uq, int q); | ||||
static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *); | static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, | ||||
struct umtx_abs_timeout *); | |||||
static int umtxq_count(struct umtx_key *key); | static int umtxq_count(struct umtx_key *key); | ||||
static struct umtx_pi *umtx_pi_alloc(int); | static struct umtx_pi *umtx_pi_alloc(int); | ||||
static void umtx_pi_free(struct umtx_pi *pi); | static void umtx_pi_free(struct umtx_pi *pi); | ||||
static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, | static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, | ||||
bool rb); | bool rb); | ||||
static void umtx_thread_cleanup(struct thread *td); | static void umtx_thread_cleanup(struct thread *td); | ||||
SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL); | SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL); | ||||
▲ Show 20 Lines • Show All 446 Lines • ▼ Show 20 Lines | |||||
tstohz(const struct timespec *tsp) | tstohz(const struct timespec *tsp) | ||||
{ | { | ||||
struct timeval tv; | struct timeval tv; | ||||
TIMESPEC_TO_TIMEVAL(&tv, tsp); | TIMESPEC_TO_TIMEVAL(&tv, tsp); | ||||
return tvtohz(&tv); | return tvtohz(&tv); | ||||
} | } | ||||
static void | void | ||||
abs_timeout_init(struct abs_timeout *timo, int clockid, int absolute, | umtx_abs_timeout_init(struct umtx_abs_timeout *timo, int clockid, | ||||
const struct timespec *timeout) | int absolute, const struct timespec *timeout) | ||||
{ | { | ||||
timo->clockid = clockid; | timo->clockid = clockid; | ||||
if (!absolute) { | if (!absolute) { | ||||
timo->is_abs_real = false; | timo->is_abs_real = false; | ||||
abs_timeout_update(timo); | umtx_abs_timeout_update(timo); | ||||
timespecadd(&timo->cur, timeout, &timo->end); | timespecadd(&timo->cur, timeout, &timo->end); | ||||
} else { | } else { | ||||
timo->end = *timeout; | timo->end = *timeout; | ||||
timo->is_abs_real = clockid == CLOCK_REALTIME || | timo->is_abs_real = clockid == CLOCK_REALTIME || | ||||
clockid == CLOCK_REALTIME_FAST || | clockid == CLOCK_REALTIME_FAST || | ||||
clockid == CLOCK_REALTIME_PRECISE; | clockid == CLOCK_REALTIME_PRECISE; | ||||
/* | /* | ||||
* If is_abs_real, umtxq_sleep will read the clock | * If is_abs_real, umtxq_sleep will read the clock | ||||
* after setting td_rtcgen; otherwise, read it here. | * after setting td_rtcgen; otherwise, read it here. | ||||
*/ | */ | ||||
if (!timo->is_abs_real) { | if (!timo->is_abs_real) { | ||||
abs_timeout_update(timo); | umtx_abs_timeout_update(timo); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
abs_timeout_init2(struct abs_timeout *timo, const struct _umtx_time *umtxtime) | umtx_abs_timeout_init2(struct umtx_abs_timeout *timo, | ||||
const struct _umtx_time *umtxtime) | |||||
Not Done Inline ActionsI would rename the rest of the abs_timeout_ functions in this go as well, but this is your call. kib: I would rename the rest of the abs_timeout_ functions in this go as well, but this is your call. | |||||
{ | { | ||||
abs_timeout_init(timo, umtxtime->_clockid, | umtx_abs_timeout_init(timo, umtxtime->_clockid, | ||||
(umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout); | (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout); | ||||
} | } | ||||
static inline void | static void | ||||
abs_timeout_update(struct abs_timeout *timo) | umtx_abs_timeout_update(struct umtx_abs_timeout *timo) | ||||
{ | { | ||||
kern_clock_gettime(curthread, timo->clockid, &timo->cur); | kern_clock_gettime(curthread, timo->clockid, &timo->cur); | ||||
} | } | ||||
static int | static int | ||||
abs_timeout_gethz(struct abs_timeout *timo) | umtx_abs_timeout_gethz(struct umtx_abs_timeout *timo) | ||||
{ | { | ||||
struct timespec tts; | struct timespec tts; | ||||
if (timespeccmp(&timo->end, &timo->cur, <=)) | if (timespeccmp(&timo->end, &timo->cur, <=)) | ||||
return (-1); | return (-1); | ||||
timespecsub(&timo->end, &timo->cur, &tts); | timespecsub(&timo->end, &timo->cur, &tts); | ||||
return (tstohz(&tts)); | return (tstohz(&tts)); | ||||
} | } | ||||
Show All 11 Lines | |||||
} | } | ||||
/* | /* | ||||
* Put thread into sleep state, before sleeping, check if | * Put thread into sleep state, before sleeping, check if | ||||
* thread was removed from umtx queue. | * thread was removed from umtx queue. | ||||
*/ | */ | ||||
static inline int | static inline int | ||||
umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *abstime) | umtxq_sleep(struct umtx_q *uq, const char *wmesg, | ||||
struct umtx_abs_timeout *abstime) | |||||
{ | { | ||||
struct umtxq_chain *uc; | struct umtxq_chain *uc; | ||||
int error, timo; | int error, timo; | ||||
if (abstime != NULL && abstime->is_abs_real) { | if (abstime != NULL && abstime->is_abs_real) { | ||||
curthread->td_rtcgen = atomic_load_acq_int(&rtc_generation); | curthread->td_rtcgen = atomic_load_acq_int(&rtc_generation); | ||||
abs_timeout_update(abstime); | umtx_abs_timeout_update(abstime); | ||||
} | } | ||||
uc = umtxq_getchain(&uq->uq_key); | uc = umtxq_getchain(&uq->uq_key); | ||||
UMTXQ_LOCKED_ASSERT(uc); | UMTXQ_LOCKED_ASSERT(uc); | ||||
for (;;) { | for (;;) { | ||||
if (!(uq->uq_flags & UQF_UMTXQ)) { | if (!(uq->uq_flags & UQF_UMTXQ)) { | ||||
error = 0; | error = 0; | ||||
break; | break; | ||||
} | } | ||||
if (abstime != NULL) { | if (abstime != NULL) { | ||||
timo = abs_timeout_gethz(abstime); | timo = umtx_abs_timeout_gethz(abstime); | ||||
if (timo < 0) { | if (timo < 0) { | ||||
error = ETIMEDOUT; | error = ETIMEDOUT; | ||||
break; | break; | ||||
} | } | ||||
} else | } else | ||||
timo = 0; | timo = 0; | ||||
error = msleep(uq, &uc->uc_lock, PCATCH | PDROP, wmesg, timo); | error = msleep(uq, &uc->uc_lock, PCATCH | PDROP, wmesg, timo); | ||||
if (error == EINTR || error == ERESTART) { | if (error == EINTR || error == ERESTART) { | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
break; | break; | ||||
} | } | ||||
if (abstime != NULL) { | if (abstime != NULL) { | ||||
if (abstime->is_abs_real) | if (abstime->is_abs_real) | ||||
curthread->td_rtcgen = | curthread->td_rtcgen = | ||||
atomic_load_acq_int(&rtc_generation); | atomic_load_acq_int(&rtc_generation); | ||||
abs_timeout_update(abstime); | umtx_abs_timeout_update(abstime); | ||||
} | } | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
} | } | ||||
curthread->td_rtcgen = 0; | curthread->td_rtcgen = 0; | ||||
return (error); | return (error); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Fetch and compare value, sleep on the address if value is not changed. | * Fetch and compare value, sleep on the address if value is not changed. | ||||
*/ | */ | ||||
static int | static int | ||||
do_wait(struct thread *td, void *addr, u_long id, | do_wait(struct thread *td, void *addr, u_long id, | ||||
struct _umtx_time *timeout, int compat32, int is_private) | struct _umtx_time *timeout, int compat32, int is_private) | ||||
{ | { | ||||
struct abs_timeout timo; | struct umtx_abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
u_long tmp; | u_long tmp; | ||||
uint32_t tmp32; | uint32_t tmp32; | ||||
int error = 0; | int error = 0; | ||||
uq = td->td_umtxq; | uq = td->td_umtxq; | ||||
if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT, | if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT, | ||||
is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0) | is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0) | ||||
return (error); | return (error); | ||||
if (timeout != NULL) | if (timeout != NULL) | ||||
abs_timeout_init2(&timo, timeout); | umtx_abs_timeout_init2(&timo, timeout); | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_insert(uq); | umtxq_insert(uq); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
if (compat32 == 0) { | if (compat32 == 0) { | ||||
error = fueword(addr, &tmp); | error = fueword(addr, &tmp); | ||||
if (error != 0) | if (error != 0) | ||||
error = EFAULT; | error = EFAULT; | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Lock PTHREAD_PRIO_NONE protocol POSIX mutex. | * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. | ||||
*/ | */ | ||||
static int | static int | ||||
do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, | do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, | ||||
struct _umtx_time *timeout, int mode) | struct _umtx_time *timeout, int mode) | ||||
{ | { | ||||
struct abs_timeout timo; | struct umtx_abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
uint32_t owner, old, id; | uint32_t owner, old, id; | ||||
int error, rv; | int error, rv; | ||||
id = td->td_tid; | id = td->td_tid; | ||||
uq = td->td_umtxq; | uq = td->td_umtxq; | ||||
error = 0; | error = 0; | ||||
if (timeout != NULL) | if (timeout != NULL) | ||||
abs_timeout_init2(&timo, timeout); | umtx_abs_timeout_init2(&timo, timeout); | ||||
/* | /* | ||||
* Care must be exercised when dealing with umtx structure. It | * Care must be exercised when dealing with umtx structure. It | ||||
* can fault on any access. | * can fault on any access. | ||||
*/ | */ | ||||
for (;;) { | for (;;) { | ||||
rv = fueword32(&m->m_owner, &owner); | rv = fueword32(&m->m_owner, &owner); | ||||
if (rv == -1) | if (rv == -1) | ||||
▲ Show 20 Lines • Show All 642 Lines • ▼ Show 20 Lines | umtx_pi_adjust(struct thread *td, u_char oldpri) | ||||
mtx_unlock(&umtx_lock); | mtx_unlock(&umtx_lock); | ||||
} | } | ||||
/* | /* | ||||
* Sleep on a PI mutex. | * Sleep on a PI mutex. | ||||
*/ | */ | ||||
static int | static int | ||||
umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner, | umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner, | ||||
const char *wmesg, struct abs_timeout *timo, bool shared) | const char *wmesg, struct umtx_abs_timeout *timo, bool shared) | ||||
{ | { | ||||
struct thread *td, *td1; | struct thread *td, *td1; | ||||
struct umtx_q *uq1; | struct umtx_q *uq1; | ||||
int error, pri; | int error, pri; | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
struct umtxq_chain *uc; | struct umtxq_chain *uc; | ||||
uc = umtxq_getchain(&pi->pi_key); | uc = umtxq_getchain(&pi->pi_key); | ||||
▲ Show 20 Lines • Show All 121 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Lock a PI mutex. | * Lock a PI mutex. | ||||
*/ | */ | ||||
static int | static int | ||||
do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, | do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, | ||||
struct _umtx_time *timeout, int try) | struct _umtx_time *timeout, int try) | ||||
{ | { | ||||
struct abs_timeout timo; | struct umtx_abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
struct umtx_pi *pi, *new_pi; | struct umtx_pi *pi, *new_pi; | ||||
uint32_t id, old_owner, owner, old; | uint32_t id, old_owner, owner, old; | ||||
int error, rv; | int error, rv; | ||||
id = td->td_tid; | id = td->td_tid; | ||||
uq = td->td_umtxq; | uq = td->td_umtxq; | ||||
if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? | if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? | ||||
TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), | TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), | ||||
&uq->uq_key)) != 0) | &uq->uq_key)) != 0) | ||||
return (error); | return (error); | ||||
if (timeout != NULL) | if (timeout != NULL) | ||||
abs_timeout_init2(&timo, timeout); | umtx_abs_timeout_init2(&timo, timeout); | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
pi = umtx_pi_lookup(&uq->uq_key); | pi = umtx_pi_lookup(&uq->uq_key); | ||||
if (pi == NULL) { | if (pi == NULL) { | ||||
new_pi = umtx_pi_alloc(M_NOWAIT); | new_pi = umtx_pi_alloc(M_NOWAIT); | ||||
if (new_pi == NULL) { | if (new_pi == NULL) { | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
new_pi = umtx_pi_alloc(M_WAITOK); | new_pi = umtx_pi_alloc(M_WAITOK); | ||||
▲ Show 20 Lines • Show All 304 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Lock a PP mutex. | * Lock a PP mutex. | ||||
*/ | */ | ||||
static int | static int | ||||
do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, | do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, | ||||
struct _umtx_time *timeout, int try) | struct _umtx_time *timeout, int try) | ||||
{ | { | ||||
struct abs_timeout timo; | struct umtx_abs_timeout timo; | ||||
struct umtx_q *uq, *uq2; | struct umtx_q *uq, *uq2; | ||||
struct umtx_pi *pi; | struct umtx_pi *pi; | ||||
uint32_t ceiling; | uint32_t ceiling; | ||||
uint32_t owner, id; | uint32_t owner, id; | ||||
int error, pri, old_inherited_pri, su, rv; | int error, pri, old_inherited_pri, su, rv; | ||||
id = td->td_tid; | id = td->td_tid; | ||||
uq = td->td_umtxq; | uq = td->td_umtxq; | ||||
if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? | if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? | ||||
TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags), | TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags), | ||||
&uq->uq_key)) != 0) | &uq->uq_key)) != 0) | ||||
return (error); | return (error); | ||||
if (timeout != NULL) | if (timeout != NULL) | ||||
abs_timeout_init2(&timo, timeout); | umtx_abs_timeout_init2(&timo, timeout); | ||||
su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); | su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); | ||||
for (;;) { | for (;;) { | ||||
old_inherited_pri = uq->uq_inherited_pri; | old_inherited_pri = uq->uq_inherited_pri; | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_busy(&uq->uq_key); | umtxq_busy(&uq->uq_key); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
▲ Show 20 Lines • Show All 373 Lines • ▼ Show 20 Lines | do_unlock_umutex(struct thread *td, struct umutex *m, bool rb) | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
static int | static int | ||||
do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, | do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, | ||||
struct timespec *timeout, u_long wflags) | struct timespec *timeout, u_long wflags) | ||||
{ | { | ||||
struct abs_timeout timo; | struct umtx_abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
uint32_t flags, clockid, hasw; | uint32_t flags, clockid, hasw; | ||||
int error; | int error; | ||||
uq = td->td_umtxq; | uq = td->td_umtxq; | ||||
error = fueword32(&cv->c_flags, &flags); | error = fueword32(&cv->c_flags, &flags); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
Show All 30 Lines | do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, | ||||
if (error == 0 && hasw == 0) | if (error == 0 && hasw == 0) | ||||
suword32(&cv->c_has_waiters, 1); | suword32(&cv->c_has_waiters, 1); | ||||
umtxq_unbusy_unlocked(&uq->uq_key); | umtxq_unbusy_unlocked(&uq->uq_key); | ||||
error = do_unlock_umutex(td, m, false); | error = do_unlock_umutex(td, m, false); | ||||
if (timeout != NULL) | if (timeout != NULL) | ||||
abs_timeout_init(&timo, clockid, (wflags & CVWAIT_ABSTIME) != 0, | umtx_abs_timeout_init(&timo, clockid, | ||||
timeout); | (wflags & CVWAIT_ABSTIME) != 0, timeout); | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
if (error == 0) { | if (error == 0) { | ||||
error = umtxq_sleep(uq, "ucond", timeout == NULL ? | error = umtxq_sleep(uq, "ucond", timeout == NULL ? | ||||
NULL : &timo); | NULL : &timo); | ||||
} | } | ||||
if ((uq->uq_flags & UQF_UMTXQ) == 0) | if ((uq->uq_flags & UQF_UMTXQ) == 0) | ||||
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines | do_cv_broadcast(struct thread *td, struct ucond *cv) | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, | do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, | ||||
struct _umtx_time *timeout) | struct _umtx_time *timeout) | ||||
{ | { | ||||
struct abs_timeout timo; | struct umtx_abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
uint32_t flags, wrflags; | uint32_t flags, wrflags; | ||||
int32_t state, oldstate; | int32_t state, oldstate; | ||||
int32_t blocked_readers; | int32_t blocked_readers; | ||||
int error, error1, rv; | int error, error1, rv; | ||||
uq = td->td_umtxq; | uq = td->td_umtxq; | ||||
error = fueword32(&rwlock->rw_flags, &flags); | error = fueword32(&rwlock->rw_flags, &flags); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); | error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
if (timeout != NULL) | if (timeout != NULL) | ||||
abs_timeout_init2(&timo, timeout); | umtx_abs_timeout_init2(&timo, timeout); | ||||
wrflags = URWLOCK_WRITE_OWNER; | wrflags = URWLOCK_WRITE_OWNER; | ||||
if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER)) | if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER)) | ||||
wrflags |= URWLOCK_WRITE_WAITERS; | wrflags |= URWLOCK_WRITE_WAITERS; | ||||
for (;;) { | for (;;) { | ||||
rv = fueword32(&rwlock->rw_state, &state); | rv = fueword32(&rwlock->rw_state, &state); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
▲ Show 20 Lines • Show All 152 Lines • ▼ Show 20 Lines | sleep: | ||||
if (error == ERESTART) | if (error == ERESTART) | ||||
error = EINTR; | error = EINTR; | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeout) | do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeout) | ||||
{ | { | ||||
struct abs_timeout timo; | struct umtx_abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
uint32_t flags; | uint32_t flags; | ||||
int32_t state, oldstate; | int32_t state, oldstate; | ||||
int32_t blocked_writers; | int32_t blocked_writers; | ||||
int32_t blocked_readers; | int32_t blocked_readers; | ||||
int error, error1, rv; | int error, error1, rv; | ||||
uq = td->td_umtxq; | uq = td->td_umtxq; | ||||
error = fueword32(&rwlock->rw_flags, &flags); | error = fueword32(&rwlock->rw_flags, &flags); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); | error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
if (timeout != NULL) | if (timeout != NULL) | ||||
abs_timeout_init2(&timo, timeout); | umtx_abs_timeout_init2(&timo, timeout); | ||||
blocked_readers = 0; | blocked_readers = 0; | ||||
for (;;) { | for (;;) { | ||||
rv = fueword32(&rwlock->rw_state, &state); | rv = fueword32(&rwlock->rw_state, &state); | ||||
if (rv == -1) { | if (rv == -1) { | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (EFAULT); | return (EFAULT); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 261 Lines • ▼ Show 20 Lines | out: | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (error); | return (error); | ||||
} | } | ||||
#if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) | #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) | ||||
static int | static int | ||||
do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout) | do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout) | ||||
{ | { | ||||
struct abs_timeout timo; | struct umtx_abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
uint32_t flags, count, count1; | uint32_t flags, count, count1; | ||||
int error, rv, rv1; | int error, rv, rv1; | ||||
uq = td->td_umtxq; | uq = td->td_umtxq; | ||||
error = fueword32(&sem->_flags, &flags); | error = fueword32(&sem->_flags, &flags); | ||||
if (error == -1) | if (error == -1) | ||||
return (EFAULT); | return (EFAULT); | ||||
error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); | error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
if (timeout != NULL) | if (timeout != NULL) | ||||
abs_timeout_init2(&timo, timeout); | umtx_abs_timeout_init2(&timo, timeout); | ||||
again: | again: | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_busy(&uq->uq_key); | umtxq_busy(&uq->uq_key); | ||||
umtxq_insert(uq); | umtxq_insert(uq); | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
rv = casueword32(&sem->_has_waiters, 0, &count1, 1); | rv = casueword32(&sem->_has_waiters, 0, &count1, 1); | ||||
if (rv == 0) | if (rv == 0) | ||||
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines | do_sem_wake(struct thread *td, struct _usem *sem) | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
return (error); | return (error); | ||||
} | } | ||||
#endif | #endif | ||||
static int | static int | ||||
do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout) | do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout) | ||||
{ | { | ||||
struct abs_timeout timo; | struct umtx_abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
uint32_t count, flags; | uint32_t count, flags; | ||||
int error, rv; | int error, rv; | ||||
uq = td->td_umtxq; | uq = td->td_umtxq; | ||||
flags = fuword32(&sem->_flags); | flags = fuword32(&sem->_flags); | ||||
if (timeout != NULL) | if (timeout != NULL) | ||||
abs_timeout_init2(&timo, timeout); | umtx_abs_timeout_init2(&timo, timeout); | ||||
again: | again: | ||||
error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); | error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
umtxq_busy(&uq->uq_key); | umtxq_busy(&uq->uq_key); | ||||
umtxq_insert(uq); | umtxq_insert(uq); | ||||
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | if ((uq->uq_flags & UQF_UMTXQ) == 0) | ||||
error = 0; | error = 0; | ||||
else { | else { | ||||
umtxq_remove(uq); | umtxq_remove(uq); | ||||
if (timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) { | if (timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) { | ||||
/* A relative timeout cannot be restarted. */ | /* A relative timeout cannot be restarted. */ | ||||
if (error == ERESTART) | if (error == ERESTART) | ||||
error = EINTR; | error = EINTR; | ||||
if (error == EINTR) { | if (error == EINTR) { | ||||
abs_timeout_update(&timo); | umtx_abs_timeout_update(&timo); | ||||
timespecsub(&timo.end, &timo.cur, | timespecsub(&timo.end, &timo.cur, | ||||
&timeout->_timeout); | &timeout->_timeout); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
umtxq_unlock(&uq->uq_key); | umtxq_unlock(&uq->uq_key); | ||||
umtx_key_release(&uq->uq_key); | umtx_key_release(&uq->uq_key); | ||||
return (error); | return (error); | ||||
▲ Show 20 Lines • Show All 1,349 Lines • Show Last 20 Lines |
I am not sure, but 'inline' is probably more important in declaration.