Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_umtx.c
Show First 20 Lines • Show All 902 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
void | void | ||||
umtx_key_release(struct umtx_key *key) | umtx_key_release(struct umtx_key *key) | ||||
{ | { | ||||
if (key->shared) | if (key->shared) | ||||
vm_object_deallocate(key->info.shared.object); | vm_object_deallocate(key->info.shared.object); | ||||
} | } | ||||
#ifdef COMPAT_FREEBSD10 | |||||
emaste: Discussed briefly on IRC with @kib, the kernel side of this existed in FreeBSD 10 but it was… | |||||
Done Inline ActionsWe provide COMPATX for syscalls removed in HEAD-X, not when the syscall use disappeared from the base. So I believe that COMPAT10 is the proper separation line. kib: We provide COMPATX for syscalls removed in HEAD-X, not when the syscall use disappeared from… | |||||
Not Done Inline ActionsMy thinking is that we could have moved it under COMPAT_FREEBSD6 in FreeBSD 7 (assuming it was 6). I think the umtx syscalls is a bit of an unusual case wrt historical practice. emaste: My thinking is that we could have moved it under `COMPAT_FREEBSD6` in FreeBSD 7 (assuming it… | |||||
Done Inline ActionsYes, but syscall was kept until 10. I agree that right action in FreeBSD 7 times was not the removal but bracing it with COMPAT6, of course. It is both umtx op that was removed, and dedicated syscalls. kib: Yes, but syscall was kept until 10. I agree that right action in FreeBSD 7 times was not the… | |||||
/* | /* | ||||
* Lock a umtx object. | |||||
*/ | |||||
static int | |||||
do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id, | |||||
const struct timespec *timeout) | |||||
{ | |||||
struct abs_timeout timo; | |||||
struct umtx_q *uq; | |||||
u_long owner; | |||||
u_long old; | |||||
int error = 0; | |||||
uq = td->td_umtxq; | |||||
if (timeout != NULL) | |||||
abs_timeout_init(&timo, CLOCK_REALTIME, 0, timeout); | |||||
/* | |||||
* Care must be exercised when dealing with umtx structure. It | |||||
* can fault on any access. | |||||
*/ | |||||
for (;;) { | |||||
/* | |||||
* Try the uncontested case. This should be done in userland. | |||||
*/ | |||||
owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id); | |||||
/* The acquire succeeded. */ | |||||
if (owner == UMTX_UNOWNED) | |||||
return (0); | |||||
/* The address was invalid. */ | |||||
if (owner == -1) | |||||
return (EFAULT); | |||||
/* If no one owns it but it is contested try to acquire it. */ | |||||
if (owner == UMTX_CONTESTED) { | |||||
owner = casuword(&umtx->u_owner, | |||||
UMTX_CONTESTED, id | UMTX_CONTESTED); | |||||
if (owner == UMTX_CONTESTED) | |||||
return (0); | |||||
/* The address was invalid. */ | |||||
if (owner == -1) | |||||
return (EFAULT); | |||||
error = thread_check_susp(td, false); | |||||
if (error != 0) | |||||
break; | |||||
/* If this failed the lock has changed, restart. */ | |||||
continue; | |||||
} | |||||
/* | |||||
* If we caught a signal, we have retried and now | |||||
* exit immediately. | |||||
*/ | |||||
if (error != 0) | |||||
break; | |||||
if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, | |||||
AUTO_SHARE, &uq->uq_key)) != 0) | |||||
return (error); | |||||
umtxq_lock(&uq->uq_key); | |||||
umtxq_busy(&uq->uq_key); | |||||
umtxq_insert(uq); | |||||
umtxq_unbusy(&uq->uq_key); | |||||
umtxq_unlock(&uq->uq_key); | |||||
/* | |||||
* Set the contested bit so that a release in user space | |||||
* knows to use the system call for unlock. If this fails | |||||
* either some one else has acquired the lock or it has been | |||||
* released. | |||||
*/ | |||||
old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED); | |||||
/* The address was invalid. */ | |||||
if (old == -1) { | |||||
umtxq_lock(&uq->uq_key); | |||||
umtxq_remove(uq); | |||||
umtxq_unlock(&uq->uq_key); | |||||
umtx_key_release(&uq->uq_key); | |||||
return (EFAULT); | |||||
} | |||||
/* | |||||
* We set the contested bit, sleep. Otherwise the lock changed | |||||
* and we need to retry or we lost a race to the thread | |||||
* unlocking the umtx. | |||||
*/ | |||||
umtxq_lock(&uq->uq_key); | |||||
if (old == owner) | |||||
error = umtxq_sleep(uq, "umtx", timeout == NULL ? NULL : | |||||
&timo); | |||||
umtxq_remove(uq); | |||||
umtxq_unlock(&uq->uq_key); | |||||
umtx_key_release(&uq->uq_key); | |||||
if (error == 0) | |||||
error = thread_check_susp(td, false); | |||||
} | |||||
if (timeout == NULL) { | |||||
/* Mutex locking is restarted if it is interrupted. */ | |||||
if (error == EINTR) | |||||
error = ERESTART; | |||||
} else { | |||||
/* Timed-locking is not restarted. */ | |||||
if (error == ERESTART) | |||||
error = EINTR; | |||||
} | |||||
return (error); | |||||
} | |||||
/* | |||||
* Unlock a umtx object. | |||||
*/ | |||||
static int | |||||
do_unlock_umtx(struct thread *td, struct umtx *umtx, u_long id) | |||||
{ | |||||
struct umtx_key key; | |||||
u_long owner; | |||||
u_long old; | |||||
int error; | |||||
int count; | |||||
/* | |||||
* Make sure we own this mtx. | |||||
*/ | |||||
owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner)); | |||||
if (owner == -1) | |||||
return (EFAULT); | |||||
if ((owner & ~UMTX_CONTESTED) != id) | |||||
return (EPERM); | |||||
/* This should be done in userland */ | |||||
if ((owner & UMTX_CONTESTED) == 0) { | |||||
old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED); | |||||
if (old == -1) | |||||
return (EFAULT); | |||||
if (old == owner) | |||||
return (0); | |||||
owner = old; | |||||
} | |||||
/* We should only ever be in here for contested locks */ | |||||
if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE, | |||||
&key)) != 0) | |||||
return (error); | |||||
umtxq_lock(&key); | |||||
umtxq_busy(&key); | |||||
count = umtxq_count(&key); | |||||
umtxq_unlock(&key); | |||||
/* | |||||
* When unlocking the umtx, it must be marked as unowned if | |||||
* there is zero or one thread only waiting for it. | |||||
* Otherwise, it must be marked as contested. | |||||
*/ | |||||
old = casuword(&umtx->u_owner, owner, | |||||
count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED); | |||||
umtxq_lock(&key); | |||||
umtxq_signal(&key,1); | |||||
umtxq_unbusy(&key); | |||||
umtxq_unlock(&key); | |||||
umtx_key_release(&key); | |||||
if (old == -1) | |||||
return (EFAULT); | |||||
if (old != owner) | |||||
return (EINVAL); | |||||
return (0); | |||||
} | |||||
#ifdef COMPAT_FREEBSD32 | |||||
/* | |||||
* Lock a umtx object. | |||||
*/ | |||||
static int | |||||
do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id, | |||||
const struct timespec *timeout) | |||||
{ | |||||
struct abs_timeout timo; | |||||
struct umtx_q *uq; | |||||
uint32_t owner; | |||||
uint32_t old; | |||||
int error = 0; | |||||
uq = td->td_umtxq; | |||||
if (timeout != NULL) | |||||
abs_timeout_init(&timo, CLOCK_REALTIME, 0, timeout); | |||||
/* | |||||
* Care must be exercised when dealing with umtx structure. It | |||||
* can fault on any access. | |||||
*/ | |||||
for (;;) { | |||||
/* | |||||
* Try the uncontested case. This should be done in userland. | |||||
*/ | |||||
owner = casuword32(m, UMUTEX_UNOWNED, id); | |||||
/* The acquire succeeded. */ | |||||
if (owner == UMUTEX_UNOWNED) | |||||
return (0); | |||||
/* The address was invalid. */ | |||||
if (owner == -1) | |||||
return (EFAULT); | |||||
/* If no one owns it but it is contested try to acquire it. */ | |||||
if (owner == UMUTEX_CONTESTED) { | |||||
owner = casuword32(m, | |||||
UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); | |||||
if (owner == UMUTEX_CONTESTED) | |||||
return (0); | |||||
/* The address was invalid. */ | |||||
if (owner == -1) | |||||
return (EFAULT); | |||||
error = thread_check_susp(td, false); | |||||
if (error != 0) | |||||
break; | |||||
/* If this failed the lock has changed, restart. */ | |||||
continue; | |||||
} | |||||
/* | |||||
* If we caught a signal, we have retried and now | |||||
* exit immediately. | |||||
*/ | |||||
if (error != 0) | |||||
return (error); | |||||
if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, | |||||
AUTO_SHARE, &uq->uq_key)) != 0) | |||||
return (error); | |||||
umtxq_lock(&uq->uq_key); | |||||
umtxq_busy(&uq->uq_key); | |||||
umtxq_insert(uq); | |||||
umtxq_unbusy(&uq->uq_key); | |||||
umtxq_unlock(&uq->uq_key); | |||||
/* | |||||
* Set the contested bit so that a release in user space | |||||
* knows to use the system call for unlock. If this fails | |||||
* either some one else has acquired the lock or it has been | |||||
* released. | |||||
*/ | |||||
old = casuword32(m, owner, owner | UMUTEX_CONTESTED); | |||||
/* The address was invalid. */ | |||||
if (old == -1) { | |||||
umtxq_lock(&uq->uq_key); | |||||
umtxq_remove(uq); | |||||
umtxq_unlock(&uq->uq_key); | |||||
umtx_key_release(&uq->uq_key); | |||||
return (EFAULT); | |||||
} | |||||
/* | |||||
* We set the contested bit, sleep. Otherwise the lock changed | |||||
* and we need to retry or we lost a race to the thread | |||||
* unlocking the umtx. | |||||
*/ | |||||
umtxq_lock(&uq->uq_key); | |||||
if (old == owner) | |||||
error = umtxq_sleep(uq, "umtx", timeout == NULL ? | |||||
NULL : &timo); | |||||
umtxq_remove(uq); | |||||
umtxq_unlock(&uq->uq_key); | |||||
umtx_key_release(&uq->uq_key); | |||||
if (error == 0) | |||||
error = thread_check_susp(td, false); | |||||
} | |||||
if (timeout == NULL) { | |||||
/* Mutex locking is restarted if it is interrupted. */ | |||||
if (error == EINTR) | |||||
error = ERESTART; | |||||
} else { | |||||
/* Timed-locking is not restarted. */ | |||||
if (error == ERESTART) | |||||
error = EINTR; | |||||
} | |||||
return (error); | |||||
} | |||||
/* | |||||
* Unlock a umtx object. | |||||
*/ | |||||
static int | |||||
do_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id) | |||||
{ | |||||
struct umtx_key key; | |||||
uint32_t owner; | |||||
uint32_t old; | |||||
int error; | |||||
int count; | |||||
/* | |||||
* Make sure we own this mtx. | |||||
*/ | |||||
owner = fuword32(m); | |||||
if (owner == -1) | |||||
return (EFAULT); | |||||
if ((owner & ~UMUTEX_CONTESTED) != id) | |||||
return (EPERM); | |||||
/* This should be done in userland */ | |||||
if ((owner & UMUTEX_CONTESTED) == 0) { | |||||
old = casuword32(m, owner, UMUTEX_UNOWNED); | |||||
if (old == -1) | |||||
return (EFAULT); | |||||
if (old == owner) | |||||
return (0); | |||||
owner = old; | |||||
} | |||||
/* We should only ever be in here for contested locks */ | |||||
if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE, | |||||
&key)) != 0) | |||||
return (error); | |||||
umtxq_lock(&key); | |||||
umtxq_busy(&key); | |||||
count = umtxq_count(&key); | |||||
umtxq_unlock(&key); | |||||
/* | |||||
* When unlocking the umtx, it must be marked as unowned if | |||||
* there is zero or one thread only waiting for it. | |||||
* Otherwise, it must be marked as contested. | |||||
*/ | |||||
old = casuword32(m, owner, | |||||
count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); | |||||
umtxq_lock(&key); | |||||
umtxq_signal(&key,1); | |||||
umtxq_unbusy(&key); | |||||
umtxq_unlock(&key); | |||||
umtx_key_release(&key); | |||||
if (old == -1) | |||||
return (EFAULT); | |||||
if (old != owner) | |||||
return (EINVAL); | |||||
return (0); | |||||
} | |||||
#endif /* COMPAT_FREEBSD32 */ | |||||
#endif /* COMPAT_FREEBSD10 */ | |||||
/* | |||||
* Fetch and compare value, sleep on the address if value is not changed. | * Fetch and compare value, sleep on the address if value is not changed. | ||||
*/ | */ | ||||
static int | static int | ||||
do_wait(struct thread *td, void *addr, u_long id, | do_wait(struct thread *td, void *addr, u_long id, | ||||
struct _umtx_time *timeout, int compat32, int is_private) | struct _umtx_time *timeout, int compat32, int is_private) | ||||
{ | { | ||||
struct abs_timeout timo; | struct abs_timeout timo; | ||||
struct umtx_q *uq; | struct umtx_q *uq; | ||||
▲ Show 20 Lines • Show All 2,472 Lines • ▼ Show 20 Lines | if (cnt > 0) { | ||||
umtxq_signal(&key, 1); | umtxq_signal(&key, 1); | ||||
} | } | ||||
umtxq_unbusy(&key); | umtxq_unbusy(&key); | ||||
umtxq_unlock(&key); | umtxq_unlock(&key); | ||||
umtx_key_release(&key); | umtx_key_release(&key); | ||||
return (error); | return (error); | ||||
} | } | ||||
#ifdef COMPAT_FREEBSD10 | |||||
int | |||||
freebsd10__umtx_lock(struct thread *td, struct freebsd10__umtx_lock_args *uap) | |||||
{ | |||||
return (do_lock_umtx(td, uap->umtx, td->td_tid, 0)); | |||||
} | |||||
int | |||||
freebsd10__umtx_unlock(struct thread *td, | |||||
struct freebsd10__umtx_unlock_args *uap) | |||||
{ | |||||
return (do_unlock_umtx(td, uap->umtx, td->td_tid)); | |||||
} | |||||
#endif | |||||
inline int | inline int | ||||
umtx_copyin_timeout(const void *uaddr, struct timespec *tsp) | umtx_copyin_timeout(const void *uaddr, struct timespec *tsp) | ||||
{ | { | ||||
int error; | int error; | ||||
error = copyin(uaddr, tsp, sizeof(*tsp)); | error = copyin(uaddr, tsp, sizeof(*tsp)); | ||||
if (error == 0) { | if (error == 0) { | ||||
if (tsp->tv_sec < 0 || | if (tsp->tv_sec < 0 || | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | umtx_copyout_timeout(void *uaddr, size_t sz, struct timespec *tsp) | ||||
* copyops. | * copyops. | ||||
*/ | */ | ||||
KASSERT(sz >= sizeof(*tsp), | KASSERT(sz >= sizeof(*tsp), | ||||
("umtx_copyops specifies incorrect sizes")); | ("umtx_copyops specifies incorrect sizes")); | ||||
return (copyout(tsp, uaddr, sizeof(*tsp))); | return (copyout(tsp, uaddr, sizeof(*tsp))); | ||||
} | } | ||||
#ifdef COMPAT_FREEBSD10 | |||||
static int | static int | ||||
__umtx_op_unimpl(struct thread *td, struct _umtx_op_args *uap, | __umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap, | ||||
const struct umtx_copyops *ops __unused) | const struct umtx_copyops *ops) | ||||
{ | { | ||||
struct timespec *ts, timeout; | |||||
int error; | |||||
/* Allow a null timespec (wait forever). */ | |||||
if (uap->uaddr2 == NULL) | |||||
ts = NULL; | |||||
else { | |||||
error = ops->copyin_timeout(uap->uaddr2, &timeout); | |||||
if (error != 0) | |||||
return (error); | |||||
ts = &timeout; | |||||
} | |||||
#ifdef COMPAT_FREEBSD32 | |||||
if (ops->compat32) | |||||
return (do_lock_umtx32(td, uap->obj, uap->val, ts)); | |||||
#endif | |||||
return (do_lock_umtx(td, uap->obj, uap->val, ts)); | |||||
} | |||||
static int | |||||
__umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap, | |||||
const struct umtx_copyops *ops) | |||||
{ | |||||
#ifdef COMPAT_FREEBSD32 | |||||
if (ops->compat32) | |||||
return (do_unlock_umtx32(td, uap->obj, uap->val)); | |||||
#endif | |||||
return (do_unlock_umtx(td, uap->obj, uap->val)); | |||||
} | |||||
#endif /* COMPAT_FREEBSD10 */ | |||||
#if !defined(COMPAT_FREEBSD10) | |||||
static int | |||||
__umtx_op_unimpl(struct thread *td __unused, struct _umtx_op_args *uap __unused, | |||||
const struct umtx_copyops *ops __unused) | |||||
{ | |||||
return (EOPNOTSUPP); | return (EOPNOTSUPP); | ||||
} | } | ||||
#endif /* COMPAT_FREEBSD10 */ | |||||
static int | static int | ||||
__umtx_op_wait(struct thread *td, struct _umtx_op_args *uap, | __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap, | ||||
const struct umtx_copyops *ops) | const struct umtx_copyops *ops) | ||||
{ | { | ||||
struct _umtx_time timeout, *tm_p; | struct _umtx_time timeout, *tm_p; | ||||
int error; | int error; | ||||
▲ Show 20 Lines • Show All 879 Lines • ▼ Show 20 Lines | umtx_copyout_timeoutx32(void *uaddr, size_t sz, struct timespec *tsp) | ||||
return (copyout(&remain32, uaddr, sizeof(remain32))); | return (copyout(&remain32, uaddr, sizeof(remain32))); | ||||
} | } | ||||
#endif /* __i386__ || __LP64__ */ | #endif /* __i386__ || __LP64__ */ | ||||
typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap, | typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap, | ||||
const struct umtx_copyops *umtx_ops); | const struct umtx_copyops *umtx_ops); | ||||
static const _umtx_op_func op_table[] = { | static const _umtx_op_func op_table[] = { | ||||
[UMTX_OP_RESERVED0] = __umtx_op_unimpl, | #ifdef COMPAT_FREEBSD10 | ||||
[UMTX_OP_RESERVED1] = __umtx_op_unimpl, | [UMTX_OP_LOCK] = __umtx_op_lock_umtx, | ||||
[UMTX_OP_UNLOCK] = __umtx_op_unlock_umtx, | |||||
#else | |||||
[UMTX_OP_LOCK] = __umtx_op_unimpl, | |||||
[UMTX_OP_UNLOCK] = __umtx_op_unimpl, | |||||
#endif | |||||
[UMTX_OP_WAIT] = __umtx_op_wait, | [UMTX_OP_WAIT] = __umtx_op_wait, | ||||
[UMTX_OP_WAKE] = __umtx_op_wake, | [UMTX_OP_WAKE] = __umtx_op_wake, | ||||
[UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex, | [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex, | ||||
[UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex, | [UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex, | ||||
[UMTX_OP_MUTEX_UNLOCK] = __umtx_op_unlock_umutex, | [UMTX_OP_MUTEX_UNLOCK] = __umtx_op_unlock_umutex, | ||||
[UMTX_OP_SET_CEILING] = __umtx_op_set_ceiling, | [UMTX_OP_SET_CEILING] = __umtx_op_set_ceiling, | ||||
[UMTX_OP_CV_WAIT] = __umtx_op_cv_wait, | [UMTX_OP_CV_WAIT] = __umtx_op_cv_wait, | ||||
[UMTX_OP_CV_SIGNAL] = __umtx_op_cv_signal, | [UMTX_OP_CV_SIGNAL] = __umtx_op_cv_signal, | ||||
▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines | #else | ||||
if ((uap->op & UMTX_OP__32BIT) != 0) | if ((uap->op & UMTX_OP__32BIT) != 0) | ||||
umtx_ops = &umtx_native_opsx32; | umtx_ops = &umtx_native_opsx32; | ||||
#endif | #endif | ||||
return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1, | return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1, | ||||
uap->uaddr2, umtx_ops)); | uap->uaddr2, umtx_ops)); | ||||
} | } | ||||
#ifdef COMPAT_FREEBSD32 | #ifdef COMPAT_FREEBSD32 | ||||
#ifdef COMPAT_FREEBSD10 | |||||
int | int | ||||
freebsd10_freebsd32_umtx_lock(struct thread *td, | |||||
struct freebsd10_freebsd32_umtx_lock_args *uap) | |||||
{ | |||||
return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL)); | |||||
} | |||||
int | |||||
freebsd10_freebsd32_umtx_unlock(struct thread *td, | |||||
struct freebsd10_freebsd32_umtx_unlock_args *uap) | |||||
{ | |||||
return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid)); | |||||
} | |||||
#endif /* COMPAT_FREEBSD10 */ | |||||
int | |||||
freebsd32__umtx_op(struct thread *td, struct freebsd32__umtx_op_args *uap) | freebsd32__umtx_op(struct thread *td, struct freebsd32__umtx_op_args *uap) | ||||
{ | { | ||||
return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr, | return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr, | ||||
uap->uaddr2, &umtx_native_ops32)); | uap->uaddr2, &umtx_native_ops32)); | ||||
} | } | ||||
#endif | #endif /* COMPAT_FREEBSD32 */ | ||||
void | void | ||||
umtx_thread_init(struct thread *td) | umtx_thread_init(struct thread *td) | ||||
{ | { | ||||
td->td_umtxq = umtxq_alloc(); | td->td_umtxq = umtxq_alloc(); | ||||
td->td_umtxq->uq_thread = td; | td->td_umtxq->uq_thread = td; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 197 Lines • Show Last 20 Lines |
Discussed briefly on IRC with @kib, the kernel side of this existed in FreeBSD 10 but it was not used by default userland for quite some time, perhaps after FreeBSD 6. Should this be COMPAT_FREEBSD6 (or whatever the actual version was) in that case?