Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_umtx.c
Context not available. | |||||
#include <machine/atomic.h> | #include <machine/atomic.h> | ||||
#include <machine/cpu.h> | #include <machine/cpu.h> | ||||
#include <compat/freebsd32/freebsd32.h> | |||||
#ifdef COMPAT_FREEBSD32 | #ifdef COMPAT_FREEBSD32 | ||||
#include <compat/freebsd32/freebsd32.h> | |||||
#include <compat/freebsd32/freebsd32_proto.h> | #include <compat/freebsd32/freebsd32_proto.h> | ||||
#endif | #endif | ||||
Context not available. | |||||
struct timespec end; | struct timespec end; | ||||
}; | }; | ||||
struct umtx_copyops { | #ifdef COMPAT_FREEBSD32 | ||||
int (*copyin_timeout)(const void *uaddr, struct timespec *tsp); | |||||
int (*copyin_umtx_time)(const void *uaddr, size_t size, | |||||
struct _umtx_time *tp); | |||||
int (*copyin_robust_lists)(const void *uaddr, size_t size, | |||||
struct umtx_robust_lists_params *rbp); | |||||
int (*copyout_timeout)(void *uaddr, size_t size, | |||||
struct timespec *tsp); | |||||
const size_t timespec_sz; | |||||
const size_t umtx_time_sz; | |||||
const bool compat32; | |||||
}; | |||||
struct umtx_time32 { | |||||
struct timespec32 _timeout; | |||||
uint32_t _flags; | |||||
uint32_t _clockid; | |||||
}; | |||||
struct umtx_robust_lists_params_compat32 { | |||||
uint32_t robust_list_offset; | |||||
uint32_t robust_priv_list_offset; | |||||
uint32_t robust_inact_offset; | |||||
}; | |||||
struct umutex32 { | |||||
volatile __lwpid_t m_owner; /* Owner of the mutex */ | |||||
__uint32_t m_flags; /* Flags of the mutex */ | |||||
__uint32_t m_ceilings[2]; /* Priority protect ceiling */ | |||||
__uint32_t m_rb_lnk; /* Robust linkage */ | |||||
__uint32_t m_pad; | |||||
__uint32_t m_spare[2]; | |||||
}; | |||||
_Static_assert(sizeof(struct umutex) == sizeof(struct umutex32), "umutex32"); | _Static_assert(sizeof(struct umutex) == sizeof(struct umutex32), "umutex32"); | ||||
_Static_assert(__offsetof(struct umutex, m_spare[0]) == | _Static_assert(__offsetof(struct umutex, m_spare[0]) == | ||||
__offsetof(struct umutex32, m_spare[0]), "m_spare32"); | __offsetof(struct umutex32, m_spare[0]), "m_spare32"); | ||||
#endif | |||||
int umtx_shm_vnobj_persistent = 0; | int umtx_shm_vnobj_persistent = 0; | ||||
SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_vnode_persistent, CTLFLAG_RWTUN, | SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_vnode_persistent, CTLFLAG_RWTUN, | ||||
Context not available. | |||||
struct umtx_robust_lists_params rb; | struct umtx_robust_lists_params rb; | ||||
int error; | int error; | ||||
if (ops->compat32) { | |||||
if ((td->td_pflags2 & TDP2_COMPAT32RB) == 0 && | |||||
(td->td_rb_list != 0 || td->td_rbp_list != 0 || | |||||
td->td_rb_inact != 0)) | |||||
return (EBUSY); | |||||
} else if ((td->td_pflags2 & TDP2_COMPAT32RB) != 0) { | |||||
return (EBUSY); | |||||
} | |||||
bzero(&rb, sizeof(rb)); | bzero(&rb, sizeof(rb)); | ||||
error = ops->copyin_robust_lists(uap->uaddr1, uap->val, &rb); | error = ops->copyin_robust_lists(uap->uaddr1, uap->val, &rb); | ||||
if (error != 0) | if (error != 0) | ||||
Context not available. | |||||
if (ops->compat32) | if (ops->compat32) | ||||
td->td_pflags2 |= TDP2_COMPAT32RB; | td->td_pflags2 |= TDP2_COMPAT32RB; | ||||
else if ((td->td_pflags2 & TDP2_COMPAT32RB) != 0) | |||||
return (EINVAL); | |||||
td->td_rb_list = rb.robust_list_offset; | td->td_rb_list = rb.robust_list_offset; | ||||
td->td_rbp_list = rb.robust_priv_list_offset; | td->td_rbp_list = rb.robust_priv_list_offset; | ||||
Context not available. | |||||
return (0); | return (0); | ||||
} | } | ||||
#if defined(__i386__) || defined(__amd64__) | #ifdef COMPAT_FREEBSD32 | ||||
/* | |||||
* Provide the standard 32-bit definitions for x86, since native/compat32 use a | |||||
* 32-bit time_t there. Other architectures just need the i386 definitions | |||||
* along with their standard compat32. | |||||
*/ | |||||
struct timespecx32 { | |||||
int64_t tv_sec; | |||||
int32_t tv_nsec; | |||||
}; | |||||
struct umtx_timex32 { | |||||
struct timespecx32 _timeout; | |||||
uint32_t _flags; | |||||
uint32_t _clockid; | |||||
}; | |||||
#ifndef __i386__ | |||||
#define timespeci386 timespec32 | |||||
#define umtx_timei386 umtx_time32 | |||||
#endif | |||||
#else /* !__i386__ && !__amd64__ */ | |||||
/* 32-bit architectures can emulate i386, so define these almost everywhere. */ | |||||
struct timespeci386 { | |||||
int32_t tv_sec; | |||||
int32_t tv_nsec; | |||||
}; | |||||
struct umtx_timei386 { | |||||
struct timespeci386 _timeout; | |||||
uint32_t _flags; | |||||
uint32_t _clockid; | |||||
}; | |||||
#if defined(__LP64__) | |||||
#define timespecx32 timespec32 | |||||
#define umtx_timex32 umtx_time32 | |||||
#endif | |||||
#endif | |||||
static int | |||||
umtx_copyin_robust_lists32(const void *uaddr, size_t size, | |||||
struct umtx_robust_lists_params *rbp) | |||||
{ | |||||
struct umtx_robust_lists_params_compat32 rb32; | |||||
int error; | |||||
if (size > sizeof(rb32)) | |||||
return (EINVAL); | |||||
bzero(&rb32, sizeof(rb32)); | |||||
error = copyin(uaddr, &rb32, size); | |||||
if (error != 0) | |||||
return (error); | |||||
CP(rb32, *rbp, robust_list_offset); | |||||
CP(rb32, *rbp, robust_priv_list_offset); | |||||
CP(rb32, *rbp, robust_inact_offset); | |||||
return (0); | |||||
} | |||||
#ifndef __i386__ | |||||
static inline int | static inline int | ||||
umtx_copyin_timeouti386(const void *uaddr, struct timespec *tsp) | umtx_copyin_timeout32(const void *uaddr, struct timespec *tsp) | ||||
{ | { | ||||
struct timespeci386 ts32; | struct timespec32 ts32; | ||||
int error; | int error; | ||||
error = copyin(uaddr, &ts32, sizeof(ts32)); | error = copyin(uaddr, &ts32, sizeof(ts32)); | ||||
Context not available. | |||||
} | } | ||||
static inline int | static inline int | ||||
umtx_copyin_umtx_timei386(const void *uaddr, size_t size, struct _umtx_time *tp) | umtx_copyin_umtx_time32(const void *uaddr, size_t size, struct _umtx_time *tp) | ||||
{ | { | ||||
struct umtx_timei386 t32; | struct umtx_time32 t32; | ||||
int error; | int error; | ||||
t32._clockid = CLOCK_REALTIME; | t32._clockid = CLOCK_REALTIME; | ||||
Context not available. | |||||
} | } | ||||
static int | static int | ||||
umtx_copyout_timeouti386(void *uaddr, size_t sz, struct timespec *tsp) | umtx_copyin_robust_lists32(const void *uaddr, size_t size, | ||||
{ | struct umtx_robust_lists_params *rbp) | ||||
struct timespeci386 remain32 = { | |||||
.tv_sec = tsp->tv_sec, | |||||
.tv_nsec = tsp->tv_nsec, | |||||
}; | |||||
/* | |||||
* Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time) | |||||
* and we're only called if sz >= sizeof(timespec) as supplied in the | |||||
* copyops. | |||||
*/ | |||||
KASSERT(sz >= sizeof(remain32), | |||||
("umtx_copyops specifies incorrect sizes")); | |||||
return (copyout(&remain32, uaddr, sizeof(remain32))); | |||||
} | |||||
#endif /* !__i386__ */ | |||||
#if defined(__i386__) || defined(__LP64__) | |||||
static inline int | |||||
umtx_copyin_timeoutx32(const void *uaddr, struct timespec *tsp) | |||||
{ | |||||
struct timespecx32 ts32; | |||||
int error; | |||||
error = copyin(uaddr, &ts32, sizeof(ts32)); | |||||
if (error == 0) { | |||||
if (ts32.tv_sec < 0 || | |||||
ts32.tv_nsec >= 1000000000 || | |||||
ts32.tv_nsec < 0) | |||||
error = EINVAL; | |||||
else { | |||||
CP(ts32, *tsp, tv_sec); | |||||
CP(ts32, *tsp, tv_nsec); | |||||
} | |||||
} | |||||
return (error); | |||||
} | |||||
static inline int | |||||
umtx_copyin_umtx_timex32(const void *uaddr, size_t size, struct _umtx_time *tp) | |||||
{ | { | ||||
struct umtx_timex32 t32; | struct umtx_robust_lists_params_compat32 rb32; | ||||
int error; | int error; | ||||
t32._clockid = CLOCK_REALTIME; | if (size > sizeof(rb32)) | ||||
t32._flags = 0; | return (EINVAL); | ||||
if (size <= sizeof(t32._timeout)) | bzero(&rb32, sizeof(rb32)); | ||||
error = copyin(uaddr, &t32._timeout, sizeof(t32._timeout)); | error = copyin(uaddr, &rb32, size); | ||||
else | |||||
error = copyin(uaddr, &t32, sizeof(t32)); | |||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
if (t32._timeout.tv_sec < 0 || | CP(rb32, *rbp, robust_list_offset); | ||||
t32._timeout.tv_nsec >= 1000000000 || t32._timeout.tv_nsec < 0) | CP(rb32, *rbp, robust_priv_list_offset); | ||||
return (EINVAL); | CP(rb32, *rbp, robust_inact_offset); | ||||
TS_CP(t32, *tp, _timeout); | |||||
CP(t32, *tp, _flags); | |||||
CP(t32, *tp, _clockid); | |||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
umtx_copyout_timeoutx32(void *uaddr, size_t sz, struct timespec *tsp) | umtx_copyout_timeout32(void *uaddr, size_t sz, struct timespec *tsp) | ||||
{ | { | ||||
struct timespecx32 remain32 = { | struct timespec32 remain32 = { | ||||
.tv_sec = tsp->tv_sec, | .tv_sec = tsp->tv_sec, | ||||
.tv_nsec = tsp->tv_nsec, | .tv_nsec = tsp->tv_nsec, | ||||
}; | }; | ||||
Context not available. | |||||
return (copyout(&remain32, uaddr, sizeof(remain32))); | return (copyout(&remain32, uaddr, sizeof(remain32))); | ||||
} | } | ||||
#endif /* __i386__ || __LP64__ */ | #endif /* COMPAT_FREEBSD32 */ | ||||
typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap, | typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap, | ||||
const struct umtx_copyops *umtx_ops); | const struct umtx_copyops *umtx_ops); | ||||
Context not available. | |||||
.umtx_time_sz = sizeof(struct _umtx_time), | .umtx_time_sz = sizeof(struct _umtx_time), | ||||
}; | }; | ||||
#ifndef __i386__ | #ifdef COMPAT_FREEBSD32 | ||||
static const struct umtx_copyops umtx_native_opsi386 = { | const struct umtx_copyops umtx_native_ops32 = { | ||||
.copyin_timeout = umtx_copyin_timeouti386, | .copyin_timeout = umtx_copyin_timeout32, | ||||
.copyin_umtx_time = umtx_copyin_umtx_timei386, | .copyin_umtx_time = umtx_copyin_umtx_time32, | ||||
.copyin_robust_lists = umtx_copyin_robust_lists32, | |||||
.copyout_timeout = umtx_copyout_timeouti386, | |||||
.timespec_sz = sizeof(struct timespeci386), | |||||
.umtx_time_sz = sizeof(struct umtx_timei386), | |||||
.compat32 = true, | |||||
}; | |||||
#endif | |||||
#if defined(__i386__) || defined(__LP64__) | |||||
/* i386 can emulate other 32-bit archs, too! */ | |||||
static const struct umtx_copyops umtx_native_opsx32 = { | |||||
.copyin_timeout = umtx_copyin_timeoutx32, | |||||
.copyin_umtx_time = umtx_copyin_umtx_timex32, | |||||
.copyin_robust_lists = umtx_copyin_robust_lists32, | .copyin_robust_lists = umtx_copyin_robust_lists32, | ||||
.copyout_timeout = umtx_copyout_timeoutx32, | .copyout_timeout = umtx_copyout_timeout32, | ||||
.timespec_sz = sizeof(struct timespecx32), | .timespec_sz = sizeof(struct timespec32), | ||||
.umtx_time_sz = sizeof(struct umtx_timex32), | .umtx_time_sz = sizeof(struct umtx_time32), | ||||
.compat32 = true, | .compat32 = true, | ||||
}; | }; | ||||
#ifdef COMPAT_FREEBSD32 | |||||
#ifdef __amd64__ | |||||
#define umtx_native_ops32 umtx_native_opsi386 | |||||
#else | |||||
#define umtx_native_ops32 umtx_native_opsx32 | |||||
#endif | #endif | ||||
#endif /* COMPAT_FREEBSD32 */ | |||||
#endif /* __i386__ || __LP64__ */ | |||||
#define UMTX_OP__FLAGS (UMTX_OP__32BIT | UMTX_OP__I386) | |||||
static int | int | ||||
kern__umtx_op(struct thread *td, void *obj, int op, unsigned long val, | kern__umtx_op(struct thread *td, void *obj, int op, unsigned long val, | ||||
void *uaddr1, void *uaddr2, const struct umtx_copyops *ops) | void *uaddr1, void *uaddr2, const struct umtx_copyops *ops) | ||||
{ | { | ||||
struct _umtx_op_args uap = { | struct _umtx_op_args uap = { | ||||
.obj = obj, | .obj = obj, | ||||
.op = op & ~UMTX_OP__FLAGS, | .op = op, | ||||
.val = val, | .val = val, | ||||
.uaddr1 = uaddr1, | .uaddr1 = uaddr1, | ||||
.uaddr2 = uaddr2 | .uaddr2 = uaddr2 | ||||
Context not available. | |||||
int | int | ||||
sys__umtx_op(struct thread *td, struct _umtx_op_args *uap) | sys__umtx_op(struct thread *td, struct _umtx_op_args *uap) | ||||
{ | { | ||||
static const struct umtx_copyops *umtx_ops; | |||||
umtx_ops = &umtx_native_ops; | |||||
#ifdef __LP64__ | |||||
if ((uap->op & (UMTX_OP__32BIT | UMTX_OP__I386)) != 0) { | |||||
if ((uap->op & UMTX_OP__I386) != 0) | |||||
umtx_ops = &umtx_native_opsi386; | |||||
else | |||||
umtx_ops = &umtx_native_opsx32; | |||||
} | |||||
#elif !defined(__i386__) | |||||
/* We consider UMTX_OP__32BIT a nop on !i386 ILP32. */ | |||||
if ((uap->op & UMTX_OP__I386) != 0) | |||||
umtx_ops = &umtx_native_opsi386; | |||||
#else | |||||
/* Likewise, UMTX_OP__I386 is a nop on i386. */ | |||||
if ((uap->op & UMTX_OP__32BIT) != 0) | |||||
umtx_ops = &umtx_native_opsx32; | |||||
#endif | |||||
return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1, | return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1, | ||||
uap->uaddr2, umtx_ops)); | uap->uaddr2, &umtx_native_ops)); | ||||
} | } | ||||
#ifdef COMPAT_FREEBSD32 | |||||
int | |||||
freebsd32__umtx_op(struct thread *td, struct freebsd32__umtx_op_args *uap) | |||||
{ | |||||
return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr, | |||||
uap->uaddr2, &umtx_native_ops32)); | |||||
} | |||||
#endif | |||||
void | void | ||||
umtx_thread_init(struct thread *td) | umtx_thread_init(struct thread *td) | ||||
{ | { | ||||
Context not available. | |||||
umtx_read_uptr(struct thread *td, uintptr_t ptr, uintptr_t *res, bool compat32) | umtx_read_uptr(struct thread *td, uintptr_t ptr, uintptr_t *res, bool compat32) | ||||
{ | { | ||||
u_long res1; | u_long res1; | ||||
#ifdef COMPAT_FREEBSD32 | |||||
uint32_t res32; | uint32_t res32; | ||||
#endif | |||||
int error; | int error; | ||||
#ifdef COMPAT_FREEBSD32 | |||||
if (compat32) { | if (compat32) { | ||||
error = fueword32((void *)ptr, &res32); | error = fueword32((void *)ptr, &res32); | ||||
if (error == 0) | if (error == 0) | ||||
res1 = res32; | res1 = res32; | ||||
} else { | } else | ||||
#endif | |||||
{ | |||||
error = fueword((void *)ptr, &res1); | error = fueword((void *)ptr, &res1); | ||||
} | } | ||||
if (error == 0) | if (error == 0) | ||||
Context not available. | |||||
umtx_read_rb_list(struct thread *td, struct umutex *m, uintptr_t *rb_list, | umtx_read_rb_list(struct thread *td, struct umutex *m, uintptr_t *rb_list, | ||||
bool compat32) | bool compat32) | ||||
{ | { | ||||
#ifdef COMPAT_FREEBSD32 | |||||
struct umutex32 m32; | struct umutex32 m32; | ||||
if (compat32) { | if (compat32) { | ||||
memcpy(&m32, m, sizeof(m32)); | memcpy(&m32, m, sizeof(m32)); | ||||
*rb_list = m32.m_rb_lnk; | *rb_list = m32.m_rb_lnk; | ||||
} else { | } else | ||||
#endif | |||||
*rb_list = m->m_rb_lnk; | *rb_list = m->m_rb_lnk; | ||||
} | |||||
} | } | ||||
static int | static int | ||||
Context not available. |