Changeset View
Standalone View
sys/kern/kern_umtx.c
Show First 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | |||||
#include <sys/sched.h> | #include <sys/sched.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/sysent.h> | #include <sys/sysent.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <sys/sysproto.h> | #include <sys/sysproto.h> | ||||
#include <sys/syscallsubr.h> | #include <sys/syscallsubr.h> | ||||
#include <sys/taskqueue.h> | #include <sys/taskqueue.h> | ||||
#include <sys/time.h> | |||||
#include <sys/eventhandler.h> | #include <sys/eventhandler.h> | ||||
#include <sys/umtx.h> | #include <sys/umtx.h> | ||||
#include <security/mac/mac_framework.h> | #include <security/mac/mac_framework.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/vm_param.h> | #include <vm/vm_param.h> | ||||
#include <vm/pmap.h> | #include <vm/pmap.h> | ||||
#include <vm/vm_map.h> | #include <vm/vm_map.h> | ||||
#include <vm/vm_object.h> | #include <vm/vm_object.h> | ||||
#include <machine/atomic.h> | |||||
#include <machine/cpu.h> | #include <machine/cpu.h> | ||||
#ifdef COMPAT_FREEBSD32 | #ifdef COMPAT_FREEBSD32 | ||||
#include <compat/freebsd32/freebsd32_proto.h> | #include <compat/freebsd32/freebsd32_proto.h> | ||||
#endif | #endif | ||||
#define _UMUTEX_TRY 1 | #define _UMUTEX_TRY 1 | ||||
#define _UMUTEX_WAIT 2 | #define _UMUTEX_WAIT 2 | ||||
▲ Show 20 Lines • Show All 124 Lines • ▼ Show 20 Lines | |||||
#define GET_SHARE(flags) \ | #define GET_SHARE(flags) \ | ||||
(((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE) | (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE) | ||||
#define BUSY_SPINS 200 | #define BUSY_SPINS 200 | ||||
struct abs_timeout { | struct abs_timeout { | ||||
int clockid; | int clockid; | ||||
bool is_realtime; | |||||
struct timespec cur; | struct timespec cur; | ||||
struct timespec end; | struct timespec end; | ||||
}; | }; | ||||
#ifdef COMPAT_FREEBSD32 | #ifdef COMPAT_FREEBSD32 | ||||
struct umutex32 { | struct umutex32 { | ||||
volatile __lwpid_t m_owner; /* Owner of the mutex */ | volatile __lwpid_t m_owner; /* Owner of the mutex */ | ||||
__uint32_t m_flags; /* Flags of the mutex */ | __uint32_t m_flags; /* Flags of the mutex */ | ||||
Show All 31 Lines | SYSCTL_INT(_debug_umtx, OID_AUTO, robust_faults_verbose, CTLFLAG_RWTUN, | ||||
""); | ""); | ||||
#ifdef UMTX_PROFILING | #ifdef UMTX_PROFILING | ||||
static long max_length; | static long max_length; | ||||
SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length"); | SYSCTL_LONG(_debug_umtx, OID_AUTO, max_length, CTLFLAG_RD, &max_length, 0, "max_length"); | ||||
static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD, 0, "umtx chain stats"); | static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD, 0, "umtx chain stats"); | ||||
#endif | #endif | ||||
static void abs_timeout_update(struct abs_timeout *timo); | |||||
static void umtx_shm_init(void); | static void umtx_shm_init(void); | ||||
static void umtxq_sysinit(void *); | static void umtxq_sysinit(void *); | ||||
static void umtxq_hash(struct umtx_key *key); | static void umtxq_hash(struct umtx_key *key); | ||||
static struct umtxq_chain *umtxq_getchain(struct umtx_key *key); | static struct umtxq_chain *umtxq_getchain(struct umtx_key *key); | ||||
static void umtxq_lock(struct umtx_key *key); | static void umtxq_lock(struct umtx_key *key); | ||||
static void umtxq_unlock(struct umtx_key *key); | static void umtxq_unlock(struct umtx_key *key); | ||||
static void umtxq_busy(struct umtx_key *key); | static void umtxq_busy(struct umtx_key *key); | ||||
static void umtxq_unbusy(struct umtx_key *key); | static void umtxq_unbusy(struct umtx_key *key); | ||||
▲ Show 20 Lines • Show All 498 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static void | static void | ||||
abs_timeout_init(struct abs_timeout *timo, int clockid, int absolute, | abs_timeout_init(struct abs_timeout *timo, int clockid, int absolute, | ||||
const struct timespec *timeout) | const struct timespec *timeout) | ||||
{ | { | ||||
timo->clockid = clockid; | timo->clockid = clockid; | ||||
timo->is_realtime = (clockid == CLOCK_REALTIME || | |||||
kib: () are not needed. | |||||
Done Inline ActionsTrue, but I like them. However, style(9) discourages them, so I'll remove them. vangyzen: True, but I like them. However, style(9) discourages them, so I'll remove them. | |||||
clockid == CLOCK_REALTIME_FAST || | |||||
clockid == CLOCK_REALTIME_PRECISE); | |||||
abs_timeout_update(timo); | |||||
if (!absolute) { | if (!absolute) { | ||||
kern_clock_gettime(curthread, clockid, &timo->end); | timo->end = timo->cur; | ||||
timo->cur = timo->end; | |||||
timespecadd(&timo->end, timeout); | timespecadd(&timo->end, timeout); | ||||
} else { | } else { | ||||
timo->end = *timeout; | timo->end = *timeout; | ||||
kern_clock_gettime(curthread, clockid, &timo->cur); | |||||
} | } | ||||
} | } | ||||
static void | static void | ||||
abs_timeout_init2(struct abs_timeout *timo, const struct _umtx_time *umtxtime) | abs_timeout_init2(struct abs_timeout *timo, const struct _umtx_time *umtxtime) | ||||
{ | { | ||||
abs_timeout_init(timo, umtxtime->_clockid, | abs_timeout_init(timo, umtxtime->_clockid, | ||||
(umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout); | (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout); | ||||
} | } | ||||
static inline void | static inline void | ||||
abs_timeout_update(struct abs_timeout *timo) | abs_timeout_update(struct abs_timeout *timo) | ||||
{ | { | ||||
if (timo->is_realtime) | |||||
curthread->td_rtcgen = atomic_load_acq_int(&rtc_generation); | |||||
kern_clock_gettime(curthread, timo->clockid, &timo->cur); | kern_clock_gettime(curthread, timo->clockid, &timo->cur); | ||||
} | } | ||||
static int | static int | ||||
abs_timeout_gethz(struct abs_timeout *timo) | abs_timeout_gethz(struct abs_timeout *timo) | ||||
{ | { | ||||
struct timespec tts; | struct timespec tts; | ||||
Show All 12 Lines | if (rb) | ||||
return (UMUTEX_RB_OWNERDEAD); | return (UMUTEX_RB_OWNERDEAD); | ||||
else if ((flags & UMUTEX_NONCONSISTENT) != 0) | else if ((flags & UMUTEX_NONCONSISTENT) != 0) | ||||
return (UMUTEX_RB_NOTRECOV); | return (UMUTEX_RB_NOTRECOV); | ||||
else | else | ||||
return (UMUTEX_UNOWNED); | return (UMUTEX_UNOWNED); | ||||
} | } | ||||
#if 1 || defined(TEST_RACE) | |||||
vangyzenAuthorUnsubmitted Done Inline ActionsThis will be removed before commit. vangyzen: This will be removed before commit. | |||||
static int umtx_rtc_race = 0; | |||||
SYSCTL_INT(_debug_umtx, OID_AUTO, rtc_race, CTLFLAG_RW, | |||||
&umtx_rtc_race, 0, ""); | |||||
#endif | |||||
/* | /* | ||||
* Put thread into sleep state, before sleeping, check if | * Put thread into sleep state, before sleeping, check if | ||||
* thread was removed from umtx queue. | * thread was removed from umtx queue. | ||||
*/ | */ | ||||
static inline int | static inline int | ||||
umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *abstime) | umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout *abstime) | ||||
{ | { | ||||
struct umtxq_chain *uc; | struct umtxq_chain *uc; | ||||
int error, timo; | int error, timo, priority; | ||||
priority = PCATCH | PDROP; | |||||
if (abstime != NULL && abstime->is_realtime) { | |||||
priority |= PRTCLK; | |||||
#if 1 || defined(TEST_RACE) | |||||
while (umtx_rtc_race) { | |||||
pause("rtcrace", hz); | |||||
} | |||||
#endif | |||||
} | |||||
uc = umtxq_getchain(&uq->uq_key); | uc = umtxq_getchain(&uq->uq_key); | ||||
UMTXQ_LOCKED_ASSERT(uc); | UMTXQ_LOCKED_ASSERT(uc); | ||||
Done Inline ActionsI understand why this reading of rtc_generation needs acquire semantic: we ensure that if we see updated rtc_generation, then we also see the updated timehands. But I do not quite see why other rtc_generation reads, in particular, in sleeping_on_old_rtc() and in sleepq_switch(), need the acquire. sleeping_on_old_rtc() happens in the same thread that incremented the variable. sleepq_switch() only needs the value of the variable, it does not order its read with other happenings in tc_windup() which did release write. Could you, please, explain your use of _acq there ? kib: I understand why this reading of rtc_generation needs acquire semantic: we ensure that if we… | |||||
Done Inline ActionsThere is no good reason. I'm embarrassed to say, it was absent-minded. Of course, you're right that only this function needs the acquire semantic. vangyzen: There is no good reason. I'm embarrassed to say, it was absent-minded. Of course, you're… | |||||
for (;;) { | for (;;) { | ||||
if (!(uq->uq_flags & UQF_UMTXQ)) | if (!(uq->uq_flags & UQF_UMTXQ)) | ||||
return (0); | return (0); | ||||
if (abstime != NULL) { | if (abstime != NULL) { | ||||
timo = abs_timeout_gethz(abstime); | timo = abs_timeout_gethz(abstime); | ||||
if (timo < 0) | if (timo < 0) | ||||
return (ETIMEDOUT); | return (ETIMEDOUT); | ||||
} else | } else | ||||
timo = 0; | timo = 0; | ||||
error = msleep(uq, &uc->uc_lock, PCATCH | PDROP, wmesg, timo); | error = msleep(uq, &uc->uc_lock, priority, wmesg, timo); | ||||
if (error != EWOULDBLOCK) { | if (error != EWOULDBLOCK && error != ERELOOKUP) { | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
break; | break; | ||||
} | } | ||||
if (abstime != NULL) | if (abstime != NULL) | ||||
abs_timeout_update(abstime); | abs_timeout_update(abstime); | ||||
umtxq_lock(&uq->uq_key); | umtxq_lock(&uq->uq_key); | ||||
} | } | ||||
return (error); | return (error); | ||||
} | } | ||||
Done Inline ActionsToo many blank lines. kib: Too many blank lines. | |||||
/* | /* | ||||
* Convert userspace address into unique logical address. | * Convert userspace address into unique logical address. | ||||
*/ | */ | ||||
int | int | ||||
umtx_key_get(const void *addr, int type, int share, struct umtx_key *key) | umtx_key_get(const void *addr, int type, int share, struct umtx_key *key) | ||||
{ | { | ||||
struct thread *td = curthread; | struct thread *td = curthread; | ||||
vm_map_t map; | vm_map_t map; | ||||
▲ Show 20 Lines • Show All 3,681 Lines • Show Last 20 Lines |
() are not needed.