Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_rwlock.c
Show First 20 Lines • Show All 346 Lines • ▼ Show 20 Lines | #ifdef ADAPTIVE_RWLOCKS | ||||
int i; | int i; | ||||
#endif | #endif | ||||
#ifdef LOCK_PROFILING | #ifdef LOCK_PROFILING | ||||
uint64_t waittime = 0; | uint64_t waittime = 0; | ||||
int contested = 0; | int contested = 0; | ||||
#endif | #endif | ||||
uintptr_t v; | uintptr_t v; | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
uintptr_t last; | |||||
uint64_t spin_cnt = 0; | uint64_t spin_cnt = 0; | ||||
uint64_t sleep_cnt = 0; | uint64_t sleep_cnt = 0; | ||||
int64_t sleep_time = 0; | int64_t sleep_time = 0; | ||||
#endif | #endif | ||||
if (SCHEDULER_STOPPED()) | if (SCHEDULER_STOPPED()) | ||||
return; | return; | ||||
▲ Show 20 Lines • Show All 159 Lines • ▼ Show 20 Lines | #endif | ||||
if (LOCK_LOG_TEST(&rw->lock_object, 0)) | if (LOCK_LOG_TEST(&rw->lock_object, 0)) | ||||
CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, | CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, | ||||
rw); | rw); | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
sleep_time -= lockstat_nsecs(); | sleep_time -= lockstat_nsecs(); | ||||
#endif | #endif | ||||
turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE); | turnstile_wait(ts, rw_owner(rw), TS_SHARED_QUEUE); | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
last = v; | |||||
avg: Perhaps it would make sense to set this vaue ("`last`") to `rw->rw_lock` before the outmost… | |||||
markjAuthorUnsubmitted Not Done Inline ActionsThat makes sense to me. markj: That makes sense to me. | |||||
sleep_time += lockstat_nsecs(); | sleep_time += lockstat_nsecs(); | ||||
sleep_cnt++; | sleep_cnt++; | ||||
#endif | #endif | ||||
if (LOCK_LOG_TEST(&rw->lock_object, 0)) | if (LOCK_LOG_TEST(&rw->lock_object, 0)) | ||||
CTR2(KTR_LOCK, "%s: %p resuming from turnstile", | CTR2(KTR_LOCK, "%s: %p resuming from turnstile", | ||||
__func__, rw); | __func__, rw); | ||||
} | } | ||||
/* | /* | ||||
* TODO: acquire "owner of record" here. Here be turnstile dragons | * TODO: acquire "owner of record" here. Here be turnstile dragons | ||||
* however. turnstiles don't like owners changing between calls to | * however. turnstiles don't like owners changing between calls to | ||||
* turnstile_wait() currently. | * turnstile_wait() currently. | ||||
*/ | */ | ||||
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, rw, contested, | if (sleep_time) | ||||
waittime, file, line); | LOCKSTAT_RECORD4(rw__block, rw, sleep_time, LOCKSTAT_READER, | ||||
(last & RW_LOCK_READ) == 0, | |||||
(last & RW_LOCK_READ) == 0 ? 0 : RW_READERS(last)); | |||||
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(rw__acquire, rw, contested, | |||||
waittime, file, line, LOCKSTAT_READER); | |||||
LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); | LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); | ||||
WITNESS_LOCK(&rw->lock_object, 0, file, line); | WITNESS_LOCK(&rw->lock_object, 0, file, line); | ||||
curthread->td_locks++; | curthread->td_locks++; | ||||
curthread->td_rw_rlocks++; | curthread->td_rw_rlocks++; | ||||
#ifdef KDTRACE_HOOKS | |||||
if (sleep_time) | |||||
LOCKSTAT_RECORD1(LS_RW_RLOCK_BLOCK, rw, sleep_time); | |||||
/* | /* | ||||
* Record only the loops spinning and not sleeping. | * Record only the loops spinning and not sleeping. | ||||
*/ | */ | ||||
if (spin_cnt > sleep_cnt) | LOCKSTAT_RECORD1(rw__spin, rw, (spin_cnt - sleep_cnt)); | ||||
avgUnsubmitted Not Done Inline ActionsAlthough Solaris does not have rw-spin, we should probably put it before rw-acquire. avg: Although Solaris does not have rw-spin, we should probably put it before rw-acquire. | |||||
avgUnsubmitted Not Done Inline Actionsit would probably make sense to pass excatly the same arguments to rw-spin as are passed to rw-block if possible (possibly modulo counts vs nsecs) avg: it would probably make sense to pass excatly the same arguments to rw-spin as are passed to rw… | |||||
markjAuthorUnsubmitted Not Done Inline ActionsOk, I'll fix both of these issues. markj: Ok, I'll fix both of these issues. | |||||
LOCKSTAT_RECORD1(LS_RW_RLOCK_SPIN, rw, (spin_cnt - sleep_cnt)); | |||||
#endif | |||||
} | } | ||||
int | int | ||||
__rw_try_rlock(volatile uintptr_t *c, const char *file, int line) | __rw_try_rlock(volatile uintptr_t *c, const char *file, int line) | ||||
{ | { | ||||
struct rwlock *rw; | struct rwlock *rw; | ||||
uintptr_t x; | uintptr_t x; | ||||
▲ Show 20 Lines • Show All 128 Lines • ▼ Show 20 Lines | for (;;) { | ||||
*/ | */ | ||||
ts = turnstile_lookup(&rw->lock_object); | ts = turnstile_lookup(&rw->lock_object); | ||||
MPASS(ts != NULL); | MPASS(ts != NULL); | ||||
turnstile_broadcast(ts, queue); | turnstile_broadcast(ts, queue); | ||||
turnstile_unpend(ts, TS_SHARED_LOCK); | turnstile_unpend(ts, TS_SHARED_LOCK); | ||||
turnstile_chain_unlock(&rw->lock_object); | turnstile_chain_unlock(&rw->lock_object); | ||||
break; | break; | ||||
} | } | ||||
LOCKSTAT_PROFILE_RELEASE_LOCK(LS_RW_RUNLOCK_RELEASE, rw); | LOCKSTAT_PROFILE_RELEASE_LOCK(rw__release, rw, LOCKSTAT_READER); | ||||
curthread->td_locks--; | curthread->td_locks--; | ||||
curthread->td_rw_rlocks--; | curthread->td_rw_rlocks--; | ||||
} | } | ||||
/* | /* | ||||
* This function is called when we are unable to obtain a write lock on the | * This function is called when we are unable to obtain a write lock on the | ||||
* first try. This means that at least one other thread holds either a | * first try. This means that at least one other thread holds either a | ||||
* read or write lock. | * read or write lock. | ||||
Show All 10 Lines | #ifdef ADAPTIVE_RWLOCKS | ||||
int i; | int i; | ||||
#endif | #endif | ||||
uintptr_t v, x; | uintptr_t v, x; | ||||
#ifdef LOCK_PROFILING | #ifdef LOCK_PROFILING | ||||
uint64_t waittime = 0; | uint64_t waittime = 0; | ||||
int contested = 0; | int contested = 0; | ||||
#endif | #endif | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
uintptr_t last; | |||||
uint64_t spin_cnt = 0; | uint64_t spin_cnt = 0; | ||||
uint64_t sleep_cnt = 0; | uint64_t sleep_cnt = 0; | ||||
int64_t sleep_time = 0; | int64_t sleep_time = 0; | ||||
#endif | #endif | ||||
if (SCHEDULER_STOPPED()) | if (SCHEDULER_STOPPED()) | ||||
return; | return; | ||||
▲ Show 20 Lines • Show All 135 Lines • ▼ Show 20 Lines | #endif | ||||
if (LOCK_LOG_TEST(&rw->lock_object, 0)) | if (LOCK_LOG_TEST(&rw->lock_object, 0)) | ||||
CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, | CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, | ||||
rw); | rw); | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
sleep_time -= lockstat_nsecs(); | sleep_time -= lockstat_nsecs(); | ||||
#endif | #endif | ||||
turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE); | turnstile_wait(ts, rw_owner(rw), TS_EXCLUSIVE_QUEUE); | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
last = v; | |||||
sleep_time += lockstat_nsecs(); | sleep_time += lockstat_nsecs(); | ||||
sleep_cnt++; | sleep_cnt++; | ||||
#endif | #endif | ||||
if (LOCK_LOG_TEST(&rw->lock_object, 0)) | if (LOCK_LOG_TEST(&rw->lock_object, 0)) | ||||
CTR2(KTR_LOCK, "%s: %p resuming from turnstile", | CTR2(KTR_LOCK, "%s: %p resuming from turnstile", | ||||
__func__, rw); | __func__, rw); | ||||
#ifdef ADAPTIVE_RWLOCKS | #ifdef ADAPTIVE_RWLOCKS | ||||
spintries = 0; | spintries = 0; | ||||
#endif | #endif | ||||
} | } | ||||
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, rw, contested, | |||||
waittime, file, line); | |||||
#ifdef KDTRACE_HOOKS | |||||
if (sleep_time) | if (sleep_time) | ||||
LOCKSTAT_RECORD1(LS_RW_WLOCK_BLOCK, rw, sleep_time); | LOCKSTAT_RECORD4(rw__block, rw, sleep_time, LOCKSTAT_WRITER, | ||||
(last & RW_LOCK_READ) == 0, | |||||
(last & RW_LOCK_READ) == 0 ? 0 : RW_READERS(last)); | |||||
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(rw__acquire, rw, contested, | |||||
waittime, file, line, LOCKSTAT_WRITER); | |||||
/* | /* | ||||
* Record only the loops spinning and not sleeping. | * Record only the loops spinning and not sleeping. | ||||
*/ | */ | ||||
if (spin_cnt > sleep_cnt) | LOCKSTAT_RECORD1(rw__spin, rw, (spin_cnt - sleep_cnt)); | ||||
LOCKSTAT_RECORD1(LS_RW_WLOCK_SPIN, rw, (spin_cnt - sleep_cnt)); | |||||
#endif | |||||
} | } | ||||
/* | /* | ||||
* This function is called if the first try at releasing a write lock failed. | * This function is called if the first try at releasing a write lock failed. | ||||
* This means that one of the 2 waiter bits must be set indicating that at | * This means that one of the 2 waiter bits must be set indicating that at | ||||
* least one thread is waiting on this lock. | * least one thread is waiting on this lock. | ||||
*/ | */ | ||||
void | void | ||||
▲ Show 20 Lines • Show All 128 Lines • ▼ Show 20 Lines | for (;;) { | ||||
} | } | ||||
turnstile_cancel(ts); | turnstile_cancel(ts); | ||||
} | } | ||||
LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line); | LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line); | ||||
if (success) { | if (success) { | ||||
curthread->td_rw_rlocks--; | curthread->td_rw_rlocks--; | ||||
WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, | WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, | ||||
file, line); | file, line); | ||||
LOCKSTAT_RECORD0(LS_RW_TRYUPGRADE_UPGRADE, rw); | LOCKSTAT_RECORD0(rw__upgrade, rw); | ||||
} | } | ||||
return (success); | return (success); | ||||
} | } | ||||
/* | /* | ||||
* Downgrade a write lock into a single read lock. | * Downgrade a write lock into a single read lock. | ||||
*/ | */ | ||||
void | void | ||||
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines | if (rwait && !wwait) { | ||||
turnstile_broadcast(ts, TS_SHARED_QUEUE); | turnstile_broadcast(ts, TS_SHARED_QUEUE); | ||||
turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); | turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); | ||||
} else | } else | ||||
turnstile_disown(ts); | turnstile_disown(ts); | ||||
turnstile_chain_unlock(&rw->lock_object); | turnstile_chain_unlock(&rw->lock_object); | ||||
out: | out: | ||||
curthread->td_rw_rlocks++; | curthread->td_rw_rlocks++; | ||||
LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line); | LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line); | ||||
LOCKSTAT_RECORD0(LS_RW_DOWNGRADE_DOWNGRADE, rw); | LOCKSTAT_RECORD0(rw__downgrade, rw); | ||||
} | } | ||||
#ifdef INVARIANT_SUPPORT | #ifdef INVARIANT_SUPPORT | ||||
#ifndef INVARIANTS | #ifndef INVARIANTS | ||||
#undef __rw_assert | #undef __rw_assert | ||||
#endif | #endif | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 125 Lines • Show Last 20 Lines |
Perhaps it would make sense to set this vaue ("last") to rw->rw_lock before the outmost loop is entered, so that the value reflects the original state of the lock? It certailny would help with arguments to rw-spin if we decide to supply them.