Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_rwlock.c
Show First 20 Lines • Show All 294 Lines • ▼ Show 20 Lines | __rw_try_wlock(volatile uintptr_t *c, const char *file, int line) | ||||
} else | } else | ||||
rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED, | rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED, | ||||
(uintptr_t)curthread); | (uintptr_t)curthread); | ||||
LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line); | LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line); | ||||
if (rval) { | if (rval) { | ||||
WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, | WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, | ||||
file, line); | file, line); | ||||
if (!rw_recursed(rw)) | |||||
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, | |||||
rw, 0, 0, file, line); | |||||
curthread->td_locks++; | curthread->td_locks++; | ||||
} | } | ||||
return (rval); | return (rval); | ||||
} | } | ||||
void | void | ||||
_rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line) | _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line) | ||||
{ | { | ||||
Show All 36 Lines | #ifdef ADAPTIVE_RWLOCKS | ||||
int i; | int i; | ||||
#endif | #endif | ||||
#ifdef LOCK_PROFILING | #ifdef LOCK_PROFILING | ||||
uint64_t waittime = 0; | uint64_t waittime = 0; | ||||
int contested = 0; | int contested = 0; | ||||
#endif | #endif | ||||
uintptr_t v; | uintptr_t v; | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
uintptr_t state; | |||||
uint64_t spin_cnt = 0; | uint64_t spin_cnt = 0; | ||||
uint64_t sleep_cnt = 0; | uint64_t sleep_cnt = 0; | ||||
int64_t sleep_time = 0; | int64_t sleep_time = 0; | ||||
int64_t all_time = 0; | |||||
#endif | #endif | ||||
if (SCHEDULER_STOPPED()) | if (SCHEDULER_STOPPED()) | ||||
return; | return; | ||||
rw = rwlock2rw(c); | rw = rwlock2rw(c); | ||||
KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), | KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), | ||||
("rw_rlock() by idle thread %p on rwlock %s @ %s:%d", | ("rw_rlock() by idle thread %p on rwlock %s @ %s:%d", | ||||
curthread, rw->lock_object.lo_name, file, line)); | curthread, rw->lock_object.lo_name, file, line)); | ||||
KASSERT(rw->rw_lock != RW_DESTROYED, | KASSERT(rw->rw_lock != RW_DESTROYED, | ||||
("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); | ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); | ||||
KASSERT(rw_wowner(rw) != curthread, | KASSERT(rw_wowner(rw) != curthread, | ||||
("rw_rlock: wlock already held for %s @ %s:%d", | ("rw_rlock: wlock already held for %s @ %s:%d", | ||||
rw->lock_object.lo_name, file, line)); | rw->lock_object.lo_name, file, line)); | ||||
WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); | WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); | ||||
#ifdef KDTRACE_HOOKS | |||||
all_time -= lockstat_nsecs(); | |||||
state = rw->rw_lock; | |||||
#endif | |||||
for (;;) { | for (;;) { | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
spin_cnt++; | spin_cnt++; | ||||
#endif | #endif | ||||
/* | /* | ||||
* Handle the easy case. If no other thread has a write | * Handle the easy case. If no other thread has a write | ||||
* lock, then try to bump up the count of read locks. Note | * lock, then try to bump up the count of read locks. Note | ||||
* that we have to preserve the current state of the | * that we have to preserve the current state of the | ||||
▲ Show 20 Lines • Show All 146 Lines • ▼ Show 20 Lines | |||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
sleep_time += lockstat_nsecs(); | sleep_time += lockstat_nsecs(); | ||||
sleep_cnt++; | sleep_cnt++; | ||||
#endif | #endif | ||||
if (LOCK_LOG_TEST(&rw->lock_object, 0)) | if (LOCK_LOG_TEST(&rw->lock_object, 0)) | ||||
CTR2(KTR_LOCK, "%s: %p resuming from turnstile", | CTR2(KTR_LOCK, "%s: %p resuming from turnstile", | ||||
__func__, rw); | __func__, rw); | ||||
} | } | ||||
#ifdef KDTRACE_HOOKS | |||||
all_time += lockstat_nsecs(); | |||||
if (sleep_time) | |||||
LOCKSTAT_RECORD4(LS_RW_RLOCK_BLOCK, rw, sleep_time, | |||||
LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, | |||||
(state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); | |||||
/* Record only the loops spinning and not sleeping. */ | |||||
if (spin_cnt > sleep_cnt) | |||||
LOCKSTAT_RECORD4(LS_RW_RLOCK_SPIN, rw, (all_time - sleep_time), | |||||
LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, | |||||
(state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); | |||||
#endif | |||||
/* | /* | ||||
* TODO: acquire "owner of record" here. Here be turnstile dragons | * TODO: acquire "owner of record" here. Here be turnstile dragons | ||||
* however. turnstiles don't like owners changing between calls to | * however. turnstiles don't like owners changing between calls to | ||||
* turnstile_wait() currently. | * turnstile_wait() currently. | ||||
*/ | */ | ||||
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, rw, contested, | LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, rw, contested, | ||||
waittime, file, line); | waittime, file, line); | ||||
LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); | LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line); | ||||
WITNESS_LOCK(&rw->lock_object, 0, file, line); | WITNESS_LOCK(&rw->lock_object, 0, file, line); | ||||
curthread->td_locks++; | curthread->td_locks++; | ||||
curthread->td_rw_rlocks++; | curthread->td_rw_rlocks++; | ||||
#ifdef KDTRACE_HOOKS | |||||
if (sleep_time) | |||||
LOCKSTAT_RECORD1(LS_RW_RLOCK_BLOCK, rw, sleep_time); | |||||
/* | |||||
* Record only the loops spinning and not sleeping. | |||||
*/ | |||||
if (spin_cnt > sleep_cnt) | |||||
LOCKSTAT_RECORD1(LS_RW_RLOCK_SPIN, rw, (spin_cnt - sleep_cnt)); | |||||
#endif | |||||
} | } | ||||
markj: Same as above - I believe spin_cnt and sleep_cnt can be removed. | |||||
Not Done Inline Actionsditto :-) avg: ditto :-) | |||||
int | int | ||||
__rw_try_rlock(volatile uintptr_t *c, const char *file, int line) | __rw_try_rlock(volatile uintptr_t *c, const char *file, int line) | ||||
{ | { | ||||
struct rwlock *rw; | struct rwlock *rw; | ||||
uintptr_t x; | uintptr_t x; | ||||
if (SCHEDULER_STOPPED()) | if (SCHEDULER_STOPPED()) | ||||
Show All 10 Lines | for (;;) { | ||||
KASSERT(rw->rw_lock != RW_DESTROYED, | KASSERT(rw->rw_lock != RW_DESTROYED, | ||||
("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line)); | ("rw_try_rlock() of destroyed rwlock @ %s:%d", file, line)); | ||||
if (!(x & RW_LOCK_READ)) | if (!(x & RW_LOCK_READ)) | ||||
break; | break; | ||||
if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) { | if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, x + RW_ONE_READER)) { | ||||
LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, | LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file, | ||||
line); | line); | ||||
WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line); | WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line); | ||||
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_RLOCK_ACQUIRE, | |||||
rw, 0, 0, file, line); | |||||
curthread->td_locks++; | curthread->td_locks++; | ||||
curthread->td_rw_rlocks++; | curthread->td_rw_rlocks++; | ||||
return (1); | return (1); | ||||
} | } | ||||
} | } | ||||
LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line); | LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line); | ||||
return (0); | return (0); | ||||
▲ Show 20 Lines • Show All 128 Lines • ▼ Show 20 Lines | #ifdef ADAPTIVE_RWLOCKS | ||||
int i; | int i; | ||||
#endif | #endif | ||||
uintptr_t v, x; | uintptr_t v, x; | ||||
#ifdef LOCK_PROFILING | #ifdef LOCK_PROFILING | ||||
uint64_t waittime = 0; | uint64_t waittime = 0; | ||||
int contested = 0; | int contested = 0; | ||||
#endif | #endif | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
uintptr_t state; | |||||
uint64_t spin_cnt = 0; | uint64_t spin_cnt = 0; | ||||
uint64_t sleep_cnt = 0; | uint64_t sleep_cnt = 0; | ||||
int64_t sleep_time = 0; | int64_t sleep_time = 0; | ||||
int64_t all_time = 0; | |||||
#endif | #endif | ||||
if (SCHEDULER_STOPPED()) | if (SCHEDULER_STOPPED()) | ||||
return; | return; | ||||
rw = rwlock2rw(c); | rw = rwlock2rw(c); | ||||
if (rw_wlocked(rw)) { | if (rw_wlocked(rw)) { | ||||
KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE, | KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE, | ||||
("%s: recursing but non-recursive rw %s @ %s:%d\n", | ("%s: recursing but non-recursive rw %s @ %s:%d\n", | ||||
__func__, rw->lock_object.lo_name, file, line)); | __func__, rw->lock_object.lo_name, file, line)); | ||||
rw->rw_recurse++; | rw->rw_recurse++; | ||||
if (LOCK_LOG_TEST(&rw->lock_object, 0)) | if (LOCK_LOG_TEST(&rw->lock_object, 0)) | ||||
CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); | CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); | ||||
return; | return; | ||||
} | } | ||||
if (LOCK_LOG_TEST(&rw->lock_object, 0)) | if (LOCK_LOG_TEST(&rw->lock_object, 0)) | ||||
CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, | CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, | ||||
rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); | rw->lock_object.lo_name, (void *)rw->rw_lock, file, line); | ||||
#ifdef KDTRACE_HOOKS | |||||
all_time -= lockstat_nsecs(); | |||||
state = rw->rw_lock; | |||||
#endif | |||||
while (!_rw_write_lock(rw, tid)) { | while (!_rw_write_lock(rw, tid)) { | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
spin_cnt++; | spin_cnt++; | ||||
#endif | #endif | ||||
#ifdef HWPMC_HOOKS | #ifdef HWPMC_HOOKS | ||||
PMC_SOFT_CALL( , , lock, failed); | PMC_SOFT_CALL( , , lock, failed); | ||||
#endif | #endif | ||||
lock_profile_obtain_lock_failed(&rw->lock_object, | lock_profile_obtain_lock_failed(&rw->lock_object, | ||||
▲ Show 20 Lines • Show All 121 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
if (LOCK_LOG_TEST(&rw->lock_object, 0)) | if (LOCK_LOG_TEST(&rw->lock_object, 0)) | ||||
CTR2(KTR_LOCK, "%s: %p resuming from turnstile", | CTR2(KTR_LOCK, "%s: %p resuming from turnstile", | ||||
__func__, rw); | __func__, rw); | ||||
#ifdef ADAPTIVE_RWLOCKS | #ifdef ADAPTIVE_RWLOCKS | ||||
spintries = 0; | spintries = 0; | ||||
#endif | #endif | ||||
} | } | ||||
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, rw, contested, | |||||
waittime, file, line); | |||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
all_time += lockstat_nsecs(); | |||||
if (sleep_time) | if (sleep_time) | ||||
LOCKSTAT_RECORD1(LS_RW_WLOCK_BLOCK, rw, sleep_time); | LOCKSTAT_RECORD4(LS_RW_WLOCK_BLOCK, rw, sleep_time, | ||||
LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0, | |||||
(state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); | |||||
/* | /* Record only the loops spinning and not sleeping. */ | ||||
* Record only the loops spinning and not sleeping. | |||||
*/ | |||||
if (spin_cnt > sleep_cnt) | if (spin_cnt > sleep_cnt) | ||||
LOCKSTAT_RECORD1(LS_RW_WLOCK_SPIN, rw, (spin_cnt - sleep_cnt)); | LOCKSTAT_RECORD4(LS_RW_WLOCK_SPIN, rw, (all_time - sleep_time), | ||||
LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, | |||||
(state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); | |||||
#endif | #endif | ||||
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, rw, contested, | |||||
waittime, file, line); | |||||
} | } | ||||
/* | /* | ||||
* This function is called if the first try at releasing a write lock failed. | * This function is called if the first try at releasing a write lock failed. | ||||
* This means that one of the 2 waiter bits must be set indicating that at | * This means that one of the 2 waiter bits must be set indicating that at | ||||
* least one thread is waiting on this lock. | * least one thread is waiting on this lock. | ||||
*/ | */ | ||||
void | void | ||||
▲ Show 20 Lines • Show All 342 Lines • Show Last 20 Lines |
Same as above - I believe spin_cnt and sleep_cnt can be removed.