Changeset View
Changeset View
Standalone View
Standalone View
head/sys/kern/kern_mutex.c
Show First 20 Lines • Show All 378 Lines • ▼ Show 20 Lines | |||||
#ifdef LOCK_PROFILING | #ifdef LOCK_PROFILING | ||||
int contested = 0; | int contested = 0; | ||||
uint64_t waittime = 0; | uint64_t waittime = 0; | ||||
#endif | #endif | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
uint64_t spin_cnt = 0; | uint64_t spin_cnt = 0; | ||||
uint64_t sleep_cnt = 0; | uint64_t sleep_cnt = 0; | ||||
int64_t sleep_time = 0; | int64_t sleep_time = 0; | ||||
int64_t all_time = 0; | |||||
#endif | #endif | ||||
if (SCHEDULER_STOPPED()) | if (SCHEDULER_STOPPED()) | ||||
return; | return; | ||||
m = mtxlock2mtx(c); | m = mtxlock2mtx(c); | ||||
if (mtx_owned(m)) { | if (mtx_owned(m)) { | ||||
Show All 14 Lines | #ifdef HWPMC_HOOKS | ||||
PMC_SOFT_CALL( , , lock, failed); | PMC_SOFT_CALL( , , lock, failed); | ||||
#endif | #endif | ||||
lock_profile_obtain_lock_failed(&m->lock_object, | lock_profile_obtain_lock_failed(&m->lock_object, | ||||
&contested, &waittime); | &contested, &waittime); | ||||
if (LOCK_LOG_TEST(&m->lock_object, opts)) | if (LOCK_LOG_TEST(&m->lock_object, opts)) | ||||
CTR4(KTR_LOCK, | CTR4(KTR_LOCK, | ||||
"_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", | "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", | ||||
m->lock_object.lo_name, (void *)m->mtx_lock, file, line); | m->lock_object.lo_name, (void *)m->mtx_lock, file, line); | ||||
#ifdef KDTRACE_HOOKS | |||||
all_time -= lockstat_nsecs(); | |||||
#endif | |||||
while (!_mtx_obtain_lock(m, tid)) { | while (!_mtx_obtain_lock(m, tid)) { | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
spin_cnt++; | spin_cnt++; | ||||
#endif | #endif | ||||
#ifdef ADAPTIVE_MUTEXES | #ifdef ADAPTIVE_MUTEXES | ||||
/* | /* | ||||
* If the owner is running on another CPU, spin until the | * If the owner is running on another CPU, spin until the | ||||
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Lines | #ifdef KDTRACE_HOOKS | ||||
sleep_time -= lockstat_nsecs(); | sleep_time -= lockstat_nsecs(); | ||||
#endif | #endif | ||||
turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE); | turnstile_wait(ts, mtx_owner(m), TS_EXCLUSIVE_QUEUE); | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
sleep_time += lockstat_nsecs(); | sleep_time += lockstat_nsecs(); | ||||
sleep_cnt++; | sleep_cnt++; | ||||
#endif | #endif | ||||
} | } | ||||
#ifdef KDTRACE_HOOKS | |||||
all_time += lockstat_nsecs(); | |||||
#endif | |||||
#ifdef KTR | #ifdef KTR | ||||
if (cont_logged) { | if (cont_logged) { | ||||
CTR4(KTR_CONTENTION, | CTR4(KTR_CONTENTION, | ||||
"contention end: %s acquired by %p at %s:%d", | "contention end: %s acquired by %p at %s:%d", | ||||
m->lock_object.lo_name, (void *)tid, file, line); | m->lock_object.lo_name, (void *)tid, file, line); | ||||
} | } | ||||
#endif | #endif | ||||
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE, m, contested, | LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE, m, contested, | ||||
waittime, file, line); | waittime, file, line); | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
if (sleep_time) | if (sleep_time) | ||||
LOCKSTAT_RECORD1(LS_MTX_LOCK_BLOCK, m, sleep_time); | LOCKSTAT_RECORD1(LS_MTX_LOCK_BLOCK, m, sleep_time); | ||||
/* | /* | ||||
* Only record the loops spinning and not sleeping. | * Only record the loops spinning and not sleeping. | ||||
*/ | */ | ||||
if (spin_cnt > sleep_cnt) | if (spin_cnt > sleep_cnt) | ||||
LOCKSTAT_RECORD1(LS_MTX_LOCK_SPIN, m, (spin_cnt - sleep_cnt)); | LOCKSTAT_RECORD1(LS_MTX_LOCK_SPIN, m, (all_time - sleep_time)); | ||||
#endif | #endif | ||||
} | } | ||||
static void | static void | ||||
_mtx_lock_spin_failed(struct mtx *m) | _mtx_lock_spin_failed(struct mtx *m) | ||||
{ | { | ||||
struct thread *td; | struct thread *td; | ||||
Show All 23 Lines | _mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts, | ||||
const char *file, int line) | const char *file, int line) | ||||
{ | { | ||||
struct mtx *m; | struct mtx *m; | ||||
int i = 0; | int i = 0; | ||||
#ifdef LOCK_PROFILING | #ifdef LOCK_PROFILING | ||||
int contested = 0; | int contested = 0; | ||||
uint64_t waittime = 0; | uint64_t waittime = 0; | ||||
#endif | #endif | ||||
#ifdef KDTRACE_HOOKS | |||||
int64_t spin_time = 0; | |||||
#endif | |||||
if (SCHEDULER_STOPPED()) | if (SCHEDULER_STOPPED()) | ||||
return; | return; | ||||
m = mtxlock2mtx(c); | m = mtxlock2mtx(c); | ||||
if (LOCK_LOG_TEST(&m->lock_object, opts)) | if (LOCK_LOG_TEST(&m->lock_object, opts)) | ||||
CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); | CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); | ||||
KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), | KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), | ||||
"spinning", "lockname:\"%s\"", m->lock_object.lo_name); | "spinning", "lockname:\"%s\"", m->lock_object.lo_name); | ||||
#ifdef HWPMC_HOOKS | #ifdef HWPMC_HOOKS | ||||
PMC_SOFT_CALL( , , lock, failed); | PMC_SOFT_CALL( , , lock, failed); | ||||
#endif | #endif | ||||
lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); | lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); | ||||
#ifdef KDTRACE_HOOKS | |||||
spin_time -= lockstat_nsecs(); | |||||
#endif | |||||
while (!_mtx_obtain_lock(m, tid)) { | while (!_mtx_obtain_lock(m, tid)) { | ||||
/* Give interrupts a chance while we spin. */ | /* Give interrupts a chance while we spin. */ | ||||
spinlock_exit(); | spinlock_exit(); | ||||
while (m->mtx_lock != MTX_UNOWNED) { | while (m->mtx_lock != MTX_UNOWNED) { | ||||
if (i++ < 10000000) { | if (i++ < 10000000) { | ||||
cpu_spinwait(); | cpu_spinwait(); | ||||
continue; | continue; | ||||
} | } | ||||
if (i < 60000000 || kdb_active || panicstr != NULL) | if (i < 60000000 || kdb_active || panicstr != NULL) | ||||
DELAY(1); | DELAY(1); | ||||
else | else | ||||
_mtx_lock_spin_failed(m); | _mtx_lock_spin_failed(m); | ||||
cpu_spinwait(); | cpu_spinwait(); | ||||
} | } | ||||
spinlock_enter(); | spinlock_enter(); | ||||
} | } | ||||
#ifdef KDTRACE_HOOKS | |||||
spin_time += lockstat_nsecs(); | |||||
#endif | |||||
if (LOCK_LOG_TEST(&m->lock_object, opts)) | if (LOCK_LOG_TEST(&m->lock_object, opts)) | ||||
CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); | CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); | ||||
KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), | KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), | ||||
"running"); | "running"); | ||||
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m, | LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, m, | ||||
contested, waittime, (file), (line)); | contested, waittime, (file), (line)); | ||||
LOCKSTAT_RECORD1(LS_MTX_SPIN_LOCK_SPIN, m, i); | LOCKSTAT_RECORD1(LS_MTX_SPIN_LOCK_SPIN, m, spin_time); | ||||
} | } | ||||
#endif /* SMP */ | #endif /* SMP */ | ||||
void | void | ||||
thread_lock_flags_(struct thread *td, int opts, const char *file, int line) | thread_lock_flags_(struct thread *td, int opts, const char *file, int line) | ||||
{ | { | ||||
struct mtx *m; | struct mtx *m; | ||||
uintptr_t tid; | uintptr_t tid; | ||||
int i; | int i; | ||||
#ifdef LOCK_PROFILING | #ifdef LOCK_PROFILING | ||||
int contested = 0; | int contested = 0; | ||||
uint64_t waittime = 0; | uint64_t waittime = 0; | ||||
#endif | #endif | ||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
uint64_t spin_cnt = 0; | int64_t spin_time = 0; | ||||
#endif | #endif | ||||
i = 0; | i = 0; | ||||
tid = (uintptr_t)curthread; | tid = (uintptr_t)curthread; | ||||
if (SCHEDULER_STOPPED()) | if (SCHEDULER_STOPPED()) | ||||
return; | return; | ||||
#ifdef KDTRACE_HOOKS | |||||
spin_time -= lockstat_nsecs(); | |||||
#endif | |||||
for (;;) { | for (;;) { | ||||
retry: | retry: | ||||
spinlock_enter(); | spinlock_enter(); | ||||
m = td->td_lock; | m = td->td_lock; | ||||
KASSERT(m->mtx_lock != MTX_DESTROYED, | KASSERT(m->mtx_lock != MTX_DESTROYED, | ||||
("thread_lock() of destroyed mutex @ %s:%d", file, line)); | ("thread_lock() of destroyed mutex @ %s:%d", file, line)); | ||||
KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, | KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, | ||||
("thread_lock() of sleep mutex %s @ %s:%d", | ("thread_lock() of sleep mutex %s @ %s:%d", | ||||
m->lock_object.lo_name, file, line)); | m->lock_object.lo_name, file, line)); | ||||
if (mtx_owned(m)) | if (mtx_owned(m)) | ||||
KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, | KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, | ||||
("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", | ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", | ||||
m->lock_object.lo_name, file, line)); | m->lock_object.lo_name, file, line)); | ||||
WITNESS_CHECKORDER(&m->lock_object, | WITNESS_CHECKORDER(&m->lock_object, | ||||
opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); | opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); | ||||
while (!_mtx_obtain_lock(m, tid)) { | while (!_mtx_obtain_lock(m, tid)) { | ||||
#ifdef KDTRACE_HOOKS | |||||
spin_cnt++; | |||||
#endif | |||||
if (m->mtx_lock == tid) { | if (m->mtx_lock == tid) { | ||||
m->mtx_recurse++; | m->mtx_recurse++; | ||||
break; | break; | ||||
} | } | ||||
#ifdef HWPMC_HOOKS | #ifdef HWPMC_HOOKS | ||||
PMC_SOFT_CALL( , , lock, failed); | PMC_SOFT_CALL( , , lock, failed); | ||||
#endif | #endif | ||||
lock_profile_obtain_lock_failed(&m->lock_object, | lock_profile_obtain_lock_failed(&m->lock_object, | ||||
Show All 12 Lines | #endif | ||||
if (m != td->td_lock) | if (m != td->td_lock) | ||||
goto retry; | goto retry; | ||||
} | } | ||||
spinlock_enter(); | spinlock_enter(); | ||||
} | } | ||||
if (m == td->td_lock) | if (m == td->td_lock) | ||||
break; | break; | ||||
__mtx_unlock_spin(m); /* does spinlock_exit() */ | __mtx_unlock_spin(m); /* does spinlock_exit() */ | ||||
} | |||||
#ifdef KDTRACE_HOOKS | #ifdef KDTRACE_HOOKS | ||||
spin_cnt++; | spin_time += lockstat_nsecs(); | ||||
#endif | #endif | ||||
} | |||||
if (m->mtx_recurse == 0) | if (m->mtx_recurse == 0) | ||||
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, | LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, | ||||
m, contested, waittime, (file), (line)); | m, contested, waittime, (file), (line)); | ||||
LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, | LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, | ||||
line); | line); | ||||
WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); | WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); | ||||
LOCKSTAT_RECORD1(LS_THREAD_LOCK_SPIN, m, spin_cnt); | LOCKSTAT_RECORD1(LS_THREAD_LOCK_SPIN, m, spin_time); | ||||
} | } | ||||
struct mtx * | struct mtx * | ||||
thread_lock_block(struct thread *td) | thread_lock_block(struct thread *td) | ||||
{ | { | ||||
struct mtx *lock; | struct mtx *lock; | ||||
THREAD_LOCK_ASSERT(td, MA_OWNED); | THREAD_LOCK_ASSERT(td, MA_OWNED); | ||||
▲ Show 20 Lines • Show All 315 Lines • Show Last 20 Lines |