Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_lock.c
Show All 34 Lines | |||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | ||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/kdb.h> | #include <sys/kdb.h> | ||||
#include <sys/ktr.h> | #include <sys/ktr.h> | ||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/lock_profile.h> | #include <sys/lock_profile.h> | ||||
#include <sys/lockmgr.h> | #include <sys/lockmgr.h> | ||||
#include <sys/lockstat.h> | |||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/sleepqueue.h> | #include <sys/sleepqueue.h> | ||||
#ifdef DEBUG_LOCKS | #ifdef DEBUG_LOCKS | ||||
#include <sys/stack.h> | #include <sys/stack.h> | ||||
#endif | #endif | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
▲ Show 20 Lines • Show All 145 Lines • ▼ Show 20 Lines | if (__predict_false(wakeup_swapper)) | ||||
kick_proc0(); | kick_proc0(); | ||||
} | } | ||||
static void | static void | ||||
lockmgr_note_shared_acquire(struct lock *lk, int contested, | lockmgr_note_shared_acquire(struct lock *lk, int contested, | ||||
uint64_t waittime, const char *file, int line, int flags) | uint64_t waittime, const char *file, int line, int flags) | ||||
{ | { | ||||
lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime, | LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(lockmgr__acquire, lk, contested, | ||||
file, line); | waittime, file, line, LOCKSTAT_READER); | ||||
LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line); | LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line); | ||||
WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line); | WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line); | ||||
TD_LOCKS_INC(curthread); | TD_LOCKS_INC(curthread); | ||||
TD_SLOCKS_INC(curthread); | TD_SLOCKS_INC(curthread); | ||||
STACK_SAVE(lk); | STACK_SAVE(lk); | ||||
} | } | ||||
static void | static void | ||||
lockmgr_note_shared_release(struct lock *lk, const char *file, int line) | lockmgr_note_shared_release(struct lock *lk, const char *file, int line) | ||||
{ | { | ||||
lock_profile_release_lock(&lk->lock_object); | LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER); | ||||
WITNESS_UNLOCK(&lk->lock_object, 0, file, line); | WITNESS_UNLOCK(&lk->lock_object, 0, file, line); | ||||
LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line); | LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line); | ||||
TD_LOCKS_DEC(curthread); | TD_LOCKS_DEC(curthread); | ||||
TD_SLOCKS_DEC(curthread); | TD_SLOCKS_DEC(curthread); | ||||
} | } | ||||
static void | static void | ||||
lockmgr_note_exclusive_acquire(struct lock *lk, int contested, | lockmgr_note_exclusive_acquire(struct lock *lk, int contested, | ||||
uint64_t waittime, const char *file, int line, int flags) | uint64_t waittime, const char *file, int line, int flags) | ||||
{ | { | ||||
lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime, | LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(lockmgr__acquire, lk, contested, | ||||
file, line); | waittime, file, line, LOCKSTAT_WRITER); | ||||
LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line); | LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line); | ||||
WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | LK_TRYWIT(flags), file, | WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | LK_TRYWIT(flags), file, | ||||
line); | line); | ||||
TD_LOCKS_INC(curthread); | TD_LOCKS_INC(curthread); | ||||
STACK_SAVE(lk); | STACK_SAVE(lk); | ||||
} | } | ||||
static void | static void | ||||
lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line) | lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line) | ||||
{ | { | ||||
lock_profile_release_lock(&lk->lock_object); | LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_WRITER); | ||||
LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file, | LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file, | ||||
line); | line); | ||||
WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line); | WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line); | ||||
TD_LOCKS_DEC(curthread); | TD_LOCKS_DEC(curthread); | ||||
} | } | ||||
static __inline struct thread * | static __inline struct thread * | ||||
lockmgr_xholder(const struct lock *lk) | lockmgr_xholder(const struct lock *lk) | ||||
▲ Show 20 Lines • Show All 299 Lines • ▼ Show 20 Lines | |||||
lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, | lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, | ||||
const char *file, int line, struct lockmgr_wait *lwa) | const char *file, int line, struct lockmgr_wait *lwa) | ||||
{ | { | ||||
uintptr_t tid, x; | uintptr_t tid, x; | ||||
int error = 0; | int error = 0; | ||||
const char *iwmesg; | const char *iwmesg; | ||||
int ipri, itimo; | int ipri, itimo; | ||||
#ifdef KDTRACE_HOOKS | |||||
uint64_t sleep_time = 0; | |||||
#endif | |||||
#ifdef LOCK_PROFILING | #ifdef LOCK_PROFILING | ||||
uint64_t waittime = 0; | uint64_t waittime = 0; | ||||
int contested = 0; | int contested = 0; | ||||
#endif | #endif | ||||
if (__predict_false(panicstr != NULL)) | if (__predict_false(panicstr != NULL)) | ||||
goto out; | goto out; | ||||
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines | if (lwa == NULL) { | ||||
itimo = lwa->itimo; | itimo = lwa->itimo; | ||||
} | } | ||||
/* | /* | ||||
* As far as we have been unable to acquire the | * As far as we have been unable to acquire the | ||||
* shared lock and the shared waiters flag is set, | * shared lock and the shared waiters flag is set, | ||||
* we will sleep. | * we will sleep. | ||||
*/ | */ | ||||
#ifdef KDTRACE_HOOKS | |||||
sleep_time -= lockstat_nsecs(&lk->lock_object); | |||||
#endif | |||||
error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, | error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, | ||||
SQ_SHARED_QUEUE); | SQ_SHARED_QUEUE); | ||||
#ifdef KDTRACE_HOOKS | |||||
sleep_time += lockstat_nsecs(&lk->lock_object); | |||||
#endif | |||||
flags &= ~LK_INTERLOCK; | flags &= ~LK_INTERLOCK; | ||||
if (error) { | if (error) { | ||||
LOCK_LOG3(lk, | LOCK_LOG3(lk, | ||||
"%s: interrupted sleep for %p with %d", | "%s: interrupted sleep for %p with %d", | ||||
__func__, lk, error); | __func__, lk, error); | ||||
break; | break; | ||||
} | } | ||||
LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", | LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", | ||||
__func__, lk); | __func__, lk); | ||||
} | } | ||||
if (error == 0) { | if (error == 0) { | ||||
#ifdef KDTRACE_HOOKS | |||||
if (sleep_time != 0) | |||||
mjg: This can and should be prefixed with LOCKSTAT_PROFILE_ENABLED(lockmgr__block) check. Reasoning… | |||||
markjAuthorUnsubmitted Done Inline ActionsI don't quite understand the claim about sleep_time. lockstat_nsecs() will return 0 unless some lockstat probes are enabled, so sleep_time should be 0 in the common case. markj: I don't quite understand the claim about sleep_time. lockstat_nsecs() will return 0 unless some… | |||||
mjgUnsubmitted Not Done Inline ActionsCorrect, that's my bad. I had a patch which always returned the time and went on autopilot here. mjg: Correct, that's my bad. I had a patch which always returned the time and went on autopilot here. | |||||
LOCKSTAT_RECORD4(lockmgr__block, lk, sleep_time, | |||||
LOCKSTAT_READER, (x & LK_SHARE) == 0, | |||||
(x & LK_SHARE) == 0 ? 0 : LK_SHARERS(x)); | |||||
#endif | |||||
#ifdef LOCK_PROFILING | #ifdef LOCK_PROFILING | ||||
lockmgr_note_shared_acquire(lk, contested, waittime, | lockmgr_note_shared_acquire(lk, contested, waittime, | ||||
file, line, flags); | file, line, flags); | ||||
#else | #else | ||||
lockmgr_note_shared_acquire(lk, 0, 0, file, line, | lockmgr_note_shared_acquire(lk, 0, 0, file, line, | ||||
flags); | flags); | ||||
#endif | #endif | ||||
} | } | ||||
out: | out: | ||||
lockmgr_exit(flags, ilk, 0); | lockmgr_exit(flags, ilk, 0); | ||||
return (error); | return (error); | ||||
} | } | ||||
static __noinline int | static __noinline int | ||||
lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, | lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, | ||||
const char *file, int line, struct lockmgr_wait *lwa) | const char *file, int line, struct lockmgr_wait *lwa) | ||||
{ | { | ||||
struct lock_class *class; | struct lock_class *class; | ||||
uintptr_t tid, x, v; | uintptr_t tid, x, v; | ||||
int error = 0; | int error = 0; | ||||
const char *iwmesg; | const char *iwmesg; | ||||
int ipri, itimo; | int ipri, itimo; | ||||
#ifdef KDTRACE_HOOKS | |||||
uint64_t sleep_time = 0; | |||||
#endif | |||||
#ifdef LOCK_PROFILING | #ifdef LOCK_PROFILING | ||||
uint64_t waittime = 0; | uint64_t waittime = 0; | ||||
int contested = 0; | int contested = 0; | ||||
#endif | #endif | ||||
if (__predict_false(panicstr != NULL)) | if (__predict_false(panicstr != NULL)) | ||||
goto out; | goto out; | ||||
▲ Show 20 Lines • Show All 123 Lines • ▼ Show 20 Lines | if (lwa == NULL) { | ||||
itimo = lwa->itimo; | itimo = lwa->itimo; | ||||
} | } | ||||
/* | /* | ||||
* As far as we have been unable to acquire the | * As far as we have been unable to acquire the | ||||
* exclusive lock and the exclusive waiters flag | * exclusive lock and the exclusive waiters flag | ||||
* is set, we will sleep. | * is set, we will sleep. | ||||
*/ | */ | ||||
#ifdef KDTRACE_HOOKS | |||||
sleep_time -= lockstat_nsecs(&lk->lock_object); | |||||
#endif | |||||
error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, | error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, | ||||
SQ_EXCLUSIVE_QUEUE); | SQ_EXCLUSIVE_QUEUE); | ||||
#ifdef KDTRACE_HOOKS | |||||
sleep_time += lockstat_nsecs(&lk->lock_object); | |||||
#endif | |||||
flags &= ~LK_INTERLOCK; | flags &= ~LK_INTERLOCK; | ||||
if (error) { | if (error) { | ||||
LOCK_LOG3(lk, | LOCK_LOG3(lk, | ||||
"%s: interrupted sleep for %p with %d", | "%s: interrupted sleep for %p with %d", | ||||
__func__, lk, error); | __func__, lk, error); | ||||
break; | break; | ||||
} | } | ||||
LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", | LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", | ||||
__func__, lk); | __func__, lk); | ||||
} | } | ||||
if (error == 0) { | if (error == 0) { | ||||
#ifdef KDTRACE_HOOKS | |||||
if (sleep_time != 0) | |||||
mjgUnsubmitted Not Done Inline Actionssee the previous comment mjg: see the previous comment | |||||
LOCKSTAT_RECORD4(lockmgr__block, lk, sleep_time, | |||||
LOCKSTAT_WRITER, (x & LK_SHARE) == 0, | |||||
(x & LK_SHARE) == 0 ? 0 : LK_SHARERS(x)); | |||||
#endif | |||||
#ifdef LOCK_PROFILING | #ifdef LOCK_PROFILING | ||||
lockmgr_note_exclusive_acquire(lk, contested, waittime, | lockmgr_note_exclusive_acquire(lk, contested, waittime, | ||||
file, line, flags); | file, line, flags); | ||||
#else | #else | ||||
lockmgr_note_exclusive_acquire(lk, 0, 0, file, line, | lockmgr_note_exclusive_acquire(lk, 0, 0, file, line, | ||||
flags); | flags); | ||||
#endif | #endif | ||||
} | } | ||||
Show All 27 Lines | lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk, | ||||
* We need to preserve waiters flags during the operation. | * We need to preserve waiters flags during the operation. | ||||
*/ | */ | ||||
if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v, | if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v, | ||||
tid | x)) { | tid | x)) { | ||||
LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file, | LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file, | ||||
line); | line); | ||||
WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE | | WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE | | ||||
LK_TRYWIT(flags), file, line); | LK_TRYWIT(flags), file, line); | ||||
LOCKSTAT_RECORD0(lockmgr__upgrade, lk); | |||||
TD_SLOCKS_DEC(curthread); | TD_SLOCKS_DEC(curthread); | ||||
goto out; | goto out; | ||||
} | } | ||||
op = flags & LK_TYPE_MASK; | op = flags & LK_TYPE_MASK; | ||||
/* | /* | ||||
* In LK_TRYUPGRADE mode, do not drop the lock, | * In LK_TRYUPGRADE mode, do not drop the lock, | ||||
▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk, | ||||
* If the lock is recursed also, then unrecurse it. | * If the lock is recursed also, then unrecurse it. | ||||
*/ | */ | ||||
if (lockmgr_xlocked_v(x) && lockmgr_recursed(lk)) { | if (lockmgr_xlocked_v(x) && lockmgr_recursed(lk)) { | ||||
LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk); | LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk); | ||||
lk->lk_recurse--; | lk->lk_recurse--; | ||||
goto out; | goto out; | ||||
} | } | ||||
if (tid != LK_KERNPROC) | if (tid != LK_KERNPROC) | ||||
lock_profile_release_lock(&lk->lock_object); | LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, | ||||
mjgUnsubmitted Not Done Inline Actionssee the previous comment mjg: see the previous comment | |||||
markjAuthorUnsubmitted Done Inline ActionsThat would disable lock_profiling too. Are you suggesting: if (tid != LK_KERNPROC) { if (LOCKSTAT_PROFILE_ENABLED(lockmgr__release)) LOCKSTAT_RECORD1(...); lock_profile_release_lock(...); } ? markj: That would disable lock_profiling too. Are you suggesting:
```
if (tid != LK_KERNPROC) {… | |||||
mjgUnsubmitted Not Done Inline ActionsOh sorry, I forgot for /these/ probes both LOCK_PROFILING and lockstat are mixed in. Since tid vast majority of the time is not LK_KERNPROC, I wanted: if (LOCKSTAT_PROFILE_ENABLED(lockmgr__release)) { if (tid != LK_KERNPROC) { ... } } to normally have only branch. However, the macro situation would require too much ugliness to be worth it as it is. mjg: Oh sorry, I forgot for /these/ probes both LOCK_PROFILING and lockstat are mixed in.
Since tid… | |||||
LOCKSTAT_WRITER); | |||||
if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) | if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) | ||||
goto out; | goto out; | ||||
sleepq_lock(&lk->lock_object); | sleepq_lock(&lk->lock_object); | ||||
x = lk->lk_lock; | x = lk->lk_lock; | ||||
v = LK_UNLOCKED; | v = LK_UNLOCKED; | ||||
▲ Show 20 Lines • Show All 164 Lines • ▼ Show 20 Lines | #endif | ||||
case LK_TRYUPGRADE: | case LK_TRYUPGRADE: | ||||
return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa)); | return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa)); | ||||
break; | break; | ||||
case LK_EXCLUSIVE: | case LK_EXCLUSIVE: | ||||
return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa)); | return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa)); | ||||
break; | break; | ||||
case LK_DOWNGRADE: | case LK_DOWNGRADE: | ||||
_lockmgr_assert(lk, KA_XLOCKED, file, line); | _lockmgr_assert(lk, KA_XLOCKED, file, line); | ||||
LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line); | |||||
WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line); | WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line); | ||||
/* | /* | ||||
* Panic if the lock is recursed. | * Panic if the lock is recursed. | ||||
*/ | */ | ||||
if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) { | if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) { | ||||
if (flags & LK_INTERLOCK) | if (flags & LK_INTERLOCK) | ||||
class->lc_unlock(ilk); | class->lc_unlock(ilk); | ||||
Show All 9 Lines | for (;;) { | ||||
x = lk->lk_lock; | x = lk->lk_lock; | ||||
MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); | MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); | ||||
x &= LK_ALL_WAITERS; | x &= LK_ALL_WAITERS; | ||||
if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, | if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x, | ||||
LK_SHARERS_LOCK(1) | x)) | LK_SHARERS_LOCK(1) | x)) | ||||
break; | break; | ||||
cpu_spinwait(); | cpu_spinwait(); | ||||
} | } | ||||
LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line); | |||||
LOCKSTAT_RECORD0(lockmgr__downgrade, lk); | |||||
break; | break; | ||||
case LK_RELEASE: | case LK_RELEASE: | ||||
_lockmgr_assert(lk, KA_LOCKED, file, line); | _lockmgr_assert(lk, KA_LOCKED, file, line); | ||||
x = lk->lk_lock; | x = lk->lk_lock; | ||||
if (__predict_true(x & LK_SHARE) != 0) { | if (__predict_true(x & LK_SHARE) != 0) { | ||||
return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line)); | return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line)); | ||||
} else { | } else { | ||||
▲ Show 20 Lines • Show All 215 Lines • ▼ Show 20 Lines | panic("%s: disown a recursed lockmgr @ %s:%d\n", | ||||
__func__, file, line); | __func__, file, line); | ||||
/* | /* | ||||
* If the owner is already LK_KERNPROC just skip the whole operation. | * If the owner is already LK_KERNPROC just skip the whole operation. | ||||
*/ | */ | ||||
if (LK_HOLDER(lk->lk_lock) != tid) | if (LK_HOLDER(lk->lk_lock) != tid) | ||||
return; | return; | ||||
lock_profile_release_lock(&lk->lock_object); | lock_profile_release_lock(&lk->lock_object); | ||||
LOCKSTAT_RECORD1(lockmgr__disown, lk, LOCKSTAT_WRITER); | |||||
LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line); | LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line); | ||||
WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line); | WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line); | ||||
TD_LOCKS_DEC(curthread); | TD_LOCKS_DEC(curthread); | ||||
STACK_SAVE(lk); | STACK_SAVE(lk); | ||||
/* | /* | ||||
* In order to preserve waiters flags, just spin. | * In order to preserve waiters flags, just spin. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 214 Lines • Show Last 20 Lines |
This can and should be prefixed with LOCKSTAT_PROFILE_ENABLED(lockmgr__block) check. Reasoning is that error is almost guaranteed to be 0 and since right now the lock always blocks on contention sleep_time will not be 0, which means there are 2 branches to evaluate in the common case. If you prefix it you can skip one branch for the common case. This can be further fixed up but should be fine enough for now.
I have cleanups pending which will reduce the surrounding branchfest.