Page MenuHomeFreeBSD
Differential D46128 Diff 141500 sys/kern/kern_lock.c

Changeset View

Standalone View

View Options

sys/kern/kern_lock.c

Show First 20 Lines • Show All 180 Lines • ▼ Show 20 Lines
}; };
static __always_inline bool lockmgr_slock_try(struct lock *lk, uintptr_t *xp, static __always_inline bool lockmgr_slock_try(struct lock *lk, uintptr_t *xp,
int flags, bool fp); int flags, bool fp);
static __always_inline bool lockmgr_sunlock_try(struct lock *lk, static __always_inline bool lockmgr_sunlock_try(struct lock *lk,
uintptr_t *xp); uintptr_t *xp);
static void static void
lockmgr_exit(u_int flags, struct lock_object *ilk, int wakeup_swapper) lockmgr_exit(u_int flags, struct lock_object *ilk)
{ {
struct lock_class *class; struct lock_class *class;
if (flags & LK_INTERLOCK) { if (flags & LK_INTERLOCK) {
class = LOCK_CLASS(ilk); class = LOCK_CLASS(ilk);
class->lc_unlock(ilk); class->lc_unlock(ilk);
} }
if (__predict_false(wakeup_swapper))
kick_proc0();
} }
static void static void
lockmgr_note_shared_acquire(struct lock *lk, int contested, lockmgr_note_shared_acquire(struct lock *lk, int contested,
uint64_t waittime, const char *file, int line, int flags) uint64_t waittime, const char *file, int line, int flags)
{ {
LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(lockmgr__acquire, lk, contested, LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(lockmgr__acquire, lk, contested,
▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines else
sleepq_wait(&lk->lock_object, pri); sleepq_wait(&lk->lock_object, pri);
GIANT_RESTORE(); GIANT_RESTORE();
if ((flags & LK_SLEEPFAIL) && error == 0) if ((flags & LK_SLEEPFAIL) && error == 0)
error = ENOLCK; error = ENOLCK;
return (error); return (error);
} }
static __inline int static __inline void
wakeupshlk(struct lock *lk, const char *file, int line) wakeupshlk(struct lock *lk, const char *file, int line)
{ {
uintptr_t v, x, orig_x; uintptr_t v, x, orig_x;
u_int realexslp; u_int realexslp;
int queue, wakeup_swapper; int queue;
wakeup_swapper = 0;
for (;;) { for (;;) {
x = lockmgr_read_value(lk); x = lockmgr_read_value(lk);
if (lockmgr_sunlock_try(lk, &x)) if (lockmgr_sunlock_try(lk, &x))
break; break;
/* /*
* We should have a sharer with waiters, so enter the hard * We should have a sharer with waiters, so enter the hard
* path in order to handle wakeups correctly. * path in order to handle wakeups correctly.
Show All 27 Lines if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
} else { } else {
lk->lk_exslpfail = 0; lk->lk_exslpfail = 0;
LOCK_LOG2(lk, LOCK_LOG2(lk,
"%s: %p has only LK_SLEEPFAIL sleepers", "%s: %p has only LK_SLEEPFAIL sleepers",
__func__, lk); __func__, lk);
LOCK_LOG2(lk, LOCK_LOG2(lk,
"%s: %p waking up threads on the exclusive queue", "%s: %p waking up threads on the exclusive queue",
__func__, lk); __func__, lk);
wakeup_swapper = sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0,
sleepq_broadcast(&lk->lock_object, SQ_EXCLUSIVE_QUEUE);
SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
queue = SQ_SHARED_QUEUE; queue = SQ_SHARED_QUEUE;
} }
} else { } else {
/* /*
* Exclusive waiters sleeping with LK_SLEEPFAIL on * Exclusive waiters sleeping with LK_SLEEPFAIL on
* and using interruptible sleeps/timeout may have * and using interruptible sleeps/timeout may have
* left spourious lk_exslpfail counts on, so clean * left spourious lk_exslpfail counts on, so clean
* it up anyway. * it up anyway.
Show All 10 Linesretry_sleepq:
x |= LK_SHARERS_LOCK(1); x |= LK_SHARERS_LOCK(1);
if (!atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, v)) { if (!atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, v)) {
orig_x = x; orig_x = x;
goto retry_sleepq; goto retry_sleepq;
} }
LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue", LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
__func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
"exclusive"); "exclusive");
wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
0, queue);
sleepq_release(&lk->lock_object); sleepq_release(&lk->lock_object);
break; break;
} }
LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER); LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER);
return (wakeup_swapper);
} }
static void static void
assert_lockmgr(const struct lock_object *lock, int what) assert_lockmgr(const struct lock_object *lock, int what)
{ {
panic("lockmgr locks do not support assertions"); panic("lockmgr locks do not support assertions");
} }
▲ Show 20 Lines • Show All 316 Lines • ▼ Show 20 Lines lockmgr_note_shared_acquire(lk, contested, waittime,
file, line, flags); file, line, flags);
#else #else
lockmgr_note_shared_acquire(lk, 0, 0, file, line, lockmgr_note_shared_acquire(lk, 0, 0, file, line,
flags); flags);
#endif #endif
} }
out: out:
lockmgr_exit(flags, ilk, 0); lockmgr_exit(flags, ilk);
return (error); return (error);
} }
static bool static bool
lockmgr_xlock_adaptive(struct lock_delay_arg *lda, struct lock *lk, uintptr_t *xp) lockmgr_xlock_adaptive(struct lock_delay_arg *lda, struct lock *lk, uintptr_t *xp)
{ {
struct thread *owner; struct thread *owner;
uintptr_t x; uintptr_t x;
▲ Show 20 Lines • Show All 221 Lines • ▼ Show 20 Lines lockmgr_note_exclusive_acquire(lk, contested, waittime,
file, line, flags); file, line, flags);
#else #else
lockmgr_note_exclusive_acquire(lk, 0, 0, file, line, lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
flags); flags);
#endif #endif
} }
out: out:
lockmgr_exit(flags, ilk, 0); lockmgr_exit(flags, ilk);
return (error); return (error);
} }
static __noinline int static __noinline int
lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk, lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk,
const char *file, int line, struct lockmgr_wait *lwa) const char *file, int line, struct lockmgr_wait *lwa)
{ {
uintptr_t tid, v, setv; uintptr_t tid, v, setv;
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines if (atomic_fcmpset_ptr(&lk->lk_lock, &v, setv)) {
goto out; goto out;
} }
} }
out_xlock: out_xlock:
error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa); error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa);
flags &= ~LK_INTERLOCK; flags &= ~LK_INTERLOCK;
out: out:
lockmgr_exit(flags, ilk, 0); lockmgr_exit(flags, ilk);
return (error); return (error);
} }
int int
lockmgr_lock_flags(struct lock *lk, u_int flags, struct lock_object *ilk, lockmgr_lock_flags(struct lock *lk, u_int flags, struct lock_object *ilk,
const char *file, int line) const char *file, int line)
{ {
struct lock_class *class; struct lock_class *class;
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines if (__predict_true(locked)) {
return (__lockmgr_args(lk, flags, ilk, LK_WMESG_DEFAULT, return (__lockmgr_args(lk, flags, ilk, LK_WMESG_DEFAULT,
LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, file, line)); LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, file, line));
} }
} }
static __noinline int static __noinline int
lockmgr_sunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk, lockmgr_sunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
const char *file, int line) const char *file, int line)
{ {
int wakeup_swapper = 0; if (!SCHEDULER_STOPPED())
kibUnsubmitted
Done
Inline Actions
if (!SCHEDULER_STOPPED())
   wakeupshlk();
lockmgr_exit();
return (0);
kib: ``` if (!SCHEDULER_STOPPED()) wakeupshlk(); lockmgr_exit(); return (0); ```
wakeupshlk(lk, file, line);
if (SCHEDULER_STOPPED()) lockmgr_exit(flags, ilk);
goto out;
wakeup_swapper = wakeupshlk(lk, file, line);
out:
lockmgr_exit(flags, ilk, wakeup_swapper);
return (0); return (0);
} }
static __noinline int static __noinline int
lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk, lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk,
const char *file, int line) const char *file, int line)
{ {
uintptr_t tid, v; uintptr_t tid, v;
int wakeup_swapper = 0;
u_int realexslp; u_int realexslp;
int queue; int queue;
if (SCHEDULER_STOPPED()) if (SCHEDULER_STOPPED())
goto out; goto out;
tid = (uintptr_t)curthread; tid = (uintptr_t)curthread;
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) {
} else { } else {
lk->lk_exslpfail = 0; lk->lk_exslpfail = 0;
LOCK_LOG2(lk, LOCK_LOG2(lk,
"%s: %p has only LK_SLEEPFAIL sleepers", "%s: %p has only LK_SLEEPFAIL sleepers",
__func__, lk); __func__, lk);
LOCK_LOG2(lk, LOCK_LOG2(lk,
"%s: %p waking up threads on the exclusive queue", "%s: %p waking up threads on the exclusive queue",
__func__, lk); __func__, lk);
wakeup_swapper = sleepq_broadcast(&lk->lock_object, sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0,
SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE); SQ_EXCLUSIVE_QUEUE);
queue = SQ_SHARED_QUEUE; queue = SQ_SHARED_QUEUE;
} }
} else { } else {
/* /*
* Exclusive waiters sleeping with LK_SLEEPFAIL * Exclusive waiters sleeping with LK_SLEEPFAIL
* on and using interruptible sleeps/timeout * on and using interruptible sleeps/timeout
* may have left spourious lk_exslpfail counts * may have left spourious lk_exslpfail counts
* on, so clean it up anyway. * on, so clean it up anyway.
*/ */
lk->lk_exslpfail = 0; lk->lk_exslpfail = 0;
queue = SQ_SHARED_QUEUE; queue = SQ_SHARED_QUEUE;
} }
LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue", LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
__func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
"exclusive"); "exclusive");
atomic_store_rel_ptr(&lk->lk_lock, v); atomic_store_rel_ptr(&lk->lk_lock, v);
wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue); sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
sleepq_release(&lk->lock_object); sleepq_release(&lk->lock_object);
out: out:
lockmgr_exit(flags, ilk, wakeup_swapper); lockmgr_exit(flags, ilk);
return (0); return (0);
} }
/* /*
* Lightweight entry points for common operations. * Lightweight entry points for common operations.
* *
* Functionality is similar to sx locks, in that none of the additional lockmgr * Functionality is similar to sx locks, in that none of the additional lockmgr
* features are supported. To be clear, these are NOT supported: * features are supported. To be clear, these are NOT supported:
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines__lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
const char *wmesg, int pri, int timo, const char *file, int line) const char *wmesg, int pri, int timo, const char *file, int line)
{ {
GIANT_DECLARE; GIANT_DECLARE;
struct lockmgr_wait lwa; struct lockmgr_wait lwa;
struct lock_class *class; struct lock_class *class;
const char *iwmesg; const char *iwmesg;
uintptr_t tid, v, x; uintptr_t tid, v, x;
u_int op, realexslp; u_int op, realexslp;
int error, ipri, itimo, queue, wakeup_swapper; int error, ipri, itimo, queue;
#ifdef LOCK_PROFILING #ifdef LOCK_PROFILING
uint64_t waittime = 0; uint64_t waittime = 0;
int contested = 0; int contested = 0;
#endif #endif
if (SCHEDULER_STOPPED()) if (SCHEDULER_STOPPED())
return (0); return (0);
Show All 35 Lines case LK_DOWNGRADE:
_lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, _lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED,
file, line); file, line);
if (flags & LK_INTERLOCK) if (flags & LK_INTERLOCK)
class->lc_unlock(ilk); class->lc_unlock(ilk);
return (0); return (0);
} }
} }
wakeup_swapper = 0;
switch (op) { switch (op) {
case LK_SHARED: case LK_SHARED:
return (lockmgr_slock_hard(lk, flags, ilk, file, line, &lwa)); return (lockmgr_slock_hard(lk, flags, ilk, file, line, &lwa));
break; break;
case LK_UPGRADE: case LK_UPGRADE:
case LK_TRYUPGRADE: case LK_TRYUPGRADE:
return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa)); return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa));
break; break;
▲ Show 20 Lines • Show All 141 Lines • ▼ Show 20 Lines#endif
v &= ~LK_SHARED_WAITERS; v &= ~LK_SHARED_WAITERS;
if (realexslp != 0) { if (realexslp != 0) {
LOCK_LOG2(lk, LOCK_LOG2(lk,
"%s: %p has only LK_SLEEPFAIL sleepers", "%s: %p has only LK_SLEEPFAIL sleepers",
__func__, lk); __func__, lk);
LOCK_LOG2(lk, LOCK_LOG2(lk,
"%s: %p waking up threads on the exclusive queue", "%s: %p waking up threads on the exclusive queue",
__func__, lk); __func__, lk);
wakeup_swapper =
sleepq_broadcast( sleepq_broadcast(
&lk->lock_object, &lk->lock_object,
SLEEPQ_LK, 0, SLEEPQ_LK, 0,
SQ_EXCLUSIVE_QUEUE); SQ_EXCLUSIVE_QUEUE);
} }
} else } else
lk->lk_exslpfail = 0; lk->lk_exslpfail = 0;
} }
if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) { if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
sleepq_release(&lk->lock_object); sleepq_release(&lk->lock_object);
continue; continue;
} }
LOCK_LOG3(lk, LOCK_LOG3(lk,
"%s: %p waking up all threads on the %s queue", "%s: %p waking up all threads on the %s queue",
__func__, lk, queue == SQ_SHARED_QUEUE ? __func__, lk, queue == SQ_SHARED_QUEUE ?
"shared" : "exclusive"); "shared" : "exclusive");
wakeup_swapper |= sleepq_broadcast( sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0,
&lk->lock_object, SLEEPQ_LK, 0, queue); queue);
/* /*
* If shared waiters have been woken up we need * If shared waiters have been woken up we need
* to wait for one of them to acquire the lock * to wait for one of them to acquire the lock
* before to set the exclusive waiters in * before to set the exclusive waiters in
* order to avoid a deadlock. * order to avoid a deadlock.
*/ */
if (queue == SQ_SHARED_QUEUE) { if (queue == SQ_SHARED_QUEUE) {
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines#endif
default: default:
if (flags & LK_INTERLOCK) if (flags & LK_INTERLOCK)
class->lc_unlock(ilk); class->lc_unlock(ilk);
panic("%s: unknown lockmgr request 0x%x\n", __func__, op); panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
} }
if (flags & LK_INTERLOCK) if (flags & LK_INTERLOCK)
class->lc_unlock(ilk); class->lc_unlock(ilk);
if (wakeup_swapper)
kick_proc0();
return (error); return (error);
} }
void void
_lockmgr_disown(struct lock *lk, const char *file, int line) _lockmgr_disown(struct lock *lk, const char *file, int line)
{ {
uintptr_t tid, x; uintptr_t tid, x;
▲ Show 20 Lines • Show All 243 Lines • Show Last 20 Lines