Differential D21355 Diff 61101 head/sys/kern/kern_lock.c

Changeset View

Standalone View

head/sys/kern/kern_lock.c

Show All 34 Lines
__FBSDID("$FreeBSD$");		__FBSDID("$FreeBSD$");

#include <sys/param.h>		#include <sys/param.h>
#include <sys/kdb.h>		#include <sys/kdb.h>
#include <sys/ktr.h>		#include <sys/ktr.h>
#include <sys/lock.h>		#include <sys/lock.h>
#include <sys/lock_profile.h>		#include <sys/lock_profile.h>
#include <sys/lockmgr.h>		#include <sys/lockmgr.h>
		#include <sys/lockstat.h>
#include <sys/mutex.h>		#include <sys/mutex.h>
#include <sys/proc.h>		#include <sys/proc.h>
#include <sys/sleepqueue.h>		#include <sys/sleepqueue.h>
#ifdef DEBUG_LOCKS		#ifdef DEBUG_LOCKS
#include <sys/stack.h>		#include <sys/stack.h>
#endif		#endif
#include <sys/sysctl.h>		#include <sys/sysctl.h>
#include <sys/systm.h>		#include <sys/systm.h>
▲ Show 20 Lines • Show All 145 Lines • ▼ Show 20 Lines	if (__predict_false(wakeup_swapper))
kick_proc0();		kick_proc0();
}		}

static void		static void
lockmgr_note_shared_acquire(struct lock *lk, int contested,		lockmgr_note_shared_acquire(struct lock *lk, int contested,
uint64_t waittime, const char *file, int line, int flags)		uint64_t waittime, const char *file, int line, int flags)
{		{

lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime,		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(lockmgr__acquire, lk, contested,
file, line);		waittime, file, line, LOCKSTAT_READER);
LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line);		LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file, line);
WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line);		WITNESS_LOCK(&lk->lock_object, LK_TRYWIT(flags), file, line);
TD_LOCKS_INC(curthread);		TD_LOCKS_INC(curthread);
TD_SLOCKS_INC(curthread);		TD_SLOCKS_INC(curthread);
STACK_SAVE(lk);		STACK_SAVE(lk);
}		}

static void		static void
lockmgr_note_shared_release(struct lock lk, const char file, int line)		lockmgr_note_shared_release(struct lock lk, const char file, int line)
{		{

lock_profile_release_lock(&lk->lock_object);		LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER);
WITNESS_UNLOCK(&lk->lock_object, 0, file, line);		WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);		LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
TD_LOCKS_DEC(curthread);		TD_LOCKS_DEC(curthread);
TD_SLOCKS_DEC(curthread);		TD_SLOCKS_DEC(curthread);
}		}

static void		static void
lockmgr_note_exclusive_acquire(struct lock *lk, int contested,		lockmgr_note_exclusive_acquire(struct lock *lk, int contested,
uint64_t waittime, const char *file, int line, int flags)		uint64_t waittime, const char *file, int line, int flags)
{		{

lock_profile_obtain_lock_success(&lk->lock_object, contested, waittime,		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(lockmgr__acquire, lk, contested,
file, line);		waittime, file, line, LOCKSTAT_WRITER);
LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);		LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE \| LK_TRYWIT(flags), file,		WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE \| LK_TRYWIT(flags), file,
line);		line);
TD_LOCKS_INC(curthread);		TD_LOCKS_INC(curthread);
STACK_SAVE(lk);		STACK_SAVE(lk);
}		}

static void		static void
lockmgr_note_exclusive_release(struct lock lk, const char file, int line)		lockmgr_note_exclusive_release(struct lock lk, const char file, int line)
{		{

lock_profile_release_lock(&lk->lock_object);		LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_WRITER);
LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file,		LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file,
line);		line);
WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);		WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
TD_LOCKS_DEC(curthread);		TD_LOCKS_DEC(curthread);
}		}

static __inline struct thread *		static __inline struct thread *
lockmgr_xholder(const struct lock *lk)		lockmgr_xholder(const struct lock *lk)
▲ Show 20 Lines • Show All 299 Lines • ▼ Show 20 Lines
lockmgr_slock_hard(struct lock lk, u_int flags, struct lock_object ilk,		lockmgr_slock_hard(struct lock lk, u_int flags, struct lock_object ilk,
const char file, int line, struct lockmgr_wait lwa)		const char file, int line, struct lockmgr_wait lwa)
{		{
uintptr_t tid, x;		uintptr_t tid, x;
int error = 0;		int error = 0;
const char *iwmesg;		const char *iwmesg;
int ipri, itimo;		int ipri, itimo;

		#ifdef KDTRACE_HOOKS
		uint64_t sleep_time = 0;
		#endif
#ifdef LOCK_PROFILING		#ifdef LOCK_PROFILING
uint64_t waittime = 0;		uint64_t waittime = 0;
int contested = 0;		int contested = 0;
#endif		#endif

if (__predict_false(panicstr != NULL))		if (__predict_false(panicstr != NULL))
goto out;		goto out;

▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines	if (lwa == NULL) {
itimo = lwa->itimo;		itimo = lwa->itimo;
}		}

/*		/*
* As far as we have been unable to acquire the		* As far as we have been unable to acquire the
* shared lock and the shared waiters flag is set,		* shared lock and the shared waiters flag is set,
* we will sleep.		* we will sleep.
*/		*/
		#ifdef KDTRACE_HOOKS
		sleep_time -= lockstat_nsecs(&lk->lock_object);
		#endif
error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,		error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
SQ_SHARED_QUEUE);		SQ_SHARED_QUEUE);
		#ifdef KDTRACE_HOOKS
		sleep_time += lockstat_nsecs(&lk->lock_object);
		#endif
flags &= ~LK_INTERLOCK;		flags &= ~LK_INTERLOCK;
if (error) {		if (error) {
LOCK_LOG3(lk,		LOCK_LOG3(lk,
"%s: interrupted sleep for %p with %d",		"%s: interrupted sleep for %p with %d",
__func__, lk, error);		__func__, lk, error);
break;		break;
}		}
LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",		LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
__func__, lk);		__func__, lk);
}		}
if (error == 0) {		if (error == 0) {
		#ifdef KDTRACE_HOOKS
		if (sleep_time != 0)
		LOCKSTAT_RECORD4(lockmgr__block, lk, sleep_time,
		LOCKSTAT_READER, (x & LK_SHARE) == 0,
		(x & LK_SHARE) == 0 ? 0 : LK_SHARERS(x));
		#endif
#ifdef LOCK_PROFILING		#ifdef LOCK_PROFILING
lockmgr_note_shared_acquire(lk, contested, waittime,		lockmgr_note_shared_acquire(lk, contested, waittime,
file, line, flags);		file, line, flags);
#else		#else
lockmgr_note_shared_acquire(lk, 0, 0, file, line,		lockmgr_note_shared_acquire(lk, 0, 0, file, line,
flags);		flags);
#endif		#endif
}		}

out:		out:
lockmgr_exit(flags, ilk, 0);		lockmgr_exit(flags, ilk, 0);
return (error);		return (error);
}		}

static __noinline int		static __noinline int
lockmgr_xlock_hard(struct lock lk, u_int flags, struct lock_object ilk,		lockmgr_xlock_hard(struct lock lk, u_int flags, struct lock_object ilk,
const char file, int line, struct lockmgr_wait lwa)		const char file, int line, struct lockmgr_wait lwa)
{		{
struct lock_class *class;		struct lock_class *class;
uintptr_t tid, x, v;		uintptr_t tid, x, v;
int error = 0;		int error = 0;
const char *iwmesg;		const char *iwmesg;
int ipri, itimo;		int ipri, itimo;

		#ifdef KDTRACE_HOOKS
		uint64_t sleep_time = 0;
		#endif
#ifdef LOCK_PROFILING		#ifdef LOCK_PROFILING
uint64_t waittime = 0;		uint64_t waittime = 0;
int contested = 0;		int contested = 0;
#endif		#endif

if (__predict_false(panicstr != NULL))		if (__predict_false(panicstr != NULL))
goto out;		goto out;

▲ Show 20 Lines • Show All 123 Lines • ▼ Show 20 Lines	if (lwa == NULL) {
itimo = lwa->itimo;		itimo = lwa->itimo;
}		}

/*		/*
* As far as we have been unable to acquire the		* As far as we have been unable to acquire the
* exclusive lock and the exclusive waiters flag		* exclusive lock and the exclusive waiters flag
* is set, we will sleep.		* is set, we will sleep.
*/		*/
		#ifdef KDTRACE_HOOKS
		sleep_time -= lockstat_nsecs(&lk->lock_object);
		#endif
error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,		error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
SQ_EXCLUSIVE_QUEUE);		SQ_EXCLUSIVE_QUEUE);
		#ifdef KDTRACE_HOOKS
		sleep_time += lockstat_nsecs(&lk->lock_object);
		#endif
flags &= ~LK_INTERLOCK;		flags &= ~LK_INTERLOCK;
if (error) {		if (error) {
LOCK_LOG3(lk,		LOCK_LOG3(lk,
"%s: interrupted sleep for %p with %d",		"%s: interrupted sleep for %p with %d",
__func__, lk, error);		__func__, lk, error);
break;		break;
}		}
LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",		LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
__func__, lk);		__func__, lk);
}		}
if (error == 0) {		if (error == 0) {
		#ifdef KDTRACE_HOOKS
		if (sleep_time != 0)
		LOCKSTAT_RECORD4(lockmgr__block, lk, sleep_time,
		LOCKSTAT_WRITER, (x & LK_SHARE) == 0,
		(x & LK_SHARE) == 0 ? 0 : LK_SHARERS(x));
		#endif
#ifdef LOCK_PROFILING		#ifdef LOCK_PROFILING
lockmgr_note_exclusive_acquire(lk, contested, waittime,		lockmgr_note_exclusive_acquire(lk, contested, waittime,
file, line, flags);		file, line, flags);
#else		#else
lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,		lockmgr_note_exclusive_acquire(lk, 0, 0, file, line,
flags);		flags);
#endif		#endif
}		}
Show All 27 Lines	lockmgr_upgrade(struct lock lk, u_int flags, struct lock_object ilk,
* We need to preserve waiters flags during the operation.		* We need to preserve waiters flags during the operation.
*/		*/
if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) \| x \| v,		if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) \| x \| v,
tid \| x)) {		tid \| x)) {
LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,		LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
line);		line);
WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE \|		WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE \|
LK_TRYWIT(flags), file, line);		LK_TRYWIT(flags), file, line);
		LOCKSTAT_RECORD0(lockmgr__upgrade, lk);
TD_SLOCKS_DEC(curthread);		TD_SLOCKS_DEC(curthread);
goto out;		goto out;
}		}

op = flags & LK_TYPE_MASK;		op = flags & LK_TYPE_MASK;

/*		/*
* In LK_TRYUPGRADE mode, do not drop the lock,		* In LK_TRYUPGRADE mode, do not drop the lock,
▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines	lockmgr_xunlock_hard(struct lock lk, uintptr_t x, u_int flags, struct lock_object ilk,
* If the lock is recursed also, then unrecurse it.		* If the lock is recursed also, then unrecurse it.
*/		*/
if (lockmgr_xlocked_v(x) && lockmgr_recursed(lk)) {		if (lockmgr_xlocked_v(x) && lockmgr_recursed(lk)) {
LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk);		LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk);
lk->lk_recurse--;		lk->lk_recurse--;
goto out;		goto out;
}		}
if (tid != LK_KERNPROC)		if (tid != LK_KERNPROC)
lock_profile_release_lock(&lk->lock_object);		LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk,
		LOCKSTAT_WRITER);

if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED))		if (x == tid && atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED))
goto out;		goto out;

sleepq_lock(&lk->lock_object);		sleepq_lock(&lk->lock_object);
x = lk->lk_lock;		x = lk->lk_lock;
v = LK_UNLOCKED;		v = LK_UNLOCKED;

▲ Show 20 Lines • Show All 164 Lines • ▼ Show 20 Lines	#endif
case LK_TRYUPGRADE:		case LK_TRYUPGRADE:
return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa));		return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa));
break;		break;
case LK_EXCLUSIVE:		case LK_EXCLUSIVE:
return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa));		return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa));
break;		break;
case LK_DOWNGRADE:		case LK_DOWNGRADE:
_lockmgr_assert(lk, KA_XLOCKED, file, line);		_lockmgr_assert(lk, KA_XLOCKED, file, line);
LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);		WITNESS_DOWNGRADE(&lk->lock_object, 0, file, line);

/*		/*
* Panic if the lock is recursed.		* Panic if the lock is recursed.
*/		*/
if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {		if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
if (flags & LK_INTERLOCK)		if (flags & LK_INTERLOCK)
class->lc_unlock(ilk);		class->lc_unlock(ilk);
Show All 9 Lines	for (;;) {
x = lk->lk_lock;		x = lk->lk_lock;
MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);		MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0);
x &= LK_ALL_WAITERS;		x &= LK_ALL_WAITERS;
if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid \| x,		if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid \| x,
LK_SHARERS_LOCK(1) \| x))		LK_SHARERS_LOCK(1) \| x))
break;		break;
cpu_spinwait();		cpu_spinwait();
}		}
		LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object, 0, 0, file, line);
		LOCKSTAT_RECORD0(lockmgr__downgrade, lk);
break;		break;
case LK_RELEASE:		case LK_RELEASE:
_lockmgr_assert(lk, KA_LOCKED, file, line);		_lockmgr_assert(lk, KA_LOCKED, file, line);
x = lk->lk_lock;		x = lk->lk_lock;

if (__predict_true(x & LK_SHARE) != 0) {		if (__predict_true(x & LK_SHARE) != 0) {
return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));		return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
} else {		} else {
▲ Show 20 Lines • Show All 215 Lines • ▼ Show 20 Lines	panic("%s: disown a recursed lockmgr @ %s:%d\n",
__func__, file, line);		__func__, file, line);

/*		/*
* If the owner is already LK_KERNPROC just skip the whole operation.		* If the owner is already LK_KERNPROC just skip the whole operation.
*/		*/
if (LK_HOLDER(lk->lk_lock) != tid)		if (LK_HOLDER(lk->lk_lock) != tid)
return;		return;
lock_profile_release_lock(&lk->lock_object);		lock_profile_release_lock(&lk->lock_object);
		LOCKSTAT_RECORD1(lockmgr__disown, lk, LOCKSTAT_WRITER);
LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);		LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);		WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
TD_LOCKS_DEC(curthread);		TD_LOCKS_DEC(curthread);
STACK_SAVE(lk);		STACK_SAVE(lk);

/*		/*
* In order to preserve waiters flags, just spin.		* In order to preserve waiters flags, just spin.
*/		*/
▲ Show 20 Lines • Show All 214 Lines • Show Last 20 Lines