Differential D2727 Diff 6131 head/sys/kern/kern_sx.c

Changeset View

Standalone View

head/sys/kern/kern_sx.c

Show First 20 Lines • Show All 282 Lines • ▼ Show 20 Lines	for (;;) {
x = sx->sx_lock;		x = sx->sx_lock;
KASSERT(x != SX_LOCK_DESTROYED,		KASSERT(x != SX_LOCK_DESTROYED,
("sx_try_slock() of destroyed sx @ %s:%d", file, line));		("sx_try_slock() of destroyed sx @ %s:%d", file, line));
if (!(x & SX_LOCK_SHARED))		if (!(x & SX_LOCK_SHARED))
break;		break;
if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) {		if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) {
LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);		LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);
WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);		WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE,
		sx, 0, 0, file, line);
curthread->td_locks++;		curthread->td_locks++;
return (1);		return (1);
}		}
}		}

LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line);		LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line);
return (0);		return (0);
}		}
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines	if (sx_xlocked(sx) &&
rval = 1;		rval = 1;
} else		} else
rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED,		rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED,
(uintptr_t)curthread);		(uintptr_t)curthread);
LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line);		LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line);
if (rval) {		if (rval) {
WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE \| LOP_TRYLOCK,		WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE \| LOP_TRYLOCK,
file, line);		file, line);
		if (!sx_recursed(sx))
		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE,
		sx, 0, 0, file, line);
curthread->td_locks++;		curthread->td_locks++;
}		}

return (rval);		return (rval);
}		}

void		void
_sx_sunlock(struct sx sx, const char file, int line)		_sx_sunlock(struct sx sx, const char file, int line)
▲ Show 20 Lines • Show All 145 Lines • ▼ Show 20 Lines
#endif		#endif
uintptr_t x;		uintptr_t x;
#ifdef LOCK_PROFILING		#ifdef LOCK_PROFILING
uint64_t waittime = 0;		uint64_t waittime = 0;
int contested = 0;		int contested = 0;
#endif		#endif
int error = 0;		int error = 0;
#ifdef KDTRACE_HOOKS		#ifdef KDTRACE_HOOKS
		uintptr_t state;
uint64_t spin_cnt = 0;		uint64_t spin_cnt = 0;
uint64_t sleep_cnt = 0;		uint64_t sleep_cnt = 0;
int64_t sleep_time = 0;		int64_t sleep_time = 0;
		int64_t all_time = 0;
#endif		#endif

if (SCHEDULER_STOPPED())		if (SCHEDULER_STOPPED())
return (0);		return (0);

/* If we already hold an exclusive lock, then recurse. */		/* If we already hold an exclusive lock, then recurse. */
if (sx_xlocked(sx)) {		if (sx_xlocked(sx)) {
KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0,		KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0,
("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n",		("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n",
sx->lock_object.lo_name, file, line));		sx->lock_object.lo_name, file, line));
sx->sx_recurse++;		sx->sx_recurse++;
atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);		atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
if (LOCK_LOG_TEST(&sx->lock_object, 0))		if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx);		CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx);
return (0);		return (0);
}		}

if (LOCK_LOG_TEST(&sx->lock_object, 0))		if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,		CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
sx->lock_object.lo_name, (void *)sx->sx_lock, file, line);		sx->lock_object.lo_name, (void *)sx->sx_lock, file, line);

		#ifdef KDTRACE_HOOKS
		all_time -= lockstat_nsecs();
		state = sx->sx_lock;
		#endif
while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) {		while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) {
#ifdef KDTRACE_HOOKS		#ifdef KDTRACE_HOOKS
spin_cnt++;		spin_cnt++;
#endif		#endif
#ifdef HWPMC_HOOKS		#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);		PMC_SOFT_CALL( , , lock, failed);
#endif		#endif
lock_profile_obtain_lock_failed(&sx->lock_object, &contested,		lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
▲ Show 20 Lines • Show All 156 Lines • ▼ Show 20 Lines	if (error) {
"%s: interruptible sleep by %p suspended by signal",		"%s: interruptible sleep by %p suspended by signal",
__func__, sx);		__func__, sx);
break;		break;
}		}
if (LOCK_LOG_TEST(&sx->lock_object, 0))		if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",		CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
__func__, sx);		__func__, sx);
}		}

GIANT_RESTORE();
if (!error)
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, sx,
contested, waittime, file, line);
#ifdef KDTRACE_HOOKS		#ifdef KDTRACE_HOOKS
		all_time += lockstat_nsecs();
if (sleep_time)		if (sleep_time)
LOCKSTAT_RECORD1(LS_SX_XLOCK_BLOCK, sx, sleep_time);		LOCKSTAT_RECORD4(LS_SX_XLOCK_BLOCK, sx, sleep_time,
		LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0,
		(state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
if (spin_cnt > sleep_cnt)		if (spin_cnt > sleep_cnt)
LOCKSTAT_RECORD1(LS_SX_XLOCK_SPIN, sx, (spin_cnt - sleep_cnt));		LOCKSTAT_RECORD4(LS_SX_XLOCK_SPIN, sx, all_time - sleep_time,
		LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0,
		(state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
#endif		#endif
		if (!error)
		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, sx,
		contested, waittime, file, line);
		GIANT_RESTORE();
return (error);		return (error);
}		}

/*		/*
* This function represents the so-called 'hard case' for sx_xunlock		* This function represents the so-called 'hard case' for sx_xunlock
* operation. All 'easy case' failures are redirected to this. Note		* operation. All 'easy case' failures are redirected to this. Note
* that ideally this would be a static function, but it needs to be		* that ideally this would be a static function, but it needs to be
* accessible from at least sx.h.		* accessible from at least sx.h.
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines
#endif		#endif
#ifdef LOCK_PROFILING		#ifdef LOCK_PROFILING
uint64_t waittime = 0;		uint64_t waittime = 0;
int contested = 0;		int contested = 0;
#endif		#endif
uintptr_t x;		uintptr_t x;
int error = 0;		int error = 0;
#ifdef KDTRACE_HOOKS		#ifdef KDTRACE_HOOKS
		uintptr_t state;
uint64_t spin_cnt = 0;		uint64_t spin_cnt = 0;
uint64_t sleep_cnt = 0;		uint64_t sleep_cnt = 0;
int64_t sleep_time = 0;		int64_t sleep_time = 0;
		int64_t all_time = 0;
#endif		#endif

if (SCHEDULER_STOPPED())		if (SCHEDULER_STOPPED())
return (0);		return (0);

		#ifdef KDTRACE_HOOKS
		state = sx->sx_lock;
		all_time -= lockstat_nsecs();
		#endif

/*		/*
* As with rwlocks, we don't make any attempt to try to block		* As with rwlocks, we don't make any attempt to try to block
* shared locks once there is an exclusive waiter.		* shared locks once there is an exclusive waiter.
*/		*/
for (;;) {		for (;;) {
#ifdef KDTRACE_HOOKS		#ifdef KDTRACE_HOOKS
spin_cnt++;		spin_cnt++;
#endif		#endif
▲ Show 20 Lines • Show All 133 Lines • ▼ Show 20 Lines	if (error) {
"%s: interruptible sleep by %p suspended by signal",		"%s: interruptible sleep by %p suspended by signal",
__func__, sx);		__func__, sx);
break;		break;
}		}
if (LOCK_LOG_TEST(&sx->lock_object, 0))		if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",		CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
__func__, sx);		__func__, sx);
}		}
if (error == 0)
LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx,
contested, waittime, file, line);
#ifdef KDTRACE_HOOKS		#ifdef KDTRACE_HOOKS
		all_time += lockstat_nsecs();
if (sleep_time)		if (sleep_time)
LOCKSTAT_RECORD1(LS_SX_XLOCK_BLOCK, sx, sleep_time);		LOCKSTAT_RECORD4(LS_SX_SLOCK_BLOCK, sx, sleep_time,
		LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0,
		(state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
if (spin_cnt > sleep_cnt)		if (spin_cnt > sleep_cnt)
LOCKSTAT_RECORD1(LS_SX_XLOCK_SPIN, sx, (spin_cnt - sleep_cnt));		LOCKSTAT_RECORD4(LS_SX_SLOCK_SPIN, sx, all_time - sleep_time,
		LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0,
		(state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state));
#endif		#endif
		if (error == 0)
		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx,
		contested, waittime, file, line);
GIANT_RESTORE();		GIANT_RESTORE();
return (error);		return (error);
}		}

/*		/*
* This function represents the so-called 'hard case' for sx_sunlock		* This function represents the so-called 'hard case' for sx_sunlock
* operation. All 'easy case' failures are redirected to this. Note		* operation. All 'easy case' failures are redirected to this. Note
* that ideally this would be a static function, but it needs to be		* that ideally this would be a static function, but it needs to be
▲ Show 20 Lines • Show All 251 Lines • Show Last 20 Lines