Index: sys/conf/NOTES =================================================================== --- sys/conf/NOTES +++ sys/conf/NOTES @@ -2841,3 +2841,6 @@ # Kernel support for stats(3). options STATS + +# Dtrace-based profiling of __predict_true/false +# options PREDICT_PROFILING Index: sys/conf/options =================================================================== --- sys/conf/options +++ sys/conf/options @@ -1012,3 +1012,6 @@ # gcov support GCOV opt_global.h LINDEBUGFS + +# Dtrace-based profiling of __predict_true/false +PREDICT_PROFILING opt_global.h Index: sys/kern/kern_sdt.c =================================================================== --- sys/kern/kern_sdt.c +++ sys/kern/kern_sdt.c @@ -33,6 +33,9 @@ #include SDT_PROVIDER_DEFINE(sdt); +#ifdef PREDICT_PROFILING +SDT_PROBE_DEFINE4(sdt, profile, predict, , "int", "int", "char *", "int"); +#endif /* * Hook for the DTrace probe function. The SDT provider will set this to @@ -54,3 +57,13 @@ printf("sdt_probe_stub: unexpectedly called\n"); kdb_backtrace(); } + +#ifdef PREDICT_PROFILING +int +sdt_predict_profile(int exp_res, int predict, const char *file, int line) +{ + + SDT_PROBE4(sdt, profile, predict, , exp_res, predict, file, line); + return (exp_res); +} +#endif Index: sys/sys/cdefs.h =================================================================== --- sys/sys/cdefs.h +++ sys/sys/cdefs.h @@ -447,11 +447,19 @@ * larger code. */ #if __GNUC_PREREQ__(2, 96) -#define __predict_true(exp) __builtin_expect((exp), 1) -#define __predict_false(exp) __builtin_expect((exp), 0) +#define __predict_true_noprobe(exp) __builtin_expect((exp), 1) +#define __predict_false_noprobe(exp) __builtin_expect((exp), 0) #else -#define __predict_true(exp) (exp) -#define __predict_false(exp) (exp) +#define __predict_true_noprobe(exp) (exp) +#define __predict_false_noprobe(exp) (exp) +#endif + +#ifdef PREDICT_PROFILING +#define __predict_true(exp) sdt_predict_profile(exp, true, __FILE__, __LINE__) +#define __predict_false(exp) sdt_predict_profile(exp, false, __FILE__, __LINE__) +#else +#define __predict_true(exp) __predict_true_noprobe(exp) +#define __predict_false(exp) __predict_false_noprobe(exp) #endif #if __GNUC_PREREQ__(4, 0) Index: sys/sys/lockstat.h =================================================================== --- sys/sys/lockstat.h +++ sys/sys/lockstat.h @@ -116,7 +116,7 @@ LOCKSTAT_RECORD1(probe, lp, a); \ } while (0) -#define LOCKSTAT_PROFILE_ENABLED(probe) __predict_false(lockstat_enabled) +#define LOCKSTAT_PROFILE_ENABLED(probe) __predict_false_noprobe(lockstat_enabled) struct lock_object; uint64_t lockstat_nsecs(struct lock_object *); Index: sys/sys/mutex.h =================================================================== --- sys/sys/mutex.h +++ sys/sys/mutex.h @@ -240,7 +240,7 @@ uintptr_t _tid = (uintptr_t)(tid); \ uintptr_t _v = MTX_UNOWNED; \ \ - if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__acquire) ||\ + if (__predict_false_noprobe(LOCKSTAT_PROFILE_ENABLED(adaptive__acquire) ||\ !_mtx_obtain_lock_fetch((mp), &_v, _tid))) \ _mtx_lock_sleep((mp), _v, (opts), (file), (line)); \ } while (0) @@ -257,7 +257,7 @@ uintptr_t _v = MTX_UNOWNED; \ \ spinlock_enter(); \ - if (__predict_false(LOCKSTAT_PROFILE_ENABLED(spin__acquire) || \ + if (__predict_false_noprobe(LOCKSTAT_PROFILE_ENABLED(spin__acquire) || \ !_mtx_obtain_lock_fetch((mp), &_v, _tid))) \ _mtx_lock_spin((mp), _v, (opts), (file), (line)); \ } while (0) @@ -308,7 +308,7 @@ #define __mtx_unlock(mp, tid, opts, file, line) do { \ uintptr_t _v = (uintptr_t)(tid); \ \ - if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__release) ||\ + if (__predict_false_noprobe(LOCKSTAT_PROFILE_ENABLED(adaptive__release) ||\ !_mtx_release_lock_fetch((mp), &_v))) \ _mtx_unlock_sleep((mp), _v, (opts), (file), (line)); \ } while (0) Index: sys/sys/param.h =================================================================== --- sys/sys/param.h +++ sys/sys/param.h @@ -40,6 +40,10 @@ #ifndef _SYS_PARAM_H_ #define _SYS_PARAM_H_ +#ifdef PREDICT_PROFILING +int sdt_predict_profile(int, int, const char *, int); +#endif + #include #define BSD 199506 /* System version (year & month). */ Index: sys/sys/rwlock.h =================================================================== --- sys/sys/rwlock.h +++ sys/sys/rwlock.h @@ -107,7 +107,7 @@ uintptr_t _tid = (uintptr_t)(tid); \ uintptr_t _v = RW_UNLOCKED; \ \ - if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__acquire) || \ + if (__predict_false_noprobe(LOCKSTAT_PROFILE_ENABLED(rw__acquire) || \ !_rw_write_lock_fetch((rw), &_v, _tid))) \ _rw_wlock_hard((rw), _v, (file), (line)); \ } while (0) @@ -116,7 +116,7 @@ #define __rw_wunlock(rw, tid, file, line) do { \ uintptr_t _v = (uintptr_t)(tid); \ \ - if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) || \ + if (__predict_false_noprobe(LOCKSTAT_PROFILE_ENABLED(rw__release) || \ !_rw_write_unlock_fetch((rw), &_v))) \ _rw_wunlock_hard((rw), _v, (file), (line)); \ } while (0) Index: sys/sys/sdt.h =================================================================== --- sys/sys/sdt.h +++ sys/sys/sdt.h @@ -165,11 +165,11 @@ #define SDT_PROBE_DECLARE(prov, mod, func, name) \ extern struct sdt_probe sdt_##prov##_##mod##_##func##_##name[1] -#define SDT_PROBES_ENABLED() __predict_false(sdt_probes_enabled) +#define SDT_PROBES_ENABLED() __predict_false_noprobe(sdt_probes_enabled) #define SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) do { \ if (SDT_PROBES_ENABLED()) { \ - if (__predict_false(sdt_##prov##_##mod##_##func##_##name->id)) \ + if (__predict_false_noprobe(sdt_##prov##_##mod##_##func##_##name->id)) \ (*sdt_probe_func)(sdt_##prov##_##mod##_##func##_##name->id, \ (uintptr_t) arg0, (uintptr_t) arg1, (uintptr_t) arg2, \ (uintptr_t) arg3, (uintptr_t) arg4); \ @@ -427,6 +427,8 @@ void sdt_probe_stub(uint32_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t); +int sdt_predict_trace(int exp_res, int predict, const char *file, int line); + SDT_PROVIDER_DECLARE(sdt); #endif /* _KERNEL */ Index: sys/sys/sx.h =================================================================== --- sys/sys/sx.h +++ sys/sys/sx.h @@ -163,7 +163,7 @@ uintptr_t v = SX_LOCK_UNLOCKED; int error = 0; - if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__acquire) || + if (__predict_false_noprobe(LOCKSTAT_PROFILE_ENABLED(sx__acquire) || !atomic_fcmpset_acq_ptr(&sx->sx_lock, &v, tid))) error = _sx_xlock_hard(sx, v, opts); @@ -176,7 +176,7 @@ { uintptr_t x = (uintptr_t)td; - if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__release) || + if (__predict_false_noprobe(LOCKSTAT_PROFILE_ENABLED(sx__release) || !atomic_fcmpset_rel_ptr(&sx->sx_lock, &x, SX_LOCK_UNLOCKED))) _sx_xunlock_hard(sx, x); } Index: sys/sys/systm.h =================================================================== --- sys/sys/systm.h +++ sys/sys/systm.h @@ -298,7 +298,7 @@ __compiler_membar(); td->td_critnest--; __compiler_membar(); - if (__predict_false(td->td_owepreempt)) + if (__predict_false_noprobe(td->td_owepreempt)) critical_exit_preempt(); }