Index: sys/kern/kern_condvar.c =================================================================== --- sys/kern/kern_condvar.c +++ sys/kern/kern_condvar.c @@ -210,13 +210,15 @@ * restarted if possible. */ int -_cv_wait_sig(struct cv *cvp, struct lock_object *lock) +_cv_wait_sig(struct cv *cvp, struct lock_object *lock, int flags) { WITNESS_SAVE_DECL(lock_witness); struct lock_class *class; struct thread *td; uintptr_t lock_state; - int rval; + int rval, sqflags; + + MPASS((flags & ~(C_USERWAIT)) == 0); td = curthread; lock_state = 0; @@ -239,8 +241,10 @@ mtx_assert(&Giant, MA_OWNED); DROP_GIANT(); - sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | - SLEEPQ_INTERRUPTIBLE, 0); + sqflags = SLEEPQ_CONDVAR | SLEEPQ_INTERRUPTIBLE; + if ((flags & C_USERWAIT) != 0) + sqflags |= SLEEPQ_USERWAIT; + sleepq_add(cvp, lock, cvp->cv_description, sqflags, 0); if (lock != &Giant.lock_object) { if (class->lc_flags & LC_SLEEPABLE) sleepq_release(cvp); @@ -277,6 +281,7 @@ struct lock_class *class; struct thread *td; int lock_state, rval; + int sqflags; td = curthread; lock_state = 0; @@ -299,7 +304,10 @@ mtx_assert(&Giant, MA_OWNED); DROP_GIANT(); - sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); + sqflags = SLEEPQ_CONDVAR; + if ((flags & C_USERWAIT) != 0) + sqflags |= SLEEPQ_USERWAIT; + sleepq_add(cvp, lock, cvp->cv_description, sqflags, 0); sleepq_set_timeout_sbt(cvp, sbt, pr, flags); if (lock != &Giant.lock_object) { if (class->lc_flags & LC_SLEEPABLE) @@ -339,6 +347,7 @@ struct lock_class *class; struct thread *td; int lock_state, rval; + int sqflags; td = curthread; lock_state = 0; @@ -361,8 +370,10 @@ mtx_assert(&Giant, MA_OWNED); DROP_GIANT(); - sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | - SLEEPQ_INTERRUPTIBLE, 0); + sqflags = SLEEPQ_CONDVAR | SLEEPQ_INTERRUPTIBLE; + if ((flags & C_USERWAIT) != 0) + sqflags |= SLEEPQ_USERWAIT; + sleepq_add(cvp, lock, cvp->cv_description, sqflags, 0); sleepq_set_timeout_sbt(cvp, sbt, pr, flags); if (lock != &Giant.lock_object) { if (class->lc_flags & LC_SLEEPABLE) Index: sys/kern/kern_event.c =================================================================== --- sys/kern/kern_event.c +++ sys/kern/kern_event.c @@ -1837,7 +1837,7 @@ error = EWOULDBLOCK; } else { kq->kq_state |= KQ_SLEEP; - error = msleep_sbt(kq, &kq->kq_lock, PSOCK | PCATCH, + error = msleep_sbt(kq, &kq->kq_lock, PSOCK | PCATCH | PUSERW, "kqread", asbt, rsbt, C_ABSOLUTE); } if (error == 0) Index: sys/kern/kern_exit.c =================================================================== --- sys/kern/kern_exit.c +++ sys/kern/kern_exit.c @@ -1338,7 +1338,8 @@ goto loop_locked; } sx_xunlock(&proctree_lock); - error = msleep(q, &q->p_mtx, PWAIT | PCATCH | PDROP, "wait", 0); + error = msleep(q, &q->p_mtx, PWAIT | PCATCH | PDROP | PUSERW, + "wait", 0); if (error) return (error); goto loop; Index: sys/kern/kern_fork.c =================================================================== --- sys/kern/kern_fork.c +++ sys/kern/kern_fork.c @@ -806,7 +806,7 @@ } else { PROC_UNLOCK(p); } - cv_timedwait(&p2->p_pwait, &p2->p_mtx, hz); + cv_timedwait_flags(&p2->p_pwait, &p2->p_mtx, hz, C_USERWAIT); } PROC_UNLOCK(p2); Index: sys/kern/kern_sig.c =================================================================== --- sys/kern/kern_sig.c +++ sys/kern/kern_sig.c @@ -1547,8 +1547,8 @@ */ (p->p_sysent->sv_set_syscall_retval)(td, EINTR); for (has_sig = 0; !has_sig;) { - while (msleep(&p->p_sigacts, &p->p_mtx, PPAUSE|PCATCH, "pause", - 0) == 0) + while (msleep(&p->p_sigacts, &p->p_mtx, PPAUSE|PCATCH|PUSERW, + "pause", 0) == 0) /* void */; thread_suspend_check(0); mtx_lock(&p->p_sigacts->ps_mtx); Index: sys/kern/kern_synch.c =================================================================== --- sys/kern/kern_synch.c +++ sys/kern/kern_synch.c @@ -138,7 +138,7 @@ struct thread *td; struct lock_class *class; uintptr_t lock_state; - int catch, pri, rval, sleepq_flags; + int catch, userw, pri, rval, sleepq_flags; WITNESS_SAVE_DECL(lock_witness); td = curthread; @@ -166,6 +166,7 @@ return (0); } catch = priority & PCATCH; + userw = priority & PUSERW; pri = priority & PRIMASK; KASSERT(!TD_ON_SLEEPQ(td), ("recursive sleep")); @@ -177,6 +178,8 @@ sleepq_flags = SLEEPQ_SLEEP; if (catch) sleepq_flags |= SLEEPQ_INTERRUPTIBLE; + if (userw) + sleepq_flags |= SLEEPQ_USERWAIT; sleepq_lock(ident); CTR5(KTR_PROC, "sleep: thread %ld (pid %ld, %s) on %s (%p)", Index: sys/kern/kern_time.c =================================================================== --- sys/kern/kern_time.c +++ sys/kern/kern_time.c @@ -560,8 +560,8 @@ if (TIMESEL(&sbt, tmp)) sbt += tc_tick_sbt; sbt += tmp; - error = tsleep_sbt(&nanowait[curcpu], PWAIT | PCATCH, "nanslp", - sbt, prec, C_ABSOLUTE); + error = tsleep_sbt(&nanowait[curcpu], PWAIT | PCATCH | PUSERW, + "nanslp", sbt, prec, C_ABSOLUTE); } while (error == 0 && is_abs_real && td->td_rtcgen == 0); td->td_rtcgen = 0; if (error != EWOULDBLOCK) { Index: sys/kern/kern_umtx.c =================================================================== --- sys/kern/kern_umtx.c +++ sys/kern/kern_umtx.c @@ -834,7 +834,7 @@ } } else timo = 0; - error = msleep(uq, &uc->uc_lock, PCATCH | PDROP, wmesg, timo); + error = msleep(uq, &uc->uc_lock, PCATCH | PDROP | PUSERW, wmesg, timo); if (error == EINTR || error == ERESTART) { umtxq_lock(&uq->uq_key); break; Index: sys/kern/subr_sleepqueue.c =================================================================== --- sys/kern/subr_sleepqueue.c +++ sys/kern/subr_sleepqueue.c @@ -177,9 +177,19 @@ static void sleepq_switch(const void *wchan, int pri); static void sleepq_timeout(void *arg); +SDT_PROVIDER_DECLARE(sched); SDT_PROBE_DECLARE(sched, , , sleep); SDT_PROBE_DECLARE(sched, , , wakeup); +SDT_PROBE_DEFINE2(sched, , sleepq, usleep, "struct thread *", + "const char *"); +SDT_PROBE_DEFINE2(sched, , sleepq, uwakeup, "struct thread *", + "const char *"); +SDT_PROBE_DEFINE2(sched, , sleepq, ksleep, "struct thread *", + "const char *"); +SDT_PROBE_DEFINE2(sched, , sleepq, kwakeup, "struct thread *", + "const char *"); + /* * Initialize SLEEPQUEUE_PROFILING specific sysctl nodes. * Note that it must happen after sleepinit() has been fully executed, so @@ -384,6 +394,9 @@ td->td_flags |= TDF_SINTR; } td->td_flags &= ~TDF_TIMEOUT; + if (flags & SLEEPQ_USERWAIT) { + td->td_flags |= TDF_USERWAIT; + } thread_unlock(td); } @@ -605,9 +618,25 @@ MPASS(td->td_sleepqueue == NULL); sched_sleep(td, pri); thread_lock_set(td, &sc->sc_lock); - SDT_PROBE0(sched, , , sleep); TD_SET_SLEEPING(td); - mi_switch(SW_VOL | SWT_SLEEPQ); + if (!SDT_PROBES_ENABLED()) { + mi_switch(SW_VOL | SWT_SLEEPQ); + } else { + SDT_PROBE0(sched, , , sleep); + if ((td->td_flags & TDF_USERWAIT) == 0) + SDT_PROBE2(sched, , sleepq, ksleep, td, + td->td_wmesg); + else + SDT_PROBE2(sched, , sleepq, usleep, td, + td->td_wmesg); + mi_switch(SW_VOL | SWT_SLEEPQ); + if ((td->td_flags & TDF_USERWAIT) == 0) + SDT_PROBE2(sched, , sleepq, kwakeup, td, + td->td_wmesg); + else + SDT_PROBE2(sched, , sleepq, uwakeup, td, + td->td_wmesg); + } KASSERT(TD_IS_RUNNING(td), ("running but not TDS_RUNNING")); CTR3(KTR_PROC, "sleepq resume: thread %p (pid %ld, %s)", (void *)td, (long)td->td_proc->p_pid, (void *)td->td_name); @@ -850,7 +879,7 @@ td->td_wmesg = NULL; td->td_wchan = NULL; - td->td_flags &= ~(TDF_SINTR | TDF_TIMEOUT); + td->td_flags &= ~(TDF_SINTR | TDF_TIMEOUT | TDF_USERWAIT); CTR3(KTR_PROC, "sleepq_wakeup: thread %p (pid %ld, %s)", (void *)td, (long)td->td_proc->p_pid, td->td_name); Index: sys/kern/subr_turnstile.c =================================================================== --- sys/kern/subr_turnstile.c +++ sys/kern/subr_turnstile.c @@ -176,6 +176,11 @@ SDT_PROBE_DEFINE2(sched, , , wakeup, "struct thread *", "struct proc *"); +SDT_PROBE_DEFINE2(sched, , turnstile, ksleep, "struct thread *", + "const char *"); +SDT_PROBE_DEFINE2(sched, , turnstile, kwakeup, "struct thread *", + "const char *"); + static inline void propagate_unlock_ts(struct turnstile *top, struct turnstile *ts) { @@ -811,10 +816,17 @@ CTR4(KTR_LOCK, "%s: td %d blocked on [%p] %s", __func__, td->td_tid, lock, lock->lo_name); - SDT_PROBE0(sched, , , sleep); - THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock); - mi_switch(SW_VOL | SWT_TURNSTILE); + if (!SDT_PROBES_ENABLED()) { + mi_switch(SW_VOL | SWT_TURNSTILE); + } else { + SDT_PROBE0(sched, , , sleep); + SDT_PROBE2(sched, , turnstile, ksleep, td, + td->td_lockname); + mi_switch(SW_VOL | SWT_TURNSTILE); + SDT_PROBE2(sched, , turnstile, kwakeup, td, + td->td_lockname); + } if (LOCK_LOG_TEST(lock, 0)) CTR4(KTR_LOCK, "%s: td %d free from blocked on [%p] %s", Index: sys/kern/sys_generic.c =================================================================== --- sys/kern/sys_generic.c +++ sys/kern/sys_generic.c @@ -1871,9 +1871,9 @@ error = EWOULDBLOCK; else if (sbt != -1) error = cv_timedwait_sig_sbt(&stp->st_wait, &stp->st_mtx, - sbt, precision, C_ABSOLUTE); + sbt, precision, C_ABSOLUTE | C_USERWAIT); else - error = cv_wait_sig(&stp->st_wait, &stp->st_mtx); + error = cv_wait_sig_flags(&stp->st_wait, &stp->st_mtx, C_USERWAIT); mtx_unlock(&stp->st_mtx); return (error); Index: sys/kern/sys_pipe.c =================================================================== --- sys/kern/sys_pipe.c +++ sys/kern/sys_pipe.c @@ -813,7 +813,7 @@ } else { rpipe->pipe_state |= PIPE_WANTR; if ((error = msleep(rpipe, PIPE_MTX(rpipe), - PRIBIO | PCATCH, + PRIBIO | PCATCH | PUSERW, "piperd", 0)) == 0) error = pipelock(rpipe, 1); } @@ -996,7 +996,7 @@ wpipe->pipe_state |= PIPE_WANTW; pipeunlock(wpipe); error = msleep(wpipe, PIPE_MTX(wpipe), - PRIBIO | PCATCH, "pipdww", 0); + PRIBIO | PCATCH | PUSERW, "pipdww", 0); if (error) goto error1; else @@ -1011,7 +1011,7 @@ wpipe->pipe_state |= PIPE_WANTW; pipeunlock(wpipe); error = msleep(wpipe, PIPE_MTX(wpipe), - PRIBIO | PCATCH, "pipdwc", 0); + PRIBIO | PCATCH | PUSERW, "pipdwc", 0); if (error) goto error1; else @@ -1033,7 +1033,7 @@ pipeselwakeup(wpipe); wpipe->pipe_state |= PIPE_WANTW; pipeunlock(wpipe); - error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH, + error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH | PUSERW, "pipdwt", 0); pipelock(wpipe, 0); if (error != 0) @@ -1174,7 +1174,7 @@ pipeselwakeup(wpipe); wpipe->pipe_state |= PIPE_WANTW; pipeunlock(wpipe); - error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH, + error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH | PUSERW, "pipbww", 0); if (error) break; @@ -1281,7 +1281,7 @@ wpipe->pipe_state |= PIPE_WANTW; pipeunlock(wpipe); error = msleep(wpipe, PIPE_MTX(rpipe), - PRIBIO | PCATCH, "pipewr", 0); + PRIBIO | PCATCH | PUSERW, "pipewr", 0); if (error != 0) break; } Index: sys/kern/tty.c =================================================================== --- sys/kern/tty.c +++ sys/kern/tty.c @@ -1542,7 +1542,7 @@ tty_lock_assert(tp, MA_OWNED|MA_NOTRECURSED); MPASS(!tty_gone(tp)); - error = cv_wait_sig(cv, tp->t_mtx); + error = cv_wait_sig_flags(cv, tp->t_mtx, C_USERWAIT); /* Bail out when the device slipped away. */ if (tty_gone(tp)) Index: sys/sys/callout.h =================================================================== --- sys/sys/callout.h +++ sys/sys/callout.h @@ -61,6 +61,7 @@ #define C_ABSOLUTE 0x0200 /* event time is absolute. */ #define C_PRECALC 0x0400 /* event time is pre-calculated. */ #define C_CATCH 0x0800 /* catch signals, used by pause_sbt(9) */ +#define C_USERWAIT 0x1000 /* catch signals, used by pause_sbt(9) */ /* Flags for callout_stop_safe() */ #define CS_DRAIN 0x0001 /* callout_drain(), wait allowed */ Index: sys/sys/condvar.h =================================================================== --- sys/sys/condvar.h +++ sys/sys/condvar.h @@ -56,7 +56,7 @@ void _cv_wait(struct cv *cvp, struct lock_object *lock); void _cv_wait_unlock(struct cv *cvp, struct lock_object *lock); -int _cv_wait_sig(struct cv *cvp, struct lock_object *lock); +int _cv_wait_sig(struct cv *cvp, struct lock_object *lock, int flags); int _cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt, sbintime_t pr, int flags); int _cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock, @@ -69,11 +69,15 @@ _cv_wait((cvp), &(lock)->lock_object) #define cv_wait_unlock(cvp, lock) \ _cv_wait_unlock((cvp), &(lock)->lock_object) +#define cv_wait_sig_flags(cvp, lock, flags) \ + _cv_wait_sig((cvp), &(lock)->lock_object, (flags)) #define cv_wait_sig(cvp, lock) \ - _cv_wait_sig((cvp), &(lock)->lock_object) -#define cv_timedwait(cvp, lock, timo) \ + cv_wait_sig_flags((cvp), (lock), 0) +#define cv_timedwait_flags(cvp, lock, timo, flags) \ _cv_timedwait_sbt((cvp), &(lock)->lock_object, \ - tick_sbt * (timo), 0, C_HARDCLOCK) + tick_sbt * (timo), 0, C_HARDCLOCK | (flags)) +#define cv_timedwait(cvp, lock, timo) \ + cv_timedwait_flags(cvp, lock, timo, 0) #define cv_timedwait_sbt(cvp, lock, sbt, pr, flags) \ _cv_timedwait_sbt((cvp), &(lock)->lock_object, (sbt), (pr), (flags)) #define cv_timedwait_sig(cvp, lock, timo) \ Index: sys/sys/param.h =================================================================== --- sys/sys/param.h +++ sys/sys/param.h @@ -226,6 +226,7 @@ #define PRIMASK 0x0ff #define PCATCH 0x100 /* OR'd with pri for tsleep to check signals */ #define PDROP 0x200 /* OR'd with pri to stop re-entry of interlock mutex */ +#define PUSERW 0x400 /* OR'd with pri to denote waiting on the user */ #define NZERO 0 /* default "nice" */ Index: sys/sys/proc.h =================================================================== --- sys/sys/proc.h +++ sys/sys/proc.h @@ -439,7 +439,7 @@ #define TDF_TIMEOUT 0x00000010 /* Timing out during sleep. */ #define TDF_IDLETD 0x00000020 /* This is a per-CPU idle thread. */ #define TDF_CANSWAP 0x00000040 /* Thread can be swapped. */ -#define TDF_UNUSED80 0x00000080 /* unused. */ +#define TDF_USERWAIT 0x00000080 /* Waiting on the user. */ #define TDF_KTH_SUSP 0x00000100 /* kthread is suspended */ #define TDF_ALLPROCSUSP 0x00000200 /* suspended by SINGLE_ALLPROC */ #define TDF_BOUNDARY 0x00000400 /* Thread suspended at user boundary */ Index: sys/sys/sleepqueue.h =================================================================== --- sys/sys/sleepqueue.h +++ sys/sys/sleepqueue.h @@ -85,6 +85,7 @@ #define SLEEPQ_LK 0x04 /* Used by a lockmgr. */ #define SLEEPQ_INTERRUPTIBLE 0x100 /* Sleep is interruptible. */ #define SLEEPQ_UNFAIR 0x200 /* Unfair wakeup order. */ +#define SLEEPQ_USERWAIT 0x400 /* Waiting on the user. */ void init_sleepqueues(void); int sleepq_abort(struct thread *td, int intrval);