Index: sys/kern/subr_syscall.c =================================================================== --- sys/kern/subr_syscall.c +++ sys/kern/subr_syscall.c @@ -42,6 +42,7 @@ #include "opt_capsicum.h" #include "opt_ktrace.h" +#include "opt_sched.h" __FBSDID("$FreeBSD$"); @@ -52,8 +53,15 @@ #include #include #endif +#include #include +#ifdef VIMAGE +#include +#endif + +static inline void _userret(struct thread *td, struct trapframe *frame); + static inline void syscallenter(struct thread *td) { @@ -207,7 +215,7 @@ /* * Handle reschedule and other end-of-syscall issues */ - userret(td, td->td_frame); + _userret(td, td->td_frame); #ifdef KTRACE if (KTRPOINT(td, KTR_SYSRET)) { @@ -241,4 +249,111 @@ if (__predict_false(td->td_pflags & TDP_RFPPWAIT)) fork_rfppwait(td); +} + +/* + * Define the code needed before returning to user mode, for trap and + * syscall. + */ +static inline void +_userret(struct thread *td, struct trapframe *frame) +{ + struct proc *p = td->td_proc; + + CTR3(KTR_SYSC, "userret: thread %p (pid %d, %s)", td, p->p_pid, + td->td_name); + KASSERT((p->p_flag & P_WEXIT) == 0, + ("Exiting process returns to usermode")); +#ifdef DIAGNOSTIC + /* + * Check that we called signotify() enough. For + * multi-threaded processes, where signal distribution might + * change due to other threads changing sigmask, the check is + * racy and cannot be performed reliably. + * If current process is vfork child, indicated by P_PPWAIT, then + * issignal() ignores stops, so we block the check to avoid + * classifying pending signals. + */ + if (p->p_numthreads == 1) { + PROC_LOCK(p); + thread_lock(td); + if ((p->p_flag & P_PPWAIT) == 0 && + (td->td_pflags & TDP_SIGFASTBLOCK) == 0) { + if (SIGPENDING(td) && (td->td_flags & + (TDF_NEEDSIGCHK | TDF_ASTPENDING)) != + (TDF_NEEDSIGCHK | TDF_ASTPENDING)) { + thread_unlock(td); + panic( + "failed to set signal flags for ast p %p td %p fl %x", + p, td, td->td_flags); + } + } + thread_unlock(td); + PROC_UNLOCK(p); + } +#endif +#ifdef KTRACE + KTRUSERRET(td); +#endif + + /* + * Charge system time if profiling. + */ + if (__predict_false(p->p_flag & P_PROFIL)) + addupc_task(td, TRAPF_PC(frame), td->td_pticks * psratio); + +#ifdef HWPMC_HOOKS + if (PMC_THREAD_HAS_SAMPLES(td)) + PMC_CALL_HOOK(td, PMC_FN_THR_USERRET, NULL); +#endif + /* + * Let the scheduler adjust our priority etc. + */ + sched_userret(td); + + /* + * Check for misbehavior. + * + * In case there is a callchain tracing ongoing because of + * hwpmc(4), skip the scheduler pinning check. + * hwpmc(4) subsystem, infact, will collect callchain informations + * at ast() checkpoint, which is past userret(). + */ + WITNESS_WARN(WARN_PANIC, NULL, "userret: returning"); + KASSERT(td->td_critnest == 0, + ("userret: Returning in a critical section")); + KASSERT(td->td_locks == 0, + ("userret: Returning with %d locks held", td->td_locks)); + KASSERT(td->td_rw_rlocks == 0, + ("userret: Returning with %d rwlocks held in read mode", + td->td_rw_rlocks)); + KASSERT(td->td_sx_slocks == 0, + ("userret: Returning with %d sx locks held in shared mode", + td->td_sx_slocks)); + KASSERT(td->td_lk_slocks == 0, + ("userret: Returning with %d lockmanager locks held in shared mode", + td->td_lk_slocks)); + KASSERT((td->td_pflags & TDP_NOFAULTING) == 0, + ("userret: Returning with pagefaults disabled")); + if (__predict_false(!THREAD_CAN_SLEEP())) { +#ifdef EPOCH_TRACE + epoch_trace_list(curthread); +#endif + KASSERT(0, ("userret: Returning with sleep disabled")); + } + KASSERT(td->td_pinned == 0 || (td->td_pflags & TDP_CALLCHAIN) != 0, + ("userret: Returning with with pinned thread")); + KASSERT(td->td_vp_reserved == NULL, + ("userret: Returning with preallocated vnode")); + KASSERT((td->td_flags & (TDF_SBDRY | TDF_SEINTR | TDF_SERESTART)) == 0, + ("userret: Returning with stop signals deferred")); + KASSERT(td->td_vslock_sz == 0, + ("userret: Returning with vslock-wired space")); +#ifdef VIMAGE + /* Unfortunately td_vnet_lpush needs VNET_DEBUG. */ + VNET_ASSERT(curvnet == NULL, + ("%s: Returning on td %p (pid %d, %s) with vnet %p set in %s", + __func__, td, p->p_pid, td->td_name, curvnet, + (td->td_vnet_lpush != NULL) ? td->td_vnet_lpush : "N/A")); +#endif } Index: sys/kern/subr_trap.c =================================================================== --- sys/kern/subr_trap.c +++ sys/kern/subr_trap.c @@ -87,6 +87,8 @@ #include +#include "subr_syscall.c" + void (*softdep_ast_cleanup)(struct thread *); /* @@ -96,104 +98,8 @@ void userret(struct thread *td, struct trapframe *frame) { - struct proc *p = td->td_proc; - CTR3(KTR_SYSC, "userret: thread %p (pid %d, %s)", td, p->p_pid, - td->td_name); - KASSERT((p->p_flag & P_WEXIT) == 0, - ("Exiting process returns to usermode")); -#ifdef DIAGNOSTIC - /* - * Check that we called signotify() enough. For - * multi-threaded processes, where signal distribution might - * change due to other threads changing sigmask, the check is - * racy and cannot be performed reliably. - * If current process is vfork child, indicated by P_PPWAIT, then - * issignal() ignores stops, so we block the check to avoid - * classifying pending signals. - */ - if (p->p_numthreads == 1) { - PROC_LOCK(p); - thread_lock(td); - if ((p->p_flag & P_PPWAIT) == 0 && - (td->td_pflags & TDP_SIGFASTBLOCK) == 0) { - if (SIGPENDING(td) && (td->td_flags & - (TDF_NEEDSIGCHK | TDF_ASTPENDING)) != - (TDF_NEEDSIGCHK | TDF_ASTPENDING)) { - thread_unlock(td); - panic( - "failed to set signal flags for ast p %p td %p fl %x", - p, td, td->td_flags); - } - } - thread_unlock(td); - PROC_UNLOCK(p); - } -#endif -#ifdef KTRACE - KTRUSERRET(td); -#endif - - /* - * Charge system time if profiling. - */ - if (__predict_false(p->p_flag & P_PROFIL)) - addupc_task(td, TRAPF_PC(frame), td->td_pticks * psratio); - -#ifdef HWPMC_HOOKS - if (PMC_THREAD_HAS_SAMPLES(td)) - PMC_CALL_HOOK(td, PMC_FN_THR_USERRET, NULL); -#endif - /* - * Let the scheduler adjust our priority etc. - */ - sched_userret(td); - - /* - * Check for misbehavior. - * - * In case there is a callchain tracing ongoing because of - * hwpmc(4), skip the scheduler pinning check. - * hwpmc(4) subsystem, infact, will collect callchain informations - * at ast() checkpoint, which is past userret(). - */ - WITNESS_WARN(WARN_PANIC, NULL, "userret: returning"); - KASSERT(td->td_critnest == 0, - ("userret: Returning in a critical section")); - KASSERT(td->td_locks == 0, - ("userret: Returning with %d locks held", td->td_locks)); - KASSERT(td->td_rw_rlocks == 0, - ("userret: Returning with %d rwlocks held in read mode", - td->td_rw_rlocks)); - KASSERT(td->td_sx_slocks == 0, - ("userret: Returning with %d sx locks held in shared mode", - td->td_sx_slocks)); - KASSERT(td->td_lk_slocks == 0, - ("userret: Returning with %d lockmanager locks held in shared mode", - td->td_lk_slocks)); - KASSERT((td->td_pflags & TDP_NOFAULTING) == 0, - ("userret: Returning with pagefaults disabled")); - if (__predict_false(!THREAD_CAN_SLEEP())) { -#ifdef EPOCH_TRACE - epoch_trace_list(curthread); -#endif - KASSERT(0, ("userret: Returning with sleep disabled")); - } - KASSERT(td->td_pinned == 0 || (td->td_pflags & TDP_CALLCHAIN) != 0, - ("userret: Returning with with pinned thread")); - KASSERT(td->td_vp_reserved == NULL, - ("userret: Returning with preallocated vnode")); - KASSERT((td->td_flags & (TDF_SBDRY | TDF_SEINTR | TDF_SERESTART)) == 0, - ("userret: Returning with stop signals deferred")); - KASSERT(td->td_vslock_sz == 0, - ("userret: Returning with vslock-wired space")); -#ifdef VIMAGE - /* Unfortunately td_vnet_lpush needs VNET_DEBUG. */ - VNET_ASSERT(curvnet == NULL, - ("%s: Returning on td %p (pid %d, %s) with vnet %p set in %s", - __func__, td, p->p_pid, td->td_name, curvnet, - (td->td_vnet_lpush != NULL) ? td->td_vnet_lpush : "N/A")); -#endif + return (_userret(td, frame)); } /*