Index: head/sys/kern/kern_exit.c =================================================================== --- head/sys/kern/kern_exit.c (revision 132371) +++ head/sys/kern/kern_exit.c (revision 132372) @@ -1,763 +1,763 @@ /* * Copyright (c) 1982, 1986, 1989, 1991, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)kern_exit.c 8.7 (Berkeley) 2/12/94 */ #include __FBSDID("$FreeBSD$"); #include "opt_compat.h" #include "opt_ktrace.h" #include "opt_mac.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* for acct_process() function prototype */ #include #include #include #include #ifdef KTRACE #include #endif #include #include #include #include #include #include #include #include /* Required to be non-static for SysVR4 emulator */ MALLOC_DEFINE(M_ZOMBIE, "zombie", "zombie proc status"); /* * exit -- * Death of process. * * MPSAFE */ void sys_exit(struct thread *td, struct sys_exit_args *uap) { exit1(td, W_EXITCODE(uap->rval, 0)); /* NOTREACHED */ } /* * Exit: deallocate address space and other resources, change proc state * to zombie, and unlink proc from allproc and parent's lists. Save exit * status and rusage for wait(). Check for child processes and orphan them. */ void exit1(struct thread *td, int rv) { struct bintime new_switchtime; struct proc *p, *nq, *q; struct tty *tp; struct vnode *ttyvp; struct vmspace *vm; struct vnode *vtmp; #ifdef KTRACE struct vnode *tracevp; struct ucred *tracecred; #endif struct plimit *plim; /* * Drop Giant if caller has it. Eventually we should warn about * being called with Giant held. */ while (mtx_owned(&Giant)) mtx_unlock(&Giant); p = td->td_proc; if (p == initproc) { printf("init died (signal %d, exit %d)\n", WTERMSIG(rv), WEXITSTATUS(rv)); panic("Going nowhere without my init!"); } /* * MUST abort all other threads before proceeding past here. */ PROC_LOCK(p); if (p->p_flag & P_SA || p->p_numthreads > 1) { retry: /* * First check if some other thread got here before us.. * if so, act apropriatly, (exit or suspend); */ thread_suspend_check(0); /* * Kill off the other threads. This requires * Some co-operation from other parts of the kernel * so it may not be instant. * With this state set: * Any thread entering the kernel from userspace will * thread_exit() in trap(). Any thread attempting to * sleep will return immediatly with EINTR or EWOULDBLOCK, * which will hopefully force them to back out to userland, * freeing resources as they go, and anything attempting * to return to userland will thread_exit() from userret(). * thread_exit() will unsuspend us when the last other * thread exits. * If there is already a thread singler after resumption, * calling thread_single will fail, in the case, we just * re-check all suspension request, the thread should * either be suspended there or exit. */ if (thread_single(SINGLE_EXIT)) goto retry; /* * All other activity in this process is now stopped. * Remove excess KSEs and KSEGRPS. XXXKSE (when we have them) * ... * Turn off threading support. */ p->p_flag &= ~P_SA; td->td_pflags &= ~TDP_SA; thread_single_end(); /* Don't need this any more. */ } p->p_flag |= P_WEXIT; PROC_UNLOCK(p); /* Are we a task leader? */ if (p == p->p_leader) { mtx_lock(&ppeers_lock); q = p->p_peers; while (q != NULL) { PROC_LOCK(q); psignal(q, SIGKILL); PROC_UNLOCK(q); q = q->p_peers; } while (p->p_peers != NULL) msleep(p, &ppeers_lock, PWAIT, "exit1", 0); mtx_unlock(&ppeers_lock); } PROC_LOCK(p); _STOPEVENT(p, S_EXIT, rv); wakeup(&p->p_stype); /* Wakeup anyone in procfs' PIOCWAIT */ PROC_UNLOCK(p); /* * Check if any loadable modules need anything done at process exit. * e.g. SYSV IPC stuff * XXX what if one of these generates an error? */ EVENTHANDLER_INVOKE(process_exit, p); MALLOC(p->p_ru, struct rusage *, sizeof(struct rusage), M_ZOMBIE, M_WAITOK); /* * If parent is waiting for us to exit or exec, * P_PPWAIT is set; we will wakeup the parent below. */ PROC_LOCK(p); stopprofclock(p); p->p_flag &= ~(P_TRACED | P_PPWAIT); SIGEMPTYSET(p->p_siglist); SIGEMPTYSET(td->td_siglist); /* * Stop the real interval timer. If the handler is currently * executing, prevent it from rearming itself and let it finish. */ if (timevalisset(&p->p_realtimer.it_value) && callout_stop(&p->p_itcallout) == 0) { timevalclear(&p->p_realtimer.it_interval); msleep(&p->p_itcallout, &p->p_mtx, PWAIT, "ritwait", 0); KASSERT(!timevalisset(&p->p_realtimer.it_value), ("realtime timer is still armed")); } PROC_UNLOCK(p); /* * Reset any sigio structures pointing to us as a result of * F_SETOWN with our pid. */ mtx_lock(&Giant); /* XXX: not sure if needed */ funsetownlst(&p->p_sigiolst); /* * Close open files and release open-file table. * This may block! */ fdfree(td); mtx_unlock(&Giant); /* * Remove ourself from our leader's peer list and wake our leader. */ mtx_lock(&ppeers_lock); if (p->p_leader->p_peers) { q = p->p_leader; while (q->p_peers != p) q = q->p_peers; q->p_peers = p->p_peers; wakeup(p->p_leader); } mtx_unlock(&ppeers_lock); mtx_lock(&Giant); /* The next two chunks should probably be moved to vmspace_exit. */ vm = p->p_vmspace; /* * Release user portion of address space. * This releases references to vnodes, * which could cause I/O if the file has been unlinked. * Need to do this early enough that we can still sleep. * Can't free the entire vmspace as the kernel stack * may be mapped within that space also. * * Processes sharing the same vmspace may exit in one order, and * get cleaned up by vmspace_exit() in a different order. The * last exiting process to reach this point releases as much of * the environment as it can, and the last process cleaned up * by vmspace_exit() (which decrements exitingcnt) cleans up the * remainder. */ ++vm->vm_exitingcnt; if (--vm->vm_refcnt == 0) { shmexit(vm); pmap_remove_pages(vmspace_pmap(vm), vm_map_min(&vm->vm_map), vm_map_max(&vm->vm_map)); (void) vm_map_remove(&vm->vm_map, vm_map_min(&vm->vm_map), vm_map_max(&vm->vm_map)); } sx_xlock(&proctree_lock); if (SESS_LEADER(p)) { struct session *sp; sp = p->p_session; if (sp->s_ttyvp) { /* * Controlling process. * Signal foreground pgrp, * drain controlling terminal * and revoke access to controlling terminal. */ if (sp->s_ttyp && (sp->s_ttyp->t_session == sp)) { tp = sp->s_ttyp; if (sp->s_ttyp->t_pgrp) { PGRP_LOCK(sp->s_ttyp->t_pgrp); pgsignal(sp->s_ttyp->t_pgrp, SIGHUP, 1); PGRP_UNLOCK(sp->s_ttyp->t_pgrp); } /* XXX tp should be locked. */ sx_xunlock(&proctree_lock); (void) ttywait(tp); sx_xlock(&proctree_lock); /* * The tty could have been revoked * if we blocked. */ if (sp->s_ttyvp) { ttyvp = sp->s_ttyvp; SESS_LOCK(p->p_session); sp->s_ttyvp = NULL; SESS_UNLOCK(p->p_session); sx_xunlock(&proctree_lock); VOP_REVOKE(ttyvp, REVOKEALL); vrele(ttyvp); sx_xlock(&proctree_lock); } } if (sp->s_ttyvp) { ttyvp = sp->s_ttyvp; SESS_LOCK(p->p_session); sp->s_ttyvp = NULL; SESS_UNLOCK(p->p_session); vrele(ttyvp); } /* * s_ttyp is not zero'd; we use this to indicate * that the session once had a controlling terminal. * (for logging and informational purposes) */ } SESS_LOCK(p->p_session); sp->s_leader = NULL; SESS_UNLOCK(p->p_session); } fixjobc(p, p->p_pgrp, 0); sx_xunlock(&proctree_lock); (void)acct_process(td); mtx_unlock(&Giant); #ifdef KTRACE /* * release trace file */ PROC_LOCK(p); mtx_lock(&ktrace_mtx); p->p_traceflag = 0; /* don't trace the vrele() */ tracevp = p->p_tracevp; p->p_tracevp = NULL; tracecred = p->p_tracecred; p->p_tracecred = NULL; mtx_unlock(&ktrace_mtx); PROC_UNLOCK(p); if (tracevp != NULL) { mtx_lock(&Giant); vrele(tracevp); mtx_unlock(&Giant); } if (tracecred != NULL) crfree(tracecred); #endif /* * Release reference to text vnode */ if ((vtmp = p->p_textvp) != NULL) { p->p_textvp = NULL; mtx_lock(&Giant); vrele(vtmp); mtx_unlock(&Giant); } /* * Release our limits structure. */ PROC_LOCK(p); plim = p->p_limit; p->p_limit = NULL; PROC_UNLOCK(p); lim_free(plim); /* * Release this thread's reference to the ucred. The actual proc * reference will stay around until the proc is harvested by * wait(). At this point the ucred is immutable (no other threads * from this proc are around that can change it) so we leave the * per-thread ucred pointer intact in case it is needed although * in theory nothing should be using it at this point. */ crfree(td->td_ucred); /* * Remove proc from allproc queue and pidhash chain. * Place onto zombproc. Unlink from parent's child list. */ sx_xlock(&allproc_lock); LIST_REMOVE(p, p_list); LIST_INSERT_HEAD(&zombproc, p, p_list); LIST_REMOVE(p, p_hash); sx_xunlock(&allproc_lock); sx_xlock(&proctree_lock); q = LIST_FIRST(&p->p_children); if (q != NULL) /* only need this if any child is S_ZOMB */ wakeup(initproc); for (; q != NULL; q = nq) { nq = LIST_NEXT(q, p_sibling); PROC_LOCK(q); proc_reparent(q, initproc); q->p_sigparent = SIGCHLD; /* * Traced processes are killed * since their existence means someone is screwing up. */ if (q->p_flag & P_TRACED) { q->p_flag &= ~P_TRACED; psignal(q, SIGKILL); } PROC_UNLOCK(q); } /* * Save exit status and final rusage info, adding in child rusage * info and self times. */ mtx_lock(&Giant); PROC_LOCK(p); p->p_xstat = rv; p->p_xthread = td; *p->p_ru = p->p_stats->p_ru; mtx_lock_spin(&sched_lock); calcru(p, &p->p_ru->ru_utime, &p->p_ru->ru_stime, NULL); mtx_unlock_spin(&sched_lock); ruadd(p->p_ru, &p->p_stats->p_cru); /* * Notify interested parties of our demise. */ KNOTE(&p->p_klist, NOTE_EXIT); mtx_unlock(&Giant); /* * Just delete all entries in the p_klist. At this point we won't * report any more events, and there are nasty race conditions that * can beat us if we don't. */ while (SLIST_FIRST(&p->p_klist)) SLIST_REMOVE_HEAD(&p->p_klist, kn_selnext); /* * Notify parent that we're gone. If parent has the PS_NOCLDWAIT * flag set, or if the handler is set to SIG_IGN, notify process * 1 instead (and hope it will handle this situation). */ PROC_LOCK(p->p_pptr); mtx_lock(&p->p_pptr->p_sigacts->ps_mtx); if (p->p_pptr->p_sigacts->ps_flag & (PS_NOCLDWAIT | PS_CLDSIGIGN)) { struct proc *pp; mtx_unlock(&p->p_pptr->p_sigacts->ps_mtx); pp = p->p_pptr; PROC_UNLOCK(pp); proc_reparent(p, initproc); p->p_sigparent = SIGCHLD; PROC_LOCK(p->p_pptr); /* * If this was the last child of our parent, notify * parent, so in case he was wait(2)ing, he will * continue. */ if (LIST_EMPTY(&pp->p_children)) wakeup(pp); } else mtx_unlock(&p->p_pptr->p_sigacts->ps_mtx); if (p->p_pptr == initproc) psignal(p->p_pptr, SIGCHLD); else if (p->p_sigparent != 0) psignal(p->p_pptr, p->p_sigparent); PROC_UNLOCK(p->p_pptr); /* * If this is a kthread, then wakeup anyone waiting for it to exit. */ if (p->p_flag & P_KTHREAD) wakeup(p); PROC_UNLOCK(p); /* * Finally, call machine-dependent code to release the remaining * resources including address space. * The address space is released by "vmspace_exitfree(p)" in * vm_waitproc(). */ cpu_exit(td); PROC_LOCK(p); PROC_LOCK(p->p_pptr); sx_xunlock(&proctree_lock); while (mtx_owned(&Giant)) mtx_unlock(&Giant); /* * We have to wait until after acquiring all locks before * changing p_state. We need to avoid any possibly context * switches while marked as a zombie including blocking on * a mutex. */ mtx_lock_spin(&sched_lock); p->p_state = PRS_ZOMBIE; critical_enter(); mtx_unlock_spin(&sched_lock); wakeup(p->p_pptr); PROC_UNLOCK(p->p_pptr); mtx_lock_spin(&sched_lock); critical_exit(); /* Do the same timestamp bookkeeping that mi_switch() would do. */ binuptime(&new_switchtime); bintime_add(&p->p_runtime, &new_switchtime); bintime_sub(&p->p_runtime, PCPU_PTR(switchtime)); PCPU_SET(switchtime, new_switchtime); PCPU_SET(switchticks, ticks); cnt.v_swtch++; - sched_exit(p->p_pptr, p); + sched_exit(p->p_pptr, td); /* * Make sure the scheduler takes this thread out of its tables etc. * This will also release this thread's reference to the ucred. * Other thread parts to release include pcb bits and such. */ thread_exit(); } #ifdef COMPAT_43 /* * MPSAFE. The dirty work is handled by kern_wait(). */ int owait(struct thread *td, struct owait_args *uap __unused) { int error, status; error = kern_wait(td, WAIT_ANY, &status, 0, NULL); if (error == 0) td->td_retval[1] = status; return (error); } #endif /* COMPAT_43 */ /* * MPSAFE. The dirty work is handled by kern_wait(). */ int wait4(struct thread *td, struct wait_args *uap) { struct rusage ru; int error, status; error = kern_wait(td, uap->pid, &status, uap->options, &ru); if (uap->status != NULL && error == 0) error = copyout(&status, uap->status, sizeof(status)); if (uap->rusage != NULL && error == 0) error = copyout(&ru, uap->rusage, sizeof(struct rusage)); return (error); } int kern_wait(struct thread *td, pid_t pid, int *status, int options, struct rusage *rusage) { int nfound; struct proc *p, *q, *t; int error; q = td->td_proc; if (pid == 0) { PROC_LOCK(q); pid = -q->p_pgid; PROC_UNLOCK(q); } if (options &~ (WUNTRACED|WNOHANG|WCONTINUED|WLINUXCLONE)) return (EINVAL); loop: nfound = 0; sx_xlock(&proctree_lock); LIST_FOREACH(p, &q->p_children, p_sibling) { PROC_LOCK(p); if (pid != WAIT_ANY && p->p_pid != pid && p->p_pgid != -pid) { PROC_UNLOCK(p); continue; } /* * This special case handles a kthread spawned by linux_clone * (see linux_misc.c). The linux_wait4 and linux_waitpid * functions need to be able to distinguish between waiting * on a process and waiting on a thread. It is a thread if * p_sigparent is not SIGCHLD, and the WLINUXCLONE option * signifies we want to wait for threads and not processes. */ if ((p->p_sigparent != SIGCHLD) ^ ((options & WLINUXCLONE) != 0)) { PROC_UNLOCK(p); continue; } nfound++; if (p->p_state == PRS_ZOMBIE) { td->td_retval[0] = p->p_pid; if (status) *status = p->p_xstat; /* convert to int */ if (rusage) bcopy(p->p_ru, rusage, sizeof(struct rusage)); /* * If we got the child via a ptrace 'attach', * we need to give it back to the old parent. */ PROC_UNLOCK(p); if (p->p_oppid && (t = pfind(p->p_oppid)) != NULL) { PROC_LOCK(p); p->p_oppid = 0; proc_reparent(p, t); PROC_UNLOCK(p); psignal(t, SIGCHLD); wakeup(t); PROC_UNLOCK(t); sx_xunlock(&proctree_lock); return (0); } /* * Remove other references to this process to ensure * we have an exclusive reference. */ sx_xlock(&allproc_lock); LIST_REMOVE(p, p_list); /* off zombproc */ sx_xunlock(&allproc_lock); LIST_REMOVE(p, p_sibling); leavepgrp(p); sx_xunlock(&proctree_lock); /* * As a side effect of this lock, we know that * all other writes to this proc are visible now, so * no more locking is needed for p. */ mtx_lock(&Giant); PROC_LOCK(p); p->p_xstat = 0; /* XXX: why? */ PROC_UNLOCK(p); PROC_LOCK(q); ruadd(&q->p_stats->p_cru, p->p_ru); PROC_UNLOCK(q); FREE(p->p_ru, M_ZOMBIE); p->p_ru = NULL; mtx_unlock(&Giant); /* * Decrement the count of procs running with this uid. */ (void)chgproccnt(p->p_ucred->cr_ruidinfo, -1, 0); /* * Free credentials, arguments, and sigacts */ crfree(p->p_ucred); p->p_ucred = NULL; pargs_drop(p->p_args); p->p_args = NULL; sigacts_free(p->p_sigacts); p->p_sigacts = NULL; /* * do any thread-system specific cleanups */ thread_wait(p); /* * Give vm and machine-dependent layer a chance * to free anything that cpu_exit couldn't * release while still running in process context. */ mtx_lock(&Giant); vm_waitproc(p); mtx_unlock(&Giant); #ifdef MAC mac_destroy_proc(p); #endif KASSERT(FIRST_THREAD_IN_PROC(p), ("kern_wait: no residual thread!")); uma_zfree(proc_zone, p); sx_xlock(&allproc_lock); nprocs--; sx_xunlock(&allproc_lock); return (0); } mtx_lock_spin(&sched_lock); if (P_SHOULDSTOP(p) && (p->p_suspcount == p->p_numthreads) && ((p->p_flag & P_WAITED) == 0) && (p->p_flag & P_TRACED || options & WUNTRACED)) { mtx_unlock_spin(&sched_lock); p->p_flag |= P_WAITED; sx_xunlock(&proctree_lock); td->td_retval[0] = p->p_pid; if (status) *status = W_STOPCODE(p->p_xstat); PROC_UNLOCK(p); return (0); } mtx_unlock_spin(&sched_lock); if (options & WCONTINUED && (p->p_flag & P_CONTINUED)) { sx_xunlock(&proctree_lock); td->td_retval[0] = p->p_pid; p->p_flag &= ~P_CONTINUED; PROC_UNLOCK(p); if (status) *status = SIGCONT; return (0); } PROC_UNLOCK(p); } if (nfound == 0) { sx_xunlock(&proctree_lock); return (ECHILD); } if (options & WNOHANG) { sx_xunlock(&proctree_lock); td->td_retval[0] = 0; return (0); } PROC_LOCK(q); sx_xunlock(&proctree_lock); error = msleep(q, &q->p_mtx, PWAIT | PCATCH, "wait", 0); PROC_UNLOCK(q); if (error) return (error); goto loop; } /* * Make process 'parent' the new parent of process 'child'. * Must be called with an exclusive hold of proctree lock. */ void proc_reparent(struct proc *child, struct proc *parent) { sx_assert(&proctree_lock, SX_XLOCKED); PROC_LOCK_ASSERT(child, MA_OWNED); if (child->p_pptr == parent) return; LIST_REMOVE(child, p_sibling); LIST_INSERT_HEAD(&parent->p_children, child, p_sibling); child->p_pptr = parent; } Index: head/sys/kern/kern_fork.c =================================================================== --- head/sys/kern/kern_fork.c (revision 132371) +++ head/sys/kern/kern_fork.c (revision 132372) @@ -1,854 +1,854 @@ /* * Copyright (c) 1982, 1986, 1989, 1991, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)kern_fork.c 8.6 (Berkeley) 4/8/94 */ #include __FBSDID("$FreeBSD$"); #include "opt_ktrace.h" #include "opt_mac.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef _SYS_SYSPROTO_H_ struct fork_args { int dummy; }; #endif static int forksleep; /* Place for fork1() to sleep on. */ /* * MPSAFE */ /* ARGSUSED */ int fork(td, uap) struct thread *td; struct fork_args *uap; { int error; struct proc *p2; error = fork1(td, RFFDG | RFPROC, 0, &p2); if (error == 0) { td->td_retval[0] = p2->p_pid; td->td_retval[1] = 0; } return (error); } /* * MPSAFE */ /* ARGSUSED */ int vfork(td, uap) struct thread *td; struct vfork_args *uap; { int error; struct proc *p2; error = fork1(td, RFFDG | RFPROC | RFPPWAIT | RFMEM, 0, &p2); if (error == 0) { td->td_retval[0] = p2->p_pid; td->td_retval[1] = 0; } return (error); } /* * MPSAFE */ int rfork(td, uap) struct thread *td; struct rfork_args *uap; { struct proc *p2; int error; /* Don't allow kernel-only flags. */ if ((uap->flags & RFKERNELONLY) != 0) return (EINVAL); error = fork1(td, uap->flags, 0, &p2); if (error == 0) { td->td_retval[0] = p2 ? p2->p_pid : 0; td->td_retval[1] = 0; } return (error); } int nprocs = 1; /* process 0 */ int lastpid = 0; SYSCTL_INT(_kern, OID_AUTO, lastpid, CTLFLAG_RD, &lastpid, 0, "Last used PID"); /* * Random component to lastpid generation. We mix in a random factor to make * it a little harder to predict. We sanity check the modulus value to avoid * doing it in critical paths. Don't let it be too small or we pointlessly * waste randomness entropy, and don't let it be impossibly large. Using a * modulus that is too big causes a LOT more process table scans and slows * down fork processing as the pidchecked caching is defeated. */ static int randompid = 0; static int sysctl_kern_randompid(SYSCTL_HANDLER_ARGS) { int error, pid; error = sysctl_wire_old_buffer(req, sizeof(int)); if (error != 0) return(error); sx_xlock(&allproc_lock); pid = randompid; error = sysctl_handle_int(oidp, &pid, 0, req); if (error == 0 && req->newptr != NULL) { if (pid < 0 || pid > PID_MAX - 100) /* out of range */ pid = PID_MAX - 100; else if (pid < 2) /* NOP */ pid = 0; else if (pid < 100) /* Make it reasonable */ pid = 100; randompid = pid; } sx_xunlock(&allproc_lock); return (error); } SYSCTL_PROC(_kern, OID_AUTO, randompid, CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_kern_randompid, "I", "Random PID modulus"); int fork1(td, flags, pages, procp) struct thread *td; int flags; int pages; struct proc **procp; { struct proc *p1, *p2, *pptr; uid_t uid; struct proc *newproc; int ok, trypid; static int curfail, pidchecked = 0; static struct timeval lastfail; struct filedesc *fd; struct filedesc_to_leader *fdtol; struct thread *td2; struct kse *ke2; struct ksegrp *kg2; struct sigacts *newsigacts; int error; /* Can't copy and clear. */ if ((flags & (RFFDG|RFCFDG)) == (RFFDG|RFCFDG)) return (EINVAL); p1 = td->td_proc; /* * Here we don't create a new process, but we divorce * certain parts of a process from itself. */ if ((flags & RFPROC) == 0) { mtx_lock(&Giant); vm_forkproc(td, NULL, NULL, flags); mtx_unlock(&Giant); /* * Close all file descriptors. */ if (flags & RFCFDG) { struct filedesc *fdtmp; FILEDESC_LOCK(td->td_proc->p_fd); fdtmp = fdinit(td->td_proc->p_fd); FILEDESC_UNLOCK(td->td_proc->p_fd); fdfree(td); p1->p_fd = fdtmp; } /* * Unshare file descriptors (from parent). */ if (flags & RFFDG) { FILEDESC_LOCK(p1->p_fd); if (p1->p_fd->fd_refcnt > 1) { struct filedesc *newfd; newfd = fdcopy(td->td_proc->p_fd); FILEDESC_UNLOCK(p1->p_fd); fdfree(td); p1->p_fd = newfd; } else FILEDESC_UNLOCK(p1->p_fd); } *procp = NULL; return (0); } /* * Note 1:1 allows for forking with one thread coming out on the * other side with the expectation that the process is about to * exec. */ if (p1->p_flag & P_SA) { /* * Idle the other threads for a second. * Since the user space is copied, it must remain stable. * In addition, all threads (from the user perspective) * need to either be suspended or in the kernel, * where they will try restart in the parent and will * be aborted in the child. */ PROC_LOCK(p1); if (thread_single(SINGLE_NO_EXIT)) { /* Abort. Someone else is single threading before us. */ PROC_UNLOCK(p1); return (ERESTART); } PROC_UNLOCK(p1); /* * All other activity in this process * is now suspended at the user boundary, * (or other safe places if we think of any). */ } /* Allocate new proc. */ newproc = uma_zalloc(proc_zone, M_WAITOK); #ifdef MAC mac_init_proc(newproc); #endif /* We have to lock the process tree while we look for a pid. */ sx_slock(&proctree_lock); /* * Although process entries are dynamically created, we still keep * a global limit on the maximum number we will create. Don't allow * a nonprivileged user to use the last ten processes; don't let root * exceed the limit. The variable nprocs is the current number of * processes, maxproc is the limit. */ sx_xlock(&allproc_lock); uid = td->td_ucred->cr_ruid; if ((nprocs >= maxproc - 10 && suser(td) != 0) || nprocs >= maxproc) { error = EAGAIN; goto fail; } /* * Increment the count of procs running with this uid. Don't allow * a nonprivileged user to exceed their current limit. */ PROC_LOCK(p1); ok = chgproccnt(td->td_ucred->cr_ruidinfo, 1, (uid != 0) ? lim_cur(p1, RLIMIT_NPROC) : 0); PROC_UNLOCK(p1); if (!ok) { error = EAGAIN; goto fail; } /* * Increment the nprocs resource before blocking can occur. There * are hard-limits as to the number of processes that can run. */ nprocs++; /* * Find an unused process ID. We remember a range of unused IDs * ready to use (from lastpid+1 through pidchecked-1). * * If RFHIGHPID is set (used during system boot), do not allocate * low-numbered pids. */ trypid = lastpid + 1; if (flags & RFHIGHPID) { if (trypid < 10) trypid = 10; } else { if (randompid) trypid += arc4random() % randompid; } retry: /* * If the process ID prototype has wrapped around, * restart somewhat above 0, as the low-numbered procs * tend to include daemons that don't exit. */ if (trypid >= PID_MAX) { trypid = trypid % PID_MAX; if (trypid < 100) trypid += 100; pidchecked = 0; } if (trypid >= pidchecked) { int doingzomb = 0; pidchecked = PID_MAX; /* * Scan the active and zombie procs to check whether this pid * is in use. Remember the lowest pid that's greater * than trypid, so we can avoid checking for a while. */ p2 = LIST_FIRST(&allproc); again: for (; p2 != NULL; p2 = LIST_NEXT(p2, p_list)) { PROC_LOCK(p2); while (p2->p_pid == trypid || (p2->p_pgrp != NULL && (p2->p_pgrp->pg_id == trypid || (p2->p_session != NULL && p2->p_session->s_sid == trypid)))) { trypid++; if (trypid >= pidchecked) { PROC_UNLOCK(p2); goto retry; } } if (p2->p_pid > trypid && pidchecked > p2->p_pid) pidchecked = p2->p_pid; if (p2->p_pgrp != NULL) { if (p2->p_pgrp->pg_id > trypid && pidchecked > p2->p_pgrp->pg_id) pidchecked = p2->p_pgrp->pg_id; if (p2->p_session != NULL && p2->p_session->s_sid > trypid && pidchecked > p2->p_session->s_sid) pidchecked = p2->p_session->s_sid; } PROC_UNLOCK(p2); } if (!doingzomb) { doingzomb = 1; p2 = LIST_FIRST(&zombproc); goto again; } } sx_sunlock(&proctree_lock); /* * RFHIGHPID does not mess with the lastpid counter during boot. */ if (flags & RFHIGHPID) pidchecked = 0; else lastpid = trypid; p2 = newproc; p2->p_state = PRS_NEW; /* protect against others */ p2->p_pid = trypid; LIST_INSERT_HEAD(&allproc, p2, p_list); LIST_INSERT_HEAD(PIDHASH(p2->p_pid), p2, p_hash); sx_xunlock(&allproc_lock); /* * Malloc things while we don't hold any locks. */ if (flags & RFSIGSHARE) newsigacts = NULL; else newsigacts = sigacts_alloc(); /* * Copy filedesc. */ if (flags & RFCFDG) { FILEDESC_LOCK(td->td_proc->p_fd); fd = fdinit(td->td_proc->p_fd); FILEDESC_UNLOCK(td->td_proc->p_fd); fdtol = NULL; } else if (flags & RFFDG) { FILEDESC_LOCK(p1->p_fd); fd = fdcopy(td->td_proc->p_fd); FILEDESC_UNLOCK(p1->p_fd); fdtol = NULL; } else { fd = fdshare(p1->p_fd); if (p1->p_fdtol == NULL) p1->p_fdtol = filedesc_to_leader_alloc(NULL, NULL, p1->p_leader); if ((flags & RFTHREAD) != 0) { /* * Shared file descriptor table and * shared process leaders. */ fdtol = p1->p_fdtol; FILEDESC_LOCK(p1->p_fd); fdtol->fdl_refcount++; FILEDESC_UNLOCK(p1->p_fd); } else { /* * Shared file descriptor table, and * different process leaders */ fdtol = filedesc_to_leader_alloc(p1->p_fdtol, p1->p_fd, p2); } } /* * Make a proc table entry for the new process. * Start by zeroing the section of proc that is zero-initialized, * then copy the section that is copied directly from the parent. */ td2 = FIRST_THREAD_IN_PROC(p2); kg2 = FIRST_KSEGRP_IN_PROC(p2); ke2 = FIRST_KSE_IN_KSEGRP(kg2); /* Allocate and switch to an alternate kstack if specified. */ if (pages != 0) vm_thread_new_altkstack(td2, pages); PROC_LOCK(p2); PROC_LOCK(p1); #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) bzero(&p2->p_startzero, (unsigned) RANGEOF(struct proc, p_startzero, p_endzero)); bzero(&ke2->ke_startzero, (unsigned) RANGEOF(struct kse, ke_startzero, ke_endzero)); bzero(&td2->td_startzero, (unsigned) RANGEOF(struct thread, td_startzero, td_endzero)); bzero(&kg2->kg_startzero, (unsigned) RANGEOF(struct ksegrp, kg_startzero, kg_endzero)); bcopy(&p1->p_startcopy, &p2->p_startcopy, (unsigned) RANGEOF(struct proc, p_startcopy, p_endcopy)); bcopy(&td->td_startcopy, &td2->td_startcopy, (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); bcopy(&td->td_ksegrp->kg_startcopy, &kg2->kg_startcopy, (unsigned) RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy)); #undef RANGEOF td2->td_sigstk = td->td_sigstk; /* Set up the thread as an active thread (as if runnable). */ ke2->ke_state = KES_THREAD; ke2->ke_thread = td2; td2->td_kse = ke2; /* * Duplicate sub-structures as needed. * Increase reference counts on shared objects. * The p_stats substruct is set in vm_forkproc. */ p2->p_flag = 0; if (p1->p_flag & P_PROFIL) startprofclock(p2); mtx_lock_spin(&sched_lock); p2->p_sflag = PS_INMEM; /* * Allow the scheduler to adjust the priority of the child and * parent while we hold the sched_lock. */ - sched_fork(p1, p2); + sched_fork(td, p2); mtx_unlock_spin(&sched_lock); p2->p_ucred = crhold(td->td_ucred); td2->td_ucred = crhold(p2->p_ucred); /* XXXKSE */ pargs_hold(p2->p_args); if (flags & RFSIGSHARE) { p2->p_sigacts = sigacts_hold(p1->p_sigacts); } else { sigacts_copy(newsigacts, p1->p_sigacts); p2->p_sigacts = newsigacts; } if (flags & RFLINUXTHPN) p2->p_sigparent = SIGUSR1; else p2->p_sigparent = SIGCHLD; p2->p_textvp = p1->p_textvp; p2->p_fd = fd; p2->p_fdtol = fdtol; /* * p_limit is copy-on-write. Bump its refcount. */ p2->p_limit = lim_hold(p1->p_limit); PROC_UNLOCK(p1); PROC_UNLOCK(p2); /* Bump references to the text vnode (for procfs) */ if (p2->p_textvp) vref(p2->p_textvp); /* * Set up linkage for kernel based threading. */ if ((flags & RFTHREAD) != 0) { mtx_lock(&ppeers_lock); p2->p_peers = p1->p_peers; p1->p_peers = p2; p2->p_leader = p1->p_leader; mtx_unlock(&ppeers_lock); PROC_LOCK(p1->p_leader); if ((p1->p_leader->p_flag & P_WEXIT) != 0) { PROC_UNLOCK(p1->p_leader); /* * The task leader is exiting, so process p1 is * going to be killed shortly. Since p1 obviously * isn't dead yet, we know that the leader is either * sending SIGKILL's to all the processes in this * task or is sleeping waiting for all the peers to * exit. We let p1 complete the fork, but we need * to go ahead and kill the new process p2 since * the task leader may not get a chance to send * SIGKILL to it. We leave it on the list so that * the task leader will wait for this new process * to commit suicide. */ PROC_LOCK(p2); psignal(p2, SIGKILL); PROC_UNLOCK(p2); } else PROC_UNLOCK(p1->p_leader); } else { p2->p_peers = NULL; p2->p_leader = p2; } sx_xlock(&proctree_lock); PGRP_LOCK(p1->p_pgrp); PROC_LOCK(p2); PROC_LOCK(p1); /* * Preserve some more flags in subprocess. P_PROFIL has already * been preserved. */ p2->p_flag |= p1->p_flag & P_SUGID; td2->td_pflags |= td->td_pflags & TDP_ALTSTACK; SESS_LOCK(p1->p_session); if (p1->p_session->s_ttyvp != NULL && p1->p_flag & P_CONTROLT) p2->p_flag |= P_CONTROLT; SESS_UNLOCK(p1->p_session); if (flags & RFPPWAIT) p2->p_flag |= P_PPWAIT; p2->p_pgrp = p1->p_pgrp; LIST_INSERT_AFTER(p1, p2, p_pglist); PGRP_UNLOCK(p1->p_pgrp); LIST_INIT(&p2->p_children); callout_init(&p2->p_itcallout, CALLOUT_MPSAFE); #ifdef KTRACE /* * Copy traceflag and tracefile if enabled. */ mtx_lock(&ktrace_mtx); KASSERT(p2->p_tracevp == NULL, ("new process has a ktrace vnode")); if (p1->p_traceflag & KTRFAC_INHERIT) { p2->p_traceflag = p1->p_traceflag; if ((p2->p_tracevp = p1->p_tracevp) != NULL) { VREF(p2->p_tracevp); KASSERT(p1->p_tracecred != NULL, ("ktrace vnode with no cred")); p2->p_tracecred = crhold(p1->p_tracecred); } } mtx_unlock(&ktrace_mtx); #endif /* * If PF_FORK is set, the child process inherits the * procfs ioctl flags from its parent. */ if (p1->p_pfsflags & PF_FORK) { p2->p_stops = p1->p_stops; p2->p_pfsflags = p1->p_pfsflags; } /* * This begins the section where we must prevent the parent * from being swapped. */ _PHOLD(p1); PROC_UNLOCK(p1); /* * Attach the new process to its parent. * * If RFNOWAIT is set, the newly created process becomes a child * of init. This effectively disassociates the child from the * parent. */ if (flags & RFNOWAIT) pptr = initproc; else pptr = p1; p2->p_pptr = pptr; LIST_INSERT_HEAD(&pptr->p_children, p2, p_sibling); sx_xunlock(&proctree_lock); /* Inform accounting that we have forked. */ p2->p_acflag = AFORK; PROC_UNLOCK(p2); /* * Finish creating the child process. It will return via a different * execution path later. (ie: directly into user mode) */ mtx_lock(&Giant); vm_forkproc(td, p2, td2, flags); if (flags == (RFFDG | RFPROC)) { cnt.v_forks++; cnt.v_forkpages += p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize; } else if (flags == (RFFDG | RFPROC | RFPPWAIT | RFMEM)) { cnt.v_vforks++; cnt.v_vforkpages += p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize; } else if (p1 == &proc0) { cnt.v_kthreads++; cnt.v_kthreadpages += p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize; } else { cnt.v_rforks++; cnt.v_rforkpages += p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize; } mtx_unlock(&Giant); /* * Both processes are set up, now check if any loadable modules want * to adjust anything. * What if they have an error? XXX */ EVENTHANDLER_INVOKE(process_fork, p1, p2, flags); /* * Set the child start time and mark the process as being complete. */ microuptime(&p2->p_stats->p_start); mtx_lock_spin(&sched_lock); p2->p_state = PRS_NORMAL; /* * If RFSTOPPED not requested, make child runnable and add to * run queue. */ if ((flags & RFSTOPPED) == 0) { TD_SET_CAN_RUN(td2); setrunqueue(td2); } mtx_unlock_spin(&sched_lock); /* * Now can be swapped. */ PROC_LOCK(p1); _PRELE(p1); /* * Tell any interested parties about the new process. */ KNOTE(&p1->p_klist, NOTE_FORK | p2->p_pid); PROC_UNLOCK(p1); /* * Preserve synchronization semantics of vfork. If waiting for * child to exec or exit, set P_PPWAIT on child, and sleep on our * proc (in case of exit). */ PROC_LOCK(p2); while (p2->p_flag & P_PPWAIT) msleep(p1, &p2->p_mtx, PWAIT, "ppwait", 0); PROC_UNLOCK(p2); /* * If other threads are waiting, let them continue now. */ if (p1->p_flag & P_SA) { PROC_LOCK(p1); thread_single_end(); PROC_UNLOCK(p1); } /* * Return child proc pointer to parent. */ *procp = p2; return (0); fail: sx_sunlock(&proctree_lock); if (ppsratecheck(&lastfail, &curfail, 1)) printf("maxproc limit exceeded by uid %i, please see tuning(7) and login.conf(5).\n", uid); sx_xunlock(&allproc_lock); #ifdef MAC mac_destroy_proc(newproc); #endif uma_zfree(proc_zone, newproc); if (p1->p_flag & P_SA) { PROC_LOCK(p1); thread_single_end(); PROC_UNLOCK(p1); } tsleep(&forksleep, PUSER, "fork", hz / 2); return (error); } /* * Handle the return of a child process from fork1(). This function * is called from the MD fork_trampoline() entry point. */ void fork_exit(callout, arg, frame) void (*callout)(void *, struct trapframe *); void *arg; struct trapframe *frame; { struct proc *p; struct thread *td; /* * Processes normally resume in mi_switch() after being * cpu_switch()'ed to, but when children start up they arrive here * instead, so we must do much the same things as mi_switch() would. */ if ((td = PCPU_GET(deadthread))) { PCPU_SET(deadthread, NULL); thread_stash(td); } td = curthread; p = td->td_proc; td->td_oncpu = PCPU_GET(cpuid); KASSERT(p->p_state == PRS_NORMAL, ("executing process is still new")); /* * Finish setting up thread glue so that it begins execution in a * non-nested critical section with sched_lock held but not recursed. */ sched_lock.mtx_lock = (uintptr_t)td; mtx_assert(&sched_lock, MA_OWNED | MA_NOTRECURSED); cpu_critical_fork_exit(); CTR3(KTR_PROC, "fork_exit: new thread %p (pid %d, %s)", td, p->p_pid, p->p_comm); mtx_unlock_spin(&sched_lock); /* * cpu_set_fork_handler intercepts this function call to * have this call a non-return function to stay in kernel mode. * initproc has its own fork handler, but it does return. */ KASSERT(callout != NULL, ("NULL callout in fork_exit")); callout(arg, frame); /* * Check if a kernel thread misbehaved and returned from its main * function. */ PROC_LOCK(p); if (p->p_flag & P_KTHREAD) { PROC_UNLOCK(p); printf("Kernel thread \"%s\" (pid %d) exited prematurely.\n", p->p_comm, p->p_pid); kthread_exit(0); } PROC_UNLOCK(p); #ifdef DIAGNOSTIC cred_free_thread(td); #endif mtx_assert(&Giant, MA_NOTOWNED); } /* * Simplified back end of syscall(), used when returning from fork() * directly into user mode. Giant is not held on entry, and must not * be held on return. This function is passed in to fork_exit() as the * first parameter and is called when returning to a new userland process. */ void fork_return(td, frame) struct thread *td; struct trapframe *frame; { userret(td, frame, 0); #ifdef KTRACE if (KTRPOINT(td, KTR_SYSRET)) ktrsysret(SYS_fork, 0, 0); #endif mtx_assert(&Giant, MA_NOTOWNED); } Index: head/sys/kern/kern_kse.c =================================================================== --- head/sys/kern/kern_kse.c (revision 132371) +++ head/sys/kern/kern_kse.c (revision 132372) @@ -1,1356 +1,1356 @@ /* * Copyright (C) 2001 Julian Elischer . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice(s), this list of conditions and the following disclaimer as * the first lines of this file unmodified other than the possible * addition of one or more copyright notices. * 2. Redistributions in binary form must reproduce the above copyright * notice(s), this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH * DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * KSEGRP related storage. */ static uma_zone_t upcall_zone; /* DEBUG ONLY */ extern int virtual_cpu; extern int thread_debug; extern int max_threads_per_proc; extern int max_groups_per_proc; extern int max_threads_hits; extern struct mtx kse_zombie_lock; #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) TAILQ_HEAD(, kse_upcall) zombie_upcalls = TAILQ_HEAD_INITIALIZER(zombie_upcalls); static int thread_update_usr_ticks(struct thread *td, int user); static void thread_alloc_spare(struct thread *td, struct thread *spare); /* move to proc.h */ extern void kse_purge(struct proc *p, struct thread *td); extern void kse_purge_group(struct thread *td); void kseinit(void); void kse_GC(void); struct kse_upcall * upcall_alloc(void) { struct kse_upcall *ku; ku = uma_zalloc(upcall_zone, M_WAITOK); bzero(ku, sizeof(*ku)); return (ku); } void upcall_free(struct kse_upcall *ku) { uma_zfree(upcall_zone, ku); } void upcall_link(struct kse_upcall *ku, struct ksegrp *kg) { mtx_assert(&sched_lock, MA_OWNED); TAILQ_INSERT_TAIL(&kg->kg_upcalls, ku, ku_link); ku->ku_ksegrp = kg; kg->kg_numupcalls++; } void upcall_unlink(struct kse_upcall *ku) { struct ksegrp *kg = ku->ku_ksegrp; mtx_assert(&sched_lock, MA_OWNED); KASSERT(ku->ku_owner == NULL, ("%s: have owner", __func__)); TAILQ_REMOVE(&kg->kg_upcalls, ku, ku_link); kg->kg_numupcalls--; upcall_stash(ku); } void upcall_remove(struct thread *td) { if (td->td_upcall) { td->td_upcall->ku_owner = NULL; upcall_unlink(td->td_upcall); td->td_upcall = 0; } } #ifndef _SYS_SYSPROTO_H_ struct kse_switchin_args { struct kse_thr_mailbox *tmbx; int flags; }; #endif int kse_switchin(struct thread *td, struct kse_switchin_args *uap) { struct kse_thr_mailbox tmbx; struct kse_upcall *ku; int error; if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) return (EINVAL); error = (uap->tmbx == NULL) ? EINVAL : 0; if (!error) error = copyin(uap->tmbx, &tmbx, sizeof(tmbx)); if (!error && (uap->flags & KSE_SWITCHIN_SETTMBX)) error = (suword(&ku->ku_mailbox->km_curthread, (long)uap->tmbx) != 0 ? EINVAL : 0); if (!error) error = set_mcontext(td, &tmbx.tm_context.uc_mcontext); if (!error) { suword32(&uap->tmbx->tm_lwp, td->td_tid); if (uap->flags & KSE_SWITCHIN_SETTMBX) { td->td_mailbox = uap->tmbx; mtx_lock_spin(&sched_lock); td->td_flags |= TDF_CAN_UNBIND; mtx_unlock_spin(&sched_lock); } if (td->td_proc->p_flag & P_TRACED) { if (tmbx.tm_dflags & TMDF_SSTEP) ptrace_single_step(td); else ptrace_clear_single_step(td); if (tmbx.tm_dflags & TMDF_DONOTRUNUSER) { mtx_lock_spin(&sched_lock); /* fuword can block, check again */ if (td->td_upcall) ku->ku_flags |= KUF_DOUPCALL; mtx_unlock_spin(&sched_lock); } } } return ((error == 0) ? EJUSTRETURN : error); } /* struct kse_thr_interrupt_args { struct kse_thr_mailbox * tmbx; int cmd; long data; }; */ int kse_thr_interrupt(struct thread *td, struct kse_thr_interrupt_args *uap) { struct proc *p; struct thread *td2; p = td->td_proc; if (!(p->p_flag & P_SA)) return (EINVAL); switch (uap->cmd) { case KSE_INTR_SENDSIG: if (uap->data < 0 || uap->data > _SIG_MAXSIG) return (EINVAL); case KSE_INTR_INTERRUPT: case KSE_INTR_RESTART: PROC_LOCK(p); mtx_lock_spin(&sched_lock); FOREACH_THREAD_IN_PROC(p, td2) { if (td2->td_mailbox == uap->tmbx) break; } if (td2 == NULL) { mtx_unlock_spin(&sched_lock); PROC_UNLOCK(p); return (ESRCH); } if (uap->cmd == KSE_INTR_SENDSIG) { if (uap->data > 0) { td2->td_flags &= ~TDF_INTERRUPT; mtx_unlock_spin(&sched_lock); tdsignal(td2, (int)uap->data, SIGTARGET_TD); } else { mtx_unlock_spin(&sched_lock); } } else { td2->td_flags |= TDF_INTERRUPT | TDF_ASTPENDING; if (TD_CAN_UNBIND(td2)) td2->td_upcall->ku_flags |= KUF_DOUPCALL; if (uap->cmd == KSE_INTR_INTERRUPT) td2->td_intrval = EINTR; else td2->td_intrval = ERESTART; if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) sleepq_abort(td2); mtx_unlock_spin(&sched_lock); } PROC_UNLOCK(p); break; case KSE_INTR_SIGEXIT: if (uap->data < 1 || uap->data > _SIG_MAXSIG) return (EINVAL); PROC_LOCK(p); sigexit(td, (int)uap->data); break; default: return (EINVAL); } return (0); } /* struct kse_exit_args { register_t dummy; }; */ int kse_exit(struct thread *td, struct kse_exit_args *uap) { struct proc *p; struct ksegrp *kg; struct kse *ke; struct kse_upcall *ku, *ku2; int error, count; p = td->td_proc; /* * Ensure that this is only called from the UTS */ if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) return (EINVAL); kg = td->td_ksegrp; count = 0; /* * Calculate the existing non-exiting upcalls in this ksegroup. * If we are the last upcall but there are still other threads, * then do not exit. We need the other threads to be able to * complete whatever they are doing. * XXX This relies on the userland knowing what to do if we return. * It may be a better choice to convert ourselves into a kse_release * ( or similar) and wait in the kernel to be needed. */ PROC_LOCK(p); mtx_lock_spin(&sched_lock); FOREACH_UPCALL_IN_GROUP(kg, ku2) { if (ku2->ku_flags & KUF_EXITING) count++; } if ((kg->kg_numupcalls - count) == 1 && (kg->kg_numthreads > 1)) { mtx_unlock_spin(&sched_lock); PROC_UNLOCK(p); return (EDEADLK); } ku->ku_flags |= KUF_EXITING; mtx_unlock_spin(&sched_lock); PROC_UNLOCK(p); /* * Mark the UTS mailbox as having been finished with. * If that fails then just go for a segfault. * XXX need to check it that can be deliverred without a mailbox. */ error = suword32(&ku->ku_mailbox->km_flags, ku->ku_mflags|KMF_DONE); PROC_LOCK(p); if (error) psignal(p, SIGSEGV); mtx_lock_spin(&sched_lock); upcall_remove(td); ke = td->td_kse; if (p->p_numthreads == 1) { kse_purge(p, td); p->p_flag &= ~P_SA; mtx_unlock_spin(&sched_lock); PROC_UNLOCK(p); } else { if (kg->kg_numthreads == 1) { /* Shutdown a group */ kse_purge_group(td); ke->ke_flags |= KEF_EXIT; } thread_stopped(p); thread_exit(); /* NOTREACHED */ } return (0); } /* * Either becomes an upcall or waits for an awakening event and * then becomes an upcall. Only error cases return. */ /* struct kse_release_args { struct timespec *timeout; }; */ int kse_release(struct thread *td, struct kse_release_args *uap) { struct proc *p; struct ksegrp *kg; struct kse_upcall *ku; struct timespec timeout; struct timeval tv; sigset_t sigset; int error; p = td->td_proc; kg = td->td_ksegrp; if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) return (EINVAL); if (uap->timeout != NULL) { if ((error = copyin(uap->timeout, &timeout, sizeof(timeout)))) return (error); TIMESPEC_TO_TIMEVAL(&tv, &timeout); } if (td->td_pflags & TDP_SA) td->td_pflags |= TDP_UPCALLING; else { ku->ku_mflags = fuword32(&ku->ku_mailbox->km_flags); if (ku->ku_mflags == -1) { PROC_LOCK(p); sigexit(td, SIGSEGV); } } PROC_LOCK(p); if (ku->ku_mflags & KMF_WAITSIGEVENT) { /* UTS wants to wait for signal event */ if (!(p->p_flag & P_SIGEVENT) && !(ku->ku_flags & KUF_DOUPCALL)) { td->td_kflags |= TDK_KSERELSIG; error = msleep(&p->p_siglist, &p->p_mtx, PPAUSE|PCATCH, "ksesigwait", (uap->timeout ? tvtohz(&tv) : 0)); td->td_kflags &= ~(TDK_KSERELSIG | TDK_WAKEUP); } p->p_flag &= ~P_SIGEVENT; sigset = p->p_siglist; PROC_UNLOCK(p); error = copyout(&sigset, &ku->ku_mailbox->km_sigscaught, sizeof(sigset)); } else { if (! kg->kg_completed && !(ku->ku_flags & KUF_DOUPCALL)) { kg->kg_upsleeps++; td->td_kflags |= TDK_KSEREL; error = msleep(&kg->kg_completed, &p->p_mtx, PPAUSE|PCATCH, "kserel", (uap->timeout ? tvtohz(&tv) : 0)); td->td_kflags &= ~(TDK_KSEREL | TDK_WAKEUP); kg->kg_upsleeps--; } PROC_UNLOCK(p); } if (ku->ku_flags & KUF_DOUPCALL) { mtx_lock_spin(&sched_lock); ku->ku_flags &= ~KUF_DOUPCALL; mtx_unlock_spin(&sched_lock); } return (0); } /* struct kse_wakeup_args { struct kse_mailbox *mbx; }; */ int kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) { struct proc *p; struct ksegrp *kg; struct kse_upcall *ku; struct thread *td2; p = td->td_proc; td2 = NULL; ku = NULL; /* KSE-enabled processes only, please. */ if (!(p->p_flag & P_SA)) return (EINVAL); PROC_LOCK(p); mtx_lock_spin(&sched_lock); if (uap->mbx) { FOREACH_KSEGRP_IN_PROC(p, kg) { FOREACH_UPCALL_IN_GROUP(kg, ku) { if (ku->ku_mailbox == uap->mbx) break; } if (ku) break; } } else { kg = td->td_ksegrp; if (kg->kg_upsleeps) { mtx_unlock_spin(&sched_lock); wakeup_one(&kg->kg_completed); PROC_UNLOCK(p); return (0); } ku = TAILQ_FIRST(&kg->kg_upcalls); } if (ku == NULL) { mtx_unlock_spin(&sched_lock); PROC_UNLOCK(p); return (ESRCH); } if ((td2 = ku->ku_owner) == NULL) { mtx_unlock_spin(&sched_lock); panic("%s: no owner", __func__); } else if (td2->td_kflags & (TDK_KSEREL | TDK_KSERELSIG)) { mtx_unlock_spin(&sched_lock); if (!(td2->td_kflags & TDK_WAKEUP)) { td2->td_kflags |= TDK_WAKEUP; if (td2->td_kflags & TDK_KSEREL) sleepq_remove(td2, &kg->kg_completed); else sleepq_remove(td2, &p->p_siglist); } } else { ku->ku_flags |= KUF_DOUPCALL; mtx_unlock_spin(&sched_lock); } PROC_UNLOCK(p); return (0); } /* * No new KSEG: first call: use current KSE, don't schedule an upcall * All other situations, do allocate max new KSEs and schedule an upcall. */ /* struct kse_create_args { struct kse_mailbox *mbx; int newgroup; }; */ int kse_create(struct thread *td, struct kse_create_args *uap) { struct kse *newke; struct ksegrp *newkg; struct ksegrp *kg; struct proc *p; struct kse_mailbox mbx; struct kse_upcall *newku; int err, ncpus, sa = 0, first = 0; struct thread *newtd; p = td->td_proc; if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) return (err); ncpus = mp_ncpus; if (virtual_cpu != 0) ncpus = virtual_cpu; if (!(mbx.km_flags & KMF_BOUND)) sa = TDP_SA; else { if (mbx.km_curthread == NULL) return (EINVAL); ncpus = 1; } PROC_LOCK(p); if (!(p->p_flag & P_SA)) { first = 1; p->p_flag |= P_SA; } PROC_UNLOCK(p); if (!sa && !uap->newgroup && !first) return (EINVAL); kg = td->td_ksegrp; if (uap->newgroup) { /* Have race condition but it is cheap */ if (p->p_numksegrps >= max_groups_per_proc) return (EPROCLIM); /* * If we want a new KSEGRP it doesn't matter whether * we have already fired up KSE mode before or not. * We put the process in KSE mode and create a new KSEGRP. */ newkg = ksegrp_alloc(); bzero(&newkg->kg_startzero, RANGEOF(struct ksegrp, kg_startzero, kg_endzero)); bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy)); PROC_LOCK(p); mtx_lock_spin(&sched_lock); if (p->p_numksegrps >= max_groups_per_proc) { mtx_unlock_spin(&sched_lock); PROC_UNLOCK(p); ksegrp_free(newkg); return (EPROCLIM); } ksegrp_link(newkg, p); - sched_fork_ksegrp(kg, newkg); + sched_fork_ksegrp(td, newkg); mtx_unlock_spin(&sched_lock); PROC_UNLOCK(p); } else { if (!first && ((td->td_pflags & TDP_SA) ^ sa) != 0) return (EINVAL); newkg = kg; } /* * Creating upcalls more than number of physical cpu does * not help performance. */ if (newkg->kg_numupcalls >= ncpus) return (EPROCLIM); if (newkg->kg_numupcalls == 0) { /* * Initialize KSE group * * For multiplxed group, create KSEs as many as physical * cpus. This increases concurrent even if userland * is not MP safe and can only run on single CPU. * In ideal world, every physical cpu should execute a thread. * If there is enough KSEs, threads in kernel can be * executed parallel on different cpus with full speed, * Concurrent in kernel shouldn't be restricted by number of * upcalls userland provides. Adding more upcall structures * only increases concurrent in userland. * * For bound thread group, because there is only thread in the * group, we only create one KSE for the group. Thread in this * kind of group will never schedule an upcall when blocked, * this intends to simulate pthread system scope thread. */ while (newkg->kg_kses < ncpus) { newke = kse_alloc(); bzero(&newke->ke_startzero, RANGEOF(struct kse, ke_startzero, ke_endzero)); #if 0 mtx_lock_spin(&sched_lock); bcopy(&ke->ke_startcopy, &newke->ke_startcopy, RANGEOF(struct kse, ke_startcopy, ke_endcopy)); mtx_unlock_spin(&sched_lock); #endif mtx_lock_spin(&sched_lock); kse_link(newke, newkg); - sched_fork_kse(td->td_kse, newke); + sched_fork_kse(td, newke); /* Add engine */ kse_reassign(newke); mtx_unlock_spin(&sched_lock); } } newku = upcall_alloc(); newku->ku_mailbox = uap->mbx; newku->ku_func = mbx.km_func; bcopy(&mbx.km_stack, &newku->ku_stack, sizeof(stack_t)); /* For the first call this may not have been set */ if (td->td_standin == NULL) thread_alloc_spare(td, NULL); PROC_LOCK(p); if (newkg->kg_numupcalls >= ncpus) { PROC_UNLOCK(p); upcall_free(newku); return (EPROCLIM); } if (first && sa) { SIGSETOR(p->p_siglist, td->td_siglist); SIGEMPTYSET(td->td_siglist); SIGFILLSET(td->td_sigmask); SIG_CANTMASK(td->td_sigmask); } mtx_lock_spin(&sched_lock); PROC_UNLOCK(p); upcall_link(newku, newkg); if (mbx.km_quantum) newkg->kg_upquantum = max(1, mbx.km_quantum/tick); /* * Each upcall structure has an owner thread, find which * one owns it. */ if (uap->newgroup) { /* * Because new ksegrp hasn't thread, * create an initial upcall thread to own it. */ newtd = thread_schedule_upcall(td, newku); } else { /* * If current thread hasn't an upcall structure, * just assign the upcall to it. */ if (td->td_upcall == NULL) { newku->ku_owner = td; td->td_upcall = newku; newtd = td; } else { /* * Create a new upcall thread to own it. */ newtd = thread_schedule_upcall(td, newku); } } mtx_unlock_spin(&sched_lock); suword32(&newku->ku_mailbox->km_lwp, newtd->td_tid); if (mbx.km_curthread) suword32(&mbx.km_curthread->tm_lwp, newtd->td_tid); if (!sa) { newtd->td_mailbox = mbx.km_curthread; newtd->td_pflags &= ~TDP_SA; if (newtd != td) { cpu_set_upcall_kse(newtd, newku); if (p->p_flag & P_TRACED) ptrace_clear_single_step(newtd); } } else { newtd->td_pflags |= TDP_SA; } if (newtd != td) { mtx_lock_spin(&sched_lock); setrunqueue(newtd); mtx_unlock_spin(&sched_lock); } return (0); } /* * Initialize global thread allocation resources. */ void kseinit(void) { upcall_zone = uma_zcreate("UPCALL", sizeof(struct kse_upcall), NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); } /* * Stash an embarasingly extra upcall into the zombie upcall queue. */ void upcall_stash(struct kse_upcall *ku) { mtx_lock_spin(&kse_zombie_lock); TAILQ_INSERT_HEAD(&zombie_upcalls, ku, ku_link); mtx_unlock_spin(&kse_zombie_lock); } /* * Reap zombie kse resource. */ void kse_GC(void) { struct kse_upcall *ku_first, *ku_next; /* * Don't even bother to lock if none at this instant, * we really don't care about the next instant.. */ if (!TAILQ_EMPTY(&zombie_upcalls)) { mtx_lock_spin(&kse_zombie_lock); ku_first = TAILQ_FIRST(&zombie_upcalls); if (ku_first) TAILQ_INIT(&zombie_upcalls); mtx_unlock_spin(&kse_zombie_lock); while (ku_first) { ku_next = TAILQ_NEXT(ku_first, ku_link); upcall_free(ku_first); ku_first = ku_next; } } } /* * Store the thread context in the UTS's mailbox. * then add the mailbox at the head of a list we are building in user space. * The list is anchored in the ksegrp structure. */ int thread_export_context(struct thread *td, int willexit) { struct proc *p; struct ksegrp *kg; uintptr_t mbx; void *addr; int error = 0, temp, sig; mcontext_t mc; p = td->td_proc; kg = td->td_ksegrp; /* * Post sync signal, or process SIGKILL and SIGSTOP. * For sync signal, it is only possible when the signal is not * caught by userland or process is being debugged. */ PROC_LOCK(p); if (td->td_flags & TDF_NEEDSIGCHK) { mtx_lock_spin(&sched_lock); td->td_flags &= ~TDF_NEEDSIGCHK; mtx_unlock_spin(&sched_lock); mtx_lock(&p->p_sigacts->ps_mtx); while ((sig = cursig(td)) != 0) postsig(sig); mtx_unlock(&p->p_sigacts->ps_mtx); } if (willexit) SIGFILLSET(td->td_sigmask); PROC_UNLOCK(p); /* Export the user/machine context. */ get_mcontext(td, &mc, 0); addr = (void *)(&td->td_mailbox->tm_context.uc_mcontext); error = copyout(&mc, addr, sizeof(mcontext_t)); if (error) goto bad; /* Exports clock ticks in kernel mode */ addr = (caddr_t)(&td->td_mailbox->tm_sticks); temp = fuword32(addr) + td->td_usticks; if (suword32(addr, temp)) { error = EFAULT; goto bad; } addr = (caddr_t)(&td->td_mailbox->tm_lwp); if (suword32(addr, 0)) { error = EFAULT; goto bad; } /* Get address in latest mbox of list pointer */ addr = (void *)(&td->td_mailbox->tm_next); /* * Put the saved address of the previous first * entry into this one */ for (;;) { mbx = (uintptr_t)kg->kg_completed; if (suword(addr, mbx)) { error = EFAULT; goto bad; } PROC_LOCK(p); if (mbx == (uintptr_t)kg->kg_completed) { kg->kg_completed = td->td_mailbox; /* * The thread context may be taken away by * other upcall threads when we unlock * process lock. it's no longer valid to * use it again in any other places. */ td->td_mailbox = NULL; PROC_UNLOCK(p); break; } PROC_UNLOCK(p); } td->td_usticks = 0; return (0); bad: PROC_LOCK(p); sigexit(td, SIGILL); return (error); } /* * Take the list of completed mailboxes for this KSEGRP and put them on this * upcall's mailbox as it's the next one going up. */ static int thread_link_mboxes(struct ksegrp *kg, struct kse_upcall *ku) { struct proc *p = kg->kg_proc; void *addr; uintptr_t mbx; addr = (void *)(&ku->ku_mailbox->km_completed); for (;;) { mbx = (uintptr_t)kg->kg_completed; if (suword(addr, mbx)) { PROC_LOCK(p); psignal(p, SIGSEGV); PROC_UNLOCK(p); return (EFAULT); } PROC_LOCK(p); if (mbx == (uintptr_t)kg->kg_completed) { kg->kg_completed = NULL; PROC_UNLOCK(p); break; } PROC_UNLOCK(p); } return (0); } /* * This function should be called at statclock interrupt time */ int thread_statclock(int user) { struct thread *td = curthread; struct ksegrp *kg = td->td_ksegrp; if (kg->kg_numupcalls == 0 || !(td->td_pflags & TDP_SA)) return (0); if (user) { /* Current always do via ast() */ td->td_pflags |= TDP_USTATCLOCK; mtx_lock_spin(&sched_lock); td->td_flags |= TDF_ASTPENDING; mtx_unlock_spin(&sched_lock); td->td_uuticks++; } else if (td->td_mailbox != NULL) td->td_usticks++; return (0); } /* * Export state clock ticks for userland */ static int thread_update_usr_ticks(struct thread *td, int user) { struct proc *p = td->td_proc; struct kse_thr_mailbox *tmbx; struct kse_upcall *ku; struct ksegrp *kg; caddr_t addr; u_int uticks; if ((ku = td->td_upcall) == NULL) return (-1); tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); if ((tmbx == NULL) || (tmbx == (void *)-1)) return (-1); if (user) { uticks = td->td_uuticks; td->td_uuticks = 0; addr = (caddr_t)&tmbx->tm_uticks; } else { uticks = td->td_usticks; td->td_usticks = 0; addr = (caddr_t)&tmbx->tm_sticks; } if (uticks) { if (suword32(addr, uticks+fuword32(addr))) { PROC_LOCK(p); psignal(p, SIGSEGV); PROC_UNLOCK(p); return (-2); } } kg = td->td_ksegrp; if (kg->kg_upquantum && ticks >= kg->kg_nextupcall) { mtx_lock_spin(&sched_lock); td->td_upcall->ku_flags |= KUF_DOUPCALL; mtx_unlock_spin(&sched_lock); } return (0); } /* * This function is intended to be used to initialize a spare thread * for upcall. Initialize thread's large data area outside sched_lock * for thread_schedule_upcall(). */ void thread_alloc_spare(struct thread *td, struct thread *spare) { if (td->td_standin) return; if (spare == NULL) spare = thread_alloc(); td->td_standin = spare; bzero(&spare->td_startzero, (unsigned)RANGEOF(struct thread, td_startzero, td_endzero)); spare->td_proc = td->td_proc; spare->td_ucred = crhold(td->td_ucred); } /* * Create a thread and schedule it for upcall on the KSE given. * Use our thread's standin so that we don't have to allocate one. */ struct thread * thread_schedule_upcall(struct thread *td, struct kse_upcall *ku) { struct thread *td2; mtx_assert(&sched_lock, MA_OWNED); /* * Schedule an upcall thread on specified kse_upcall, * the kse_upcall must be free. * td must have a spare thread. */ KASSERT(ku->ku_owner == NULL, ("%s: upcall has owner", __func__)); if ((td2 = td->td_standin) != NULL) { td->td_standin = NULL; } else { panic("no reserve thread when scheduling an upcall"); return (NULL); } CTR3(KTR_PROC, "thread_schedule_upcall: thread %p (pid %d, %s)", td2, td->td_proc->p_pid, td->td_proc->p_comm); bcopy(&td->td_startcopy, &td2->td_startcopy, (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); thread_link(td2, ku->ku_ksegrp); /* inherit parts of blocked thread's context as a good template */ cpu_set_upcall(td2, td); /* Let the new thread become owner of the upcall */ ku->ku_owner = td2; td2->td_upcall = ku; td2->td_flags = 0; td2->td_pflags = TDP_SA|TDP_UPCALLING; td2->td_kse = NULL; td2->td_state = TDS_CAN_RUN; td2->td_inhibitors = 0; SIGFILLSET(td2->td_sigmask); SIG_CANTMASK(td2->td_sigmask); sched_fork_thread(td, td2); return (td2); /* bogus.. should be a void function */ } /* * It is only used when thread generated a trap and process is being * debugged. */ void thread_signal_add(struct thread *td, int sig) { struct proc *p; siginfo_t siginfo; struct sigacts *ps; int error; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); ps = p->p_sigacts; mtx_assert(&ps->ps_mtx, MA_OWNED); cpu_thread_siginfo(sig, 0, &siginfo); mtx_unlock(&ps->ps_mtx); SIGADDSET(td->td_sigmask, sig); PROC_UNLOCK(p); error = copyout(&siginfo, &td->td_mailbox->tm_syncsig, sizeof(siginfo)); if (error) { PROC_LOCK(p); sigexit(td, SIGSEGV); } PROC_LOCK(p); mtx_lock(&ps->ps_mtx); } void thread_switchout(struct thread *td) { struct kse_upcall *ku; struct thread *td2; mtx_assert(&sched_lock, MA_OWNED); /* * If the outgoing thread is in threaded group and has never * scheduled an upcall, decide whether this is a short * or long term event and thus whether or not to schedule * an upcall. * If it is a short term event, just suspend it in * a way that takes its KSE with it. * Select the events for which we want to schedule upcalls. * For now it's just sleep. * XXXKSE eventually almost any inhibition could do. */ if (TD_CAN_UNBIND(td) && (td->td_standin) && TD_ON_SLEEPQ(td)) { /* * Release ownership of upcall, and schedule an upcall * thread, this new upcall thread becomes the owner of * the upcall structure. */ ku = td->td_upcall; ku->ku_owner = NULL; td->td_upcall = NULL; td->td_flags &= ~TDF_CAN_UNBIND; td2 = thread_schedule_upcall(td, ku); setrunqueue(td2); } } /* * Setup done on the thread when it enters the kernel. */ void thread_user_enter(struct proc *p, struct thread *td) { struct ksegrp *kg; struct kse_upcall *ku; struct kse_thr_mailbox *tmbx; uint32_t flags; /* * First check that we shouldn't just abort. * But check if we are the single thread first! */ if (__predict_false(p->p_flag & P_SINGLE_EXIT)) { PROC_LOCK(p); mtx_lock_spin(&sched_lock); thread_stopped(p); thread_exit(); /* NOTREACHED */ } if (!(td->td_pflags & TDP_SA)) return; /* * If we are doing a syscall in a KSE environment, * note where our mailbox is. */ kg = td->td_ksegrp; ku = td->td_upcall; KASSERT(ku != NULL, ("no upcall owned")); KASSERT(ku->ku_owner == td, ("wrong owner")); KASSERT(!TD_CAN_UNBIND(td), ("can unbind")); ku->ku_mflags = fuword32((void *)&ku->ku_mailbox->km_flags); tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); if ((tmbx == NULL) || (tmbx == (void *)-1L) || (ku->ku_mflags & KMF_NOUPCALL)) { td->td_mailbox = NULL; } else { if (td->td_standin == NULL) thread_alloc_spare(td, NULL); flags = fuword32(&tmbx->tm_flags); /* * On some architectures, TP register points to thread * mailbox but not points to kse mailbox, and userland * can not atomically clear km_curthread, but can * use TP register, and set TMF_NOUPCALL in thread * flag to indicate a critical region. */ if (flags & TMF_NOUPCALL) { td->td_mailbox = NULL; } else { td->td_mailbox = tmbx; mtx_lock_spin(&sched_lock); td->td_flags |= TDF_CAN_UNBIND; mtx_unlock_spin(&sched_lock); if (__predict_false(p->p_flag & P_TRACED)) { flags = fuword32(&tmbx->tm_dflags); if (flags & TMDF_DONOTRUNUSER) { mtx_lock_spin(&sched_lock); /* fuword can block, check again */ if (td->td_upcall) ku->ku_flags |= KUF_DOUPCALL; mtx_unlock_spin(&sched_lock); } } } } } /* * The extra work we go through if we are a threaded process when we * return to userland. * * If we are a KSE process and returning to user mode, check for * extra work to do before we return (e.g. for more syscalls * to complete first). If we were in a critical section, we should * just return to let it finish. Same if we were in the UTS (in * which case the mailbox's context's busy indicator will be set). * The only traps we suport will have set the mailbox. * We will clear it here. */ int thread_userret(struct thread *td, struct trapframe *frame) { struct kse_upcall *ku; struct ksegrp *kg, *kg2; struct proc *p; struct timespec ts; int error = 0, upcalls, uts_crit; p = td->td_proc; kg = td->td_ksegrp; ku = td->td_upcall; /* Nothing to do with bound thread */ if (!(td->td_pflags & TDP_SA)) return (0); /* * Stat clock interrupt hit in userland, it * is returning from interrupt, charge thread's * userland time for UTS. */ if (td->td_pflags & TDP_USTATCLOCK) { thread_update_usr_ticks(td, 1); td->td_pflags &= ~TDP_USTATCLOCK; } /* * Check if we should unbind and schedule upcall * after returned from interrupt or etcs, this * is usually true when process is being debugged. */ if (td->td_mailbox == NULL && ku != NULL && !(td->td_pflags & TDP_UPCALLING) && (kg->kg_completed || ku->ku_flags & KUF_DOUPCALL)) thread_user_enter(p, td); uts_crit = (td->td_mailbox == NULL); /* * Optimisation: * This thread has not started any upcall. * If there is no work to report other than ourself, * then it can return direct to userland. */ if (TD_CAN_UNBIND(td)) { mtx_lock_spin(&sched_lock); td->td_flags &= ~TDF_CAN_UNBIND; if ((td->td_flags & TDF_NEEDSIGCHK) == 0 && (kg->kg_completed == NULL) && (ku->ku_flags & KUF_DOUPCALL) == 0 && (kg->kg_upquantum && ticks < kg->kg_nextupcall)) { mtx_unlock_spin(&sched_lock); thread_update_usr_ticks(td, 0); nanotime(&ts); error = copyout(&ts, (caddr_t)&ku->ku_mailbox->km_timeofday, sizeof(ts)); td->td_mailbox = 0; ku->ku_mflags = 0; if (error) goto out; return (0); } mtx_unlock_spin(&sched_lock); thread_export_context(td, 0); /* * There is something to report, and we own an upcall * strucuture, we can go to userland. * Turn ourself into an upcall thread. */ td->td_pflags |= TDP_UPCALLING; } else if (td->td_mailbox && (ku == NULL)) { thread_export_context(td, 1); PROC_LOCK(p); /* * There are upcall threads waiting for * work to do, wake one of them up. * XXXKSE Maybe wake all of them up. */ if (kg->kg_upsleeps) wakeup_one(&kg->kg_completed); mtx_lock_spin(&sched_lock); thread_stopped(p); thread_exit(); /* NOTREACHED */ } KASSERT(ku != NULL, ("upcall is NULL\n")); KASSERT(TD_CAN_UNBIND(td) == 0, ("can unbind")); if (p->p_numthreads > max_threads_per_proc) { max_threads_hits++; PROC_LOCK(p); mtx_lock_spin(&sched_lock); p->p_maxthrwaits++; while (p->p_numthreads > max_threads_per_proc) { upcalls = 0; FOREACH_KSEGRP_IN_PROC(p, kg2) { if (kg2->kg_numupcalls == 0) upcalls++; else upcalls += kg2->kg_numupcalls; } if (upcalls >= max_threads_per_proc) break; mtx_unlock_spin(&sched_lock); if (msleep(&p->p_numthreads, &p->p_mtx, PPAUSE|PCATCH, "maxthreads", 0)) { mtx_lock_spin(&sched_lock); break; } else { mtx_lock_spin(&sched_lock); } } p->p_maxthrwaits--; mtx_unlock_spin(&sched_lock); PROC_UNLOCK(p); } if (td->td_pflags & TDP_UPCALLING) { uts_crit = 0; kg->kg_nextupcall = ticks+kg->kg_upquantum; /* * There is no more work to do and we are going to ride * this thread up to userland as an upcall. * Do the last parts of the setup needed for the upcall. */ CTR3(KTR_PROC, "userret: upcall thread %p (pid %d, %s)", td, td->td_proc->p_pid, td->td_proc->p_comm); td->td_pflags &= ~TDP_UPCALLING; if (ku->ku_flags & KUF_DOUPCALL) { mtx_lock_spin(&sched_lock); ku->ku_flags &= ~KUF_DOUPCALL; mtx_unlock_spin(&sched_lock); } /* * Set user context to the UTS */ if (!(ku->ku_mflags & KMF_NOUPCALL)) { cpu_set_upcall_kse(td, ku); if (p->p_flag & P_TRACED) ptrace_clear_single_step(td); error = suword32(&ku->ku_mailbox->km_lwp, td->td_tid); if (error) goto out; error = suword(&ku->ku_mailbox->km_curthread, 0); if (error) goto out; } /* * Unhook the list of completed threads. * anything that completes after this gets to * come in next time. * Put the list of completed thread mailboxes on * this KSE's mailbox. */ if (!(ku->ku_mflags & KMF_NOCOMPLETED) && (error = thread_link_mboxes(kg, ku)) != 0) goto out; } if (!uts_crit) { nanotime(&ts); error = copyout(&ts, &ku->ku_mailbox->km_timeofday, sizeof(ts)); } out: if (error) { /* * Things are going to be so screwed we should just kill * the process. * how do we do that? */ PROC_LOCK(td->td_proc); psignal(td->td_proc, SIGSEGV); PROC_UNLOCK(td->td_proc); } else { /* * Optimisation: * Ensure that we have a spare thread available, * for when we re-enter the kernel. */ if (td->td_standin == NULL) thread_alloc_spare(td, NULL); } ku->ku_mflags = 0; /* * Clear thread mailbox first, then clear system tick count. * The order is important because thread_statclock() use * mailbox pointer to see if it is an userland thread or * an UTS kernel thread. */ td->td_mailbox = NULL; td->td_usticks = 0; return (error); /* go sync */ } int thread_upcall_check(struct thread *td) { PROC_LOCK_ASSERT(td->td_proc, MA_OWNED); if (td->td_kflags & TDK_WAKEUP) return (1); else return (0); } /* * called after ptrace resumed a process, force all * virtual CPUs to schedule upcall for SA process, * because debugger may have changed something in userland, * we should notice UTS as soon as possible. */ void thread_continued(struct proc *p) { struct ksegrp *kg; struct kse_upcall *ku; struct thread *td; PROC_LOCK_ASSERT(p, MA_OWNED); mtx_assert(&sched_lock, MA_OWNED); if (!(p->p_flag & P_SA)) return; if (p->p_flag & P_TRACED) { FOREACH_KSEGRP_IN_PROC(p, kg) { td = TAILQ_FIRST(&kg->kg_threads); if (td == NULL) continue; /* not a SA group, nothing to do */ if (!(td->td_pflags & TDP_SA)) continue; FOREACH_UPCALL_IN_GROUP(kg, ku) { ku->ku_flags |= KUF_DOUPCALL; if (TD_IS_SUSPENDED(ku->ku_owner)) { thread_unsuspend_one(ku->ku_owner); } } } } } Index: head/sys/kern/kern_thr.c =================================================================== --- head/sys/kern/kern_thr.c (revision 132371) +++ head/sys/kern/kern_thr.c (revision 132372) @@ -1,314 +1,314 @@ /* * Copyright (c) 2003, Jeffrey Roberson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Back end support functions. */ void thr_exit1(void) { struct ksegrp *kg; struct thread *td; struct kse *ke; struct proc *p; td = curthread; p = td->td_proc; kg = td->td_ksegrp; ke = td->td_kse; mtx_assert(&sched_lock, MA_OWNED); PROC_LOCK_ASSERT(p, MA_OWNED); KASSERT(!mtx_owned(&Giant), ("dying thread owns giant")); /* * Shutting down last thread in the proc. This will actually * call exit() in the trampoline when it returns. */ if (p->p_numthreads == 1) { PROC_UNLOCK(p); return; } /* * XXX Undelivered process wide signals should be reposted to the * proc. */ /* Clean up cpu resources. */ cpu_thread_exit(td); /* Unlink the thread from the process and kseg. */ thread_unlink(td); ke->ke_state = KES_UNQUEUED; ke->ke_thread = NULL; kse_unlink(ke); - sched_exit_kse(TAILQ_NEXT(ke, ke_kglist), ke); + sched_exit_kse(TAILQ_NEXT(ke, ke_kglist), td); /* * If we were stopped while waiting for all threads to exit and this * is the last thread wakeup the exiting thread. */ if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) if (p->p_numthreads == 1) thread_unsuspend_one(p->p_singlethread); PROC_UNLOCK(p); td->td_kse = NULL; td->td_state = TDS_INACTIVE; #if 0 td->td_proc = NULL; #endif td->td_ksegrp = NULL; td->td_last_kse = NULL; sched_exit_thread(TAILQ_NEXT(td, td_kglist), td); thread_stash(td); cpu_throw(td, choosethread()); } #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) /* * System call interface. */ int thr_create(struct thread *td, struct thr_create_args *uap) /* ucontext_t *ctx, long *id, int flags */ { struct kse *ke0; struct thread *td0; ucontext_t ctx; long id; int error; if ((error = copyin(uap->ctx, &ctx, sizeof(ctx)))) return (error); /* Initialize our td. */ td0 = thread_alloc(); /* * Try the copyout as soon as we allocate the td so we don't have to * tear things down in a failure case below. */ id = td0->td_tid; if ((error = copyout(&id, uap->id, sizeof(long)))) { thread_free(td0); return (error); } bzero(&td0->td_startzero, (unsigned)RANGEOF(struct thread, td_startzero, td_endzero)); bcopy(&td->td_startcopy, &td0->td_startcopy, (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); td0->td_proc = td->td_proc; PROC_LOCK(td->td_proc); td0->td_sigmask = td->td_sigmask; PROC_UNLOCK(td->td_proc); td0->td_ucred = crhold(td->td_ucred); /* Initialize our kse structure. */ ke0 = kse_alloc(); bzero(&ke0->ke_startzero, RANGEOF(struct kse, ke_startzero, ke_endzero)); /* Set up our machine context. */ cpu_set_upcall(td0, td); error = set_mcontext(td0, &ctx.uc_mcontext); if (error != 0) { kse_free(ke0); thread_free(td0); goto out; } /* Link the thread and kse into the ksegrp and make it runnable. */ mtx_lock_spin(&sched_lock); thread_link(td0, td->td_ksegrp); kse_link(ke0, td->td_ksegrp); /* Bind this thread and kse together. */ td0->td_kse = ke0; ke0->ke_thread = td0; - sched_fork_kse(td->td_kse, ke0); + sched_fork_kse(td, ke0); sched_fork_thread(td, td0); TD_SET_CAN_RUN(td0); if ((uap->flags & THR_SUSPENDED) == 0) setrunqueue(td0); mtx_unlock_spin(&sched_lock); out: return (error); } int thr_self(struct thread *td, struct thr_self_args *uap) /* long *id */ { long id; int error; id = td->td_tid; if ((error = copyout(&id, uap->id, sizeof(long)))) return (error); return (0); } int thr_exit(struct thread *td, struct thr_exit_args *uap) /* NULL */ { struct proc *p; p = td->td_proc; PROC_LOCK(p); mtx_lock_spin(&sched_lock); /* * This unlocks proc and doesn't return unless this is the last * thread. */ thr_exit1(); mtx_unlock_spin(&sched_lock); return (0); } int thr_kill(struct thread *td, struct thr_kill_args *uap) /* long id, int sig */ { struct thread *ttd; struct proc *p; int error; p = td->td_proc; error = 0; PROC_LOCK(p); FOREACH_THREAD_IN_PROC(p, ttd) { if (ttd->td_tid == uap->id) break; } if (ttd == NULL) { error = ESRCH; goto out; } if (uap->sig == 0) goto out; if (!_SIG_VALID(uap->sig)) { error = EINVAL; goto out; } tdsignal(ttd, uap->sig, SIGTARGET_TD); out: PROC_UNLOCK(p); return (error); } int thr_suspend(struct thread *td, struct thr_suspend_args *uap) /* const struct timespec *timeout */ { struct timespec ts; struct timeval tv; int error; int hz; hz = 0; error = 0; if (uap->timeout != NULL) { error = copyin((const void *)uap->timeout, (void *)&ts, sizeof(struct timespec)); if (error != 0) return (error); if (ts.tv_nsec < 0 || ts.tv_nsec > 1000000000) return (EINVAL); if (ts.tv_sec == 0 && ts.tv_nsec == 0) return (ETIMEDOUT); TIMESPEC_TO_TIMEVAL(&tv, &ts); hz = tvtohz(&tv); } PROC_LOCK(td->td_proc); if ((td->td_flags & TDF_THRWAKEUP) == 0) error = msleep((void *)td, &td->td_proc->p_mtx, td->td_priority | PCATCH, "lthr", hz); mtx_lock_spin(&sched_lock); td->td_flags &= ~TDF_THRWAKEUP; mtx_unlock_spin(&sched_lock); PROC_UNLOCK(td->td_proc); return (error == EWOULDBLOCK ? ETIMEDOUT : error); } int thr_wake(struct thread *td, struct thr_wake_args *uap) /* long id */ { struct thread *ttd; PROC_LOCK(td->td_proc); FOREACH_THREAD_IN_PROC(td->td_proc, ttd) { if (ttd->td_tid == uap->id) break; } if (ttd == NULL) { PROC_UNLOCK(td->td_proc); return (ESRCH); } mtx_lock_spin(&sched_lock); ttd->td_flags |= TDF_THRWAKEUP; mtx_unlock_spin(&sched_lock); wakeup_one((void *)ttd); PROC_UNLOCK(td->td_proc); return (0); } Index: head/sys/kern/kern_thread.c =================================================================== --- head/sys/kern/kern_thread.c (revision 132371) +++ head/sys/kern/kern_thread.c (revision 132372) @@ -1,1108 +1,1108 @@ /* * Copyright (C) 2001 Julian Elischer . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice(s), this list of conditions and the following disclaimer as * the first lines of this file unmodified other than the possible * addition of one or more copyright notices. * 2. Redistributions in binary form must reproduce the above copyright * notice(s), this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH * DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * KSEGRP related storage. */ static uma_zone_t ksegrp_zone; static uma_zone_t kse_zone; static uma_zone_t thread_zone; /* DEBUG ONLY */ SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); static int thread_debug = 0; SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, &thread_debug, 0, "thread debug"); int max_threads_per_proc = 1500; SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, &max_threads_per_proc, 0, "Limit on threads per proc"); int max_groups_per_proc = 500; SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, &max_groups_per_proc, 0, "Limit on thread groups per proc"); int max_threads_hits; SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, &max_threads_hits, 0, ""); int virtual_cpu; #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); TAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses); TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); struct mtx kse_zombie_lock; MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); void kse_purge(struct proc *p, struct thread *td); void kse_purge_group(struct thread *td); /* move to proc.h */ extern void kseinit(void); extern void kse_GC(void); static int sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) { int error, new_val; int def_val; def_val = mp_ncpus; if (virtual_cpu == 0) new_val = def_val; else new_val = virtual_cpu; error = sysctl_handle_int(oidp, &new_val, 0, req); if (error != 0 || req->newptr == NULL) return (error); if (new_val < 0) return (EINVAL); virtual_cpu = new_val; return (0); } /* DEBUG ONLY */ SYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", "debug virtual cpus"); /* * Thread ID allocator. The allocator keeps track of assigned IDs by * using a bitmap. The bitmap is created in parts. The parts are linked * together. */ typedef u_long tid_bitmap_word; #define TID_IDS_PER_PART 1024 #define TID_IDS_PER_IDX (sizeof(tid_bitmap_word) << 3) #define TID_BITMAP_SIZE (TID_IDS_PER_PART / TID_IDS_PER_IDX) #define TID_MIN (PID_MAX + 1) struct tid_bitmap_part { STAILQ_ENTRY(tid_bitmap_part) bmp_next; tid_bitmap_word bmp_bitmap[TID_BITMAP_SIZE]; lwpid_t bmp_base; int bmp_free; }; static STAILQ_HEAD(, tid_bitmap_part) tid_bitmap = STAILQ_HEAD_INITIALIZER(tid_bitmap); static uma_zone_t tid_zone; struct mtx tid_lock; MTX_SYSINIT(tid_lock, &tid_lock, "TID lock", MTX_DEF); /* * Prepare a thread for use. */ static void thread_ctor(void *mem, int size, void *arg) { struct thread *td; td = (struct thread *)mem; td->td_state = TDS_INACTIVE; td->td_oncpu = NOCPU; /* * Note that td_critnest begins life as 1 because the thread is not * running and is thereby implicitly waiting to be on the receiving * end of a context switch. A context switch must occur inside a * critical section, and in fact, includes hand-off of the sched_lock. * After a context switch to a newly created thread, it will release * sched_lock for the first time, and its td_critnest will hit 0 for * the first time. This happens on the far end of a context switch, * and when it context switches away from itself, it will in fact go * back into a critical section, and hand off the sched lock to the * next thread. */ td->td_critnest = 1; } /* * Reclaim a thread after use. */ static void thread_dtor(void *mem, int size, void *arg) { struct thread *td; td = (struct thread *)mem; #ifdef INVARIANTS /* Verify that this thread is in a safe state to free. */ switch (td->td_state) { case TDS_INHIBITED: case TDS_RUNNING: case TDS_CAN_RUN: case TDS_RUNQ: /* * We must never unlink a thread that is in one of * these states, because it is currently active. */ panic("bad state for thread unlinking"); /* NOTREACHED */ case TDS_INACTIVE: break; default: panic("bad thread state"); /* NOTREACHED */ } #endif } /* * Initialize type-stable parts of a thread (when newly created). */ static void thread_init(void *mem, int size) { struct thread *td; struct tid_bitmap_part *bmp, *new; int bit, idx; td = (struct thread *)mem; mtx_lock(&tid_lock); STAILQ_FOREACH(bmp, &tid_bitmap, bmp_next) { if (bmp->bmp_free) break; } /* Create a new bitmap if we run out of free bits. */ if (bmp == NULL) { mtx_unlock(&tid_lock); new = uma_zalloc(tid_zone, M_WAITOK); mtx_lock(&tid_lock); bmp = STAILQ_LAST(&tid_bitmap, tid_bitmap_part, bmp_next); if (bmp == NULL || bmp->bmp_free < TID_IDS_PER_PART/2) { /* 1=free, 0=assigned. This way we can use ffsl(). */ memset(new->bmp_bitmap, ~0U, sizeof(new->bmp_bitmap)); new->bmp_base = (bmp == NULL) ? TID_MIN : bmp->bmp_base + TID_IDS_PER_PART; new->bmp_free = TID_IDS_PER_PART; STAILQ_INSERT_TAIL(&tid_bitmap, new, bmp_next); bmp = new; new = NULL; } } else new = NULL; /* We have a bitmap with available IDs. */ idx = 0; while (idx < TID_BITMAP_SIZE && bmp->bmp_bitmap[idx] == 0UL) idx++; bit = ffsl(bmp->bmp_bitmap[idx]) - 1; td->td_tid = bmp->bmp_base + idx * TID_IDS_PER_IDX + bit; bmp->bmp_bitmap[idx] &= ~(1UL << bit); bmp->bmp_free--; mtx_unlock(&tid_lock); if (new != NULL) uma_zfree(tid_zone, new); vm_thread_new(td, 0); cpu_thread_setup(td); td->td_sleepqueue = sleepq_alloc(); td->td_turnstile = turnstile_alloc(); td->td_sched = (struct td_sched *)&td[1]; } /* * Tear down type-stable parts of a thread (just before being discarded). */ static void thread_fini(void *mem, int size) { struct thread *td; struct tid_bitmap_part *bmp; lwpid_t tid; int bit, idx; td = (struct thread *)mem; turnstile_free(td->td_turnstile); sleepq_free(td->td_sleepqueue); vm_thread_dispose(td); STAILQ_FOREACH(bmp, &tid_bitmap, bmp_next) { if (td->td_tid >= bmp->bmp_base && td->td_tid < bmp->bmp_base + TID_IDS_PER_PART) break; } KASSERT(bmp != NULL, ("No TID bitmap?")); mtx_lock(&tid_lock); tid = td->td_tid - bmp->bmp_base; idx = tid / TID_IDS_PER_IDX; bit = 1UL << (tid % TID_IDS_PER_IDX); bmp->bmp_bitmap[idx] |= bit; bmp->bmp_free++; mtx_unlock(&tid_lock); } /* * Initialize type-stable parts of a kse (when newly created). */ static void kse_init(void *mem, int size) { struct kse *ke; ke = (struct kse *)mem; ke->ke_sched = (struct ke_sched *)&ke[1]; } /* * Initialize type-stable parts of a ksegrp (when newly created). */ static void ksegrp_init(void *mem, int size) { struct ksegrp *kg; kg = (struct ksegrp *)mem; kg->kg_sched = (struct kg_sched *)&kg[1]; } /* * KSE is linked into kse group. */ void kse_link(struct kse *ke, struct ksegrp *kg) { struct proc *p = kg->kg_proc; TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist); kg->kg_kses++; ke->ke_state = KES_UNQUEUED; ke->ke_proc = p; ke->ke_ksegrp = kg; ke->ke_thread = NULL; ke->ke_oncpu = NOCPU; ke->ke_flags = 0; } void kse_unlink(struct kse *ke) { struct ksegrp *kg; mtx_assert(&sched_lock, MA_OWNED); kg = ke->ke_ksegrp; TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); if (ke->ke_state == KES_IDLE) { TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); kg->kg_idle_kses--; } --kg->kg_kses; /* * Aggregate stats from the KSE */ kse_stash(ke); } void ksegrp_link(struct ksegrp *kg, struct proc *p) { TAILQ_INIT(&kg->kg_threads); TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ TAILQ_INIT(&kg->kg_slpq); /* links with td_runq */ TAILQ_INIT(&kg->kg_kseq); /* all kses in ksegrp */ TAILQ_INIT(&kg->kg_iq); /* all idle kses in ksegrp */ TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ kg->kg_proc = p; /* * the following counters are in the -zero- section * and may not need clearing */ kg->kg_numthreads = 0; kg->kg_runnable = 0; kg->kg_kses = 0; kg->kg_runq_kses = 0; /* XXXKSE change name */ kg->kg_idle_kses = 0; kg->kg_numupcalls = 0; /* link it in now that it's consistent */ p->p_numksegrps++; TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); } void ksegrp_unlink(struct ksegrp *kg) { struct proc *p; mtx_assert(&sched_lock, MA_OWNED); KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); KASSERT((kg->kg_kses == 0), ("ksegrp_unlink: residual kses")); KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); p = kg->kg_proc; TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); p->p_numksegrps--; /* * Aggregate stats from the KSE */ ksegrp_stash(kg); } /* * For a newly created process, * link up all the structures and its initial threads etc. */ void proc_linkup(struct proc *p, struct ksegrp *kg, struct kse *ke, struct thread *td) { TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ TAILQ_INIT(&p->p_threads); /* all threads in proc */ TAILQ_INIT(&p->p_suspended); /* Threads suspended */ p->p_numksegrps = 0; p->p_numthreads = 0; ksegrp_link(kg, p); kse_link(ke, kg); thread_link(td, kg); } /* * Initialize global thread allocation resources. */ void threadinit(void) { thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), thread_ctor, thread_dtor, thread_init, thread_fini, UMA_ALIGN_CACHE, 0); tid_zone = uma_zcreate("TID", sizeof(struct tid_bitmap_part), NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), NULL, NULL, ksegrp_init, NULL, UMA_ALIGN_CACHE, 0); kse_zone = uma_zcreate("KSE", sched_sizeof_kse(), NULL, NULL, kse_init, NULL, UMA_ALIGN_CACHE, 0); kseinit(); } /* * Stash an embarasingly extra thread into the zombie thread queue. */ void thread_stash(struct thread *td) { mtx_lock_spin(&kse_zombie_lock); TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); mtx_unlock_spin(&kse_zombie_lock); } /* * Stash an embarasingly extra kse into the zombie kse queue. */ void kse_stash(struct kse *ke) { mtx_lock_spin(&kse_zombie_lock); TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq); mtx_unlock_spin(&kse_zombie_lock); } /* * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. */ void ksegrp_stash(struct ksegrp *kg) { mtx_lock_spin(&kse_zombie_lock); TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); mtx_unlock_spin(&kse_zombie_lock); } /* * Reap zombie kse resource. */ void thread_reap(void) { struct thread *td_first, *td_next; struct kse *ke_first, *ke_next; struct ksegrp *kg_first, * kg_next; /* * Don't even bother to lock if none at this instant, * we really don't care about the next instant.. */ if ((!TAILQ_EMPTY(&zombie_threads)) || (!TAILQ_EMPTY(&zombie_kses)) || (!TAILQ_EMPTY(&zombie_ksegrps))) { mtx_lock_spin(&kse_zombie_lock); td_first = TAILQ_FIRST(&zombie_threads); ke_first = TAILQ_FIRST(&zombie_kses); kg_first = TAILQ_FIRST(&zombie_ksegrps); if (td_first) TAILQ_INIT(&zombie_threads); if (ke_first) TAILQ_INIT(&zombie_kses); if (kg_first) TAILQ_INIT(&zombie_ksegrps); mtx_unlock_spin(&kse_zombie_lock); while (td_first) { td_next = TAILQ_NEXT(td_first, td_runq); if (td_first->td_ucred) crfree(td_first->td_ucred); thread_free(td_first); td_first = td_next; } while (ke_first) { ke_next = TAILQ_NEXT(ke_first, ke_procq); kse_free(ke_first); ke_first = ke_next; } while (kg_first) { kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); ksegrp_free(kg_first); kg_first = kg_next; } } kse_GC(); } /* * Allocate a ksegrp. */ struct ksegrp * ksegrp_alloc(void) { return (uma_zalloc(ksegrp_zone, M_WAITOK)); } /* * Allocate a kse. */ struct kse * kse_alloc(void) { return (uma_zalloc(kse_zone, M_WAITOK)); } /* * Allocate a thread. */ struct thread * thread_alloc(void) { thread_reap(); /* check if any zombies to get */ return (uma_zalloc(thread_zone, M_WAITOK)); } /* * Deallocate a ksegrp. */ void ksegrp_free(struct ksegrp *td) { uma_zfree(ksegrp_zone, td); } /* * Deallocate a kse. */ void kse_free(struct kse *td) { uma_zfree(kse_zone, td); } /* * Deallocate a thread. */ void thread_free(struct thread *td) { cpu_thread_clean(td); uma_zfree(thread_zone, td); } /* * Discard the current thread and exit from its context. * Always called with scheduler locked. * * Because we can't free a thread while we're operating under its context, * push the current thread into our CPU's deadthread holder. This means * we needn't worry about someone else grabbing our context before we * do a cpu_throw(). This may not be needed now as we are under schedlock. * Maybe we can just do a thread_stash() as thr_exit1 does. */ /* XXX * libthr expects its thread exit to return for the last * thread, meaning that the program is back to non-threaded * mode I guess. Because we do this (cpu_throw) unconditionally * here, they have their own version of it. (thr_exit1()) * that doesn't do it all if this was the last thread. * It is also called from thread_suspend_check(). * Of course in the end, they end up coming here through exit1 * anyhow.. After fixing 'thr' to play by the rules we should be able * to merge these two functions together. */ void thread_exit(void) { struct thread *td; struct kse *ke; struct proc *p; struct ksegrp *kg; td = curthread; kg = td->td_ksegrp; p = td->td_proc; ke = td->td_kse; mtx_assert(&sched_lock, MA_OWNED); KASSERT(p != NULL, ("thread exiting without a process")); KASSERT(ke != NULL, ("thread exiting without a kse")); KASSERT(kg != NULL, ("thread exiting without a kse group")); PROC_LOCK_ASSERT(p, MA_OWNED); CTR1(KTR_PROC, "thread_exit: thread %p", td); mtx_assert(&Giant, MA_NOTOWNED); if (td->td_standin != NULL) { thread_stash(td->td_standin); td->td_standin = NULL; } cpu_thread_exit(td); /* XXXSMP */ /* * The last thread is left attached to the process * So that the whole bundle gets recycled. Skip * all this stuff. */ if (p->p_numthreads > 1) { thread_unlink(td); if (p->p_maxthrwaits) wakeup(&p->p_numthreads); /* * The test below is NOT true if we are the * sole exiting thread. P_STOPPED_SNGL is unset * in exit1() after it is the only survivor. */ if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { if (p->p_numthreads == p->p_suspcount) { thread_unsuspend_one(p->p_singlethread); } } /* * Because each upcall structure has an owner thread, * owner thread exits only when process is in exiting * state, so upcall to userland is no longer needed, * deleting upcall structure is safe here. * So when all threads in a group is exited, all upcalls * in the group should be automatically freed. */ if (td->td_upcall) upcall_remove(td); sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); - sched_exit_kse(FIRST_KSE_IN_PROC(p), ke); + sched_exit_kse(FIRST_KSE_IN_PROC(p), td); ke->ke_state = KES_UNQUEUED; ke->ke_thread = NULL; /* * Decide what to do with the KSE attached to this thread. */ if (ke->ke_flags & KEF_EXIT) { kse_unlink(ke); if (kg->kg_kses == 0) { - sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), kg); + sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td); ksegrp_unlink(kg); } } else kse_reassign(ke); PROC_UNLOCK(p); td->td_kse = NULL; #if 0 td->td_proc = NULL; #endif td->td_ksegrp = NULL; td->td_last_kse = NULL; PCPU_SET(deadthread, td); } else { PROC_UNLOCK(p); } td->td_state = TDS_INACTIVE; /* XXX Shouldn't cpu_throw() here. */ mtx_assert(&sched_lock, MA_OWNED); cpu_throw(td, choosethread()); panic("I'm a teapot!"); /* NOTREACHED */ } /* * Do any thread specific cleanups that may be needed in wait() * called with Giant, proc and schedlock not held. */ void thread_wait(struct proc *p) { struct thread *td; mtx_assert(&Giant, MA_NOTOWNED); KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); KASSERT((p->p_numksegrps == 1), ("Multiple ksegrps in wait1()")); FOREACH_THREAD_IN_PROC(p, td) { if (td->td_standin != NULL) { thread_free(td->td_standin); td->td_standin = NULL; } cpu_thread_clean(td); } thread_reap(); /* check for zombie threads etc. */ } /* * Link a thread to a process. * set up anything that needs to be initialized for it to * be used by the process. * * Note that we do not link to the proc's ucred here. * The thread is linked as if running but no KSE assigned. */ void thread_link(struct thread *td, struct ksegrp *kg) { struct proc *p; p = kg->kg_proc; td->td_state = TDS_INACTIVE; td->td_proc = p; td->td_ksegrp = kg; td->td_last_kse = NULL; td->td_flags = 0; td->td_kflags = 0; td->td_kse = NULL; LIST_INIT(&td->td_contested); callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); p->p_numthreads++; kg->kg_numthreads++; } void thread_unlink(struct thread *td) { struct proc *p = td->td_proc; struct ksegrp *kg = td->td_ksegrp; mtx_assert(&sched_lock, MA_OWNED); TAILQ_REMOVE(&p->p_threads, td, td_plist); p->p_numthreads--; TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); kg->kg_numthreads--; /* could clear a few other things here */ } /* * Purge a ksegrp resource. When a ksegrp is preparing to * exit, it calls this function. */ void kse_purge_group(struct thread *td) { struct ksegrp *kg; struct kse *ke; kg = td->td_ksegrp; KASSERT(kg->kg_numthreads == 1, ("%s: bad thread number", __func__)); while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { KASSERT(ke->ke_state == KES_IDLE, ("%s: wrong idle KSE state", __func__)); kse_unlink(ke); } KASSERT((kg->kg_kses == 1), ("%s: ksegrp still has %d KSEs", __func__, kg->kg_kses)); KASSERT((kg->kg_numupcalls == 0), ("%s: ksegrp still has %d upcall datas", __func__, kg->kg_numupcalls)); } /* * Purge a process's KSE resource. When a process is preparing to * exit, it calls kse_purge to release any extra KSE resources in * the process. */ void kse_purge(struct proc *p, struct thread *td) { struct ksegrp *kg; struct kse *ke; KASSERT(p->p_numthreads == 1, ("bad thread number")); while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) { TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); p->p_numksegrps--; /* * There is no ownership for KSE, after all threads * in the group exited, it is possible that some KSEs * were left in idle queue, gc them now. */ while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { KASSERT(ke->ke_state == KES_IDLE, ("%s: wrong idle KSE state", __func__)); TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); kg->kg_idle_kses--; TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); kg->kg_kses--; kse_stash(ke); } KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) || ((kg->kg_kses == 1) && (kg == td->td_ksegrp)), ("ksegrp has wrong kg_kses: %d", kg->kg_kses)); KASSERT((kg->kg_numupcalls == 0), ("%s: ksegrp still has %d upcall datas", __func__, kg->kg_numupcalls)); if (kg != td->td_ksegrp) ksegrp_stash(kg); } TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp); p->p_numksegrps++; } /* * Enforce single-threading. * * Returns 1 if the caller must abort (another thread is waiting to * exit the process or similar). Process is locked! * Returns 0 when you are successfully the only thread running. * A process has successfully single threaded in the suspend mode when * There are no threads in user mode. Threads in the kernel must be * allowed to continue until they get to the user boundary. They may even * copy out their return values and data before suspending. They may however be * accellerated in reaching the user boundary as we will wake up * any sleeping threads that are interruptable. (PCATCH). */ int thread_single(int force_exit) { struct thread *td; struct thread *td2; struct proc *p; int remaining; td = curthread; p = td->td_proc; mtx_assert(&Giant, MA_NOTOWNED); PROC_LOCK_ASSERT(p, MA_OWNED); KASSERT((td != NULL), ("curthread is NULL")); if ((p->p_flag & P_SA) == 0 && p->p_numthreads == 1) return (0); /* Is someone already single threading? */ if (p->p_singlethread) return (1); if (force_exit == SINGLE_EXIT) { p->p_flag |= P_SINGLE_EXIT; } else p->p_flag &= ~P_SINGLE_EXIT; p->p_flag |= P_STOPPED_SINGLE; mtx_lock_spin(&sched_lock); p->p_singlethread = td; if (force_exit == SINGLE_EXIT) remaining = p->p_numthreads; else remaining = p->p_numthreads - p->p_suspcount; while (remaining != 1) { FOREACH_THREAD_IN_PROC(p, td2) { if (td2 == td) continue; td2->td_flags |= TDF_ASTPENDING; if (TD_IS_INHIBITED(td2)) { if (force_exit == SINGLE_EXIT) { if (td->td_flags & TDF_DBSUSPEND) td->td_flags &= ~TDF_DBSUSPEND; if (TD_IS_SUSPENDED(td2)) { thread_unsuspend_one(td2); } if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) { sleepq_abort(td2); } } else { if (TD_IS_SUSPENDED(td2)) continue; /* * maybe other inhibitted states too? * XXXKSE Is it totally safe to * suspend a non-interruptable thread? */ if (td2->td_inhibitors & (TDI_SLEEPING | TDI_SWAPPED)) thread_suspend_one(td2); } } } if (force_exit == SINGLE_EXIT) remaining = p->p_numthreads; else remaining = p->p_numthreads - p->p_suspcount; /* * Maybe we suspended some threads.. was it enough? */ if (remaining == 1) break; /* * Wake us up when everyone else has suspended. * In the mean time we suspend as well. */ thread_suspend_one(td); PROC_UNLOCK(p); mi_switch(SW_VOL, NULL); mtx_unlock_spin(&sched_lock); PROC_LOCK(p); mtx_lock_spin(&sched_lock); if (force_exit == SINGLE_EXIT) remaining = p->p_numthreads; else remaining = p->p_numthreads - p->p_suspcount; } if (force_exit == SINGLE_EXIT) { if (td->td_upcall) upcall_remove(td); kse_purge(p, td); } mtx_unlock_spin(&sched_lock); return (0); } /* * Called in from locations that can safely check to see * whether we have to suspend or at least throttle for a * single-thread event (e.g. fork). * * Such locations include userret(). * If the "return_instead" argument is non zero, the thread must be able to * accept 0 (caller may continue), or 1 (caller must abort) as a result. * * The 'return_instead' argument tells the function if it may do a * thread_exit() or suspend, or whether the caller must abort and back * out instead. * * If the thread that set the single_threading request has set the * P_SINGLE_EXIT bit in the process flags then this call will never return * if 'return_instead' is false, but will exit. * * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 *---------------+--------------------+--------------------- * 0 | returns 0 | returns 0 or 1 * | when ST ends | immediatly *---------------+--------------------+--------------------- * 1 | thread exits | returns 1 * | | immediatly * 0 = thread_exit() or suspension ok, * other = return error instead of stopping the thread. * * While a full suspension is under effect, even a single threading * thread would be suspended if it made this call (but it shouldn't). * This call should only be made from places where * thread_exit() would be safe as that may be the outcome unless * return_instead is set. */ int thread_suspend_check(int return_instead) { struct thread *td; struct proc *p; td = curthread; p = td->td_proc; mtx_assert(&Giant, MA_NOTOWNED); PROC_LOCK_ASSERT(p, MA_OWNED); while (P_SHOULDSTOP(p) || ((p->p_flag & P_TRACED) && (td->td_flags & TDF_DBSUSPEND))) { if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { KASSERT(p->p_singlethread != NULL, ("singlethread not set")); /* * The only suspension in action is a * single-threading. Single threader need not stop. * XXX Should be safe to access unlocked * as it can only be set to be true by us. */ if (p->p_singlethread == td) return (0); /* Exempt from stopping. */ } if (return_instead) return (1); mtx_lock_spin(&sched_lock); thread_stopped(p); /* * If the process is waiting for us to exit, * this thread should just suicide. * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. */ if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { if (p->p_flag & P_SA) thread_exit(); else thr_exit1(); } /* * When a thread suspends, it just * moves to the processes's suspend queue * and stays there. */ thread_suspend_one(td); if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { if (p->p_numthreads == p->p_suspcount) { thread_unsuspend_one(p->p_singlethread); } } PROC_UNLOCK(p); mi_switch(SW_INVOL, NULL); mtx_unlock_spin(&sched_lock); PROC_LOCK(p); } return (0); } void thread_suspend_one(struct thread *td) { struct proc *p = td->td_proc; mtx_assert(&sched_lock, MA_OWNED); PROC_LOCK_ASSERT(p, MA_OWNED); KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); p->p_suspcount++; TD_SET_SUSPENDED(td); TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); /* * Hack: If we are suspending but are on the sleep queue * then we are in msleep or the cv equivalent. We * want to look like we have two Inhibitors. * May already be set.. doesn't matter. */ if (TD_ON_SLEEPQ(td)) TD_SET_SLEEPING(td); } void thread_unsuspend_one(struct thread *td) { struct proc *p = td->td_proc; mtx_assert(&sched_lock, MA_OWNED); PROC_LOCK_ASSERT(p, MA_OWNED); TAILQ_REMOVE(&p->p_suspended, td, td_runq); TD_CLR_SUSPENDED(td); p->p_suspcount--; setrunnable(td); } /* * Allow all threads blocked by single threading to continue running. */ void thread_unsuspend(struct proc *p) { struct thread *td; mtx_assert(&sched_lock, MA_OWNED); PROC_LOCK_ASSERT(p, MA_OWNED); if (!P_SHOULDSTOP(p)) { while ((td = TAILQ_FIRST(&p->p_suspended))) { thread_unsuspend_one(td); } } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && (p->p_numthreads == p->p_suspcount)) { /* * Stopping everything also did the job for the single * threading request. Now we've downgraded to single-threaded, * let it continue. */ thread_unsuspend_one(p->p_singlethread); } } void thread_single_end(void) { struct thread *td; struct proc *p; td = curthread; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT); mtx_lock_spin(&sched_lock); p->p_singlethread = NULL; /* * If there are other threads they mey now run, * unless of course there is a blanket 'stop order' * on the process. The single threader must be allowed * to continue however as this is a bad place to stop. */ if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { while (( td = TAILQ_FIRST(&p->p_suspended))) { thread_unsuspend_one(td); } } mtx_unlock_spin(&sched_lock); } Index: head/sys/kern/sched_4bsd.c =================================================================== --- head/sys/kern/sched_4bsd.c (revision 132371) +++ head/sys/kern/sched_4bsd.c (revision 132372) @@ -1,899 +1,899 @@ /*- * Copyright (c) 1982, 1986, 1990, 1991, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #define KTR_4BSD 0x0 /* * INVERSE_ESTCPU_WEIGHT is only suitable for statclock() frequencies in * the range 100-256 Hz (approximately). */ #define ESTCPULIM(e) \ min((e), INVERSE_ESTCPU_WEIGHT * (NICE_WEIGHT * (PRIO_MAX - PRIO_MIN) - \ RQ_PPQ) + INVERSE_ESTCPU_WEIGHT - 1) #ifdef SMP #define INVERSE_ESTCPU_WEIGHT (8 * smp_cpus) #else #define INVERSE_ESTCPU_WEIGHT 8 /* 1 / (priorities per estcpu level). */ #endif #define NICE_WEIGHT 1 /* Priorities per nice level. */ struct ke_sched { int ske_cpticks; /* (j) Ticks of cpu time. */ struct runq *ske_runq; /* runq the kse is currently on */ }; #define ke_runq ke_sched->ske_runq #define KEF_BOUND KEF_SCHED1 #define SKE_RUNQ_PCPU(ke) \ ((ke)->ke_runq != 0 && (ke)->ke_runq != &runq) /* * KSE_CAN_MIGRATE macro returns true if the kse can migrate between * cpus. */ #define KSE_CAN_MIGRATE(ke) \ ((ke)->ke_thread->td_pinned == 0 && ((ke)->ke_flags & KEF_BOUND) == 0) static struct ke_sched ke_sched; struct ke_sched *kse0_sched = &ke_sched; struct kg_sched *ksegrp0_sched = NULL; struct p_sched *proc0_sched = NULL; struct td_sched *thread0_sched = NULL; static int sched_tdcnt; /* Total runnable threads in the system. */ static int sched_quantum; /* Roundrobin scheduling quantum in ticks. */ #define SCHED_QUANTUM (hz / 10) /* Default sched quantum */ static struct callout roundrobin_callout; static void setup_runqs(void); static void roundrobin(void *arg); static void schedcpu(void); static void schedcpu_thread(void); static void sched_setup(void *dummy); static void maybe_resched(struct thread *td); static void updatepri(struct ksegrp *kg); static void resetpriority(struct ksegrp *kg); static struct kproc_desc sched_kp = { "schedcpu", schedcpu_thread, NULL }; SYSINIT(schedcpu, SI_SUB_RUN_SCHEDULER, SI_ORDER_FIRST, kproc_start, &sched_kp) SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL) /* * Global run queue. */ static struct runq runq; #ifdef SMP /* * Per-CPU run queues */ static struct runq runq_pcpu[MAXCPU]; #endif static void setup_runqs(void) { #ifdef SMP int i; for (i = 0; i < MAXCPU; ++i) runq_init(&runq_pcpu[i]); #endif runq_init(&runq); } static int sysctl_kern_quantum(SYSCTL_HANDLER_ARGS) { int error, new_val; new_val = sched_quantum * tick; error = sysctl_handle_int(oidp, &new_val, 0, req); if (error != 0 || req->newptr == NULL) return (error); if (new_val < tick) return (EINVAL); sched_quantum = new_val / tick; hogticks = 2 * sched_quantum; return (0); } SYSCTL_NODE(_kern, OID_AUTO, sched, CTLFLAG_RD, 0, "SCHED"); #define SCHD_NAME "4bsd" #define SCHD_NAME_LEN 4 SYSCTL_STRING(_kern_sched, OID_AUTO, name, CTLFLAG_RD, SCHD_NAME, SCHD_NAME_LEN, "System is using the 4BSD scheduler"); SYSCTL_PROC(_kern_sched, OID_AUTO, quantum, CTLTYPE_INT|CTLFLAG_RW, 0, sizeof sched_quantum, sysctl_kern_quantum, "I", "Roundrobin scheduling quantum in microseconds"); /* * Arrange to reschedule if necessary, taking the priorities and * schedulers into account. */ static void maybe_resched(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); if (td->td_priority < curthread->td_priority && curthread->td_kse) curthread->td_flags |= TDF_NEEDRESCHED; } /* * Force switch among equal priority processes every 100ms. * We don't actually need to force a context switch of the current process. * The act of firing the event triggers a context switch to softclock() and * then switching back out again which is equivalent to a preemption, thus * no further work is needed on the local CPU. */ /* ARGSUSED */ static void roundrobin(void *arg) { #ifdef SMP mtx_lock_spin(&sched_lock); forward_roundrobin(); mtx_unlock_spin(&sched_lock); #endif callout_reset(&roundrobin_callout, sched_quantum, roundrobin, NULL); } /* * Constants for digital decay and forget: * 90% of (kg_estcpu) usage in 5 * loadav time * 95% of (ke_pctcpu) usage in 60 seconds (load insensitive) * Note that, as ps(1) mentions, this can let percentages * total over 100% (I've seen 137.9% for 3 processes). * * Note that schedclock() updates kg_estcpu and p_cpticks asynchronously. * * We wish to decay away 90% of kg_estcpu in (5 * loadavg) seconds. * That is, the system wants to compute a value of decay such * that the following for loop: * for (i = 0; i < (5 * loadavg); i++) * kg_estcpu *= decay; * will compute * kg_estcpu *= 0.1; * for all values of loadavg: * * Mathematically this loop can be expressed by saying: * decay ** (5 * loadavg) ~= .1 * * The system computes decay as: * decay = (2 * loadavg) / (2 * loadavg + 1) * * We wish to prove that the system's computation of decay * will always fulfill the equation: * decay ** (5 * loadavg) ~= .1 * * If we compute b as: * b = 2 * loadavg * then * decay = b / (b + 1) * * We now need to prove two things: * 1) Given factor ** (5 * loadavg) ~= .1, prove factor == b/(b+1) * 2) Given b/(b+1) ** power ~= .1, prove power == (5 * loadavg) * * Facts: * For x close to zero, exp(x) =~ 1 + x, since * exp(x) = 0! + x**1/1! + x**2/2! + ... . * therefore exp(-1/b) =~ 1 - (1/b) = (b-1)/b. * For x close to zero, ln(1+x) =~ x, since * ln(1+x) = x - x**2/2 + x**3/3 - ... -1 < x < 1 * therefore ln(b/(b+1)) = ln(1 - 1/(b+1)) =~ -1/(b+1). * ln(.1) =~ -2.30 * * Proof of (1): * Solve (factor)**(power) =~ .1 given power (5*loadav): * solving for factor, * ln(factor) =~ (-2.30/5*loadav), or * factor =~ exp(-1/((5/2.30)*loadav)) =~ exp(-1/(2*loadav)) = * exp(-1/b) =~ (b-1)/b =~ b/(b+1). QED * * Proof of (2): * Solve (factor)**(power) =~ .1 given factor == (b/(b+1)): * solving for power, * power*ln(b/(b+1)) =~ -2.30, or * power =~ 2.3 * (b + 1) = 4.6*loadav + 2.3 =~ 5*loadav. QED * * Actual power values for the implemented algorithm are as follows: * loadav: 1 2 3 4 * power: 5.68 10.32 14.94 19.55 */ /* calculations for digital decay to forget 90% of usage in 5*loadav sec */ #define loadfactor(loadav) (2 * (loadav)) #define decay_cpu(loadfac, cpu) (((loadfac) * (cpu)) / ((loadfac) + FSCALE)) /* decay 95% of `ke_pctcpu' in 60 seconds; see CCPU_SHIFT before changing */ static fixpt_t ccpu = 0.95122942450071400909 * FSCALE; /* exp(-1/20) */ SYSCTL_INT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0, ""); /* * If `ccpu' is not equal to `exp(-1/20)' and you still want to use the * faster/more-accurate formula, you'll have to estimate CCPU_SHIFT below * and possibly adjust FSHIFT in "param.h" so that (FSHIFT >= CCPU_SHIFT). * * To estimate CCPU_SHIFT for exp(-1/20), the following formula was used: * 1 - exp(-1/20) ~= 0.0487 ~= 0.0488 == 1 (fixed pt, *11* bits). * * If you don't want to bother with the faster/more-accurate formula, you * can set CCPU_SHIFT to (FSHIFT + 1) which will use a slower/less-accurate * (more general) method of calculating the %age of CPU used by a process. */ #define CCPU_SHIFT 11 /* * Recompute process priorities, every hz ticks. * MP-safe, called without the Giant mutex. */ /* ARGSUSED */ static void schedcpu(void) { register fixpt_t loadfac = loadfactor(averunnable.ldavg[0]); struct thread *td; struct proc *p; struct kse *ke; struct ksegrp *kg; int awake, realstathz; realstathz = stathz ? stathz : hz; sx_slock(&allproc_lock); FOREACH_PROC_IN_SYSTEM(p) { /* * Prevent state changes and protect run queue. */ mtx_lock_spin(&sched_lock); /* * Increment time in/out of memory. We ignore overflow; with * 16-bit int's (remember them?) overflow takes 45 days. */ p->p_swtime++; FOREACH_KSEGRP_IN_PROC(p, kg) { awake = 0; FOREACH_KSE_IN_GROUP(kg, ke) { /* * Increment sleep time (if sleeping). We * ignore overflow, as above. */ /* * The kse slptimes are not touched in wakeup * because the thread may not HAVE a KSE. */ if (ke->ke_state == KES_ONRUNQ) { awake = 1; ke->ke_flags &= ~KEF_DIDRUN; } else if ((ke->ke_state == KES_THREAD) && (TD_IS_RUNNING(ke->ke_thread))) { awake = 1; /* Do not clear KEF_DIDRUN */ } else if (ke->ke_flags & KEF_DIDRUN) { awake = 1; ke->ke_flags &= ~KEF_DIDRUN; } /* * ke_pctcpu is only for ps and ttyinfo(). * Do it per kse, and add them up at the end? * XXXKSE */ ke->ke_pctcpu = (ke->ke_pctcpu * ccpu) >> FSHIFT; /* * If the kse has been idle the entire second, * stop recalculating its priority until * it wakes up. */ if (ke->ke_sched->ske_cpticks == 0) continue; #if (FSHIFT >= CCPU_SHIFT) ke->ke_pctcpu += (realstathz == 100) ? ((fixpt_t) ke->ke_sched->ske_cpticks) << (FSHIFT - CCPU_SHIFT) : 100 * (((fixpt_t) ke->ke_sched->ske_cpticks) << (FSHIFT - CCPU_SHIFT)) / realstathz; #else ke->ke_pctcpu += ((FSCALE - ccpu) * (ke->ke_sched->ske_cpticks * FSCALE / realstathz)) >> FSHIFT; #endif ke->ke_sched->ske_cpticks = 0; } /* end of kse loop */ /* * If there are ANY running threads in this KSEGRP, * then don't count it as sleeping. */ if (awake) { if (kg->kg_slptime > 1) { /* * In an ideal world, this should not * happen, because whoever woke us * up from the long sleep should have * unwound the slptime and reset our * priority before we run at the stale * priority. Should KASSERT at some * point when all the cases are fixed. */ updatepri(kg); } kg->kg_slptime = 0; } else kg->kg_slptime++; if (kg->kg_slptime > 1) continue; kg->kg_estcpu = decay_cpu(loadfac, kg->kg_estcpu); resetpriority(kg); FOREACH_THREAD_IN_GROUP(kg, td) { if (td->td_priority >= PUSER) { sched_prio(td, kg->kg_user_pri); } } } /* end of ksegrp loop */ mtx_unlock_spin(&sched_lock); } /* end of process loop */ sx_sunlock(&allproc_lock); } /* * Main loop for a kthread that executes schedcpu once a second. */ static void schedcpu_thread(void) { int nowake; for (;;) { schedcpu(); tsleep(&nowake, curthread->td_priority, "-", hz); } } /* * Recalculate the priority of a process after it has slept for a while. * For all load averages >= 1 and max kg_estcpu of 255, sleeping for at * least six times the loadfactor will decay kg_estcpu to zero. */ static void updatepri(struct ksegrp *kg) { register fixpt_t loadfac; register unsigned int newcpu; loadfac = loadfactor(averunnable.ldavg[0]); if (kg->kg_slptime > 5 * loadfac) kg->kg_estcpu = 0; else { newcpu = kg->kg_estcpu; kg->kg_slptime--; /* was incremented in schedcpu() */ while (newcpu && --kg->kg_slptime) newcpu = decay_cpu(loadfac, newcpu); kg->kg_estcpu = newcpu; } resetpriority(kg); } /* * Compute the priority of a process when running in user mode. * Arrange to reschedule if the resulting priority is better * than that of the current process. */ static void resetpriority(struct ksegrp *kg) { register unsigned int newpriority; struct thread *td; if (kg->kg_pri_class == PRI_TIMESHARE) { newpriority = PUSER + kg->kg_estcpu / INVERSE_ESTCPU_WEIGHT + NICE_WEIGHT * (kg->kg_proc->p_nice - PRIO_MIN); newpriority = min(max(newpriority, PRI_MIN_TIMESHARE), PRI_MAX_TIMESHARE); kg->kg_user_pri = newpriority; } FOREACH_THREAD_IN_GROUP(kg, td) { maybe_resched(td); /* XXXKSE silly */ } } /* ARGSUSED */ static void sched_setup(void *dummy) { setup_runqs(); if (sched_quantum == 0) sched_quantum = SCHED_QUANTUM; hogticks = 2 * sched_quantum; callout_init(&roundrobin_callout, CALLOUT_MPSAFE); /* Kick off timeout driven events by calling first time. */ roundrobin(NULL); /* Account for thread0. */ sched_tdcnt++; } /* External interfaces start here */ int sched_runnable(void) { #ifdef SMP return runq_check(&runq) + runq_check(&runq_pcpu[PCPU_GET(cpuid)]); #else return runq_check(&runq); #endif } int sched_rr_interval(void) { if (sched_quantum == 0) sched_quantum = SCHED_QUANTUM; return (sched_quantum); } /* * We adjust the priority of the current process. The priority of * a process gets worse as it accumulates CPU time. The cpu usage * estimator (kg_estcpu) is increased here. resetpriority() will * compute a different priority each time kg_estcpu increases by * INVERSE_ESTCPU_WEIGHT * (until MAXPRI is reached). The cpu usage estimator ramps up * quite quickly when the process is running (linearly), and decays * away exponentially, at a rate which is proportionally slower when * the system is busy. The basic principle is that the system will * 90% forget that the process used a lot of CPU time in 5 * loadav * seconds. This causes the system to favor processes which haven't * run much recently, and to round-robin among other processes. */ void sched_clock(struct thread *td) { struct ksegrp *kg; struct kse *ke; mtx_assert(&sched_lock, MA_OWNED); kg = td->td_ksegrp; ke = td->td_kse; ke->ke_sched->ske_cpticks++; kg->kg_estcpu = ESTCPULIM(kg->kg_estcpu + 1); if ((kg->kg_estcpu % INVERSE_ESTCPU_WEIGHT) == 0) { resetpriority(kg); if (td->td_priority >= PUSER) td->td_priority = kg->kg_user_pri; } } /* * charge childs scheduling cpu usage to parent. * * XXXKSE assume only one thread & kse & ksegrp keep estcpu in each ksegrp. * Charge it to the ksegrp that did the wait since process estcpu is sum of * all ksegrps, this is strictly as expected. Assume that the child process * aggregated all the estcpu into the 'built-in' ksegrp. */ void -sched_exit(struct proc *p, struct proc *p1) +sched_exit(struct proc *p, struct thread *td) { - sched_exit_kse(FIRST_KSE_IN_PROC(p), FIRST_KSE_IN_PROC(p1)); - sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), FIRST_KSEGRP_IN_PROC(p1)); - sched_exit_thread(FIRST_THREAD_IN_PROC(p), FIRST_THREAD_IN_PROC(p1)); + sched_exit_kse(FIRST_KSE_IN_PROC(p), td); + sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td); + sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); } void -sched_exit_kse(struct kse *ke, struct kse *child) +sched_exit_kse(struct kse *ke, struct thread *child) { } void -sched_exit_ksegrp(struct ksegrp *kg, struct ksegrp *child) +sched_exit_ksegrp(struct ksegrp *kg, struct thread *childtd) { mtx_assert(&sched_lock, MA_OWNED); - kg->kg_estcpu = ESTCPULIM(kg->kg_estcpu + child->kg_estcpu); + kg->kg_estcpu = ESTCPULIM(kg->kg_estcpu + childtd->td_ksegrp->kg_estcpu); } void sched_exit_thread(struct thread *td, struct thread *child) { if ((child->td_proc->p_flag & P_NOLOAD) == 0) sched_tdcnt--; } void -sched_fork(struct proc *p, struct proc *p1) +sched_fork(struct thread *td, struct proc *p1) { - sched_fork_kse(FIRST_KSE_IN_PROC(p), FIRST_KSE_IN_PROC(p1)); - sched_fork_ksegrp(FIRST_KSEGRP_IN_PROC(p), FIRST_KSEGRP_IN_PROC(p1)); - sched_fork_thread(FIRST_THREAD_IN_PROC(p), FIRST_THREAD_IN_PROC(p1)); + sched_fork_kse(td, FIRST_KSE_IN_PROC(p1)); + sched_fork_ksegrp(td, FIRST_KSEGRP_IN_PROC(p1)); + sched_fork_thread(td, FIRST_THREAD_IN_PROC(p1)); } void -sched_fork_kse(struct kse *ke, struct kse *child) +sched_fork_kse(struct thread *td, struct kse *child) { child->ke_sched->ske_cpticks = 0; } void -sched_fork_ksegrp(struct ksegrp *kg, struct ksegrp *child) +sched_fork_ksegrp(struct thread *td, struct ksegrp *child) { mtx_assert(&sched_lock, MA_OWNED); - child->kg_estcpu = kg->kg_estcpu; + child->kg_estcpu = td->td_ksegrp->kg_estcpu; } void sched_fork_thread(struct thread *td, struct thread *child) { } void sched_nice(struct proc *p, int nice) { struct ksegrp *kg; PROC_LOCK_ASSERT(p, MA_OWNED); mtx_assert(&sched_lock, MA_OWNED); p->p_nice = nice; FOREACH_KSEGRP_IN_PROC(p, kg) { resetpriority(kg); } } void sched_class(struct ksegrp *kg, int class) { mtx_assert(&sched_lock, MA_OWNED); kg->kg_pri_class = class; } /* * Adjust the priority of a thread. * This may include moving the thread within the KSEGRP, * changing the assignment of a kse to the thread, * and moving a KSE in the system run queue. */ void sched_prio(struct thread *td, u_char prio) { mtx_assert(&sched_lock, MA_OWNED); if (TD_ON_RUNQ(td)) { adjustrunqueue(td, prio); } else { td->td_priority = prio; } } void sched_sleep(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); td->td_ksegrp->kg_slptime = 0; td->td_base_pri = td->td_priority; } void sched_switch(struct thread *td, struct thread *newtd) { struct kse *ke; struct proc *p; ke = td->td_kse; p = td->td_proc; mtx_assert(&sched_lock, MA_OWNED); KASSERT((ke->ke_state == KES_THREAD), ("sched_switch: kse state?")); if ((p->p_flag & P_NOLOAD) == 0) sched_tdcnt--; if (newtd != NULL && (newtd->td_proc->p_flag & P_NOLOAD) == 0) sched_tdcnt++; td->td_lastcpu = td->td_oncpu; td->td_last_kse = ke; td->td_flags &= ~TDF_NEEDRESCHED; td->td_pflags &= ~TDP_OWEPREEMPT; td->td_oncpu = NOCPU; /* * At the last moment, if this thread is still marked RUNNING, * then put it back on the run queue as it has not been suspended * or stopped or any thing else similar. We never put the idle * threads on the run queue, however. */ if (td == PCPU_GET(idlethread)) TD_SET_CAN_RUN(td); else if (TD_IS_RUNNING(td)) { /* Put us back on the run queue (kse and all). */ setrunqueue(td); } else if (p->p_flag & P_SA) { /* * We will not be on the run queue. So we must be * sleeping or similar. As it's available, * someone else can use the KSE if they need it. */ kse_reassign(ke); } if (newtd == NULL) newtd = choosethread(); if (td != newtd) cpu_switch(td, newtd); sched_lock.mtx_lock = (uintptr_t)td; td->td_oncpu = PCPU_GET(cpuid); } void sched_wakeup(struct thread *td) { struct ksegrp *kg; mtx_assert(&sched_lock, MA_OWNED); kg = td->td_ksegrp; if (kg->kg_slptime > 1) updatepri(kg); kg->kg_slptime = 0; setrunqueue(td); } void sched_add(struct thread *td) { struct kse *ke; ke = td->td_kse; mtx_assert(&sched_lock, MA_OWNED); KASSERT((ke->ke_thread != NULL), ("sched_add: No thread on KSE")); KASSERT((ke->ke_thread->td_kse != NULL), ("sched_add: No KSE on thread")); KASSERT(ke->ke_state != KES_ONRUNQ, ("sched_add: kse %p (%s) already in run queue", ke, ke->ke_proc->p_comm)); KASSERT(ke->ke_proc->p_sflag & PS_INMEM, ("sched_add: process swapped out")); #ifdef SMP /* * Only try to preempt if the thread is unpinned or pinned to the * current CPU. */ if (KSE_CAN_MIGRATE(ke) || ke->ke_runq == &runq_pcpu[PCPU_GET(cpuid)]) #endif if (maybe_preempt(td)) return; ke->ke_ksegrp->kg_runq_kses++; ke->ke_state = KES_ONRUNQ; #ifdef SMP if (KSE_CAN_MIGRATE(ke)) { CTR1(KTR_4BSD, "adding kse:%p to gbl runq", ke); ke->ke_runq = &runq; } else { CTR1(KTR_4BSD, "adding kse:%p to pcpu runq", ke); if (!SKE_RUNQ_PCPU(ke)) ke->ke_runq = &runq_pcpu[PCPU_GET(cpuid)]; } #else ke->ke_runq = &runq; #endif if ((td->td_proc->p_flag & P_NOLOAD) == 0) sched_tdcnt++; runq_add(ke->ke_runq, ke); maybe_resched(td); } void sched_rem(struct thread *td) { struct kse *ke; ke = td->td_kse; KASSERT(ke->ke_proc->p_sflag & PS_INMEM, ("sched_rem: process swapped out")); KASSERT((ke->ke_state == KES_ONRUNQ), ("sched_rem: KSE not on run queue")); mtx_assert(&sched_lock, MA_OWNED); if ((td->td_proc->p_flag & P_NOLOAD) == 0) sched_tdcnt--; runq_remove(ke->ke_sched->ske_runq, ke); ke->ke_state = KES_THREAD; ke->ke_ksegrp->kg_runq_kses--; } struct kse * sched_choose(void) { struct kse *ke; struct runq *rq; #ifdef SMP struct kse *kecpu; rq = &runq; ke = runq_choose(&runq); kecpu = runq_choose(&runq_pcpu[PCPU_GET(cpuid)]); if (ke == NULL || (kecpu != NULL && kecpu->ke_thread->td_priority < ke->ke_thread->td_priority)) { CTR2(KTR_4BSD, "choosing kse %p from pcpu runq %d", kecpu, PCPU_GET(cpuid)); ke = kecpu; rq = &runq_pcpu[PCPU_GET(cpuid)]; } else { CTR1(KTR_4BSD, "choosing kse %p from main runq", ke); } #else rq = &runq; ke = runq_choose(&runq); #endif if (ke != NULL) { runq_remove(rq, ke); ke->ke_state = KES_THREAD; KASSERT((ke->ke_thread != NULL), ("sched_choose: No thread on KSE")); KASSERT((ke->ke_thread->td_kse != NULL), ("sched_choose: No KSE on thread")); KASSERT(ke->ke_proc->p_sflag & PS_INMEM, ("sched_choose: process swapped out")); } return (ke); } void sched_userret(struct thread *td) { struct ksegrp *kg; /* * XXX we cheat slightly on the locking here to avoid locking in * the usual case. Setting td_priority here is essentially an * incomplete workaround for not setting it properly elsewhere. * Now that some interrupt handlers are threads, not setting it * properly elsewhere can clobber it in the window between setting * it here and returning to user mode, so don't waste time setting * it perfectly here. */ kg = td->td_ksegrp; if (td->td_priority != kg->kg_user_pri) { mtx_lock_spin(&sched_lock); td->td_priority = kg->kg_user_pri; mtx_unlock_spin(&sched_lock); } } void sched_bind(struct thread *td, int cpu) { struct kse *ke; mtx_assert(&sched_lock, MA_OWNED); KASSERT(TD_IS_RUNNING(td), ("sched_bind: cannot bind non-running thread")); ke = td->td_kse; ke->ke_flags |= KEF_BOUND; #ifdef SMP ke->ke_runq = &runq_pcpu[cpu]; if (PCPU_GET(cpuid) == cpu) return; ke->ke_state = KES_THREAD; mi_switch(SW_VOL, NULL); #endif } void sched_unbind(struct thread* td) { mtx_assert(&sched_lock, MA_OWNED); td->td_kse->ke_flags &= ~KEF_BOUND; } int sched_load(void) { return (sched_tdcnt); } int sched_sizeof_kse(void) { return (sizeof(struct kse) + sizeof(struct ke_sched)); } int sched_sizeof_ksegrp(void) { return (sizeof(struct ksegrp)); } int sched_sizeof_proc(void) { return (sizeof(struct proc)); } int sched_sizeof_thread(void) { return (sizeof(struct thread)); } fixpt_t sched_pctcpu(struct thread *td) { struct kse *ke; ke = td->td_kse; if (ke == NULL) ke = td->td_last_kse; if (ke) return (ke->ke_pctcpu); return (0); } Index: head/sys/kern/sched_ule.c =================================================================== --- head/sys/kern/sched_ule.c (revision 132371) +++ head/sys/kern/sched_ule.c (revision 132372) @@ -1,1775 +1,1778 @@ /*- * Copyright (c) 2002-2003, Jeffrey Roberson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef KTRACE #include #include #endif #include #include #define KTR_ULE KTR_NFS /* decay 95% of `p_pctcpu' in 60 seconds; see CCPU_SHIFT before changing */ /* XXX This is bogus compatability crap for ps */ static fixpt_t ccpu = 0.95122942450071400909 * FSCALE; /* exp(-1/20) */ SYSCTL_INT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0, ""); static void sched_setup(void *dummy); SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL) static SYSCTL_NODE(_kern, OID_AUTO, sched, CTLFLAG_RW, 0, "SCHED"); #define ULE_NAME "ule" #define ULE_NAME_LEN 3 SYSCTL_STRING(_kern_sched, OID_AUTO, name, CTLFLAG_RD, ULE_NAME, ULE_NAME_LEN, "System is using the ULE scheduler"); static int slice_min = 1; SYSCTL_INT(_kern_sched, OID_AUTO, slice_min, CTLFLAG_RW, &slice_min, 0, ""); static int slice_max = 10; SYSCTL_INT(_kern_sched, OID_AUTO, slice_max, CTLFLAG_RW, &slice_max, 0, ""); int realstathz; int tickincr = 1; /* * These datastructures are allocated within their parent datastructure but * are scheduler specific. */ struct ke_sched { int ske_slice; struct runq *ske_runq; /* The following variables are only used for pctcpu calculation */ int ske_ltick; /* Last tick that we were running on */ int ske_ftick; /* First tick that we were running on */ int ske_ticks; /* Tick count */ /* CPU that we have affinity for. */ u_char ske_cpu; }; #define ke_slice ke_sched->ske_slice #define ke_runq ke_sched->ske_runq #define ke_ltick ke_sched->ske_ltick #define ke_ftick ke_sched->ske_ftick #define ke_ticks ke_sched->ske_ticks #define ke_cpu ke_sched->ske_cpu #define ke_assign ke_procq.tqe_next #define KEF_ASSIGNED KEF_SCHED0 /* KSE is being migrated. */ #define KEF_BOUND KEF_SCHED1 /* KSE can not migrate. */ struct kg_sched { int skg_slptime; /* Number of ticks we vol. slept */ int skg_runtime; /* Number of ticks we were running */ }; #define kg_slptime kg_sched->skg_slptime #define kg_runtime kg_sched->skg_runtime struct td_sched { int std_slptime; }; #define td_slptime td_sched->std_slptime struct td_sched td_sched; struct ke_sched ke_sched; struct kg_sched kg_sched; struct ke_sched *kse0_sched = &ke_sched; struct kg_sched *ksegrp0_sched = &kg_sched; struct p_sched *proc0_sched = NULL; struct td_sched *thread0_sched = &td_sched; /* * The priority is primarily determined by the interactivity score. Thus, we * give lower(better) priorities to kse groups that use less CPU. The nice * value is then directly added to this to allow nice to have some effect * on latency. * * PRI_RANGE: Total priority range for timeshare threads. * PRI_NRESV: Number of nice values. * PRI_BASE: The start of the dynamic range. */ #define SCHED_PRI_RANGE (PRI_MAX_TIMESHARE - PRI_MIN_TIMESHARE + 1) #define SCHED_PRI_NRESV ((PRIO_MAX - PRIO_MIN) + 1) #define SCHED_PRI_NHALF (SCHED_PRI_NRESV / 2) #define SCHED_PRI_BASE (PRI_MIN_TIMESHARE) #define SCHED_PRI_INTERACT(score) \ ((score) * SCHED_PRI_RANGE / SCHED_INTERACT_MAX) /* * These determine the interactivity of a process. * * SLP_RUN_MAX: Maximum amount of sleep time + run time we'll accumulate * before throttling back. * SLP_RUN_FORK: Maximum slp+run time to inherit at fork time. * INTERACT_MAX: Maximum interactivity value. Smaller is better. * INTERACT_THRESH: Threshhold for placement on the current runq. */ #define SCHED_SLP_RUN_MAX ((hz * 5) << 10) #define SCHED_SLP_RUN_FORK ((hz / 2) << 10) #define SCHED_INTERACT_MAX (100) #define SCHED_INTERACT_HALF (SCHED_INTERACT_MAX / 2) #define SCHED_INTERACT_THRESH (30) /* * These parameters and macros determine the size of the time slice that is * granted to each thread. * * SLICE_MIN: Minimum time slice granted, in units of ticks. * SLICE_MAX: Maximum time slice granted. * SLICE_RANGE: Range of available time slices scaled by hz. * SLICE_SCALE: The number slices granted per val in the range of [0, max]. * SLICE_NICE: Determine the amount of slice granted to a scaled nice. * SLICE_NTHRESH: The nice cutoff point for slice assignment. */ #define SCHED_SLICE_MIN (slice_min) #define SCHED_SLICE_MAX (slice_max) #define SCHED_SLICE_INTERACTIVE (slice_max) #define SCHED_SLICE_NTHRESH (SCHED_PRI_NHALF - 1) #define SCHED_SLICE_RANGE (SCHED_SLICE_MAX - SCHED_SLICE_MIN + 1) #define SCHED_SLICE_SCALE(val, max) (((val) * SCHED_SLICE_RANGE) / (max)) #define SCHED_SLICE_NICE(nice) \ (SCHED_SLICE_MAX - SCHED_SLICE_SCALE((nice), SCHED_SLICE_NTHRESH)) /* * This macro determines whether or not the kse belongs on the current or * next run queue. */ #define SCHED_INTERACTIVE(kg) \ (sched_interact_score(kg) < SCHED_INTERACT_THRESH) #define SCHED_CURR(kg, ke) \ (ke->ke_thread->td_priority < kg->kg_user_pri || \ SCHED_INTERACTIVE(kg)) /* * Cpu percentage computation macros and defines. * * SCHED_CPU_TIME: Number of seconds to average the cpu usage across. * SCHED_CPU_TICKS: Number of hz ticks to average the cpu usage across. */ #define SCHED_CPU_TIME 10 #define SCHED_CPU_TICKS (hz * SCHED_CPU_TIME) /* * kseq - per processor runqs and statistics. */ struct kseq { struct runq ksq_idle; /* Queue of IDLE threads. */ struct runq ksq_timeshare[2]; /* Run queues for !IDLE. */ struct runq *ksq_next; /* Next timeshare queue. */ struct runq *ksq_curr; /* Current queue. */ int ksq_load_timeshare; /* Load for timeshare. */ int ksq_load; /* Aggregate load. */ short ksq_nice[SCHED_PRI_NRESV]; /* KSEs in each nice bin. */ short ksq_nicemin; /* Least nice. */ #ifdef SMP int ksq_transferable; LIST_ENTRY(kseq) ksq_siblings; /* Next in kseq group. */ struct kseq_group *ksq_group; /* Our processor group. */ volatile struct kse *ksq_assigned; /* assigned by another CPU. */ #else int ksq_sysload; /* For loadavg, !ITHD load. */ #endif }; #ifdef SMP /* * kseq groups are groups of processors which can cheaply share threads. When * one processor in the group goes idle it will check the runqs of the other * processors in its group prior to halting and waiting for an interrupt. * These groups are suitable for SMT (Symetric Multi-Threading) and not NUMA. * In a numa environment we'd want an idle bitmap per group and a two tiered * load balancer. */ struct kseq_group { int ksg_cpus; /* Count of CPUs in this kseq group. */ cpumask_t ksg_cpumask; /* Mask of cpus in this group. */ cpumask_t ksg_idlemask; /* Idle cpus in this group. */ cpumask_t ksg_mask; /* Bit mask for first cpu. */ int ksg_load; /* Total load of this group. */ int ksg_transferable; /* Transferable load of this group. */ LIST_HEAD(, kseq) ksg_members; /* Linked list of all members. */ }; #endif /* * One kse queue per processor. */ #ifdef SMP static cpumask_t kseq_idle; static int ksg_maxid; static struct kseq kseq_cpu[MAXCPU]; static struct kseq_group kseq_groups[MAXCPU]; static int bal_tick; static int gbal_tick; #define KSEQ_SELF() (&kseq_cpu[PCPU_GET(cpuid)]) #define KSEQ_CPU(x) (&kseq_cpu[(x)]) #define KSEQ_ID(x) ((x) - kseq_cpu) #define KSEQ_GROUP(x) (&kseq_groups[(x)]) #else /* !SMP */ static struct kseq kseq_cpu; #define KSEQ_SELF() (&kseq_cpu) #define KSEQ_CPU(x) (&kseq_cpu) #endif static void sched_add_internal(struct thread *td, int preemptive); static void sched_slice(struct kse *ke); static void sched_priority(struct ksegrp *kg); static int sched_interact_score(struct ksegrp *kg); static void sched_interact_update(struct ksegrp *kg); static void sched_interact_fork(struct ksegrp *kg); static void sched_pctcpu_update(struct kse *ke); /* Operations on per processor queues */ static struct kse * kseq_choose(struct kseq *kseq); static void kseq_setup(struct kseq *kseq); static void kseq_load_add(struct kseq *kseq, struct kse *ke); static void kseq_load_rem(struct kseq *kseq, struct kse *ke); static __inline void kseq_runq_add(struct kseq *kseq, struct kse *ke); static __inline void kseq_runq_rem(struct kseq *kseq, struct kse *ke); static void kseq_nice_add(struct kseq *kseq, int nice); static void kseq_nice_rem(struct kseq *kseq, int nice); void kseq_print(int cpu); #ifdef SMP static int kseq_transfer(struct kseq *ksq, struct kse *ke, int class); static struct kse *runq_steal(struct runq *rq); static void sched_balance(void); static void sched_balance_groups(void); static void sched_balance_group(struct kseq_group *ksg); static void sched_balance_pair(struct kseq *high, struct kseq *low); static void kseq_move(struct kseq *from, int cpu); static int kseq_idled(struct kseq *kseq); static void kseq_notify(struct kse *ke, int cpu); static void kseq_assign(struct kseq *); static struct kse *kseq_steal(struct kseq *kseq, int stealidle); /* * On P4 Xeons the round-robin interrupt delivery is broken. As a result of * this, we can't pin interrupts to the cpu that they were delivered to, * otherwise all ithreads only run on CPU 0. */ #ifdef __i386__ #define KSE_CAN_MIGRATE(ke, class) \ ((ke)->ke_thread->td_pinned == 0 && ((ke)->ke_flags & KEF_BOUND) == 0) #else /* !__i386__ */ #define KSE_CAN_MIGRATE(ke, class) \ ((class) != PRI_ITHD && (ke)->ke_thread->td_pinned == 0 && \ ((ke)->ke_flags & KEF_BOUND) == 0) #endif /* !__i386__ */ #endif void kseq_print(int cpu) { struct kseq *kseq; int i; kseq = KSEQ_CPU(cpu); printf("kseq:\n"); printf("\tload: %d\n", kseq->ksq_load); printf("\tload TIMESHARE: %d\n", kseq->ksq_load_timeshare); #ifdef SMP printf("\tload transferable: %d\n", kseq->ksq_transferable); #endif printf("\tnicemin:\t%d\n", kseq->ksq_nicemin); printf("\tnice counts:\n"); for (i = 0; i < SCHED_PRI_NRESV; i++) if (kseq->ksq_nice[i]) printf("\t\t%d = %d\n", i - SCHED_PRI_NHALF, kseq->ksq_nice[i]); } static __inline void kseq_runq_add(struct kseq *kseq, struct kse *ke) { #ifdef SMP if (KSE_CAN_MIGRATE(ke, PRI_BASE(ke->ke_ksegrp->kg_pri_class))) { kseq->ksq_transferable++; kseq->ksq_group->ksg_transferable++; } #endif runq_add(ke->ke_runq, ke); } static __inline void kseq_runq_rem(struct kseq *kseq, struct kse *ke) { #ifdef SMP if (KSE_CAN_MIGRATE(ke, PRI_BASE(ke->ke_ksegrp->kg_pri_class))) { kseq->ksq_transferable--; kseq->ksq_group->ksg_transferable--; } #endif runq_remove(ke->ke_runq, ke); } static void kseq_load_add(struct kseq *kseq, struct kse *ke) { int class; mtx_assert(&sched_lock, MA_OWNED); class = PRI_BASE(ke->ke_ksegrp->kg_pri_class); if (class == PRI_TIMESHARE) kseq->ksq_load_timeshare++; kseq->ksq_load++; if (class != PRI_ITHD && (ke->ke_proc->p_flag & P_NOLOAD) == 0) #ifdef SMP kseq->ksq_group->ksg_load++; #else kseq->ksq_sysload++; #endif if (ke->ke_ksegrp->kg_pri_class == PRI_TIMESHARE) CTR6(KTR_ULE, "Add kse %p to %p (slice: %d, pri: %d, nice: %d(%d))", ke, ke->ke_runq, ke->ke_slice, ke->ke_thread->td_priority, ke->ke_proc->p_nice, kseq->ksq_nicemin); if (ke->ke_ksegrp->kg_pri_class == PRI_TIMESHARE) kseq_nice_add(kseq, ke->ke_proc->p_nice); } static void kseq_load_rem(struct kseq *kseq, struct kse *ke) { int class; mtx_assert(&sched_lock, MA_OWNED); class = PRI_BASE(ke->ke_ksegrp->kg_pri_class); if (class == PRI_TIMESHARE) kseq->ksq_load_timeshare--; if (class != PRI_ITHD && (ke->ke_proc->p_flag & P_NOLOAD) == 0) #ifdef SMP kseq->ksq_group->ksg_load--; #else kseq->ksq_sysload--; #endif kseq->ksq_load--; ke->ke_runq = NULL; if (ke->ke_ksegrp->kg_pri_class == PRI_TIMESHARE) kseq_nice_rem(kseq, ke->ke_proc->p_nice); } static void kseq_nice_add(struct kseq *kseq, int nice) { mtx_assert(&sched_lock, MA_OWNED); /* Normalize to zero. */ kseq->ksq_nice[nice + SCHED_PRI_NHALF]++; if (nice < kseq->ksq_nicemin || kseq->ksq_load_timeshare == 1) kseq->ksq_nicemin = nice; } static void kseq_nice_rem(struct kseq *kseq, int nice) { int n; mtx_assert(&sched_lock, MA_OWNED); /* Normalize to zero. */ n = nice + SCHED_PRI_NHALF; kseq->ksq_nice[n]--; KASSERT(kseq->ksq_nice[n] >= 0, ("Negative nice count.")); /* * If this wasn't the smallest nice value or there are more in * this bucket we can just return. Otherwise we have to recalculate * the smallest nice. */ if (nice != kseq->ksq_nicemin || kseq->ksq_nice[n] != 0 || kseq->ksq_load_timeshare == 0) return; for (; n < SCHED_PRI_NRESV; n++) if (kseq->ksq_nice[n]) { kseq->ksq_nicemin = n - SCHED_PRI_NHALF; return; } } #ifdef SMP /* * sched_balance is a simple CPU load balancing algorithm. It operates by * finding the least loaded and most loaded cpu and equalizing their load * by migrating some processes. * * Dealing only with two CPUs at a time has two advantages. Firstly, most * installations will only have 2 cpus. Secondly, load balancing too much at * once can have an unpleasant effect on the system. The scheduler rarely has * enough information to make perfect decisions. So this algorithm chooses * algorithm simplicity and more gradual effects on load in larger systems. * * It could be improved by considering the priorities and slices assigned to * each task prior to balancing them. There are many pathological cases with * any approach and so the semi random algorithm below may work as well as any. * */ static void sched_balance(void) { struct kseq_group *high; struct kseq_group *low; struct kseq_group *ksg; int cnt; int i; if (smp_started == 0) goto out; low = high = NULL; i = random() % (ksg_maxid + 1); for (cnt = 0; cnt <= ksg_maxid; cnt++) { ksg = KSEQ_GROUP(i); /* * Find the CPU with the highest load that has some * threads to transfer. */ if ((high == NULL || ksg->ksg_load > high->ksg_load) && ksg->ksg_transferable) high = ksg; if (low == NULL || ksg->ksg_load < low->ksg_load) low = ksg; if (++i > ksg_maxid) i = 0; } if (low != NULL && high != NULL && high != low) sched_balance_pair(LIST_FIRST(&high->ksg_members), LIST_FIRST(&low->ksg_members)); out: bal_tick = ticks + (random() % (hz * 2)); } static void sched_balance_groups(void) { int i; mtx_assert(&sched_lock, MA_OWNED); if (smp_started) for (i = 0; i <= ksg_maxid; i++) sched_balance_group(KSEQ_GROUP(i)); gbal_tick = ticks + (random() % (hz * 2)); } static void sched_balance_group(struct kseq_group *ksg) { struct kseq *kseq; struct kseq *high; struct kseq *low; int load; if (ksg->ksg_transferable == 0) return; low = NULL; high = NULL; LIST_FOREACH(kseq, &ksg->ksg_members, ksq_siblings) { load = kseq->ksq_load; if (high == NULL || load > high->ksq_load) high = kseq; if (low == NULL || load < low->ksq_load) low = kseq; } if (high != NULL && low != NULL && high != low) sched_balance_pair(high, low); } static void sched_balance_pair(struct kseq *high, struct kseq *low) { int transferable; int high_load; int low_load; int move; int diff; int i; /* * If we're transfering within a group we have to use this specific * kseq's transferable count, otherwise we can steal from other members * of the group. */ if (high->ksq_group == low->ksq_group) { transferable = high->ksq_transferable; high_load = high->ksq_load; low_load = low->ksq_load; } else { transferable = high->ksq_group->ksg_transferable; high_load = high->ksq_group->ksg_load; low_load = low->ksq_group->ksg_load; } if (transferable == 0) return; /* * Determine what the imbalance is and then adjust that to how many * kses we actually have to give up (transferable). */ diff = high_load - low_load; move = diff / 2; if (diff & 0x1) move++; move = min(move, transferable); for (i = 0; i < move; i++) kseq_move(high, KSEQ_ID(low)); return; } static void kseq_move(struct kseq *from, int cpu) { struct kseq *kseq; struct kseq *to; struct kse *ke; kseq = from; to = KSEQ_CPU(cpu); ke = kseq_steal(kseq, 1); if (ke == NULL) { struct kseq_group *ksg; ksg = kseq->ksq_group; LIST_FOREACH(kseq, &ksg->ksg_members, ksq_siblings) { if (kseq == from || kseq->ksq_transferable == 0) continue; ke = kseq_steal(kseq, 1); break; } if (ke == NULL) panic("kseq_move: No KSEs available with a " "transferable count of %d\n", ksg->ksg_transferable); } if (kseq == to) return; ke->ke_state = KES_THREAD; kseq_runq_rem(kseq, ke); kseq_load_rem(kseq, ke); kseq_notify(ke, cpu); } static int kseq_idled(struct kseq *kseq) { struct kseq_group *ksg; struct kseq *steal; struct kse *ke; ksg = kseq->ksq_group; /* * If we're in a cpu group, try and steal kses from another cpu in * the group before idling. */ if (ksg->ksg_cpus > 1 && ksg->ksg_transferable) { LIST_FOREACH(steal, &ksg->ksg_members, ksq_siblings) { if (steal == kseq || steal->ksq_transferable == 0) continue; ke = kseq_steal(steal, 0); if (ke == NULL) continue; ke->ke_state = KES_THREAD; kseq_runq_rem(steal, ke); kseq_load_rem(steal, ke); ke->ke_cpu = PCPU_GET(cpuid); sched_add_internal(ke->ke_thread, 0); return (0); } } /* * We only set the idled bit when all of the cpus in the group are * idle. Otherwise we could get into a situation where a KSE bounces * back and forth between two idle cores on seperate physical CPUs. */ ksg->ksg_idlemask |= PCPU_GET(cpumask); if (ksg->ksg_idlemask != ksg->ksg_cpumask) return (1); atomic_set_int(&kseq_idle, ksg->ksg_mask); return (1); } static void kseq_assign(struct kseq *kseq) { struct kse *nke; struct kse *ke; do { (volatile struct kse *)ke = kseq->ksq_assigned; } while(!atomic_cmpset_ptr(&kseq->ksq_assigned, ke, NULL)); for (; ke != NULL; ke = nke) { nke = ke->ke_assign; ke->ke_flags &= ~KEF_ASSIGNED; sched_add_internal(ke->ke_thread, 0); } } static void kseq_notify(struct kse *ke, int cpu) { struct kseq *kseq; struct thread *td; struct pcpu *pcpu; ke->ke_cpu = cpu; ke->ke_flags |= KEF_ASSIGNED; kseq = KSEQ_CPU(cpu); /* * Place a KSE on another cpu's queue and force a resched. */ do { (volatile struct kse *)ke->ke_assign = kseq->ksq_assigned; } while(!atomic_cmpset_ptr(&kseq->ksq_assigned, ke->ke_assign, ke)); pcpu = pcpu_find(cpu); td = pcpu->pc_curthread; if (ke->ke_thread->td_priority < td->td_priority || td == pcpu->pc_idlethread) { td->td_flags |= TDF_NEEDRESCHED; ipi_selected(1 << cpu, IPI_AST); } } static struct kse * runq_steal(struct runq *rq) { struct rqhead *rqh; struct rqbits *rqb; struct kse *ke; int word; int bit; mtx_assert(&sched_lock, MA_OWNED); rqb = &rq->rq_status; for (word = 0; word < RQB_LEN; word++) { if (rqb->rqb_bits[word] == 0) continue; for (bit = 0; bit < RQB_BPW; bit++) { if ((rqb->rqb_bits[word] & (1ul << bit)) == 0) continue; rqh = &rq->rq_queues[bit + (word << RQB_L2BPW)]; TAILQ_FOREACH(ke, rqh, ke_procq) { if (KSE_CAN_MIGRATE(ke, PRI_BASE(ke->ke_ksegrp->kg_pri_class))) return (ke); } } } return (NULL); } static struct kse * kseq_steal(struct kseq *kseq, int stealidle) { struct kse *ke; /* * Steal from next first to try to get a non-interactive task that * may not have run for a while. */ if ((ke = runq_steal(kseq->ksq_next)) != NULL) return (ke); if ((ke = runq_steal(kseq->ksq_curr)) != NULL) return (ke); if (stealidle) return (runq_steal(&kseq->ksq_idle)); return (NULL); } int kseq_transfer(struct kseq *kseq, struct kse *ke, int class) { struct kseq_group *ksg; int cpu; if (smp_started == 0) return (0); cpu = 0; ksg = kseq->ksq_group; /* * If there are any idle groups, give them our extra load. The * threshold at which we start to reassign kses has a large impact * on the overall performance of the system. Tuned too high and * some CPUs may idle. Too low and there will be excess migration * and context switches. */ if (ksg->ksg_load > (ksg->ksg_cpus * 2) && kseq_idle) { /* * Multiple cpus could find this bit simultaneously * but the race shouldn't be terrible. */ cpu = ffs(kseq_idle); if (cpu) atomic_clear_int(&kseq_idle, 1 << (cpu - 1)); } /* * If another cpu in this group has idled, assign a thread over * to them after checking to see if there are idled groups. */ if (cpu == 0 && kseq->ksq_load > 1 && ksg->ksg_idlemask) { cpu = ffs(ksg->ksg_idlemask); if (cpu) ksg->ksg_idlemask &= ~(1 << (cpu - 1)); } /* * Now that we've found an idle CPU, migrate the thread. */ if (cpu) { cpu--; ke->ke_runq = NULL; kseq_notify(ke, cpu); return (1); } return (0); } #endif /* SMP */ /* * Pick the highest priority task we have and return it. */ static struct kse * kseq_choose(struct kseq *kseq) { struct kse *ke; struct runq *swap; mtx_assert(&sched_lock, MA_OWNED); swap = NULL; for (;;) { ke = runq_choose(kseq->ksq_curr); if (ke == NULL) { /* * We already swapped once and didn't get anywhere. */ if (swap) break; swap = kseq->ksq_curr; kseq->ksq_curr = kseq->ksq_next; kseq->ksq_next = swap; continue; } /* * If we encounter a slice of 0 the kse is in a * TIMESHARE kse group and its nice was too far out * of the range that receives slices. */ if (ke->ke_slice == 0) { runq_remove(ke->ke_runq, ke); sched_slice(ke); ke->ke_runq = kseq->ksq_next; runq_add(ke->ke_runq, ke); continue; } return (ke); } return (runq_choose(&kseq->ksq_idle)); } static void kseq_setup(struct kseq *kseq) { runq_init(&kseq->ksq_timeshare[0]); runq_init(&kseq->ksq_timeshare[1]); runq_init(&kseq->ksq_idle); kseq->ksq_curr = &kseq->ksq_timeshare[0]; kseq->ksq_next = &kseq->ksq_timeshare[1]; kseq->ksq_load = 0; kseq->ksq_load_timeshare = 0; } static void sched_setup(void *dummy) { #ifdef SMP int balance_groups; int i; #endif slice_min = (hz/100); /* 10ms */ slice_max = (hz/7); /* ~140ms */ #ifdef SMP balance_groups = 0; /* * Initialize the kseqs. */ for (i = 0; i < MAXCPU; i++) { struct kseq *ksq; ksq = &kseq_cpu[i]; ksq->ksq_assigned = NULL; kseq_setup(&kseq_cpu[i]); } if (smp_topology == NULL) { struct kseq_group *ksg; struct kseq *ksq; for (i = 0; i < MAXCPU; i++) { ksq = &kseq_cpu[i]; ksg = &kseq_groups[i]; /* * Setup a kseq group with one member. */ ksq->ksq_transferable = 0; ksq->ksq_group = ksg; ksg->ksg_cpus = 1; ksg->ksg_idlemask = 0; ksg->ksg_cpumask = ksg->ksg_mask = 1 << i; ksg->ksg_load = 0; ksg->ksg_transferable = 0; LIST_INIT(&ksg->ksg_members); LIST_INSERT_HEAD(&ksg->ksg_members, ksq, ksq_siblings); } } else { struct kseq_group *ksg; struct cpu_group *cg; int j; for (i = 0; i < smp_topology->ct_count; i++) { cg = &smp_topology->ct_group[i]; ksg = &kseq_groups[i]; /* * Initialize the group. */ ksg->ksg_idlemask = 0; ksg->ksg_load = 0; ksg->ksg_transferable = 0; ksg->ksg_cpus = cg->cg_count; ksg->ksg_cpumask = cg->cg_mask; LIST_INIT(&ksg->ksg_members); /* * Find all of the group members and add them. */ for (j = 0; j < MAXCPU; j++) { if ((cg->cg_mask & (1 << j)) != 0) { if (ksg->ksg_mask == 0) ksg->ksg_mask = 1 << j; kseq_cpu[j].ksq_transferable = 0; kseq_cpu[j].ksq_group = ksg; LIST_INSERT_HEAD(&ksg->ksg_members, &kseq_cpu[j], ksq_siblings); } } if (ksg->ksg_cpus > 1) balance_groups = 1; } ksg_maxid = smp_topology->ct_count - 1; } /* * Stagger the group and global load balancer so they do not * interfere with each other. */ bal_tick = ticks + hz; if (balance_groups) gbal_tick = ticks + (hz / 2); #else kseq_setup(KSEQ_SELF()); #endif mtx_lock_spin(&sched_lock); kseq_load_add(KSEQ_SELF(), &kse0); mtx_unlock_spin(&sched_lock); } /* * Scale the scheduling priority according to the "interactivity" of this * process. */ static void sched_priority(struct ksegrp *kg) { int pri; if (kg->kg_pri_class != PRI_TIMESHARE) return; pri = SCHED_PRI_INTERACT(sched_interact_score(kg)); pri += SCHED_PRI_BASE; pri += kg->kg_proc->p_nice; if (pri > PRI_MAX_TIMESHARE) pri = PRI_MAX_TIMESHARE; else if (pri < PRI_MIN_TIMESHARE) pri = PRI_MIN_TIMESHARE; kg->kg_user_pri = pri; return; } /* * Calculate a time slice based on the properties of the kseg and the runq * that we're on. This is only for PRI_TIMESHARE ksegrps. */ static void sched_slice(struct kse *ke) { struct kseq *kseq; struct ksegrp *kg; kg = ke->ke_ksegrp; kseq = KSEQ_CPU(ke->ke_cpu); /* * Rationale: * KSEs in interactive ksegs get the minimum slice so that we * quickly notice if it abuses its advantage. * * KSEs in non-interactive ksegs are assigned a slice that is * based on the ksegs nice value relative to the least nice kseg * on the run queue for this cpu. * * If the KSE is less nice than all others it gets the maximum * slice and other KSEs will adjust their slice relative to * this when they first expire. * * There is 20 point window that starts relative to the least * nice kse on the run queue. Slice size is determined by * the kse distance from the last nice ksegrp. * * If the kse is outside of the window it will get no slice * and will be reevaluated each time it is selected on the * run queue. The exception to this is nice 0 ksegs when * a nice -20 is running. They are always granted a minimum * slice. */ if (!SCHED_INTERACTIVE(kg)) { int nice; nice = kg->kg_proc->p_nice + (0 - kseq->ksq_nicemin); if (kseq->ksq_load_timeshare == 0 || kg->kg_proc->p_nice < kseq->ksq_nicemin) ke->ke_slice = SCHED_SLICE_MAX; else if (nice <= SCHED_SLICE_NTHRESH) ke->ke_slice = SCHED_SLICE_NICE(nice); else if (kg->kg_proc->p_nice == 0) ke->ke_slice = SCHED_SLICE_MIN; else ke->ke_slice = 0; } else ke->ke_slice = SCHED_SLICE_INTERACTIVE; CTR6(KTR_ULE, "Sliced %p(%d) (nice: %d, nicemin: %d, load: %d, interactive: %d)", ke, ke->ke_slice, kg->kg_proc->p_nice, kseq->ksq_nicemin, kseq->ksq_load_timeshare, SCHED_INTERACTIVE(kg)); return; } /* * This routine enforces a maximum limit on the amount of scheduling history * kept. It is called after either the slptime or runtime is adjusted. * This routine will not operate correctly when slp or run times have been * adjusted to more than double their maximum. */ static void sched_interact_update(struct ksegrp *kg) { int sum; sum = kg->kg_runtime + kg->kg_slptime; if (sum < SCHED_SLP_RUN_MAX) return; /* * If we have exceeded by more than 1/5th then the algorithm below * will not bring us back into range. Dividing by two here forces * us into the range of [3/5 * SCHED_INTERACT_MAX, SCHED_INTERACT_MAX] */ if (sum > (SCHED_SLP_RUN_MAX / 5) * 6) { kg->kg_runtime /= 2; kg->kg_slptime /= 2; return; } kg->kg_runtime = (kg->kg_runtime / 5) * 4; kg->kg_slptime = (kg->kg_slptime / 5) * 4; } static void sched_interact_fork(struct ksegrp *kg) { int ratio; int sum; sum = kg->kg_runtime + kg->kg_slptime; if (sum > SCHED_SLP_RUN_FORK) { ratio = sum / SCHED_SLP_RUN_FORK; kg->kg_runtime /= ratio; kg->kg_slptime /= ratio; } } static int sched_interact_score(struct ksegrp *kg) { int div; if (kg->kg_runtime > kg->kg_slptime) { div = max(1, kg->kg_runtime / SCHED_INTERACT_HALF); return (SCHED_INTERACT_HALF + (SCHED_INTERACT_HALF - (kg->kg_slptime / div))); } if (kg->kg_slptime > kg->kg_runtime) { div = max(1, kg->kg_slptime / SCHED_INTERACT_HALF); return (kg->kg_runtime / div); } /* * This can happen if slptime and runtime are 0. */ return (0); } /* * This is only somewhat accurate since given many processes of the same * priority they will switch when their slices run out, which will be * at most SCHED_SLICE_MAX. */ int sched_rr_interval(void) { return (SCHED_SLICE_MAX); } static void sched_pctcpu_update(struct kse *ke) { /* * Adjust counters and watermark for pctcpu calc. */ if (ke->ke_ltick > ticks - SCHED_CPU_TICKS) { /* * Shift the tick count out so that the divide doesn't * round away our results. */ ke->ke_ticks <<= 10; ke->ke_ticks = (ke->ke_ticks / (ticks - ke->ke_ftick)) * SCHED_CPU_TICKS; ke->ke_ticks >>= 10; } else ke->ke_ticks = 0; ke->ke_ltick = ticks; ke->ke_ftick = ke->ke_ltick - SCHED_CPU_TICKS; } void sched_prio(struct thread *td, u_char prio) { struct kse *ke; ke = td->td_kse; mtx_assert(&sched_lock, MA_OWNED); if (TD_ON_RUNQ(td)) { /* * If the priority has been elevated due to priority * propagation, we may have to move ourselves to a new * queue. We still call adjustrunqueue below in case kse * needs to fix things up. */ if (prio < td->td_priority && ke && (ke->ke_flags & KEF_ASSIGNED) == 0 && ke->ke_runq != KSEQ_CPU(ke->ke_cpu)->ksq_curr) { runq_remove(ke->ke_runq, ke); ke->ke_runq = KSEQ_CPU(ke->ke_cpu)->ksq_curr; runq_add(ke->ke_runq, ke); } adjustrunqueue(td, prio); } else td->td_priority = prio; } void sched_switch(struct thread *td, struct thread *newtd) { struct kse *ke; mtx_assert(&sched_lock, MA_OWNED); ke = td->td_kse; td->td_last_kse = ke; td->td_lastcpu = td->td_oncpu; td->td_oncpu = NOCPU; td->td_flags &= ~TDF_NEEDRESCHED; td->td_pflags &= ~TDP_OWEPREEMPT; /* * If the KSE has been assigned it may be in the process of switching * to the new cpu. This is the case in sched_bind(). */ if ((ke->ke_flags & KEF_ASSIGNED) == 0) { if (td == PCPU_GET(idlethread)) TD_SET_CAN_RUN(td); else if (TD_IS_RUNNING(td)) { kseq_load_rem(KSEQ_CPU(ke->ke_cpu), ke); setrunqueue(td); } else { if (ke->ke_runq) { kseq_load_rem(KSEQ_CPU(ke->ke_cpu), ke); } else if ((td->td_flags & TDF_IDLETD) == 0) kdb_backtrace(); /* * We will not be on the run queue. So we must be * sleeping or similar. */ if (td->td_proc->p_flag & P_SA) kse_reassign(ke); } } if (newtd == NULL) newtd = choosethread(); else kseq_load_add(KSEQ_SELF(), newtd->td_kse); if (td != newtd) cpu_switch(td, newtd); sched_lock.mtx_lock = (uintptr_t)td; td->td_oncpu = PCPU_GET(cpuid); } void sched_nice(struct proc *p, int nice) { struct ksegrp *kg; struct kse *ke; struct thread *td; struct kseq *kseq; PROC_LOCK_ASSERT(p, MA_OWNED); mtx_assert(&sched_lock, MA_OWNED); /* * We need to adjust the nice counts for running KSEs. */ FOREACH_KSEGRP_IN_PROC(p, kg) { if (kg->kg_pri_class == PRI_TIMESHARE) { FOREACH_KSE_IN_GROUP(kg, ke) { if (ke->ke_runq == NULL) continue; kseq = KSEQ_CPU(ke->ke_cpu); kseq_nice_rem(kseq, p->p_nice); kseq_nice_add(kseq, nice); } } } p->p_nice = nice; FOREACH_KSEGRP_IN_PROC(p, kg) { sched_priority(kg); FOREACH_THREAD_IN_GROUP(kg, td) td->td_flags |= TDF_NEEDRESCHED; } } void sched_sleep(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); td->td_slptime = ticks; td->td_base_pri = td->td_priority; CTR2(KTR_ULE, "sleep kse %p (tick: %d)", td->td_kse, td->td_slptime); } void sched_wakeup(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); /* * Let the kseg know how long we slept for. This is because process * interactivity behavior is modeled in the kseg. */ if (td->td_slptime) { struct ksegrp *kg; int hzticks; kg = td->td_ksegrp; hzticks = (ticks - td->td_slptime) << 10; if (hzticks >= SCHED_SLP_RUN_MAX) { kg->kg_slptime = SCHED_SLP_RUN_MAX; kg->kg_runtime = 1; } else { kg->kg_slptime += hzticks; sched_interact_update(kg); } sched_priority(kg); if (td->td_kse) sched_slice(td->td_kse); CTR2(KTR_ULE, "wakeup kse %p (%d ticks)", td->td_kse, hzticks); td->td_slptime = 0; } setrunqueue(td); } /* * Penalize the parent for creating a new child and initialize the child's * priority. */ void -sched_fork(struct proc *p, struct proc *p1) +sched_fork(struct thread *td, struct proc *p1) { mtx_assert(&sched_lock, MA_OWNED); - p1->p_nice = p->p_nice; - sched_fork_ksegrp(FIRST_KSEGRP_IN_PROC(p), FIRST_KSEGRP_IN_PROC(p1)); - sched_fork_kse(FIRST_KSE_IN_PROC(p), FIRST_KSE_IN_PROC(p1)); - sched_fork_thread(FIRST_THREAD_IN_PROC(p), FIRST_THREAD_IN_PROC(p1)); + p1->p_nice = td->td_proc->p_nice; + sched_fork_ksegrp(td, FIRST_KSEGRP_IN_PROC(p1)); + sched_fork_kse(td, FIRST_KSE_IN_PROC(p1)); + sched_fork_thread(td, FIRST_THREAD_IN_PROC(p1)); } void -sched_fork_kse(struct kse *ke, struct kse *child) +sched_fork_kse(struct thread *td, struct kse *child) { + struct kse *ke = td->td_kse; + child->ke_slice = 1; /* Attempt to quickly learn interactivity. */ child->ke_cpu = ke->ke_cpu; child->ke_runq = NULL; /* Grab our parents cpu estimation information. */ child->ke_ticks = ke->ke_ticks; child->ke_ltick = ke->ke_ltick; child->ke_ftick = ke->ke_ftick; } void -sched_fork_ksegrp(struct ksegrp *kg, struct ksegrp *child) +sched_fork_ksegrp(struct thread *td, struct ksegrp *child) { + struct ksegrp *kg = td->td_ksegrp; PROC_LOCK_ASSERT(child->kg_proc, MA_OWNED); child->kg_slptime = kg->kg_slptime; child->kg_runtime = kg->kg_runtime; child->kg_user_pri = kg->kg_user_pri; sched_interact_fork(child); kg->kg_runtime += tickincr << 10; sched_interact_update(kg); CTR6(KTR_ULE, "sched_fork_ksegrp: %d(%d, %d) - %d(%d, %d)", kg->kg_proc->p_pid, kg->kg_slptime, kg->kg_runtime, child->kg_proc->p_pid, child->kg_slptime, child->kg_runtime); } void sched_fork_thread(struct thread *td, struct thread *child) { } void sched_class(struct ksegrp *kg, int class) { struct kseq *kseq; struct kse *ke; int nclass; int oclass; mtx_assert(&sched_lock, MA_OWNED); if (kg->kg_pri_class == class) return; nclass = PRI_BASE(class); oclass = PRI_BASE(kg->kg_pri_class); FOREACH_KSE_IN_GROUP(kg, ke) { if (ke->ke_state != KES_ONRUNQ && ke->ke_state != KES_THREAD) continue; kseq = KSEQ_CPU(ke->ke_cpu); #ifdef SMP /* * On SMP if we're on the RUNQ we must adjust the transferable * count because could be changing to or from an interrupt * class. */ if (ke->ke_state == KES_ONRUNQ) { if (KSE_CAN_MIGRATE(ke, oclass)) { kseq->ksq_transferable--; kseq->ksq_group->ksg_transferable--; } if (KSE_CAN_MIGRATE(ke, nclass)) { kseq->ksq_transferable++; kseq->ksq_group->ksg_transferable++; } } #endif if (oclass == PRI_TIMESHARE) { kseq->ksq_load_timeshare--; kseq_nice_rem(kseq, kg->kg_proc->p_nice); } if (nclass == PRI_TIMESHARE) { kseq->ksq_load_timeshare++; kseq_nice_add(kseq, kg->kg_proc->p_nice); } } kg->kg_pri_class = class; } /* * Return some of the child's priority and interactivity to the parent. */ void -sched_exit(struct proc *p, struct proc *child) +sched_exit(struct proc *p, struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); - sched_exit_kse(FIRST_KSE_IN_PROC(p), FIRST_KSE_IN_PROC(child)); - sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), FIRST_KSEGRP_IN_PROC(child)); + sched_exit_kse(FIRST_KSE_IN_PROC(p), td); + sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td); } void -sched_exit_kse(struct kse *ke, struct kse *child) +sched_exit_kse(struct kse *ke, struct thread *td) { - kseq_load_rem(KSEQ_CPU(child->ke_cpu), child); + kseq_load_rem(KSEQ_CPU(td->td_kse->ke_cpu), td->td_kse); } void -sched_exit_ksegrp(struct ksegrp *kg, struct ksegrp *child) +sched_exit_ksegrp(struct ksegrp *kg, struct thread *td) { - /* kg->kg_slptime += child->kg_slptime; */ - kg->kg_runtime += child->kg_runtime; + /* kg->kg_slptime += td->td_ksegrp->kg_slptime; */ + kg->kg_runtime += td->td_ksegrp->kg_runtime; sched_interact_update(kg); } void sched_exit_thread(struct thread *td, struct thread *child) { } void sched_clock(struct thread *td) { struct kseq *kseq; struct ksegrp *kg; struct kse *ke; mtx_assert(&sched_lock, MA_OWNED); #ifdef SMP if (ticks == bal_tick) sched_balance(); if (ticks == gbal_tick) sched_balance_groups(); #endif /* * sched_setup() apparently happens prior to stathz being set. We * need to resolve the timers earlier in the boot so we can avoid * calculating this here. */ if (realstathz == 0) { realstathz = stathz ? stathz : hz; tickincr = hz / realstathz; /* * XXX This does not work for values of stathz that are much * larger than hz. */ if (tickincr == 0) tickincr = 1; } ke = td->td_kse; kg = ke->ke_ksegrp; /* Adjust ticks for pctcpu */ ke->ke_ticks++; ke->ke_ltick = ticks; /* Go up to one second beyond our max and then trim back down */ if (ke->ke_ftick + SCHED_CPU_TICKS + hz < ke->ke_ltick) sched_pctcpu_update(ke); if (td->td_flags & TDF_IDLETD) return; CTR4(KTR_ULE, "Tick kse %p (slice: %d, slptime: %d, runtime: %d)", ke, ke->ke_slice, kg->kg_slptime >> 10, kg->kg_runtime >> 10); /* * We only do slicing code for TIMESHARE ksegrps. */ if (kg->kg_pri_class != PRI_TIMESHARE) return; /* * We used a tick charge it to the ksegrp so that we can compute our * interactivity. */ kg->kg_runtime += tickincr << 10; sched_interact_update(kg); /* * We used up one time slice. */ if (--ke->ke_slice > 0) return; /* * We're out of time, recompute priorities and requeue. */ kseq = KSEQ_SELF(); kseq_load_rem(kseq, ke); sched_priority(kg); sched_slice(ke); if (SCHED_CURR(kg, ke)) ke->ke_runq = kseq->ksq_curr; else ke->ke_runq = kseq->ksq_next; kseq_load_add(kseq, ke); td->td_flags |= TDF_NEEDRESCHED; } int sched_runnable(void) { struct kseq *kseq; int load; load = 1; kseq = KSEQ_SELF(); #ifdef SMP if (kseq->ksq_assigned) { mtx_lock_spin(&sched_lock); kseq_assign(kseq); mtx_unlock_spin(&sched_lock); } #endif if ((curthread->td_flags & TDF_IDLETD) != 0) { if (kseq->ksq_load > 0) goto out; } else if (kseq->ksq_load - 1 > 0) goto out; load = 0; out: return (load); } void sched_userret(struct thread *td) { struct ksegrp *kg; kg = td->td_ksegrp; if (td->td_priority != kg->kg_user_pri) { mtx_lock_spin(&sched_lock); td->td_priority = kg->kg_user_pri; mtx_unlock_spin(&sched_lock); } } struct kse * sched_choose(void) { struct kseq *kseq; struct kse *ke; mtx_assert(&sched_lock, MA_OWNED); kseq = KSEQ_SELF(); #ifdef SMP restart: if (kseq->ksq_assigned) kseq_assign(kseq); #endif ke = kseq_choose(kseq); if (ke) { #ifdef SMP if (ke->ke_ksegrp->kg_pri_class == PRI_IDLE) if (kseq_idled(kseq) == 0) goto restart; #endif kseq_runq_rem(kseq, ke); ke->ke_state = KES_THREAD; if (ke->ke_ksegrp->kg_pri_class == PRI_TIMESHARE) { CTR4(KTR_ULE, "Run kse %p from %p (slice: %d, pri: %d)", ke, ke->ke_runq, ke->ke_slice, ke->ke_thread->td_priority); } return (ke); } #ifdef SMP if (kseq_idled(kseq) == 0) goto restart; #endif return (NULL); } void sched_add(struct thread *td) { sched_add_internal(td, 1); } static void sched_add_internal(struct thread *td, int preemptive) { struct kseq *kseq; struct ksegrp *kg; struct kse *ke; int class; mtx_assert(&sched_lock, MA_OWNED); ke = td->td_kse; kg = td->td_ksegrp; if (ke->ke_flags & KEF_ASSIGNED) return; kseq = KSEQ_SELF(); KASSERT((ke->ke_thread != NULL), ("sched_add: No thread on KSE")); KASSERT((ke->ke_thread->td_kse != NULL), ("sched_add: No KSE on thread")); KASSERT(ke->ke_state != KES_ONRUNQ, ("sched_add: kse %p (%s) already in run queue", ke, ke->ke_proc->p_comm)); KASSERT(ke->ke_proc->p_sflag & PS_INMEM, ("sched_add: process swapped out")); KASSERT(ke->ke_runq == NULL, ("sched_add: KSE %p is still assigned to a run queue", ke)); class = PRI_BASE(kg->kg_pri_class); switch (class) { case PRI_ITHD: case PRI_REALTIME: ke->ke_runq = kseq->ksq_curr; ke->ke_slice = SCHED_SLICE_MAX; ke->ke_cpu = PCPU_GET(cpuid); break; case PRI_TIMESHARE: if (SCHED_CURR(kg, ke)) ke->ke_runq = kseq->ksq_curr; else ke->ke_runq = kseq->ksq_next; break; case PRI_IDLE: /* * This is for priority prop. */ if (ke->ke_thread->td_priority < PRI_MIN_IDLE) ke->ke_runq = kseq->ksq_curr; else ke->ke_runq = &kseq->ksq_idle; ke->ke_slice = SCHED_SLICE_MIN; break; default: panic("Unknown pri class."); break; } #ifdef SMP if (ke->ke_cpu != PCPU_GET(cpuid)) { ke->ke_runq = NULL; kseq_notify(ke, ke->ke_cpu); return; } /* * If we had been idle, clear our bit in the group and potentially * the global bitmap. If not, see if we should transfer this thread. */ if ((class == PRI_TIMESHARE || class == PRI_REALTIME) && (kseq->ksq_group->ksg_idlemask & PCPU_GET(cpumask)) != 0) { /* * Check to see if our group is unidling, and if so, remove it * from the global idle mask. */ if (kseq->ksq_group->ksg_idlemask == kseq->ksq_group->ksg_cpumask) atomic_clear_int(&kseq_idle, kseq->ksq_group->ksg_mask); /* * Now remove ourselves from the group specific idle mask. */ kseq->ksq_group->ksg_idlemask &= ~PCPU_GET(cpumask); } else if (kseq->ksq_load > 1 && KSE_CAN_MIGRATE(ke, class)) if (kseq_transfer(kseq, ke, class)) return; #endif if (td->td_priority < curthread->td_priority) curthread->td_flags |= TDF_NEEDRESCHED; #ifdef SMP /* * Only try to preempt if the thread is unpinned or pinned to the * current CPU. */ if (KSE_CAN_MIGRATE(ke, class) || ke->ke_cpu == PCPU_GET(cpuid)) #endif if (preemptive && maybe_preempt(td)) return; ke->ke_ksegrp->kg_runq_kses++; ke->ke_state = KES_ONRUNQ; kseq_runq_add(kseq, ke); kseq_load_add(kseq, ke); } void sched_rem(struct thread *td) { struct kseq *kseq; struct kse *ke; ke = td->td_kse; /* * It is safe to just return here because sched_rem() is only ever * used in places where we're immediately going to add the * kse back on again. In that case it'll be added with the correct * thread and priority when the caller drops the sched_lock. */ if (ke->ke_flags & KEF_ASSIGNED) return; mtx_assert(&sched_lock, MA_OWNED); KASSERT((ke->ke_state == KES_ONRUNQ), ("sched_rem: KSE not on run queue")); ke->ke_state = KES_THREAD; ke->ke_ksegrp->kg_runq_kses--; kseq = KSEQ_CPU(ke->ke_cpu); kseq_runq_rem(kseq, ke); kseq_load_rem(kseq, ke); } fixpt_t sched_pctcpu(struct thread *td) { fixpt_t pctcpu; struct kse *ke; pctcpu = 0; ke = td->td_kse; if (ke == NULL) return (0); mtx_lock_spin(&sched_lock); if (ke->ke_ticks) { int rtick; /* * Don't update more frequently than twice a second. Allowing * this causes the cpu usage to decay away too quickly due to * rounding errors. */ if (ke->ke_ftick + SCHED_CPU_TICKS < ke->ke_ltick || ke->ke_ltick < (ticks - (hz / 2))) sched_pctcpu_update(ke); /* How many rtick per second ? */ rtick = min(ke->ke_ticks / SCHED_CPU_TIME, SCHED_CPU_TICKS); pctcpu = (FSCALE * ((FSCALE * rtick)/realstathz)) >> FSHIFT; } ke->ke_proc->p_swtime = ke->ke_ltick - ke->ke_ftick; mtx_unlock_spin(&sched_lock); return (pctcpu); } void sched_bind(struct thread *td, int cpu) { struct kse *ke; mtx_assert(&sched_lock, MA_OWNED); ke = td->td_kse; ke->ke_flags |= KEF_BOUND; #ifdef SMP if (PCPU_GET(cpuid) == cpu) return; /* sched_rem without the runq_remove */ ke->ke_state = KES_THREAD; ke->ke_ksegrp->kg_runq_kses--; kseq_load_rem(KSEQ_CPU(ke->ke_cpu), ke); kseq_notify(ke, cpu); /* When we return from mi_switch we'll be on the correct cpu. */ mi_switch(SW_VOL, NULL); #endif } void sched_unbind(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); td->td_kse->ke_flags &= ~KEF_BOUND; } int sched_load(void) { #ifdef SMP int total; int i; total = 0; for (i = 0; i <= ksg_maxid; i++) total += KSEQ_GROUP(i)->ksg_load; return (total); #else return (KSEQ_SELF()->ksq_sysload); #endif } int sched_sizeof_kse(void) { return (sizeof(struct kse) + sizeof(struct ke_sched)); } int sched_sizeof_ksegrp(void) { return (sizeof(struct ksegrp) + sizeof(struct kg_sched)); } int sched_sizeof_proc(void) { return (sizeof(struct proc)); } int sched_sizeof_thread(void) { return (sizeof(struct thread) + sizeof(struct td_sched)); } Index: head/sys/sys/sched.h =================================================================== --- head/sys/sys/sched.h (revision 132371) +++ head/sys/sys/sched.h (revision 132372) @@ -1,122 +1,122 @@ /*- * Copyright (c) 2002, Jeffrey Roberson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_SCHED_H_ #define _SYS_SCHED_H_ /* * General scheduling info. * * sched_load: * Total runnable non-ithread threads in the system. * * sched_runnable: * Runnable threads for this processor. */ int sched_load(void); int sched_rr_interval(void); int sched_runnable(void); /* * Proc related scheduling hooks. */ -void sched_exit(struct proc *p, struct proc *child); -void sched_fork(struct proc *p, struct proc *child); +void sched_exit(struct proc *p, struct thread *childtd); +void sched_fork(struct thread *td, struct proc *child); /* * KSE Groups contain scheduling priority information. They record the * behavior of groups of KSEs and threads. */ void sched_class(struct ksegrp *kg, int class); -void sched_exit_ksegrp(struct ksegrp *kg, struct ksegrp *child); -void sched_fork_ksegrp(struct ksegrp *kg, struct ksegrp *child); +void sched_exit_ksegrp(struct ksegrp *kg, struct thread *childtd); +void sched_fork_ksegrp(struct thread *td, struct ksegrp *child); void sched_nice(struct proc *p, int nice); /* * Threads are switched in and out, block on resources, have temporary * priorities inherited from their ksegs, and use up cpu time. */ void sched_exit_thread(struct thread *td, struct thread *child); void sched_fork_thread(struct thread *td, struct thread *child); fixpt_t sched_pctcpu(struct thread *td); void sched_prio(struct thread *td, u_char prio); void sched_sleep(struct thread *td); void sched_switch(struct thread *td, struct thread *newtd); void sched_userret(struct thread *td); void sched_wakeup(struct thread *td); /* * Threads are moved on and off of run queues */ void sched_add(struct thread *td); struct kse *sched_choose(void); /* XXX Should be thread * */ void sched_clock(struct thread *td); void sched_rem(struct thread *td); /* * Binding makes cpu affinity permanent while pinning is used to temporarily * hold a thread on a particular CPU. */ void sched_bind(struct thread *td, int cpu); static __inline void sched_pin(void); void sched_unbind(struct thread *td); static __inline void sched_unpin(void); /* * These interfaces will eventually be removed. */ -void sched_exit_kse(struct kse *ke, struct kse *child); -void sched_fork_kse(struct kse *ke, struct kse *child); +void sched_exit_kse(struct kse *ke, struct thread *childtd); +void sched_fork_kse(struct thread *td, struct kse *child); /* * These procedures tell the process data structure allocation code how * many bytes to actually allocate. */ int sched_sizeof_kse(void); int sched_sizeof_ksegrp(void); int sched_sizeof_proc(void); int sched_sizeof_thread(void); extern struct ke_sched *kse0_sched; extern struct kg_sched *ksegrp0_sched; extern struct p_sched *proc0_sched; extern struct td_sched *thread0_sched; static __inline void sched_pin(void) { curthread->td_pinned++; } static __inline void sched_unpin(void) { curthread->td_pinned--; } #endif /* !_SYS_SCHED_H_ */