Page MenuHomeFreeBSD
Differential D54831 Diff 170318 sys/kern/sched_4bsd.c

Changeset View

Standalone View

View Options

sys/kern/sched_4bsd.c

Show All 28 Lines
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE. * SUCH DAMAGE.
*/ */
#include <sys/cdefs.h>
#include "opt_hwpmc_hooks.h" #include "opt_hwpmc_hooks.h"
#include "opt_hwt_hooks.h" #include "opt_hwt_hooks.h"
#include "opt_sched.h" #include "opt_sched.h"
#include <sys/param.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/cpuset.h> #include <sys/cpuset.h>
#include <sys/kernel.h> #include <sys/kernel.h>
#include <sys/ktr.h> #include <sys/ktr.h>
#include <sys/lock.h> #include <sys/lock.h>
#include <sys/kthread.h> #include <sys/kthread.h>
#include <sys/mutex.h> #include <sys/mutex.h>
#include <sys/proc.h> #include <sys/proc.h>
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines
static int realstathz = 127; /* stathz is sometimes 0 and run off of hz. */ static int realstathz = 127; /* stathz is sometimes 0 and run off of hz. */
static int sched_tdcnt; /* Total runnable threads in the system. */ static int sched_tdcnt; /* Total runnable threads in the system. */
static int sched_slice = 12; /* Thread run time before rescheduling. */ static int sched_slice = 12; /* Thread run time before rescheduling. */
static void setup_runqs(void); static void setup_runqs(void);
static void schedcpu(void); static void schedcpu(void);
static void schedcpu_thread(void); static void schedcpu_thread(void);
static void sched_priority(struct thread *td, u_char prio); static void sched_priority(struct thread *td, u_char prio);
static void sched_setup(void *dummy);
static void maybe_resched(struct thread *td); static void maybe_resched(struct thread *td);
static void updatepri(struct thread *td); static void updatepri(struct thread *td);
static void resetpriority(struct thread *td); static void resetpriority(struct thread *td);
static void resetpriority_thread(struct thread *td); static void resetpriority_thread(struct thread *td);
#ifdef SMP #ifdef SMP
static int sched_pickcpu(struct thread *td); static int sched_pickcpu(struct thread *td);
static int forward_wakeup(int cpunum); static int forward_wakeup(int cpunum);
static void kick_other_cpu(int pri, int cpuid); static void kick_other_cpu(int pri, int cpuid);
#endif #endif
static struct kproc_desc sched_kp = { static struct kproc_desc sched_kp = {
"schedcpu", "schedcpu",
schedcpu_thread, schedcpu_thread,
NULL NULL
}; };
SYSINIT(schedcpu, SI_SUB_LAST, SI_ORDER_FIRST, kproc_start,
&sched_kp);
SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL);
static void sched_initticks(void *dummy); static void
SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks, sched_4bsd_schedcpu(void)
NULL); {
kproc_start(&sched_kp);
}
/* /*
* Global run queue. * Global run queue.
*/ */
static struct runq runq; static struct runq runq;
#ifdef SMP #ifdef SMP
/* /*
Show All 20 Lines#ifdef SMP
for (i = 0; i < MAXCPU; ++i) for (i = 0; i < MAXCPU; ++i)
runq_init(&runq_pcpu[i]); runq_init(&runq_pcpu[i]);
#endif #endif
runq_init(&runq); runq_init(&runq);
} }
static int static int
sysctl_kern_quantum(SYSCTL_HANDLER_ARGS) sysctl_kern_4bsd_quantum(SYSCTL_HANDLER_ARGS)
{ {
int error, new_val, period; int error, new_val, period;
period = 1000000 / realstathz; period = 1000000 / realstathz;
new_val = period * sched_slice; new_val = period * sched_slice;
error = sysctl_handle_int(oidp, &new_val, 0, req); error = sysctl_handle_int(oidp, &new_val, 0, req);
if (error != 0 || req->newptr == NULL) if (error != 0 || req->newptr == NULL)
return (error); return (error);
if (new_val <= 0) if (new_val <= 0)
return (EINVAL); return (EINVAL);
sched_slice = imax(1, (new_val + period / 2) / period); sched_slice = imax(1, (new_val + period / 2) / period);
hogticks = imax(1, (2 * hz * sched_slice + realstathz / 2) / hogticks = imax(1, (2 * hz * sched_slice + realstathz / 2) /
realstathz); realstathz);
return (0); return (0);
} }
SYSCTL_NODE(_kern, OID_AUTO, sched, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, SYSCTL_NODE(_kern_sched, OID_AUTO, 4bsd, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"Scheduler"); "4BSD Scheduler");
SYSCTL_STRING(_kern_sched, OID_AUTO, name, CTLFLAG_RD, "4BSD", 0, SYSCTL_PROC(_kern_sched_4bsd, OID_AUTO, quantum,
"Scheduler name");
SYSCTL_PROC(_kern_sched, OID_AUTO, quantum,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
sysctl_kern_quantum, "I", sysctl_kern_4bsd_quantum, "I",
"Quantum for timeshare threads in microseconds"); "Quantum for timeshare threads in microseconds");
SYSCTL_INT(_kern_sched, OID_AUTO, slice, CTLFLAG_RW, &sched_slice, 0, SYSCTL_INT(_kern_sched_4bsd, OID_AUTO, slice, CTLFLAG_RW, &sched_slice, 0,
"Quantum for timeshare threads in stathz ticks"); "Quantum for timeshare threads in stathz ticks");
#ifdef SMP #ifdef SMP
/* Enable forwarding of wakeups to all other cpus */ /* Enable forwarding of wakeups to all other cpus */
static SYSCTL_NODE(_kern_sched, OID_AUTO, ipiwakeup, static SYSCTL_NODE(_kern_sched_4bsd, OID_AUTO, ipiwakeup,
CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
"Kernel SMP"); "Kernel SMP");
static int runq_fuzz = 1; static int runq_fuzz = 1;
SYSCTL_INT(_kern_sched, OID_AUTO, runq_fuzz, CTLFLAG_RW, &runq_fuzz, 0, ""); SYSCTL_INT(_kern_sched_4bsd, OID_AUTO, runq_fuzz, CTLFLAG_RW,
&runq_fuzz, 0, "");
static int forward_wakeup_enabled = 1; static int forward_wakeup_enabled = 1;
SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, enabled, CTLFLAG_RW, SYSCTL_INT(_kern_sched_4bsd_ipiwakeup, OID_AUTO, enabled, CTLFLAG_RW,
&forward_wakeup_enabled, 0, &forward_wakeup_enabled, 0,
"Forwarding of wakeup to idle CPUs"); "Forwarding of wakeup to idle CPUs");
static int forward_wakeups_requested = 0; static int forward_wakeups_requested = 0;
SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, requested, CTLFLAG_RD, SYSCTL_INT(_kern_sched_4bsd_ipiwakeup, OID_AUTO, requested, CTLFLAG_RD,
&forward_wakeups_requested, 0, &forward_wakeups_requested, 0,
"Requests for Forwarding of wakeup to idle CPUs"); "Requests for Forwarding of wakeup to idle CPUs");
static int forward_wakeups_delivered = 0; static int forward_wakeups_delivered = 0;
SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, delivered, CTLFLAG_RD, SYSCTL_INT(_kern_sched_4bsd_ipiwakeup, OID_AUTO, delivered, CTLFLAG_RD,
&forward_wakeups_delivered, 0, &forward_wakeups_delivered, 0,
"Completed Forwarding of wakeup to idle CPUs"); "Completed Forwarding of wakeup to idle CPUs");
static int forward_wakeup_use_mask = 1; static int forward_wakeup_use_mask = 1;
SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, usemask, CTLFLAG_RW, SYSCTL_INT(_kern_sched_4bsd_ipiwakeup, OID_AUTO, usemask, CTLFLAG_RW,
&forward_wakeup_use_mask, 0, &forward_wakeup_use_mask, 0,
"Use the mask of idle cpus"); "Use the mask of idle cpus");
static int forward_wakeup_use_loop = 0; static int forward_wakeup_use_loop = 0;
SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, useloop, CTLFLAG_RW, SYSCTL_INT(_kern_sched_4bsd_ipiwakeup, OID_AUTO, useloop, CTLFLAG_RW,
&forward_wakeup_use_loop, 0, &forward_wakeup_use_loop, 0,
"Use a loop to find idle cpus"); "Use a loop to find idle cpus");
#endif #endif
#if 0 #if 0
static int sched_followon = 0; static int sched_followon = 0;
SYSCTL_INT(_kern_sched, OID_AUTO, followon, CTLFLAG_RW, SYSCTL_INT(_kern_sched_4bsd, OID_AUTO, followon, CTLFLAG_RW,
&sched_followon, 0, &sched_followon, 0,
"allow threads to share a quantum"); "allow threads to share a quantum");
#endif #endif
SDT_PROVIDER_DEFINE(sched); SDT_PROVIDER_DEFINE(sched);
SDT_PROBE_DEFINE3(sched, , , change__pri, "struct thread *", SDT_PROBE_DEFINE3(sched, , , change__pri, "struct thread *",
"struct proc *", "uint8_t"); "struct proc *", "uint8_t");
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines
} }
/* /*
* This function is called when a thread is about to be put on run queue * This function is called when a thread is about to be put on run queue
* because it has been made runnable or its priority has been adjusted. It * because it has been made runnable or its priority has been adjusted. It
* determines if the new thread should preempt the current thread. If so, * determines if the new thread should preempt the current thread. If so,
* it sets td_owepreempt to request a preemption. * it sets td_owepreempt to request a preemption.
*/ */
int static int
maybe_preempt(struct thread *td) maybe_preempt(struct thread *td)
{ {
#ifdef PREEMPTION #ifdef PREEMPTION
struct thread *ctd; struct thread *ctd;
int cpri, pri; int cpri, pri;
/* /*
* The new thread should not preempt the current thread if any of the * The new thread should not preempt the current thread if any of the
▲ Show 20 Lines • Show All 102 Lines • ▼ Show 20 Lines
* loadav: 1 2 3 4 * loadav: 1 2 3 4
* power: 5.68 10.32 14.94 19.55 * power: 5.68 10.32 14.94 19.55
*/ */
/* calculations for digital decay to forget 90% of usage in 5*loadav sec */ /* calculations for digital decay to forget 90% of usage in 5*loadav sec */
#define loadfactor(loadav) (2 * (loadav)) #define loadfactor(loadav) (2 * (loadav))
#define decay_cpu(loadfac, cpu) (((loadfac) * (cpu)) / ((loadfac) + FSCALE)) #define decay_cpu(loadfac, cpu) (((loadfac) * (cpu)) / ((loadfac) + FSCALE))
/* decay 95% of `ts_pctcpu' in 60 seconds; see CCPU_SHIFT before changing */ extern fixpt_t ccpu;
static fixpt_t ccpu = 0.95122942450071400909 * FSCALE; /* exp(-1/20) */
SYSCTL_UINT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0,
"Decay factor used for updating %CPU");
/* /*
* If `ccpu' is not equal to `exp(-1/20)' and you still want to use the * If `ccpu' is not equal to `exp(-1/20)' and you still want to use the
* faster/more-accurate formula, you'll have to estimate CCPU_SHIFT below * faster/more-accurate formula, you'll have to estimate CCPU_SHIFT below
* and possibly adjust FSHIFT in "param.h" so that (FSHIFT >= CCPU_SHIFT). * and possibly adjust FSHIFT in "param.h" so that (FSHIFT >= CCPU_SHIFT).
* *
* To estimate CCPU_SHIFT for exp(-1/20), the following formula was used: * To estimate CCPU_SHIFT for exp(-1/20), the following formula was used:
* 1 - exp(-1/20) ~= 0.0487 ~= 0.0488 == 1 (fixed pt, *11* bits). * 1 - exp(-1/20) ~= 0.0487 ~= 0.0488 == 1 (fixed pt, *11* bits).
▲ Show 20 Lines • Show All 179 Lines • ▼ Show 20 Lines if (td->td_priority < PRI_MIN_TIMESHARE ||
return; return;
/* XXX the whole needresched thing is broken, but not silly. */ /* XXX the whole needresched thing is broken, but not silly. */
maybe_resched(td); maybe_resched(td);
sched_prio(td, td->td_user_pri); sched_prio(td, td->td_user_pri);
} }
/* ARGSUSED */
static void static void
sched_setup(void *dummy) sched_4bsd_setup(void)
{ {
/*
* Decay 95% of `ts_pctcpu' in 60 seconds; see CCPU_SHIFT
* before changing.
*/
ccpu = 0.95122942450071400909 * FSCALE; /* exp(-1/20) */
setup_runqs(); setup_runqs();
/* Account for thread0. */ /* Account for thread0. */
sched_load_add(); sched_load_add();
} }
/* /*
* This routine determines time constants after stathz and hz are setup. * This routine determines time constants after stathz and hz are setup.
*/ */
static void static void
sched_initticks(void *dummy) sched_4bsd_initticks(void)
{ {
realstathz = stathz ? stathz : hz; realstathz = stathz ? stathz : hz;
sched_slice = realstathz / 10; /* ~100ms */ sched_slice = realstathz / 10; /* ~100ms */
hogticks = imax(1, (2 * hz * sched_slice + realstathz / 2) / hogticks = imax(1, (2 * hz * sched_slice + realstathz / 2) /
realstathz); realstathz);
} }
/* External interfaces start here */ /* External interfaces start here */
/* /*
* Very early in the boot some setup of scheduler-specific * Very early in the boot some setup of scheduler-specific
* parts of proc0 and of some scheduler resources needs to be done. * parts of proc0 and of some scheduler resources needs to be done.
* Called from: * Called from:
* proc0_init() * proc0_init()
*/ */
void static void
schedinit(void) sched_4bsd_init(void)
{ {
/* /*
* Set up the scheduler specific parts of thread0. * Set up the scheduler specific parts of thread0.
*/ */
thread0.td_lock = &sched_lock; thread0.td_lock = &sched_lock;
td_get_sched(&thread0)->ts_slice = sched_slice; td_get_sched(&thread0)->ts_slice = sched_slice;
mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN); mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN);
} }
void static void
schedinit_ap(void) sched_4bsd_init_ap(void)
{ {
/* Nothing needed. */ /* Nothing needed. */
} }
bool static bool
sched_runnable(void) sched_4bsd_runnable(void)
{ {
#ifdef SMP #ifdef SMP
return (runq_not_empty(&runq) || return (runq_not_empty(&runq) ||
runq_not_empty(&runq_pcpu[PCPU_GET(cpuid)])); runq_not_empty(&runq_pcpu[PCPU_GET(cpuid)]));
#else #else
return (runq_not_empty(&runq)); return (runq_not_empty(&runq));
#endif #endif
} }
int static int
sched_rr_interval(void) sched_4bsd_rr_interval(void)
{ {
/* Convert sched_slice from stathz to hz. */ /* Convert sched_slice from stathz to hz. */
return (imax(1, (sched_slice * hz + realstathz / 2) / realstathz)); return (imax(1, (sched_slice * hz + realstathz / 2) / realstathz));
} }
SCHED_STAT_DEFINE(ithread_demotions, "Interrupt thread priority demotions"); SCHED_STAT_DEFINE(ithread_demotions, "Interrupt thread priority demotions");
SCHED_STAT_DEFINE(ithread_preemptions, SCHED_STAT_DEFINE(ithread_preemptions,
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines if (!TD_IS_IDLETHREAD(td) && --ts->ts_slice <= 0) {
} }
} }
stat = DPCPU_PTR(idlestat); stat = DPCPU_PTR(idlestat);
stat->oldidlecalls = stat->idlecalls; stat->oldidlecalls = stat->idlecalls;
stat->idlecalls = 0; stat->idlecalls = 0;
} }
void static void
sched_clock(struct thread *td, int cnt) sched_4bsd_clock(struct thread *td, int cnt)
{ {
for ( ; cnt > 0; cnt--) for ( ; cnt > 0; cnt--)
sched_clock_tick(td); sched_clock_tick(td);
} }
/* /*
* Charge child's scheduling CPU usage to parent. * Charge child's scheduling CPU usage to parent.
*/ */
void static void
sched_exit(struct proc *p, struct thread *td) sched_4bsd_exit(struct proc *p, struct thread *td)
{ {
KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "proc exit", KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "proc exit",
"prio:%d", td->td_priority); "prio:%d", td->td_priority);
PROC_LOCK_ASSERT(p, MA_OWNED); PROC_LOCK_ASSERT(p, MA_OWNED);
sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); sched_exit_thread(FIRST_THREAD_IN_PROC(p), td);
} }
void static void
sched_exit_thread(struct thread *td, struct thread *child) sched_4bsd_exit_thread(struct thread *td, struct thread *child)
{ {
KTR_STATE1(KTR_SCHED, "thread", sched_tdname(child), "exit", KTR_STATE1(KTR_SCHED, "thread", sched_tdname(child), "exit",
"prio:%d", child->td_priority); "prio:%d", child->td_priority);
thread_lock(td); thread_lock(td);
td_get_sched(td)->ts_estcpu = ESTCPULIM(td_get_sched(td)->ts_estcpu + td_get_sched(td)->ts_estcpu = ESTCPULIM(td_get_sched(td)->ts_estcpu +
td_get_sched(child)->ts_estcpu); td_get_sched(child)->ts_estcpu);
thread_unlock(td); thread_unlock(td);
thread_lock(child); thread_lock(child);
if ((child->td_flags & TDF_NOLOAD) == 0) if ((child->td_flags & TDF_NOLOAD) == 0)
sched_load_rem(); sched_load_rem();
thread_unlock(child); thread_unlock(child);
} }
void static void
sched_fork(struct thread *td, struct thread *childtd) sched_4bsd_fork(struct thread *td, struct thread *childtd)
{ {
sched_fork_thread(td, childtd); sched_fork_thread(td, childtd);
} }
void static void
sched_fork_thread(struct thread *td, struct thread *childtd) sched_4bsd_fork_thread(struct thread *td, struct thread *childtd)
{ {
struct td_sched *ts, *tsc; struct td_sched *ts, *tsc;
childtd->td_oncpu = NOCPU; childtd->td_oncpu = NOCPU;
childtd->td_lastcpu = NOCPU; childtd->td_lastcpu = NOCPU;
childtd->td_lock = &sched_lock; childtd->td_lock = &sched_lock;
childtd->td_cpuset = cpuset_ref(td->td_cpuset); childtd->td_cpuset = cpuset_ref(td->td_cpuset);
childtd->td_domain.dr_policy = td->td_cpuset->cs_domain; childtd->td_domain.dr_policy = td->td_cpuset->cs_domain;
childtd->td_priority = childtd->td_base_pri; childtd->td_priority = childtd->td_base_pri;
ts = td_get_sched(childtd); ts = td_get_sched(childtd);
bzero(ts, sizeof(*ts)); bzero(ts, sizeof(*ts));
tsc = td_get_sched(td); tsc = td_get_sched(td);
ts->ts_estcpu = tsc->ts_estcpu; ts->ts_estcpu = tsc->ts_estcpu;
ts->ts_flags |= (tsc->ts_flags & TSF_AFFINITY); ts->ts_flags |= (tsc->ts_flags & TSF_AFFINITY);
ts->ts_slice = 1; ts->ts_slice = 1;
} }
void static void
sched_nice(struct proc *p, int nice) sched_4bsd_nice(struct proc *p, int nice)
{ {
struct thread *td; struct thread *td;
PROC_LOCK_ASSERT(p, MA_OWNED); PROC_LOCK_ASSERT(p, MA_OWNED);
p->p_nice = nice; p->p_nice = nice;
FOREACH_THREAD_IN_PROC(p, td) { FOREACH_THREAD_IN_PROC(p, td) {
thread_lock(td); thread_lock(td);
resetpriority(td); resetpriority(td);
resetpriority_thread(td); resetpriority_thread(td);
thread_unlock(td); thread_unlock(td);
} }
} }
void static void
sched_class(struct thread *td, int class) sched_4bsd_class(struct thread *td, int class)
{ {
THREAD_LOCK_ASSERT(td, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED);
td->td_pri_class = class; td->td_pri_class = class;
} }
/* /*
* Adjust the priority of a thread. * Adjust the priority of a thread.
*/ */
Show All 21 Lines if (TD_ON_RUNQ(td) && td->td_rqindex != RQ_PRI_TO_QUEUE_IDX(prio)) {
sched_add(td, SRQ_BORING | SRQ_HOLDTD); sched_add(td, SRQ_BORING | SRQ_HOLDTD);
} }
} }
/* /*
* Update a thread's priority when it is lent another thread's * Update a thread's priority when it is lent another thread's
* priority. * priority.
*/ */
void static void
sched_lend_prio(struct thread *td, u_char prio) sched_4bsd_lend_prio(struct thread *td, u_char prio)
{ {
td->td_flags |= TDF_BORROWING; td->td_flags |= TDF_BORROWING;
sched_priority(td, prio); sched_priority(td, prio);
} }
/* /*
* Restore a thread's priority when priority propagation is * Restore a thread's priority when priority propagation is
* over. The prio argument is the minimum priority the thread * over. The prio argument is the minimum priority the thread
* needs to have to satisfy other possible priority lending * needs to have to satisfy other possible priority lending
* requests. If the thread's regulary priority is less * requests. If the thread's regulary priority is less
* important than prio the thread will keep a priority boost * important than prio the thread will keep a priority boost
* of prio. * of prio.
*/ */
void static void
sched_unlend_prio(struct thread *td, u_char prio) sched_4bsd_unlend_prio(struct thread *td, u_char prio)
{ {
u_char base_pri; u_char base_pri;
if (td->td_base_pri >= PRI_MIN_TIMESHARE && if (td->td_base_pri >= PRI_MIN_TIMESHARE &&
td->td_base_pri <= PRI_MAX_TIMESHARE) td->td_base_pri <= PRI_MAX_TIMESHARE)
base_pri = td->td_user_pri; base_pri = td->td_user_pri;
else else
base_pri = td->td_base_pri; base_pri = td->td_base_pri;
if (prio >= base_pri) { if (prio >= base_pri) {
td->td_flags &= ~TDF_BORROWING; td->td_flags &= ~TDF_BORROWING;
sched_prio(td, base_pri); sched_prio(td, base_pri);
} else } else
sched_lend_prio(td, prio); sched_lend_prio(td, prio);
} }
void static void
sched_prio(struct thread *td, u_char prio) sched_4bsd_prio(struct thread *td, u_char prio)
{ {
u_char oldprio; u_char oldprio;
/* First, update the base priority. */ /* First, update the base priority. */
td->td_base_pri = prio; td->td_base_pri = prio;
/* /*
* If the thread is borrowing another thread's priority, don't ever * If the thread is borrowing another thread's priority, don't ever
Show All 9 Linessched_4bsd_prio(struct thread *td, u_char prio)
/* /*
* If the thread is on a turnstile, then let the turnstile update * If the thread is on a turnstile, then let the turnstile update
* its state. * its state.
*/ */
if (TD_ON_LOCK(td) && oldprio != prio) if (TD_ON_LOCK(td) && oldprio != prio)
turnstile_adjust(td, oldprio); turnstile_adjust(td, oldprio);
} }
void static void
sched_ithread_prio(struct thread *td, u_char prio) sched_4bsd_ithread_prio(struct thread *td, u_char prio)
{ {
THREAD_LOCK_ASSERT(td, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED);
MPASS(td->td_pri_class == PRI_ITHD); MPASS(td->td_pri_class == PRI_ITHD);
td->td_base_ithread_pri = prio; td->td_base_ithread_pri = prio;
sched_prio(td, prio); sched_prio(td, prio);
} }
void static void
sched_user_prio(struct thread *td, u_char prio) sched_4bsd_user_prio(struct thread *td, u_char prio)
{ {
THREAD_LOCK_ASSERT(td, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED);
td->td_base_user_pri = prio; td->td_base_user_pri = prio;
if (td->td_lend_user_pri <= prio) if (td->td_lend_user_pri <= prio)
return; return;
td->td_user_pri = prio; td->td_user_pri = prio;
} }
void static void
sched_lend_user_prio(struct thread *td, u_char prio) sched_4bsd_lend_user_prio(struct thread *td, u_char prio)
{ {
THREAD_LOCK_ASSERT(td, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED);
td->td_lend_user_pri = prio; td->td_lend_user_pri = prio;
td->td_user_pri = min(prio, td->td_base_user_pri); td->td_user_pri = min(prio, td->td_base_user_pri);
if (td->td_priority > td->td_user_pri) if (td->td_priority > td->td_user_pri)
sched_prio(td, td->td_user_pri); sched_prio(td, td->td_user_pri);
else if (td->td_priority != td->td_user_pri) else if (td->td_priority != td->td_user_pri)
ast_sched_locked(td, TDA_SCHED); ast_sched_locked(td, TDA_SCHED);
} }
/* /*
* Like the above but first check if there is anything to do. * Like the above but first check if there is anything to do.
*/ */
void static void
sched_lend_user_prio_cond(struct thread *td, u_char prio) sched_4bsd_lend_user_prio_cond(struct thread *td, u_char prio)
{ {
if (td->td_lend_user_pri == prio) if (td->td_lend_user_pri == prio)
return; return;
thread_lock(td); thread_lock(td);
sched_lend_user_prio(td, prio); sched_lend_user_prio(td, prio);
thread_unlock(td); thread_unlock(td);
} }
void static void
sched_sleep(struct thread *td, int pri) sched_4bsd_sleep(struct thread *td, int pri)
{ {
THREAD_LOCK_ASSERT(td, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED);
td->td_slptick = ticks; td->td_slptick = ticks;
td_get_sched(td)->ts_slptime = 0; td_get_sched(td)->ts_slptime = 0;
if (pri != 0 && PRI_BASE(td->td_pri_class) == PRI_TIMESHARE) if (pri != 0 && PRI_BASE(td->td_pri_class) == PRI_TIMESHARE)
sched_prio(td, pri); sched_prio(td, pri);
} }
void static void
sched_switch(struct thread *td, int flags) sched_4bsd_sswitch(struct thread *td, int flags)
{ {
struct thread *newtd; struct thread *newtd;
struct mtx *tmtx; struct mtx *tmtx;
int preempted; int preempted;
tmtx = &sched_lock; tmtx = &sched_lock;
THREAD_LOCK_ASSERT(td, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED);
▲ Show 20 Lines • Show All 115 Lines • ▼ Show 20 Lines if (td->td_flags & TDF_IDLETD)
CPU_SET(PCPU_GET(cpuid), &idle_cpus_mask); CPU_SET(PCPU_GET(cpuid), &idle_cpus_mask);
#endif #endif
sched_lock.mtx_lock = (uintptr_t)td; sched_lock.mtx_lock = (uintptr_t)td;
td->td_oncpu = PCPU_GET(cpuid); td->td_oncpu = PCPU_GET(cpuid);
spinlock_enter(); spinlock_enter();
mtx_unlock_spin(&sched_lock); mtx_unlock_spin(&sched_lock);
} }
void static void
sched_wakeup(struct thread *td, int srqflags) sched_4bsd_wakeup(struct thread *td, int srqflags)
{ {
struct td_sched *ts; struct td_sched *ts;
THREAD_LOCK_ASSERT(td, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED);
ts = td_get_sched(td); ts = td_get_sched(td);
if (ts->ts_slptime > 1) { if (ts->ts_slptime > 1) {
updatepri(td); updatepri(td);
resetpriority(td); resetpriority(td);
▲ Show 20 Lines • Show All 158 Lines • ▼ Show 20 Lines else if (runq_length[cpu] < runq_length[best])
best = cpu; best = cpu;
} }
KASSERT(best != NOCPU, ("no valid CPUs")); KASSERT(best != NOCPU, ("no valid CPUs"));
return (best); return (best);
} }
#endif #endif
void static void
sched_add(struct thread *td, int flags) sched_4bsd_add(struct thread *td, int flags)
#ifdef SMP #ifdef SMP
{ {
cpuset_t tidlemsk; cpuset_t tidlemsk;
struct td_sched *ts; struct td_sched *ts;
u_int cpu, cpuid; u_int cpu, cpuid;
int forwarded = 0; int forwarded = 0;
int single_cpu = 0; int single_cpu = 0;
▲ Show 20 Lines • Show All 130 Lines • ▼ Show 20 Lines#else /* SMP */
runq_add(ts->ts_runq, td, flags); runq_add(ts->ts_runq, td, flags);
if (!maybe_preempt(td)) if (!maybe_preempt(td))
maybe_resched(td); maybe_resched(td);
if ((flags & SRQ_HOLDTD) == 0) if ((flags & SRQ_HOLDTD) == 0)
thread_unlock(td); thread_unlock(td);
} }
#endif /* SMP */ #endif /* SMP */
void static void
sched_rem(struct thread *td) sched_4bsd_rem(struct thread *td)
{ {
struct td_sched *ts; struct td_sched *ts;
ts = td_get_sched(td); ts = td_get_sched(td);
KASSERT(td->td_flags & TDF_INMEM, KASSERT(td->td_flags & TDF_INMEM,
("sched_rem: thread swapped out")); ("sched_rem: thread swapped out"));
KASSERT(TD_ON_RUNQ(td), KASSERT(TD_ON_RUNQ(td),
("sched_rem: thread not on run queue")); ("sched_rem: thread not on run queue"));
Show All 12 Lines#endif
runq_remove(ts->ts_runq, td); runq_remove(ts->ts_runq, td);
TD_SET_CAN_RUN(td); TD_SET_CAN_RUN(td);
} }
/* /*
* Select threads to run. Note that running threads still consume a * Select threads to run. Note that running threads still consume a
* slot. * slot.
*/ */
struct thread * static struct thread *
sched_choose(void) sched_4bsd_choose(void)
{ {
struct thread *td; struct thread *td;
struct runq *rq; struct runq *rq;
mtx_assert(&sched_lock, MA_OWNED); mtx_assert(&sched_lock, MA_OWNED);
#ifdef SMP #ifdef SMP
struct thread *tdcpu; struct thread *tdcpu;
Show All 27 Lines#endif
KASSERT(td->td_flags & TDF_INMEM, KASSERT(td->td_flags & TDF_INMEM,
("sched_choose: thread swapped out")); ("sched_choose: thread swapped out"));
return (td); return (td);
} }
return (PCPU_GET(idlethread)); return (PCPU_GET(idlethread));
} }
void static void
sched_preempt(struct thread *td) sched_4bsd_preempt(struct thread *td)
{ {
int flags; int flags;
SDT_PROBE2(sched, , , surrender, td, td->td_proc); SDT_PROBE2(sched, , , surrender, td, td->td_proc);
if (td->td_critnest > 1) { if (td->td_critnest > 1) {
td->td_owepreempt = 1; td->td_owepreempt = 1;
} else { } else {
thread_lock(td); thread_lock(td);
flags = SW_INVOL | SW_PREEMPT; flags = SW_INVOL | SW_PREEMPT;
flags |= TD_IS_IDLETHREAD(td) ? SWT_REMOTEWAKEIDLE : flags |= TD_IS_IDLETHREAD(td) ? SWT_REMOTEWAKEIDLE :
SWT_REMOTEPREEMPT; SWT_REMOTEPREEMPT;
mi_switch(flags); mi_switch(flags);
} }
} }
void static void
sched_userret_slowpath(struct thread *td) sched_4bsd_userret_slowpath(struct thread *td)
{ {
thread_lock(td); thread_lock(td);
td->td_priority = td->td_user_pri; td->td_priority = td->td_user_pri;
td->td_base_pri = td->td_user_pri; td->td_base_pri = td->td_user_pri;
thread_unlock(td); thread_unlock(td);
} }
void static void
sched_bind(struct thread *td, int cpu) sched_4bsd_bind(struct thread *td, int cpu)
{ {
#ifdef SMP #ifdef SMP
struct td_sched *ts = td_get_sched(td); struct td_sched *ts = td_get_sched(td);
#endif #endif
THREAD_LOCK_ASSERT(td, MA_OWNED|MA_NOTRECURSED); THREAD_LOCK_ASSERT(td, MA_OWNED|MA_NOTRECURSED);
KASSERT(td == curthread, ("sched_bind: can only bind curthread")); KASSERT(td == curthread, ("sched_bind: can only bind curthread"));
td->td_flags |= TDF_BOUND; td->td_flags |= TDF_BOUND;
#ifdef SMP #ifdef SMP
ts->ts_runq = &runq_pcpu[cpu]; ts->ts_runq = &runq_pcpu[cpu];
if (PCPU_GET(cpuid) == cpu) if (PCPU_GET(cpuid) == cpu)
return; return;
mi_switch(SW_VOL | SWT_BIND); mi_switch(SW_VOL | SWT_BIND);
thread_lock(td); thread_lock(td);
#endif #endif
} }
void static void
sched_unbind(struct thread* td) sched_4bsd_unbind(struct thread* td)
{ {
THREAD_LOCK_ASSERT(td, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED);
KASSERT(td == curthread, ("sched_unbind: can only bind curthread")); KASSERT(td == curthread, ("sched_unbind: can only bind curthread"));
td->td_flags &= ~TDF_BOUND; td->td_flags &= ~TDF_BOUND;
} }
int static int
sched_is_bound(struct thread *td) sched_4bsd_is_bound(struct thread *td)
{ {
THREAD_LOCK_ASSERT(td, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED);
return (td->td_flags & TDF_BOUND); return (td->td_flags & TDF_BOUND);
} }
void static void
sched_relinquish(struct thread *td) sched_4bsd_relinquish(struct thread *td)
{ {
thread_lock(td); thread_lock(td);
mi_switch(SW_VOL | SWT_RELINQUISH); mi_switch(SW_VOL | SWT_RELINQUISH);
} }
int static int
sched_load(void) sched_4bsd_load(void)
{ {
return (sched_tdcnt); return (sched_tdcnt);
} }
int static int
sched_sizeof_proc(void) sched_4bsd_sizeof_proc(void)
{ {
return (sizeof(struct proc)); return (sizeof(struct proc));
} }
int static int
sched_sizeof_thread(void) sched_4bsd_sizeof_thread(void)
{ {
return (sizeof(struct thread) + sizeof(struct td_sched)); return (sizeof(struct thread) + sizeof(struct td_sched));
} }
fixpt_t static fixpt_t
sched_pctcpu(struct thread *td) sched_4bsd_pctcpu(struct thread *td)
{ {
struct td_sched *ts; struct td_sched *ts;
THREAD_LOCK_ASSERT(td, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED);
ts = td_get_sched(td); ts = td_get_sched(td);
return (ts->ts_pctcpu); return (ts->ts_pctcpu);
} }
#ifdef RACCT static u_int
jrtc27Unsubmitted
Done
Inline Actions

This has been unused since c72188d85a793c7610208beafb83af544de6e3b7, no need to keep it and add a stub for ULE

jrtc27: This has been unused since c72188d85a793c7610208beafb83af544de6e3b7, no need to keep it and add…
/* sched_4bsd_estcpu(struct thread *td)
* Calculates the contribution to the thread cpu usage for the latest
* (unfinished) second.
*/
fixpt_t
sched_pctcpu_delta(struct thread *td)
{ {
struct td_sched *ts;
fixpt_t delta;
int realstathz;
THREAD_LOCK_ASSERT(td, MA_OWNED);
ts = td_get_sched(td);
delta = 0;
realstathz = stathz ? stathz : hz;
if (ts->ts_cpticks != 0) {
#if (FSHIFT >= CCPU_SHIFT)
delta = (realstathz == 100)
? ((fixpt_t) ts->ts_cpticks) <<
(FSHIFT - CCPU_SHIFT) :
100 * (((fixpt_t) ts->ts_cpticks)
<< (FSHIFT - CCPU_SHIFT)) / realstathz;
#else
delta = ((FSCALE - ccpu) *
(ts->ts_cpticks *
FSCALE / realstathz)) >> FSHIFT;
#endif
}
return (delta);
}
#endif
u_int
sched_estcpu(struct thread *td)
{
return (td_get_sched(td)->ts_estcpu); return (td_get_sched(td)->ts_estcpu);
} }
/* /*
* The actual idle process. * The actual idle process.
*/ */
void static void
sched_idletd(void *dummy) sched_4bsd_idletd(void *dummy)
{ {
struct pcpuidlestat *stat; struct pcpuidlestat *stat;
THREAD_NO_SLEEPING(); THREAD_NO_SLEEPING();
stat = DPCPU_PTR(idlestat); stat = DPCPU_PTR(idlestat);
for (;;) { for (;;) {
mtx_assert(&Giant, MA_NOTOWNED); mtx_assert(&Giant, MA_NOTOWNED);
Show All 24 Lines
#endif #endif
cpu_throw(td, newtd); /* doesn't return */ cpu_throw(td, newtd); /* doesn't return */
} }
/* /*
* A CPU is entering for the first time. * A CPU is entering for the first time.
*/ */
void static void
sched_ap_entry(void) sched_4bsd_ap_entry(void)
{ {
/* /*
* Correct spinlock nesting. The idle thread context that we are * Correct spinlock nesting. The idle thread context that we are
* borrowing was created so that it would start out with a single * borrowing was created so that it would start out with a single
* spin lock (sched_lock) held in fork_trampoline(). Since we've * spin lock (sched_lock) held in fork_trampoline(). Since we've
* explicitly acquired locks in this function, the nesting count * explicitly acquired locks in this function, the nesting count
* is now 2 rather than 1. Since we are nested, calling * is now 2 rather than 1. Since we are nested, calling
* spinlock_exit() will simply adjust the counts without allowing * spinlock_exit() will simply adjust the counts without allowing
* spin lock using code to interrupt us. * spin lock using code to interrupt us.
*/ */
mtx_lock_spin(&sched_lock); mtx_lock_spin(&sched_lock);
spinlock_exit(); spinlock_exit();
PCPU_SET(switchtime, cpu_ticks()); PCPU_SET(switchtime, cpu_ticks());
PCPU_SET(switchticks, ticks); PCPU_SET(switchticks, ticks);
sched_throw_tail(NULL); sched_throw_tail(NULL);
} }
/* /*
* A thread is exiting. * A thread is exiting.
*/ */
void static void
sched_throw(struct thread *td) sched_4bsd_throw(struct thread *td)
{ {
MPASS(td != NULL); MPASS(td != NULL);
MPASS(td->td_lock == &sched_lock); MPASS(td->td_lock == &sched_lock);
lock_profile_release_lock(&sched_lock.lock_object, true); lock_profile_release_lock(&sched_lock.lock_object, true);
td->td_lastcpu = td->td_oncpu; td->td_lastcpu = td->td_oncpu;
td->td_oncpu = NOCPU; td->td_oncpu = NOCPU;
sched_throw_tail(td); sched_throw_tail(td);
} }
void static void
sched_fork_exit(struct thread *td) sched_4bsd_fork_exit(struct thread *td)
{ {
/* /*
* Finish setting up thread glue so that it begins execution in a * Finish setting up thread glue so that it begins execution in a
* non-nested critical section with sched_lock held but not recursed. * non-nested critical section with sched_lock held but not recursed.
*/ */
td->td_oncpu = PCPU_GET(cpuid); td->td_oncpu = PCPU_GET(cpuid);
sched_lock.mtx_lock = (uintptr_t)td; sched_lock.mtx_lock = (uintptr_t)td;
lock_profile_obtain_lock_success(&sched_lock.lock_object, true, lock_profile_obtain_lock_success(&sched_lock.lock_object, true,
0, 0, __FILE__, __LINE__); 0, 0, __FILE__, __LINE__);
THREAD_LOCK_ASSERT(td, MA_OWNED | MA_NOTRECURSED); THREAD_LOCK_ASSERT(td, MA_OWNED | MA_NOTRECURSED);
KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "running", KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "running",
"prio:%d", td->td_priority); "prio:%d", td->td_priority);
SDT_PROBE0(sched, , , on__cpu); SDT_PROBE0(sched, , , on__cpu);
} }
char * static char *
sched_tdname(struct thread *td) sched_4bsd_tdname(struct thread *td)
{ {
#ifdef KTR #ifdef KTR
struct td_sched *ts; struct td_sched *ts;
ts = td_get_sched(td); ts = td_get_sched(td);
if (ts->ts_name[0] == '\0') if (ts->ts_name[0] == '\0')
snprintf(ts->ts_name, sizeof(ts->ts_name), snprintf(ts->ts_name, sizeof(ts->ts_name),
"%s tid %d", td->td_name, td->td_tid); "%s tid %d", td->td_name, td->td_tid);
return (ts->ts_name); return (ts->ts_name);
#else #else
return (td->td_name); return (td->td_name);
#endif #endif
} }
#ifdef KTR static void
jrtc27Unsubmitted
Done
Inline Actions

This and the corresponding ULE change don't build on non-KTR kernels, which is most of them (powerpc's DPAA is the only non-LINT one), since ts_name isn't defined in the corresponding struct for !KTR. Either the slot and shim need to be conditional too or it should be made a stub in each for !KTR (and the callers can drop the #ifdef). Did you test build this at all?

jrtc27: This and the corresponding ULE change don't build on non-KTR kernels, which is most of them…
kibAuthorUnsubmitted
Done
Inline Actions

I built and run-time tested by debug config on amd64.
I started tinderbox after posting the patch, and now restarted it for the latest update.

The patch is still in the state of discussion, so I did not see it reasonable to put that resources on it.
Thank you for taking a try and finding the issues.

kib: I built and run-time tested by debug config on amd64. I started tinderbox after posting the…
void sched_4bsd_clear_tdname(struct thread *td)
sched_clear_tdname(struct thread *td)
{ {
#ifdef KTR
struct td_sched *ts; struct td_sched *ts;
ts = td_get_sched(td); ts = td_get_sched(td);
ts->ts_name[0] = '\0'; ts->ts_name[0] = '\0';
jrtc27Unsubmitted
Done
Inline Actions

This one's still missing a KTR #ifdef, no?

jrtc27: This one's still missing a KTR #ifdef, no?
}
#endif #endif
}
void static void
sched_affinity(struct thread *td) sched_4bsd_affinity(struct thread *td)
{ {
#ifdef SMP #ifdef SMP
struct td_sched *ts; struct td_sched *ts;
int cpu; int cpu;
THREAD_LOCK_ASSERT(td, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED);
/* /*
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines case TDS_RUNNING:
if (td != curthread) if (td != curthread)
ipi_cpu(cpu, IPI_AST); ipi_cpu(cpu, IPI_AST);
break; break;
default: default:
break; break;
} }
#endif #endif
} }
static bool
sched_4bsd_do_timer_accounting(void)
{
#ifdef SMP
/*
* Don't do any accounting for the disabled HTT cores, since it
* will provide misleading numbers for the userland.
*
* No locking is necessary here, since even if we lose the race
* when hlt_cpus_mask changes it is not a big deal, really.
*
* Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask
* and unlike other schedulers it actually schedules threads to
* those CPUs.
*/
return (!CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask));
#else
return (true);
#endif
}
static int
sched_4bsd_find_l2_neighbor(int cpu)
{
return (-1);
}
struct sched_instance sched_4bsd_instance = {
#define SLOT(name) .name = sched_4bsd_##name
SLOT(load),
SLOT(rr_interval),
SLOT(runnable),
SLOT(exit),
SLOT(fork),
SLOT(fork_exit),
SLOT(class),
SLOT(nice),
SLOT(ap_entry),
SLOT(exit_thread),
SLOT(estcpu),
SLOT(fork_thread),
SLOT(ithread_prio),
SLOT(lend_prio),
SLOT(lend_user_prio),
SLOT(lend_user_prio_cond),
SLOT(pctcpu),
SLOT(prio),
SLOT(sleep),
SLOT(sswitch),
SLOT(throw),
SLOT(unlend_prio),
SLOT(user_prio),
SLOT(userret_slowpath),
SLOT(add),
SLOT(choose),
SLOT(clock),
SLOT(idletd),
SLOT(preempt),
SLOT(relinquish),
SLOT(rem),
SLOT(wakeup),
SLOT(bind),
SLOT(unbind),
SLOT(is_bound),
SLOT(affinity),
SLOT(sizeof_proc),
SLOT(sizeof_thread),
SLOT(tdname),
SLOT(clear_tdname),
SLOT(do_timer_accounting),
SLOT(find_l2_neighbor),
SLOT(init),
SLOT(init_ap),
SLOT(setup),
SLOT(initticks),
SLOT(schedcpu),
#undef SLOT
};
DECLARE_SCHEDULER(fourbsd_sched_selector, "4BSD", &sched_4bsd_instance);