Differential D54831 Diff 170318 sys/kern/sched_ule.c

Changeset View

Standalone View

sys/kern/sched_ule.c

Show All 31 Lines
* performance under load even on uni-processor systems.		* performance under load even on uni-processor systems.
*		*
* etymology:		* etymology:
* ULE is the last three letters in schedule. It owes its name to a		* ULE is the last three letters in schedule. It owes its name to a
* generic user created for a scheduling system by Paul Mikesell at		* generic user created for a scheduling system by Paul Mikesell at
* Isilon Systems and a general lack of creativity on the part of the author.		* Isilon Systems and a general lack of creativity on the part of the author.
*/		*/

#include <sys/cdefs.h>
#include "opt_hwpmc_hooks.h"		#include "opt_hwpmc_hooks.h"
#include "opt_hwt_hooks.h"		#include "opt_hwt_hooks.h"
#include "opt_sched.h"		#include "opt_sched.h"

#include <sys/param.h>
#include <sys/systm.h>		#include <sys/systm.h>
#include <sys/kdb.h>		#include <sys/kdb.h>
#include <sys/kernel.h>		#include <sys/kernel.h>
#include <sys/ktr.h>		#include <sys/ktr.h>
#include <sys/limits.h>		#include <sys/limits.h>
#include <sys/lock.h>		#include <sys/lock.h>
#include <sys/mutex.h>		#include <sys/mutex.h>
#include <sys/proc.h>		#include <sys/proc.h>
▲ Show 20 Lines • Show All 347 Lines • ▼ Show 20 Lines	static inline struct thread runq_steal_idle(struct runq const rq,
int cpu);		int cpu);
static struct thread tdq_steal(struct tdq , int);		static struct thread tdq_steal(struct tdq , int);

static int sched_pickcpu(struct thread *, int);		static int sched_pickcpu(struct thread *, int);
static void sched_balance(void);		static void sched_balance(void);
static bool sched_balance_pair(struct tdq , struct tdq );		static bool sched_balance_pair(struct tdq , struct tdq );
static inline struct tdq sched_setcpu(struct thread , int, int);		static inline struct tdq sched_setcpu(struct thread , int, int);
static inline void thread_unblock_switch(struct thread , struct mtx );		static inline void thread_unblock_switch(struct thread , struct mtx );
static int sysctl_kern_sched_topology_spec(SYSCTL_HANDLER_ARGS);		static int sysctl_kern_sched_ule_topology_spec(SYSCTL_HANDLER_ARGS);
static int sysctl_kern_sched_topology_spec_internal(struct sbuf *sb,		static int sysctl_kern_sched_ule_topology_spec_internal(struct sbuf *sb,
struct cpu_group *cg, int indent);		struct cpu_group *cg, int indent);
#endif		#endif

static void sched_setup(void *dummy);
SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL);

static void sched_initticks(void *dummy);
SYSINIT(sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks,
NULL);

SDT_PROVIDER_DEFINE(sched);		SDT_PROVIDER_DEFINE(sched);

SDT_PROBE_DEFINE3(sched, , , change__pri, "struct thread *",		SDT_PROBE_DEFINE3(sched, , , change__pri, "struct thread *",
"struct proc *", "uint8_t");		"struct proc *", "uint8_t");
SDT_PROBE_DEFINE3(sched, , , dequeue, "struct thread *",		SDT_PROBE_DEFINE3(sched, , , dequeue, "struct thread *",
"struct proc ", "void ");		"struct proc ", "void ");
SDT_PROBE_DEFINE4(sched, , , enqueue, "struct thread *",		SDT_PROBE_DEFINE4(sched, , , enqueue, "struct thread *",
"struct proc ", "void ", "int");		"struct proc ", "void ", "int");
▲ Show 20 Lines • Show All 1,208 Lines • ▼ Show 20 Lines
}		}
#endif		#endif

/*		/*
* Setup the thread queues and initialize the topology based on MD		* Setup the thread queues and initialize the topology based on MD
* information.		* information.
*/		*/
static void		static void
sched_setup(void *dummy)		sched_ule_setup(void)
{		{
struct tdq *tdq;		struct tdq *tdq;

#ifdef SMP		#ifdef SMP
sched_setup_smp();		sched_setup_smp();
#else		#else
tdq_setup(TDQ_SELF(), 0);		tdq_setup(TDQ_SELF(), 0);
#endif		#endif
tdq = TDQ_SELF();		tdq = TDQ_SELF();

/* Add thread0's load since it's running. */		/* Add thread0's load since it's running. */
TDQ_LOCK(tdq);		TDQ_LOCK(tdq);
thread0.td_lock = TDQ_LOCKPTR(tdq);		thread0.td_lock = TDQ_LOCKPTR(tdq);
tdq_load_add(tdq, &thread0);		tdq_load_add(tdq, &thread0);
tdq->tdq_curthread = &thread0;		tdq->tdq_curthread = &thread0;
tdq->tdq_lowpri = thread0.td_priority;		tdq->tdq_lowpri = thread0.td_priority;
TDQ_UNLOCK(tdq);		TDQ_UNLOCK(tdq);
}		}

/*		/*
* This routine determines time constants after stathz and hz are setup.		* This routine determines time constants after stathz and hz are setup.
*/		*/
/* ARGSUSED */		/* ARGSUSED */
static void		static void
sched_initticks(void *dummy)		sched_ule_initticks(void)
{		{
int incr;		int incr;

realstathz = stathz ? stathz : hz;		realstathz = stathz ? stathz : hz;
sched_slice = realstathz / SCHED_SLICE_DEFAULT_DIVISOR;		sched_slice = realstathz / SCHED_SLICE_DEFAULT_DIVISOR;
sched_slice_min = sched_slice / SCHED_SLICE_MIN_DIVISOR;		sched_slice_min = sched_slice / SCHED_SLICE_MIN_DIVISOR;
hogticks = imax(1, (2 * hz * sched_slice + realstathz / 2) /		hogticks = imax(1, (2 * hz * sched_slice + realstathz / 2) /
realstathz);		realstathz);
▲ Show 20 Lines • Show All 207 Lines • ▼ Show 20 Lines	if (sum > SCHED_SLP_RUN_FORK) {
ts->ts_runtime /= ratio;		ts->ts_runtime /= ratio;
ts->ts_slptime /= ratio;		ts->ts_slptime /= ratio;
}		}
}		}

/*		/*
* Called from proc0_init() to setup the scheduler fields.		* Called from proc0_init() to setup the scheduler fields.
*/		*/
void		static void
schedinit(void)		sched_ule_init(void)
{		{
struct td_sched *ts0;		struct td_sched *ts0;

/*		/*
* Set up the scheduler specific parts of thread0.		* Set up the scheduler specific parts of thread0.
*/		*/
ts0 = td_get_sched(&thread0);		ts0 = td_get_sched(&thread0);
ts0->ts_ftick = (u_int)ticks;		ts0->ts_ftick = (u_int)ticks;
ts0->ts_ltick = ts0->ts_ftick;		ts0->ts_ltick = ts0->ts_ftick;
ts0->ts_slice = 0;		ts0->ts_slice = 0;
ts0->ts_cpu = curcpu; /* set valid CPU number */		ts0->ts_cpu = curcpu; /* set valid CPU number */
}		}

/*		/*
* schedinit_ap() is needed prior to calling sched_throw(NULL) to ensure that		* schedinit_ap() is needed prior to calling sched_throw(NULL) to ensure that
* the pcpu requirements are met for any calls in the period between curthread		* the pcpu requirements are met for any calls in the period between curthread
* initialization and sched_throw(). One can safely add threads to the queue		* initialization and sched_throw(). One can safely add threads to the queue
* before sched_throw(), for instance, as long as the thread lock is setup		* before sched_throw(), for instance, as long as the thread lock is setup
* correctly.		* correctly.
*		*
* TDQ_SELF() relies on the below sched pcpu setting; it may be used only		* TDQ_SELF() relies on the below sched pcpu setting; it may be used only
* after schedinit_ap().		* after schedinit_ap().
*/		*/
void		static void
schedinit_ap(void)		sched_ule_init_ap(void)
{		{

#ifdef SMP		#ifdef SMP
PCPU_SET(sched, DPCPU_PTR(tdq));		PCPU_SET(sched, DPCPU_PTR(tdq));
#endif		#endif
PCPU_GET(idlethread)->td_lock = TDQ_LOCKPTR(TDQ_SELF());		PCPU_GET(idlethread)->td_lock = TDQ_LOCKPTR(TDQ_SELF());
}		}

/*		/*
* This is only somewhat accurate since given many processes of the same		* This is only somewhat accurate since given many processes of the same
* priority they will switch when their slices run out, which will be		* priority they will switch when their slices run out, which will be
* at most sched_slice stathz ticks.		* at most sched_slice stathz ticks.
*/		*/
int		static int
sched_rr_interval(void)		sched_ule_rr_interval(void)
{		{

/* Convert sched_slice from stathz to hz. */		/* Convert sched_slice from stathz to hz. */
return (imax(1, (sched_slice * hz + realstathz / 2) / realstathz));		return (imax(1, (sched_slice * hz + realstathz / 2) / realstathz));
}		}

/*		/*
* Update the percent cpu tracking information when it is requested or the total		* Update the percent cpu tracking information when it is requested or the total
▲ Show 20 Lines • Show All 102 Lines • ▼ Show 20 Lines	sched_thread_priority(struct thread *td, u_char prio)
}		}
td->td_priority = prio;		td->td_priority = prio;
}		}

/*		/*
* Update a thread's priority when it is lent another thread's		* Update a thread's priority when it is lent another thread's
* priority.		* priority.
*/		*/
void		static void
sched_lend_prio(struct thread *td, u_char prio)		sched_ule_lend_prio(struct thread *td, u_char prio)
{		{

td->td_flags \|= TDF_BORROWING;		td->td_flags \|= TDF_BORROWING;
sched_thread_priority(td, prio);		sched_thread_priority(td, prio);
}		}

/*		/*
* Restore a thread's priority when priority propagation is		* Restore a thread's priority when priority propagation is
* over. The prio argument is the minimum priority the thread		* over. The prio argument is the minimum priority the thread
* needs to have to satisfy other possible priority lending		* needs to have to satisfy other possible priority lending
* requests. If the thread's regular priority is less		* requests. If the thread's regular priority is less
* important than prio, the thread will keep a priority boost		* important than prio, the thread will keep a priority boost
* of prio.		* of prio.
*/		*/
void		static void
sched_unlend_prio(struct thread *td, u_char prio)		sched_ule_unlend_prio(struct thread *td, u_char prio)
{		{
u_char base_pri;		u_char base_pri;

if (td->td_base_pri >= PRI_MIN_TIMESHARE &&		if (td->td_base_pri >= PRI_MIN_TIMESHARE &&
td->td_base_pri <= PRI_MAX_TIMESHARE)		td->td_base_pri <= PRI_MAX_TIMESHARE)
base_pri = td->td_user_pri;		base_pri = td->td_user_pri;
else		else
base_pri = td->td_base_pri;		base_pri = td->td_base_pri;
if (prio >= base_pri) {		if (prio >= base_pri) {
td->td_flags &= ~TDF_BORROWING;		td->td_flags &= ~TDF_BORROWING;
sched_thread_priority(td, base_pri);		sched_thread_priority(td, base_pri);
} else		} else
sched_lend_prio(td, prio);		sched_lend_prio(td, prio);
}		}

/*		/*
* Standard entry for setting the priority to an absolute value.		* Standard entry for setting the priority to an absolute value.
*/		*/
void		static void
sched_prio(struct thread *td, u_char prio)		sched_ule_prio(struct thread *td, u_char prio)
{		{
u_char oldprio;		u_char oldprio;

/* First, update the base priority. */		/* First, update the base priority. */
td->td_base_pri = prio;		td->td_base_pri = prio;

/*		/*
* If the thread is borrowing another thread's priority, don't		* If the thread is borrowing another thread's priority, don't
Show All 12 Lines	sched_ule_prio(struct thread *td, u_char prio)
*/		*/
if (TD_ON_LOCK(td) && oldprio != prio)		if (TD_ON_LOCK(td) && oldprio != prio)
turnstile_adjust(td, oldprio);		turnstile_adjust(td, oldprio);
}		}

/*		/*
* Set the base interrupt thread priority.		* Set the base interrupt thread priority.
*/		*/
void		static void
sched_ithread_prio(struct thread *td, u_char prio)		sched_ule_ithread_prio(struct thread *td, u_char prio)
{		{
THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
MPASS(td->td_pri_class == PRI_ITHD);		MPASS(td->td_pri_class == PRI_ITHD);
td->td_base_ithread_pri = prio;		td->td_base_ithread_pri = prio;
sched_prio(td, prio);		sched_prio(td, prio);
}		}

/*		/*
* Set the base user priority, does not effect current running priority.		* Set the base user priority, does not effect current running priority.
*/		*/
void		static void
sched_user_prio(struct thread *td, u_char prio)		sched_ule_user_prio(struct thread *td, u_char prio)
{		{

td->td_base_user_pri = prio;		td->td_base_user_pri = prio;
if (td->td_lend_user_pri <= prio)		if (td->td_lend_user_pri <= prio)
return;		return;
td->td_user_pri = prio;		td->td_user_pri = prio;
}		}

void		static void
sched_lend_user_prio(struct thread *td, u_char prio)		sched_ule_lend_user_prio(struct thread *td, u_char prio)
{		{

THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
td->td_lend_user_pri = prio;		td->td_lend_user_pri = prio;
td->td_user_pri = min(prio, td->td_base_user_pri);		td->td_user_pri = min(prio, td->td_base_user_pri);
if (td->td_priority > td->td_user_pri)		if (td->td_priority > td->td_user_pri)
sched_prio(td, td->td_user_pri);		sched_prio(td, td->td_user_pri);
else if (td->td_priority != td->td_user_pri)		else if (td->td_priority != td->td_user_pri)
ast_sched_locked(td, TDA_SCHED);		ast_sched_locked(td, TDA_SCHED);
}		}

/*		/*
* Like the above but first check if there is anything to do.		* Like the above but first check if there is anything to do.
*/		*/
void		static void
sched_lend_user_prio_cond(struct thread *td, u_char prio)		sched_ule_lend_user_prio_cond(struct thread *td, u_char prio)
{		{

if (td->td_lend_user_pri == prio)		if (td->td_lend_user_pri == prio)
return;		return;

thread_lock(td);		thread_lock(td);
sched_lend_user_prio(td, prio);		sched_lend_user_prio(td, prio);
thread_unlock(td);		thread_unlock(td);
▲ Show 20 Lines • Show All 154 Lines • ▼ Show 20 Lines
}		}

/*		/*
* Switch threads. This function has to handle threads coming in while		* Switch threads. This function has to handle threads coming in while
* blocked for some reason, running, or idle. It also must deal with		* blocked for some reason, running, or idle. It also must deal with
* migrating a thread from one queue to another as running threads may		* migrating a thread from one queue to another as running threads may
* be assigned elsewhere via binding.		* be assigned elsewhere via binding.
*/		*/
void		static void
sched_switch(struct thread *td, int flags)		sched_ule_sswitch(struct thread *td, int flags)
{		{
struct thread *newtd;		struct thread *newtd;
struct tdq *tdq;		struct tdq *tdq;
struct td_sched *ts;		struct td_sched *ts;
struct mtx *mtx;		struct mtx *mtx;
int srqflag;		int srqflag;
int cpuid, preempted;		int cpuid, preempted;
#ifdef SMP		#ifdef SMP
▲ Show 20 Lines • Show All 121 Lines • ▼ Show 20 Lines	#endif

KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "running",		KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "running",
"prio:%d", td->td_priority);		"prio:%d", td->td_priority);
}		}

/*		/*
* Adjust thread priorities as a result of a nice request.		* Adjust thread priorities as a result of a nice request.
*/		*/
void		static void
sched_nice(struct proc *p, int nice)		sched_ule_nice(struct proc *p, int nice)
{		{
struct thread *td;		struct thread *td;

PROC_LOCK_ASSERT(p, MA_OWNED);		PROC_LOCK_ASSERT(p, MA_OWNED);

p->p_nice = nice;		p->p_nice = nice;
FOREACH_THREAD_IN_PROC(p, td) {		FOREACH_THREAD_IN_PROC(p, td) {
thread_lock(td);		thread_lock(td);
sched_priority(td);		sched_priority(td);
sched_prio(td, td->td_base_user_pri);		sched_prio(td, td->td_base_user_pri);
thread_unlock(td);		thread_unlock(td);
}		}
}		}

/*		/*
* Record the sleep time for the interactivity scorer.		* Record the sleep time for the interactivity scorer.
*/		*/
void		static void
sched_sleep(struct thread *td, int prio)		sched_ule_sleep(struct thread *td, int prio)
{		{

THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);

td->td_slptick = ticks;		td->td_slptick = ticks;
if (PRI_BASE(td->td_pri_class) != PRI_TIMESHARE)		if (PRI_BASE(td->td_pri_class) != PRI_TIMESHARE)
return;		return;
if (static_boost == 1 && prio)		if (static_boost == 1 && prio)
sched_prio(td, prio);		sched_prio(td, prio);
else if (static_boost && td->td_priority > static_boost)		else if (static_boost && td->td_priority > static_boost)
sched_prio(td, static_boost);		sched_prio(td, static_boost);
}		}

/*		/*
* Schedule a thread to resume execution and record how long it voluntarily		* Schedule a thread to resume execution and record how long it voluntarily
* slept. We also update the pctcpu, interactivity, and priority.		* slept. We also update the pctcpu, interactivity, and priority.
*		*
* Requires the thread lock on entry, drops on exit.		* Requires the thread lock on entry, drops on exit.
*/		*/
void		static void
sched_wakeup(struct thread *td, int srqflags)		sched_ule_wakeup(struct thread *td, int srqflags)
{		{
struct td_sched *ts;		struct td_sched *ts;
int slptick;		int slptick;

THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
ts = td_get_sched(td);		ts = td_get_sched(td);

/*		/*
Show All 22 Lines	sched_ule_wakeup(struct thread *td, int srqflags)
ts->ts_slice = 0;		ts->ts_slice = 0;
sched_add(td, SRQ_BORING \| srqflags);		sched_add(td, SRQ_BORING \| srqflags);
}		}

/*		/*
* Penalize the parent for creating a new child and initialize the child's		* Penalize the parent for creating a new child and initialize the child's
* priority.		* priority.
*/		*/
void		static void
sched_fork(struct thread td, struct thread child)		sched_ule_fork(struct thread td, struct thread child)
{		{
THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
sched_pctcpu_update(td_get_sched(td), 1);		sched_pctcpu_update(td_get_sched(td), 1);
sched_fork_thread(td, child);		sched_fork_thread(td, child);
/*		/*
* Penalize the parent and child for forking.		* Penalize the parent and child for forking.
*/		*/
sched_interact_fork(child);		sched_interact_fork(child);
sched_priority(child);		sched_priority(child);
td_get_sched(td)->ts_runtime += tickincr;		td_get_sched(td)->ts_runtime += tickincr;
sched_interact_update(td);		sched_interact_update(td);
sched_priority(td);		sched_priority(td);
}		}

/*		/*
* Fork a new thread, may be within the same process.		* Fork a new thread, may be within the same process.
*/		*/
void		static void
sched_fork_thread(struct thread td, struct thread child)		sched_ule_fork_thread(struct thread td, struct thread child)
{		{
struct td_sched *ts;		struct td_sched *ts;
struct td_sched *ts2;		struct td_sched *ts2;
struct tdq *tdq;		struct tdq *tdq;

tdq = TDQ_SELF();		tdq = TDQ_SELF();
THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
/*		/*
Show All 28 Lines
#ifdef KTR		#ifdef KTR
bzero(ts2->ts_name, sizeof(ts2->ts_name));		bzero(ts2->ts_name, sizeof(ts2->ts_name));
#endif		#endif
}		}

/*		/*
* Adjust the priority class of a thread.		* Adjust the priority class of a thread.
*/		*/
void		static void
sched_class(struct thread *td, int class)		sched_ule_class(struct thread *td, int class)
{		{

THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
if (td->td_pri_class == class)		if (td->td_pri_class == class)
return;		return;
td->td_pri_class = class;		td->td_pri_class = class;
}		}

/*		/*
* Return some of the child's priority and interactivity to the parent.		* Return some of the child's priority and interactivity to the parent.
*/		*/
void		static void
sched_exit(struct proc p, struct thread child)		sched_ule_exit(struct proc p, struct thread child)
{		{
struct thread *td;		struct thread *td;

KTR_STATE1(KTR_SCHED, "thread", sched_tdname(child), "proc exit",		KTR_STATE1(KTR_SCHED, "thread", sched_tdname(child), "proc exit",
"prio:%d", child->td_priority);		"prio:%d", child->td_priority);
PROC_LOCK_ASSERT(p, MA_OWNED);		PROC_LOCK_ASSERT(p, MA_OWNED);
td = FIRST_THREAD_IN_PROC(p);		td = FIRST_THREAD_IN_PROC(p);
sched_exit_thread(td, child);		sched_exit_thread(td, child);
}		}

/*		/*
* Penalize another thread for the time spent on this one. This helps to		* Penalize another thread for the time spent on this one. This helps to
* worsen the priority and interactivity of processes which schedule batch		* worsen the priority and interactivity of processes which schedule batch
* jobs such as make. This has little effect on the make process itself but		* jobs such as make. This has little effect on the make process itself but
* causes new processes spawned by it to receive worse scores immediately.		* causes new processes spawned by it to receive worse scores immediately.
*/		*/
void		static void
sched_exit_thread(struct thread td, struct thread child)		sched_ule_exit_thread(struct thread td, struct thread child)
{		{

KTR_STATE1(KTR_SCHED, "thread", sched_tdname(child), "thread exit",		KTR_STATE1(KTR_SCHED, "thread", sched_tdname(child), "thread exit",
"prio:%d", child->td_priority);		"prio:%d", child->td_priority);
/*		/*
* Give the child's runtime to the parent without returning the		* Give the child's runtime to the parent without returning the
* sleep time as a penalty to the parent. This causes shells that		* sleep time as a penalty to the parent. This causes shells that
* launch expensive things to mark their children as expensive.		* launch expensive things to mark their children as expensive.
*/		*/
thread_lock(td);		thread_lock(td);
td_get_sched(td)->ts_runtime += td_get_sched(child)->ts_runtime;		td_get_sched(td)->ts_runtime += td_get_sched(child)->ts_runtime;
sched_interact_update(td);		sched_interact_update(td);
sched_priority(td);		sched_priority(td);
thread_unlock(td);		thread_unlock(td);
}		}

void		static void
sched_preempt(struct thread *td)		sched_ule_preempt(struct thread *td)
{		{
struct tdq *tdq;		struct tdq *tdq;
int flags;		int flags;

SDT_PROBE2(sched, , , surrender, td, td->td_proc);		SDT_PROBE2(sched, , , surrender, td, td->td_proc);

thread_lock(td);		thread_lock(td);
tdq = TDQ_SELF();		tdq = TDQ_SELF();
Show All 13 Lines	sched_ule_preempt(struct thread *td)
}		}
thread_unlock(td);		thread_unlock(td);
}		}

/*		/*
* Fix priorities on return to user-space. Priorities may be elevated due		* Fix priorities on return to user-space. Priorities may be elevated due
* to static priorities in msleep() or similar.		* to static priorities in msleep() or similar.
*/		*/
void		static void
sched_userret_slowpath(struct thread *td)		sched_ule_userret_slowpath(struct thread *td)
{		{

thread_lock(td);		thread_lock(td);
td->td_priority = td->td_user_pri;		td->td_priority = td->td_user_pri;
td->td_base_pri = td->td_user_pri;		td->td_base_pri = td->td_user_pri;
tdq_setlowpri(TDQ_SELF(), td);		tdq_setlowpri(TDQ_SELF(), td);
thread_unlock(td);		thread_unlock(td);
}		}
Show All 13 Lines	if (PRI_BASE(td->td_pri_class) == PRI_ITHD)
return (sched_slice);		return (sched_slice);
return (tdq_slice(tdq));		return (tdq_slice(tdq));
}		}

/*		/*
* Handle a stathz tick. This is really only relevant for timeshare		* Handle a stathz tick. This is really only relevant for timeshare
* and interrupt threads.		* and interrupt threads.
*/		*/
void		static void
sched_clock(struct thread *td, int cnt)		sched_ule_clock(struct thread *td, int cnt)
{		{
struct tdq *tdq;		struct tdq *tdq;
struct td_sched *ts;		struct td_sched *ts;

THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
tdq = TDQ_SELF();		tdq = TDQ_SELF();
#ifdef SMP		#ifdef SMP
/*		/*
▲ Show 20 Lines • Show All 68 Lines • ▼ Show 20 Lines	if (PRI_BASE(td->td_pri_class) == PRI_ITHD) {
}		}
} else {		} else {
ast_sched_locked(td, TDA_SCHED);		ast_sched_locked(td, TDA_SCHED);
td->td_flags \|= TDF_SLICEEND;		td->td_flags \|= TDF_SLICEEND;
}		}
}		}
}		}

u_int		static u_int
sched_estcpu(struct thread *td __unused)		sched_ule_estcpu(struct thread *td __unused)
{		{

return (0);		return (0);
}		}

/*		/*
* Return whether the current CPU has runnable tasks. Used for in-kernel		* Return whether the current CPU has runnable tasks. Used for in-kernel
* cooperative idle threads.		* cooperative idle threads.
*/		*/
bool		static bool
sched_runnable(void)		sched_ule_runnable(void)
{		{
struct tdq *tdq;		struct tdq *tdq;

tdq = TDQ_SELF();		tdq = TDQ_SELF();
return (TDQ_LOAD(tdq) > (TD_IS_IDLETHREAD(curthread) ? 0 : 1));		return (TDQ_LOAD(tdq) > (TD_IS_IDLETHREAD(curthread) ? 0 : 1));
}		}

/*		/*
* Choose the highest priority thread to run. The thread is removed from		* Choose the highest priority thread to run. The thread is removed from
* the run-queue while running however the load remains.		* the run-queue while running however the load remains.
*/		*/
struct thread *		static struct thread *
sched_choose(void)		sched_ule_choose(void)
{		{
struct thread *td;		struct thread *td;
struct tdq *tdq;		struct tdq *tdq;

tdq = TDQ_SELF();		tdq = TDQ_SELF();
TDQ_LOCK_ASSERT(tdq, MA_OWNED);		TDQ_LOCK_ASSERT(tdq, MA_OWNED);
td = tdq_choose(tdq);		td = tdq_choose(tdq);
if (td != NULL) {		if (td != NULL) {
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines
}		}

/*		/*
* Select the target thread queue and add a thread to it. Request		* Select the target thread queue and add a thread to it. Request
* preemption or IPI a remote processor if required.		* preemption or IPI a remote processor if required.
*		*
* Requires the thread lock on entry, drops on exit.		* Requires the thread lock on entry, drops on exit.
*/		*/
void		static void
sched_add(struct thread *td, int flags)		sched_ule_add(struct thread *td, int flags)
{		{
struct tdq *tdq;		struct tdq *tdq;
#ifdef SMP		#ifdef SMP
int cpu, lowpri;		int cpu, lowpri;
#endif		#endif

KTR_STATE2(KTR_SCHED, "thread", sched_tdname(td), "runq add",		KTR_STATE2(KTR_SCHED, "thread", sched_tdname(td), "runq add",
"prio:%d", td->td_priority, KTR_ATTR_LINKED,		"prio:%d", td->td_priority, KTR_ATTR_LINKED,
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines	if (!(flags & SRQ_HOLDTD))
thread_unlock(td);		thread_unlock(td);
}		}

/*		/*
* Remove a thread from a run-queue without running it. This is used		* Remove a thread from a run-queue without running it. This is used
* when we're stealing a thread from a remote queue. Otherwise all threads		* when we're stealing a thread from a remote queue. Otherwise all threads
* exit by calling sched_exit_thread() and sched_throw() themselves.		* exit by calling sched_exit_thread() and sched_throw() themselves.
*/		*/
void		static void
sched_rem(struct thread *td)		sched_ule_rem(struct thread *td)
{		{
struct tdq *tdq;		struct tdq *tdq;

KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "runq rem",		KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "runq rem",
"prio:%d", td->td_priority);		"prio:%d", td->td_priority);
SDT_PROBE3(sched, , , dequeue, td, td->td_proc, NULL);		SDT_PROBE3(sched, , , dequeue, td, td->td_proc, NULL);
tdq = TDQ_CPU(td_get_sched(td)->ts_cpu);		tdq = TDQ_CPU(td_get_sched(td)->ts_cpu);
TDQ_LOCK_ASSERT(tdq, MA_OWNED);		TDQ_LOCK_ASSERT(tdq, MA_OWNED);
MPASS(td->td_lock == TDQ_LOCKPTR(tdq));		MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
KASSERT(TD_ON_RUNQ(td),		KASSERT(TD_ON_RUNQ(td),
("sched_rem: thread not on run queue"));		("sched_rem: thread not on run queue"));
tdq_runq_rem(tdq, td);		tdq_runq_rem(tdq, td);
tdq_load_rem(tdq, td);		tdq_load_rem(tdq, td);
TD_SET_CAN_RUN(td);		TD_SET_CAN_RUN(td);
if (td->td_priority == tdq->tdq_lowpri)		if (td->td_priority == tdq->tdq_lowpri)
tdq_setlowpri(tdq, NULL);		tdq_setlowpri(tdq, NULL);
}		}

/*		/*
* Fetch cpu utilization information. Updates on demand.		* Fetch cpu utilization information. Updates on demand.
*/		*/
fixpt_t		static fixpt_t
sched_pctcpu(struct thread *td)		sched_ule_pctcpu(struct thread *td)
{		{
struct td_sched *ts;		struct td_sched *ts;
u_int len;		u_int len;
fixpt_t pctcpu;		fixpt_t pctcpu;

THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
ts = td_get_sched(td);		ts = td_get_sched(td);
sched_pctcpu_update(ts, TD_IS_RUNNING(td));		sched_pctcpu_update(ts, TD_IS_RUNNING(td));
len = SCHED_TICK_LENGTH(ts);		len = SCHED_TICK_LENGTH(ts);
pctcpu = ((FSHIFT >= SCHED_TICK_SHIFT ? /* Resolved at compile-time. */		pctcpu = ((FSHIFT >= SCHED_TICK_SHIFT ? /* Resolved at compile-time. */
(SCHED_TICK_RUN_SHIFTED(ts) << (FSHIFT - SCHED_TICK_SHIFT)) :		(SCHED_TICK_RUN_SHIFTED(ts) << (FSHIFT - SCHED_TICK_SHIFT)) :
(SCHED_TICK_RUN_SHIFTED(ts) >> (SCHED_TICK_SHIFT - FSHIFT))) +		(SCHED_TICK_RUN_SHIFTED(ts) >> (SCHED_TICK_SHIFT - FSHIFT))) +
len / 2) / len;		len / 2) / len;
return (pctcpu);		return (pctcpu);
}		}

/*		/*
* Enforce affinity settings for a thread. Called after adjustments to		* Enforce affinity settings for a thread. Called after adjustments to
* cpumask.		* cpumask.
*/		*/
void		static void
sched_affinity(struct thread *td)		sched_ule_affinity(struct thread *td)
{		{
#ifdef SMP		#ifdef SMP
struct td_sched *ts;		struct td_sched *ts;

THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
ts = td_get_sched(td);		ts = td_get_sched(td);
if (THREAD_CAN_SCHED(td, ts->ts_cpu))		if (THREAD_CAN_SCHED(td, ts->ts_cpu))
return;		return;
Show All 13 Lines	#ifdef SMP
if (td != curthread)		if (td != curthread)
ipi_cpu(ts->ts_cpu, IPI_PREEMPT);		ipi_cpu(ts->ts_cpu, IPI_PREEMPT);
#endif		#endif
}		}

/*		/*
* Bind a thread to a target cpu.		* Bind a thread to a target cpu.
*/		*/
void		static void
sched_bind(struct thread *td, int cpu)		sched_ule_bind(struct thread *td, int cpu)
{		{
struct td_sched *ts;		struct td_sched *ts;

THREAD_LOCK_ASSERT(td, MA_OWNED\|MA_NOTRECURSED);		THREAD_LOCK_ASSERT(td, MA_OWNED\|MA_NOTRECURSED);
KASSERT(td == curthread, ("sched_bind: can only bind curthread"));		KASSERT(td == curthread, ("sched_bind: can only bind curthread"));
ts = td_get_sched(td);		ts = td_get_sched(td);
if (ts->ts_flags & TSF_BOUND)		if (ts->ts_flags & TSF_BOUND)
sched_unbind(td);		sched_unbind(td);
KASSERT(THREAD_CAN_MIGRATE(td), ("%p must be migratable", td));		KASSERT(THREAD_CAN_MIGRATE(td), ("%p must be migratable", td));
ts->ts_flags \|= TSF_BOUND;		ts->ts_flags \|= TSF_BOUND;
sched_pin();		sched_pin();
if (PCPU_GET(cpuid) == cpu)		if (PCPU_GET(cpuid) == cpu)
return;		return;
ts->ts_cpu = cpu;		ts->ts_cpu = cpu;
/* When we return from mi_switch we'll be on the correct cpu. */		/* When we return from mi_switch we'll be on the correct cpu. */
mi_switch(SW_VOL \| SWT_BIND);		mi_switch(SW_VOL \| SWT_BIND);
thread_lock(td);		thread_lock(td);
}		}

/*		/*
* Release a bound thread.		* Release a bound thread.
*/		*/
void		static void
sched_unbind(struct thread *td)		sched_ule_unbind(struct thread *td)
{		{
struct td_sched *ts;		struct td_sched *ts;

THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
KASSERT(td == curthread, ("sched_unbind: can only bind curthread"));		KASSERT(td == curthread, ("sched_unbind: can only bind curthread"));
ts = td_get_sched(td);		ts = td_get_sched(td);
if ((ts->ts_flags & TSF_BOUND) == 0)		if ((ts->ts_flags & TSF_BOUND) == 0)
return;		return;
ts->ts_flags &= ~TSF_BOUND;		ts->ts_flags &= ~TSF_BOUND;
sched_unpin();		sched_unpin();
}		}

int		static int
sched_is_bound(struct thread *td)		sched_ule_is_bound(struct thread *td)
{		{
THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
return (td_get_sched(td)->ts_flags & TSF_BOUND);		return (td_get_sched(td)->ts_flags & TSF_BOUND);
}		}

/*		/*
* Basic yield call.		* Basic yield call.
*/		*/
void		static void
sched_relinquish(struct thread *td)		sched_ule_relinquish(struct thread *td)
{		{
thread_lock(td);		thread_lock(td);
mi_switch(SW_VOL \| SWT_RELINQUISH);		mi_switch(SW_VOL \| SWT_RELINQUISH);
}		}

/*		/*
* Return the total system load.		* Return the total system load.
*/		*/
int		static int
sched_load(void)		sched_ule_load(void)
{		{
#ifdef SMP		#ifdef SMP
int total;		int total;
int i;		int i;

total = 0;		total = 0;
CPU_FOREACH(i)		CPU_FOREACH(i)
total += atomic_load_int(&TDQ_CPU(i)->tdq_sysload);		total += atomic_load_int(&TDQ_CPU(i)->tdq_sysload);
return (total);		return (total);
#else		#else
return (atomic_load_int(&TDQ_SELF()->tdq_sysload));		return (atomic_load_int(&TDQ_SELF()->tdq_sysload));
#endif		#endif
}		}

int		static int
sched_sizeof_proc(void)		sched_ule_sizeof_proc(void)
{		{
return (sizeof(struct proc));		return (sizeof(struct proc));
}		}

int		static int
sched_sizeof_thread(void)		sched_ule_sizeof_thread(void)
{		{
return (sizeof(struct thread) + sizeof(struct td_sched));		return (sizeof(struct thread) + sizeof(struct td_sched));
}		}

#ifdef SMP		#ifdef SMP
#define TDQ_IDLESPIN(tdq) \		#define TDQ_IDLESPIN(tdq) \
((tdq)->tdq_cg != NULL && ((tdq)->tdq_cg->cg_flags & CG_FLAG_THREAD) == 0)		((tdq)->tdq_cg != NULL && ((tdq)->tdq_cg->cg_flags & CG_FLAG_THREAD) == 0)
#else		#else
#define TDQ_IDLESPIN(tdq) 1		#define TDQ_IDLESPIN(tdq) 1
#endif		#endif

/*		/*
* The actual idle process.		* The actual idle process.
*/		*/
void		static void
sched_idletd(void *dummy)		sched_ule_idletd(void *dummy)
{		{
struct thread *td;		struct thread *td;
struct tdq *tdq;		struct tdq *tdq;
int oldswitchcnt, switchcnt;		int oldswitchcnt, switchcnt;
int i;		int i;

mtx_assert(&Giant, MA_NOTOWNED);		mtx_assert(&Giant, MA_NOTOWNED);
td = curthread;		td = curthread;
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines	sched_throw_grab(struct tdq *tdq)
KASSERT(curthread->td_md.md_spinlock_count == 1,		KASSERT(curthread->td_md.md_spinlock_count == 1,
("invalid count %d", curthread->td_md.md_spinlock_count));		("invalid count %d", curthread->td_md.md_spinlock_count));
return (newtd);		return (newtd);
}		}

/*		/*
* A CPU is entering for the first time.		* A CPU is entering for the first time.
*/		*/
void		static void
sched_ap_entry(void)		sched_ule_ap_entry(void)
{		{
struct thread *newtd;		struct thread *newtd;
struct tdq *tdq;		struct tdq *tdq;

tdq = TDQ_SELF();		tdq = TDQ_SELF();

/* This should have been setup in schedinit_ap(). */		/* This should have been setup in schedinit_ap(). */
THREAD_LOCKPTR_ASSERT(curthread, TDQ_LOCKPTR(tdq));		THREAD_LOCKPTR_ASSERT(curthread, TDQ_LOCKPTR(tdq));
Show All 12 Lines	#endif

/* doesn't return */		/* doesn't return */
cpu_throw(NULL, newtd);		cpu_throw(NULL, newtd);
}		}

/*		/*
* A thread is exiting.		* A thread is exiting.
*/		*/
void		static void
sched_throw(struct thread *td)		sched_ule_throw(struct thread *td)
{		{
struct thread *newtd;		struct thread *newtd;
struct tdq *tdq;		struct tdq *tdq;

tdq = TDQ_SELF();		tdq = TDQ_SELF();

MPASS(td != NULL);		MPASS(td != NULL);
THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
Show All 13 Lines	#endif
/* doesn't return */		/* doesn't return */
cpu_switch(td, newtd, TDQ_LOCKPTR(tdq));		cpu_switch(td, newtd, TDQ_LOCKPTR(tdq));
}		}

/*		/*
* This is called from fork_exit(). Just acquire the correct locks and		* This is called from fork_exit(). Just acquire the correct locks and
* let fork do the rest of the work.		* let fork do the rest of the work.
*/		*/
void		static void
sched_fork_exit(struct thread *td)		sched_ule_fork_exit(struct thread *td)
{		{
struct tdq *tdq;		struct tdq *tdq;
int cpuid;		int cpuid;

/*		/*
* Finish setting up thread glue so that it begins execution in a		* Finish setting up thread glue so that it begins execution in a
* non-nested critical section with the scheduler lock held.		* non-nested critical section with the scheduler lock held.
*/		*/
KASSERT(curthread->td_md.md_spinlock_count == 1,		KASSERT(curthread->td_md.md_spinlock_count == 1,
("invalid count %d", curthread->td_md.md_spinlock_count));		("invalid count %d", curthread->td_md.md_spinlock_count));
cpuid = PCPU_GET(cpuid);		cpuid = PCPU_GET(cpuid);
tdq = TDQ_SELF();		tdq = TDQ_SELF();
TDQ_LOCK(tdq);		TDQ_LOCK(tdq);
spinlock_exit();		spinlock_exit();
MPASS(td->td_lock == TDQ_LOCKPTR(tdq));		MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
td->td_oncpu = cpuid;		td->td_oncpu = cpuid;
KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "running",		KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "running",
"prio:%d", td->td_priority);		"prio:%d", td->td_priority);
SDT_PROBE0(sched, , , on__cpu);		SDT_PROBE0(sched, , , on__cpu);
}		}

/*		/*
* Create on first use to catch odd startup conditions.		* Create on first use to catch odd startup conditions.
*/		*/
char *		static char *
sched_tdname(struct thread *td)		sched_ule_tdname(struct thread *td)
{		{
#ifdef KTR		#ifdef KTR
struct td_sched *ts;		struct td_sched *ts;

ts = td_get_sched(td);		ts = td_get_sched(td);
if (ts->ts_name[0] == '\0')		if (ts->ts_name[0] == '\0')
snprintf(ts->ts_name, sizeof(ts->ts_name),		snprintf(ts->ts_name, sizeof(ts->ts_name),
"%s tid %d", td->td_name, td->td_tid);		"%s tid %d", td->td_name, td->td_tid);
return (ts->ts_name);		return (ts->ts_name);
#else		#else
return (td->td_name);		return (td->td_name);
#endif		#endif
}		}

#ifdef KTR		static void
void		sched_ule_clear_tdname(struct thread *td)
sched_clear_tdname(struct thread *td)
{		{
		#ifdef KTR
struct td_sched *ts;		struct td_sched *ts;

ts = td_get_sched(td);		ts = td_get_sched(td);
ts->ts_name[0] = '\0';		ts->ts_name[0] = '\0';
}
#endif		#endif
		}

		minsoochoo0122_proton.meUnsubmitted Done Inline Actions `__unused`? minsoochoo0122_proton.me: `__unused`?
		kibAuthorUnsubmitted Done Inline Actions What do you mean? It is used in the sense that its pointer is taken by the slots table. kib: What do you mean? It is used in the sense that its pointer is taken by the slots table.
		minsoochoo0122_proton.meUnsubmitted Done Inline Actions I mean the parameter. Is it considered as used if the function pointer is stored in the slots table? minsoochoo0122_proton.me: I mean the parameter. Is it considered as used if the function pointer is stored in the slots…
		kibAuthorUnsubmitted Done Inline Actions Well, there is no compiler' complain, so why add this? kib: Well, there is no compiler' complain, so why add this?
		minsoochoo0122_proton.meUnsubmitted Done Inline Actions Interesting... I see `__unused` everywhere in sys so I thought compiler was strict on this. But if not this should be fine... minsoochoo0122_proton.me: Interesting... I see `__unused` everywhere in sys so I thought compiler was strict on this. But…
		static void
		sched_ule_schedcpu(void)
		{
		}

		static bool
		sched_ule_do_timer_accounting(void)
		{
		return (true);
		}

		static int
		sched_ule_find_child_with_core(int cpu, struct cpu_group *grp)
		{
		int i;

		if (grp->cg_children == 0)
		return (-1);

		MPASS(grp->cg_child);
		for (i = 0; i < grp->cg_children; i++) {
		if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask))
		return (i);
		}

		return (-1);
		}

		static int
		sched_ule_find_l2_neighbor(int cpu)
		{
		struct cpu_group *grp;
		int i;

		grp = cpu_top;
		if (grp == NULL)
		return (-1);

		/*
		* Find the smallest CPU group that contains the given core.
		*/
		i = 0;
		while ((i = sched_ule_find_child_with_core(cpu, grp)) != -1) {
		/*
		* If the smallest group containing the given CPU has less
		* than two members, we conclude the given CPU has no
		* L2 neighbor.
		*/
		if (grp->cg_child[i].cg_count <= 1)
		return (-1);
		grp = &grp->cg_child[i];
		}

		/* Must share L2. */
		if (grp->cg_level > CG_SHARE_L2 \|\| grp->cg_level == CG_SHARE_NONE)
		return (-1);

		/*
		* Select the first member of the set that isn't the reference
		* CPU, which at this point is guaranteed to exist.
		*/
		for (i = 0; i < CPU_SETSIZE; i++) {
		if (CPU_ISSET(i, &grp->cg_mask) && i != cpu)
		return (i);
		}

		/* Should never be reached */
		return (-1);
		}

		struct sched_instance sched_ule_instance = {
		#define SLOT(name) .name = sched_ule_##name
		SLOT(load),
		SLOT(rr_interval),
		SLOT(runnable),
		SLOT(exit),
		SLOT(fork),
		SLOT(fork_exit),
		SLOT(class),
		SLOT(nice),
		SLOT(ap_entry),
		SLOT(exit_thread),
		SLOT(estcpu),
		SLOT(fork_thread),
		SLOT(ithread_prio),
		SLOT(lend_prio),
		SLOT(lend_user_prio),
		SLOT(lend_user_prio_cond),
		SLOT(pctcpu),
		SLOT(prio),
		SLOT(sleep),
		SLOT(sswitch),
		SLOT(throw),
		SLOT(unlend_prio),
		SLOT(user_prio),
		SLOT(userret_slowpath),
		SLOT(add),
		SLOT(choose),
		SLOT(clock),
		SLOT(idletd),
		SLOT(preempt),
		SLOT(relinquish),
		SLOT(rem),
		SLOT(wakeup),
		SLOT(bind),
		SLOT(unbind),
		SLOT(is_bound),
		SLOT(affinity),
		SLOT(sizeof_proc),
		SLOT(sizeof_thread),
		SLOT(tdname),
		SLOT(clear_tdname),
		SLOT(do_timer_accounting),
		SLOT(find_l2_neighbor),
		SLOT(init),
		SLOT(init_ap),
		SLOT(setup),
		SLOT(initticks),
		SLOT(schedcpu),
		#undef SLOT
		};
		DECLARE_SCHEDULER(ule_sched_selector, "ULE", &sched_ule_instance);

#ifdef SMP		#ifdef SMP

/*		/*
* Build the CPU topology dump string. Is recursively called to collect		* Build the CPU topology dump string. Is recursively called to collect
* the topology tree.		* the topology tree.
*/		*/
static int		static int
sysctl_kern_sched_topology_spec_internal(struct sbuf sb, struct cpu_group cg,		sysctl_kern_sched_ule_topology_spec_internal(struct sbuf *sb,
int indent)		struct cpu_group *cg, int indent)
{		{
char cpusetbuf[CPUSETBUFSIZ];		char cpusetbuf[CPUSETBUFSIZ];
int i, first;		int i, first;

sbuf_printf(sb, "%*s<group level=\"%d\" cache-level=\"%d\">\n", indent,		sbuf_printf(sb, "%*s<group level=\"%d\" cache-level=\"%d\">\n", indent,
"", 1 + indent / 2, cg->cg_level);		"", 1 + indent / 2, cg->cg_level);
sbuf_printf(sb, "%*s <cpu count=\"%d\" mask=\"%s\">", indent, "",		sbuf_printf(sb, "%*s <cpu count=\"%d\" mask=\"%s\">", indent, "",
cg->cg_count, cpusetobj_strprint(cpusetbuf, &cg->cg_mask));		cg->cg_count, cpusetobj_strprint(cpusetbuf, &cg->cg_mask));
Show All 20 Lines	if (cg->cg_flags != 0) {
if ((cg->cg_flags & CG_FLAG_NODE) != 0)		if ((cg->cg_flags & CG_FLAG_NODE) != 0)
sbuf_cat(sb, "<flag name=\"NODE\">NUMA node</flag>");		sbuf_cat(sb, "<flag name=\"NODE\">NUMA node</flag>");
sbuf_cat(sb, "</flags>\n");		sbuf_cat(sb, "</flags>\n");
}		}

if (cg->cg_children > 0) {		if (cg->cg_children > 0) {
sbuf_printf(sb, "%*s <children>\n", indent, "");		sbuf_printf(sb, "%*s <children>\n", indent, "");
for (i = 0; i < cg->cg_children; i++)		for (i = 0; i < cg->cg_children; i++)
sysctl_kern_sched_topology_spec_internal(sb,		sysctl_kern_sched_ule_topology_spec_internal(sb,
&cg->cg_child[i], indent+2);		&cg->cg_child[i], indent+2);
sbuf_printf(sb, "%*s </children>\n", indent, "");		sbuf_printf(sb, "%*s </children>\n", indent, "");
}		}
sbuf_printf(sb, "%*s</group>\n", indent, "");		sbuf_printf(sb, "%*s</group>\n", indent, "");
return (0);		return (0);
}		}

/*		/*
* Sysctl handler for retrieving topology dump. It's a wrapper for		* Sysctl handler for retrieving topology dump. It's a wrapper for
* the recursive sysctl_kern_smp_topology_spec_internal().		* the recursive sysctl_kern_smp_topology_spec_internal().
*/		*/
static int		static int
sysctl_kern_sched_topology_spec(SYSCTL_HANDLER_ARGS)		sysctl_kern_sched_ule_topology_spec(SYSCTL_HANDLER_ARGS)
{		{
struct sbuf *topo;		struct sbuf *topo;
int err;		int err;

KASSERT(cpu_top != NULL, ("cpu_top isn't initialized"));		if (cpu_top == NULL)
		return (ENOTTY);

topo = sbuf_new_for_sysctl(NULL, NULL, 512, req);		topo = sbuf_new_for_sysctl(NULL, NULL, 512, req);
if (topo == NULL)		if (topo == NULL)
return (ENOMEM);		return (ENOMEM);

sbuf_cat(topo, "<groups>\n");		sbuf_cat(topo, "<groups>\n");
err = sysctl_kern_sched_topology_spec_internal(topo, cpu_top, 1);		err = sysctl_kern_sched_ule_topology_spec_internal(topo, cpu_top, 1);
sbuf_cat(topo, "</groups>\n");		sbuf_cat(topo, "</groups>\n");

if (err == 0) {		if (err == 0) {
err = sbuf_finish(topo);		err = sbuf_finish(topo);
}		}
sbuf_delete(topo);		sbuf_delete(topo);
return (err);		return (err);
}		}
Show All 14 Lines	if (new_val <= 0)
return (EINVAL);		return (EINVAL);
sched_slice = imax(1, (new_val + period / 2) / period);		sched_slice = imax(1, (new_val + period / 2) / period);
sched_slice_min = sched_slice / SCHED_SLICE_MIN_DIVISOR;		sched_slice_min = sched_slice / SCHED_SLICE_MIN_DIVISOR;
hogticks = imax(1, (2 * hz * sched_slice + realstathz / 2) /		hogticks = imax(1, (2 * hz * sched_slice + realstathz / 2) /
realstathz);		realstathz);
return (0);		return (0);
}		}

SYSCTL_NODE(_kern, OID_AUTO, sched, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,		SYSCTL_NODE(_kern_sched, OID_AUTO, ule, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
"Scheduler");		"ULE Scheduler");
SYSCTL_STRING(_kern_sched, OID_AUTO, name, CTLFLAG_RD, "ULE", 0,
"Scheduler name");		SYSCTL_PROC(_kern_sched_ule, OID_AUTO, quantum,
SYSCTL_PROC(_kern_sched, OID_AUTO, quantum,
CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_MPSAFE, NULL, 0,		CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_MPSAFE, NULL, 0,
sysctl_kern_quantum, "I",		sysctl_kern_quantum, "I",
"Quantum for timeshare threads in microseconds");		"Quantum for timeshare threads in microseconds");
SYSCTL_INT(_kern_sched, OID_AUTO, slice, CTLFLAG_RW, &sched_slice, 0,		SYSCTL_INT(_kern_sched_ule, OID_AUTO, slice, CTLFLAG_RW, &sched_slice, 0,
"Quantum for timeshare threads in stathz ticks");		"Quantum for timeshare threads in stathz ticks");
SYSCTL_UINT(_kern_sched, OID_AUTO, interact, CTLFLAG_RWTUN, &sched_interact, 0,		SYSCTL_UINT(_kern_sched_ule, OID_AUTO, interact, CTLFLAG_RWTUN, &sched_interact, 0,
"Interactivity score threshold");		"Interactivity score threshold");
SYSCTL_INT(_kern_sched, OID_AUTO, preempt_thresh, CTLFLAG_RWTUN,		SYSCTL_INT(_kern_sched_ule, OID_AUTO, preempt_thresh, CTLFLAG_RWTUN,
&preempt_thresh, 0,		&preempt_thresh, 0,
"Maximal (lowest) priority for preemption");		"Maximal (lowest) priority for preemption");
SYSCTL_INT(_kern_sched, OID_AUTO, static_boost, CTLFLAG_RWTUN, &static_boost, 0,		SYSCTL_INT(_kern_sched_ule, OID_AUTO, static_boost, CTLFLAG_RWTUN,
		&static_boost, 0,
"Assign static kernel priorities to sleeping threads");		"Assign static kernel priorities to sleeping threads");
SYSCTL_INT(_kern_sched, OID_AUTO, idlespins, CTLFLAG_RWTUN, &sched_idlespins, 0,		SYSCTL_INT(_kern_sched_ule, OID_AUTO, idlespins, CTLFLAG_RWTUN,
		&sched_idlespins, 0,
"Number of times idle thread will spin waiting for new work");		"Number of times idle thread will spin waiting for new work");
SYSCTL_INT(_kern_sched, OID_AUTO, idlespinthresh, CTLFLAG_RW,		SYSCTL_INT(_kern_sched_ule, OID_AUTO, idlespinthresh, CTLFLAG_RW,
&sched_idlespinthresh, 0,		&sched_idlespinthresh, 0,
"Threshold before we will permit idle thread spinning");		"Threshold before we will permit idle thread spinning");
#ifdef SMP		#ifdef SMP
SYSCTL_INT(_kern_sched, OID_AUTO, affinity, CTLFLAG_RW, &affinity, 0,		SYSCTL_INT(_kern_sched_ule, OID_AUTO, affinity, CTLFLAG_RW, &affinity, 0,
"Number of hz ticks to keep thread affinity for");		"Number of hz ticks to keep thread affinity for");
SYSCTL_INT(_kern_sched, OID_AUTO, balance, CTLFLAG_RWTUN, &rebalance, 0,		SYSCTL_INT(_kern_sched_ule, OID_AUTO, balance, CTLFLAG_RWTUN, &rebalance, 0,
"Enables the long-term load balancer");		"Enables the long-term load balancer");
SYSCTL_INT(_kern_sched, OID_AUTO, balance_interval, CTLFLAG_RW,		SYSCTL_INT(_kern_sched_ule, OID_AUTO, balance_interval, CTLFLAG_RW,
&balance_interval, 0,		&balance_interval, 0,
"Average period in stathz ticks to run the long-term balancer");		"Average period in stathz ticks to run the long-term balancer");
SYSCTL_INT(_kern_sched, OID_AUTO, steal_idle, CTLFLAG_RWTUN, &steal_idle, 0,		SYSCTL_INT(_kern_sched_ule, OID_AUTO, steal_idle, CTLFLAG_RWTUN,
		&steal_idle, 0,
"Attempts to steal work from other cores before idling");		"Attempts to steal work from other cores before idling");
SYSCTL_INT(_kern_sched, OID_AUTO, steal_thresh, CTLFLAG_RWTUN, &steal_thresh, 0,		SYSCTL_INT(_kern_sched_ule, OID_AUTO, steal_thresh, CTLFLAG_RWTUN,
		&steal_thresh, 0,
"Minimum load on remote CPU before we'll steal");		"Minimum load on remote CPU before we'll steal");
SYSCTL_INT(_kern_sched, OID_AUTO, trysteal_limit, CTLFLAG_RWTUN,		SYSCTL_INT(_kern_sched_ule, OID_AUTO, trysteal_limit, CTLFLAG_RWTUN,
&trysteal_limit, 0,		&trysteal_limit, 0,
"Topological distance limit for stealing threads in sched_switch()");		"Topological distance limit for stealing threads in sched_switch()");
SYSCTL_INT(_kern_sched, OID_AUTO, always_steal, CTLFLAG_RWTUN, &always_steal, 0,		SYSCTL_INT(_kern_sched_ule, OID_AUTO, always_steal, CTLFLAG_RWTUN,
		&always_steal, 0,
"Always run the stealer from the idle thread");		"Always run the stealer from the idle thread");
SYSCTL_PROC(_kern_sched, OID_AUTO, topology_spec, CTLTYPE_STRING \|		SYSCTL_PROC(_kern_sched_ule, OID_AUTO, topology_spec, CTLTYPE_STRING \|
CTLFLAG_MPSAFE \| CTLFLAG_RD, NULL, 0, sysctl_kern_sched_topology_spec, "A",		CTLFLAG_MPSAFE \| CTLFLAG_RD, NULL, 0,
		sysctl_kern_sched_ule_topology_spec, "A",
"XML dump of detected CPU topology");		"XML dump of detected CPU topology");
#endif		#endif

/* ps compat. All cpu percentages from ULE are weighted. */
static int ccpu = 0;
SYSCTL_INT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0,
"Decay factor used for updating %CPU in 4BSD scheduler");