Differential D22626 Diff 65687 head/sys/kern/sched_4bsd.c

Changeset View

Standalone View

head/sys/kern/sched_4bsd.c

Show First 20 Lines • Show All 840 Lines • ▼ Show 20 Lines	SDT_PROBE4(sched, , , lend__pri, td, td->td_proc, prio,
curthread);		curthread);
}		}
THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
if (td->td_priority == prio)		if (td->td_priority == prio)
return;		return;
td->td_priority = prio;		td->td_priority = prio;
if (TD_ON_RUNQ(td) && td->td_rqindex != (prio / RQ_PPQ)) {		if (TD_ON_RUNQ(td) && td->td_rqindex != (prio / RQ_PPQ)) {
sched_rem(td);		sched_rem(td);
sched_add(td, SRQ_BORING);		sched_add(td, SRQ_BORING \| SRQ_HOLDTD);
}		}
}		}

/*		/*
* Update a thread's priority when it is lent another thread's		* Update a thread's priority when it is lent another thread's
* priority.		* priority.
*/		*/
void		void
▲ Show 20 Lines • Show All 117 Lines • ▼ Show 20 Lines
void		void
sched_switch(struct thread td, struct thread newtd, int flags)		sched_switch(struct thread td, struct thread newtd, int flags)
{		{
struct mtx *tmtx;		struct mtx *tmtx;
struct td_sched *ts;		struct td_sched *ts;
struct proc *p;		struct proc *p;
int preempted;		int preempted;

tmtx = NULL;		tmtx = &sched_lock;
ts = td_get_sched(td);		ts = td_get_sched(td);
p = td->td_proc;		p = td->td_proc;

THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);

/*
* Switch to the sched lock to fix things up and pick
* a new thread.
* Block the td_lock in order to avoid breaking the critical path.
*/
if (td->td_lock != &sched_lock) {
mtx_lock_spin(&sched_lock);
tmtx = thread_lock_block(td);
}

if ((td->td_flags & TDF_NOLOAD) == 0)
sched_load_rem();

td->td_lastcpu = td->td_oncpu;		td->td_lastcpu = td->td_oncpu;
preempted = (td->td_flags & TDF_SLICEEND) == 0 &&		preempted = (td->td_flags & TDF_SLICEEND) == 0 &&
(flags & SW_PREEMPT) != 0;		(flags & SW_PREEMPT) != 0;
td->td_flags &= ~(TDF_NEEDRESCHED \| TDF_SLICEEND);		td->td_flags &= ~(TDF_NEEDRESCHED \| TDF_SLICEEND);
td->td_owepreempt = 0;		td->td_owepreempt = 0;
td->td_oncpu = NOCPU;		td->td_oncpu = NOCPU;

/*		/*
* At the last moment, if this thread is still marked RUNNING,		* At the last moment, if this thread is still marked RUNNING,
* then put it back on the run queue as it has not been suspended		* then put it back on the run queue as it has not been suspended
* or stopped or any thing else similar. We never put the idle		* or stopped or any thing else similar. We never put the idle
* threads on the run queue, however.		* threads on the run queue, however.
*/		*/
if (td->td_flags & TDF_IDLETD) {		if (td->td_flags & TDF_IDLETD) {
TD_SET_CAN_RUN(td);		TD_SET_CAN_RUN(td);
#ifdef SMP		#ifdef SMP
CPU_CLR(PCPU_GET(cpuid), &idle_cpus_mask);		CPU_CLR(PCPU_GET(cpuid), &idle_cpus_mask);
#endif		#endif
} else {		} else {
if (TD_IS_RUNNING(td)) {		if (TD_IS_RUNNING(td)) {
/* Put us back on the run queue. */		/* Put us back on the run queue. */
sched_add(td, preempted ?		sched_add(td, preempted ?
SRQ_OURSELF\|SRQ_YIELDING\|SRQ_PREEMPTED :		SRQ_HOLDTD\|SRQ_OURSELF\|SRQ_YIELDING\|SRQ_PREEMPTED :
SRQ_OURSELF\|SRQ_YIELDING);		SRQ_HOLDTD\|SRQ_OURSELF\|SRQ_YIELDING);
}		}
}		}

		/*
		* Switch to the sched lock to fix things up and pick
		* a new thread. Block the td_lock in order to avoid
		* breaking the critical path.
		*/
		if (td->td_lock != &sched_lock) {
		mtx_lock_spin(&sched_lock);
		tmtx = thread_lock_block(td);
		mtx_unlock_spin(tmtx);
		}

		if ((td->td_flags & TDF_NOLOAD) == 0)
		sched_load_rem();

if (newtd) {		if (newtd) {
/*		/*
* The thread we are about to run needs to be counted		* The thread we are about to run needs to be counted
* as if it had been added to the run queue and selected.		* as if it had been added to the run queue and selected.
* It came from:		* It came from:
* * A preemption		* * A preemption
* * An upcall		* * An upcall
* * A followon		* * A followon
*/		*/
KASSERT((newtd->td_inhibitors == 0),		KASSERT((newtd->td_inhibitors == 0),
("trying to run inhibited thread"));		("trying to run inhibited thread"));
newtd->td_flags \|= TDF_DIDRUN;		newtd->td_flags \|= TDF_DIDRUN;
TD_SET_RUNNING(newtd);		TD_SET_RUNNING(newtd);
if ((newtd->td_flags & TDF_NOLOAD) == 0)		if ((newtd->td_flags & TDF_NOLOAD) == 0)
sched_load_add();		sched_load_add();
} else {		} else {
newtd = choosethread();		newtd = choosethread();
MPASS(newtd->td_lock == &sched_lock);
}		}

		MPASS(newtd->td_lock == &sched_lock);

#if (KTR_COMPILE & KTR_SCHED) != 0		#if (KTR_COMPILE & KTR_SCHED) != 0
if (TD_IS_IDLETHREAD(td))		if (TD_IS_IDLETHREAD(td))
KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "idle",		KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "idle",
"prio:%d", td->td_priority);		"prio:%d", td->td_priority);
else		else
KTR_STATE3(KTR_SCHED, "thread", sched_tdname(td), KTDSTATE(td),		KTR_STATE3(KTR_SCHED, "thread", sched_tdname(td), KTDSTATE(td),
"prio:%d", td->td_priority, "wmesg:\"%s\"", td->td_wmesg,		"prio:%d", td->td_priority, "wmesg:\"%s\"", td->td_wmesg,
"lockname:\"%s\"", td->td_lockname);		"lockname:\"%s\"", td->td_lockname);
Show All 14 Lines	#ifdef KDTRACE_HOOKS
* If DTrace has set the active vtime enum to anything		* If DTrace has set the active vtime enum to anything
* other than INACTIVE (0), then it should have set the		* other than INACTIVE (0), then it should have set the
* function to call.		* function to call.
*/		*/
if (dtrace_vtime_active)		if (dtrace_vtime_active)
(*dtrace_vtime_switch_func)(newtd);		(*dtrace_vtime_switch_func)(newtd);
#endif		#endif

cpu_switch(td, newtd, tmtx != NULL ? tmtx : td->td_lock);		cpu_switch(td, newtd, tmtx);
lock_profile_obtain_lock_success(&sched_lock.lock_object,		lock_profile_obtain_lock_success(&sched_lock.lock_object,
0, 0, __FILE__, __LINE__);		0, 0, __FILE__, __LINE__);
/*		/*
* Where am I? What year is it?		* Where am I? What year is it?
* We are in the same thread that went to sleep above,		* We are in the same thread that went to sleep above,
* but any amount of time may have passed. All our context		* but any amount of time may have passed. All our context
* will still be available as will local variables.		* will still be available as will local variables.
* PCPU values however may have changed as we may have		* PCPU values however may have changed as we may have
* changed CPU so don't trust cached values of them.		* changed CPU so don't trust cached values of them.
* New threads will go to fork_exit() instead of here		* New threads will go to fork_exit() instead of here
* so if you change things here you may need to change		* so if you change things here you may need to change
* things there too.		* things there too.
*		*
* If the thread above was exiting it will never wake		* If the thread above was exiting it will never wake
* up again here, so either it has saved everything it		* up again here, so either it has saved everything it
* needed to, or the thread_wait() or wait() will		* needed to, or the thread_wait() or wait() will
* need to reap it.		* need to reap it.
*/		*/

SDT_PROBE0(sched, , , on__cpu);		SDT_PROBE0(sched, , , on__cpu);
#ifdef HWPMC_HOOKS		#ifdef HWPMC_HOOKS
if (PMC_PROC_IS_USING_PMCS(td->td_proc))		if (PMC_PROC_IS_USING_PMCS(td->td_proc))
PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN);		PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN);
#endif		#endif
} else		} else {
		td->td_lock = &sched_lock;
SDT_PROBE0(sched, , , remain__cpu);		SDT_PROBE0(sched, , , remain__cpu);
		}

KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "running",		KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "running",
"prio:%d", td->td_priority);		"prio:%d", td->td_priority);

#ifdef SMP		#ifdef SMP
if (td->td_flags & TDF_IDLETD)		if (td->td_flags & TDF_IDLETD)
CPU_SET(PCPU_GET(cpuid), &idle_cpus_mask);		CPU_SET(PCPU_GET(cpuid), &idle_cpus_mask);
#endif		#endif
sched_lock.mtx_lock = (uintptr_t)td;		sched_lock.mtx_lock = (uintptr_t)td;
td->td_oncpu = PCPU_GET(cpuid);		td->td_oncpu = PCPU_GET(cpuid);
MPASS(td->td_lock == &sched_lock);		MPASS(td->td_lock == &sched_lock);
}		}

void		void
sched_wakeup(struct thread *td)		sched_wakeup(struct thread *td, int srqflags)
{		{
struct td_sched *ts;		struct td_sched *ts;

THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
ts = td_get_sched(td);		ts = td_get_sched(td);
td->td_flags &= ~TDF_CANSWAP;		td->td_flags &= ~TDF_CANSWAP;
if (ts->ts_slptime > 1) {		if (ts->ts_slptime > 1) {
updatepri(td);		updatepri(td);
resetpriority(td);		resetpriority(td);
}		}
td->td_slptick = 0;		td->td_slptick = 0;
ts->ts_slptime = 0;		ts->ts_slptime = 0;
ts->ts_slice = sched_slice;		ts->ts_slice = sched_slice;
sched_add(td, SRQ_BORING);		sched_add(td, srqflags);
}		}

#ifdef SMP		#ifdef SMP
static int		static int
forward_wakeup(int cpunum)		forward_wakeup(int cpunum)
{		{
struct pcpu *pc;		struct pcpu *pc;
cpuset_t dontuse, map, map2;		cpuset_t dontuse, map, map2;
▲ Show 20 Lines • Show All 169 Lines • ▼ Show 20 Lines	#ifdef SMP


/*		/*
* Now that the thread is moving to the run-queue, set the lock		* Now that the thread is moving to the run-queue, set the lock
* to the scheduler's lock.		* to the scheduler's lock.
*/		*/
if (td->td_lock != &sched_lock) {		if (td->td_lock != &sched_lock) {
mtx_lock_spin(&sched_lock);		mtx_lock_spin(&sched_lock);
		if ((flags & SRQ_HOLD) != 0)
		td->td_lock = &sched_lock;
		else
thread_lock_set(td, &sched_lock);		thread_lock_set(td, &sched_lock);

}		}
TD_SET_RUNQ(td);		TD_SET_RUNQ(td);

/*		/*
* If SMP is started and the thread is pinned or otherwise limited to		* If SMP is started and the thread is pinned or otherwise limited to
* a specific set of CPUs, queue the thread to a per-CPU run queue.		* a specific set of CPUs, queue the thread to a per-CPU run queue.
* Otherwise, queue the thread to the global run queue.		* Otherwise, queue the thread to the global run queue.
*		*
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines	if (!single_cpu) {
forwarded = forward_wakeup(cpu);		forwarded = forward_wakeup(cpu);
}		}

if (!forwarded) {		if (!forwarded) {
if (!maybe_preempt(td))		if (!maybe_preempt(td))
maybe_resched(td);		maybe_resched(td);
}		}
}		}
		if ((flags & SRQ_HOLDTD) == 0)
		thread_unlock(td);
}		}
#else /* SMP */		#else /* SMP */
{		{
struct td_sched *ts;		struct td_sched *ts;

ts = td_get_sched(td);		ts = td_get_sched(td);
THREAD_LOCK_ASSERT(td, MA_OWNED);		THREAD_LOCK_ASSERT(td, MA_OWNED);
KASSERT((td->td_inhibitors == 0),		KASSERT((td->td_inhibitors == 0),
Show All 11 Lines	SDT_PROBE4(sched, , , enqueue, td, td->td_proc, NULL,
flags & SRQ_PREEMPTED);		flags & SRQ_PREEMPTED);

/*		/*
* Now that the thread is moving to the run-queue, set the lock		* Now that the thread is moving to the run-queue, set the lock
* to the scheduler's lock.		* to the scheduler's lock.
*/		*/
if (td->td_lock != &sched_lock) {		if (td->td_lock != &sched_lock) {
mtx_lock_spin(&sched_lock);		mtx_lock_spin(&sched_lock);
		if ((flags & SRQ_HOLD) != 0)
		td->td_lock = &sched_lock;
		else
thread_lock_set(td, &sched_lock);		thread_lock_set(td, &sched_lock);
}		}
TD_SET_RUNQ(td);		TD_SET_RUNQ(td);
CTR2(KTR_RUNQ, "sched_add: adding td_sched:%p (td:%p) to runq", ts, td);		CTR2(KTR_RUNQ, "sched_add: adding td_sched:%p (td:%p) to runq", ts, td);
ts->ts_runq = &runq;		ts->ts_runq = &runq;

if ((td->td_flags & TDF_NOLOAD) == 0)		if ((td->td_flags & TDF_NOLOAD) == 0)
sched_load_add();		sched_load_add();
runq_add(ts->ts_runq, td, flags);		runq_add(ts->ts_runq, td, flags);
if (!maybe_preempt(td))		if (!maybe_preempt(td))
maybe_resched(td);		maybe_resched(td);
		if ((flags & SRQ_HOLDTD) == 0)
		thread_unlock(td);
}		}
#endif /* SMP */		#endif /* SMP */

void		void
sched_rem(struct thread *td)		sched_rem(struct thread *td)
{		{
struct td_sched *ts;		struct td_sched *ts;

▲ Show 20 Lines • Show All 342 Lines • ▼ Show 20 Lines	case TDS_RUNQ:
* then nothing needs to be done.		* then nothing needs to be done.
*/		*/
if (ts->ts_runq != &runq &&		if (ts->ts_runq != &runq &&
THREAD_CAN_SCHED(td, ts->ts_runq - runq_pcpu))		THREAD_CAN_SCHED(td, ts->ts_runq - runq_pcpu))
return;		return;

/* Put this thread on a valid per-CPU runqueue. */		/* Put this thread on a valid per-CPU runqueue. */
sched_rem(td);		sched_rem(td);
sched_add(td, SRQ_BORING);		sched_add(td, SRQ_HOLDTD \| SRQ_BORING);
break;		break;
case TDS_RUNNING:		case TDS_RUNNING:
/*		/*
* See if our current CPU is in the set. If not, force a		* See if our current CPU is in the set. If not, force a
* context switch.		* context switch.
*/		*/
if (THREAD_CAN_SCHED(td, td->td_oncpu))		if (THREAD_CAN_SCHED(td, td->td_oncpu))
return;		return;
Show All 10 Lines