diff --git a/sys/arm/arm/mp_machdep.c b/sys/arm/arm/mp_machdep.c --- a/sys/arm/arm/mp_machdep.c +++ b/sys/arm/arm/mp_machdep.c @@ -217,7 +217,7 @@ CTR0(KTR_SMP, "go into scheduler"); /* Enter the scheduler */ - sched_throw(NULL); + sched_ap_entry(); panic("scheduler returned us to %s", __func__); /* NOTREACHED */ diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c --- a/sys/arm64/arm64/mp_machdep.c +++ b/sys/arm64/arm64/mp_machdep.c @@ -293,7 +293,7 @@ MPASS(PCPU_GET(curpcb) == NULL); /* Enter the scheduler */ - sched_throw(NULL); + sched_ap_entry(); panic("scheduler returned us to init_secondary"); /* NOTREACHED */ diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c --- a/sys/kern/sched_4bsd.c +++ b/sys/kern/sched_4bsd.c @@ -1662,12 +1662,22 @@ } } +static void +sched_throw_tail(struct thread *td) +{ + + mtx_assert(&sched_lock, MA_OWNED); + KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count")); + cpu_throw(td, choosethread()); /* doesn't return */ +} + /* - * A CPU is entering for the first time or a thread is exiting. + * A CPU is entering for the first time. */ void -sched_throw(struct thread *td) +sched_ap_entry(void) { + /* * Correct spinlock nesting. The idle thread context that we are * borrowing was created so that it would start out with a single @@ -1677,20 +1687,29 @@ * spinlock_exit() will simply adjust the counts without allowing * spin lock using code to interrupt us. */ - if (td == NULL) { - mtx_lock_spin(&sched_lock); - spinlock_exit(); - PCPU_SET(switchtime, cpu_ticks()); - PCPU_SET(switchticks, ticks); - } else { - lock_profile_release_lock(&sched_lock.lock_object, true); - MPASS(td->td_lock == &sched_lock); - td->td_lastcpu = td->td_oncpu; - td->td_oncpu = NOCPU; - } - mtx_assert(&sched_lock, MA_OWNED); - KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count")); - cpu_throw(td, choosethread()); /* doesn't return */ + mtx_lock_spin(&sched_lock); + spinlock_exit(); + PCPU_SET(switchtime, cpu_ticks()); + PCPU_SET(switchticks, ticks); + + sched_throw_tail(NULL); +} + +/* + * A thread is exiting. + */ +void +sched_throw(struct thread *td) +{ + + MPASS(td != NULL); + MPASS(td->td_lock == &sched_lock); + + lock_profile_release_lock(&sched_lock.lock_object, true); + td->td_lastcpu = td->td_oncpu; + td->td_oncpu = NOCPU; + + sched_throw_tail(td); } void diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c --- a/sys/kern/sched_ule.c +++ b/sys/kern/sched_ule.c @@ -2985,39 +2985,74 @@ } /* - * A CPU is entering for the first time or a thread is exiting. + * sched_throw_grab() chooses a thread from the queue to switch to + * next. It returns with the tdq lock dropped in a spinlock section to + * keep interrupts disabled until the CPU is running in a proper threaded + * context. */ -void -sched_throw(struct thread *td) +static struct thread * +sched_throw_grab(struct tdq *tdq) { struct thread *newtd; - struct tdq *tdq; - tdq = TDQ_SELF(); - if (__predict_false(td == NULL)) { - TDQ_LOCK(tdq); - /* Correct spinlock nesting. */ - spinlock_exit(); - PCPU_SET(switchtime, cpu_ticks()); - PCPU_SET(switchticks, ticks); - } else { - THREAD_LOCK_ASSERT(td, MA_OWNED); - THREAD_LOCKPTR_ASSERT(td, TDQ_LOCKPTR(tdq)); - tdq_load_rem(tdq, td); - td->td_lastcpu = td->td_oncpu; - td->td_oncpu = NOCPU; - thread_lock_block(td); - } newtd = choosethread(); spinlock_enter(); TDQ_UNLOCK(tdq); KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count %d", curthread->td_md.md_spinlock_count)); + return (newtd); +} + +/* + * A CPU is entering for the first time. + */ +void +sched_ap_entry(void) +{ + struct thread *newtd; + struct tdq *tdq; + + tdq = TDQ_SELF(); + + /* This should have been setup in schedinit_ap(). */ + THREAD_LOCKPTR_ASSERT(curthread, TDQ_LOCKPTR(tdq)); + + TDQ_LOCK(tdq); + /* Correct spinlock nesting. */ + spinlock_exit(); + PCPU_SET(switchtime, cpu_ticks()); + PCPU_SET(switchticks, ticks); + + newtd = sched_throw_grab(tdq); + /* doesn't return */ - if (__predict_false(td == NULL)) - cpu_throw(td, newtd); /* doesn't return */ - else - cpu_switch(td, newtd, TDQ_LOCKPTR(tdq)); + cpu_throw(NULL, newtd); +} + +/* + * A thread is exiting. + */ +void +sched_throw(struct thread *td) +{ + struct thread *newtd; + struct tdq *tdq; + + tdq = TDQ_SELF(); + + MPASS(td != NULL); + THREAD_LOCK_ASSERT(td, MA_OWNED); + THREAD_LOCKPTR_ASSERT(td, TDQ_LOCKPTR(tdq)); + + tdq_load_rem(tdq, td); + td->td_lastcpu = td->td_oncpu; + td->td_oncpu = NOCPU; + thread_lock_block(td); + + newtd = sched_throw_grab(tdq); + + /* doesn't return */ + cpu_switch(td, newtd, TDQ_LOCKPTR(tdq)); } /* diff --git a/sys/mips/mips/mp_machdep.c b/sys/mips/mips/mp_machdep.c --- a/sys/mips/mips/mp_machdep.c +++ b/sys/mips/mips/mp_machdep.c @@ -335,7 +335,7 @@ cpu_initclocks_ap(); /* enter the scheduler */ - sched_throw(NULL); + sched_ap_entry(); panic("scheduler returned us to %s", __func__); /* NOTREACHED */ diff --git a/sys/powerpc/powerpc/mp_machdep.c b/sys/powerpc/powerpc/mp_machdep.c --- a/sys/powerpc/powerpc/mp_machdep.c +++ b/sys/powerpc/powerpc/mp_machdep.c @@ -112,7 +112,7 @@ cpu_initclocks_ap(); /* Announce ourselves awake, and enter the scheduler */ - sched_throw(NULL); + sched_ap_entry(); } void diff --git a/sys/riscv/riscv/mp_machdep.c b/sys/riscv/riscv/mp_machdep.c --- a/sys/riscv/riscv/mp_machdep.c +++ b/sys/riscv/riscv/mp_machdep.c @@ -291,7 +291,7 @@ MPASS(PCPU_GET(curpcb) == NULL); /* Enter the scheduler */ - sched_throw(NULL); + sched_ap_entry(); panic("scheduler returned us to init_secondary"); /* NOTREACHED */ diff --git a/sys/sys/sched.h b/sys/sys/sched.h --- a/sys/sys/sched.h +++ b/sys/sys/sched.h @@ -91,6 +91,7 @@ * Threads are switched in and out, block on resources, have temporary * priorities inherited from their procs, and use up cpu time. */ +void sched_ap_entry(void); void sched_exit_thread(struct thread *td, struct thread *child); u_int sched_estcpu(struct thread *td); void sched_fork_thread(struct thread *td, struct thread *child); diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c --- a/sys/x86/x86/mp_x86.c +++ b/sys/x86/x86/mp_x86.c @@ -1099,7 +1099,7 @@ */ MPASS(PCPU_GET(curpcb) == NULL); - sched_throw(NULL); + sched_ap_entry(); panic("scheduler returned us to %s", __func__); /* NOTREACHED */