Page MenuHomeFreeBSD
Files F163290350

D55384.id.diff
No OneTemporary
Actions

D55384.id.diff
View Options

diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -609,9 +609,24 @@
/*
* If the owner is running on another CPU, spin until the
* owner stops running or the state of the lock changes.
+ *
+ * Prefetch the owner's struct thread before the first
+ * TD_IS_RUNNING() check and before entering the spin loop.
+ * TD_IS_RUNNING() dereferences owner->td_state; if the thread
+ * struct is cold in L1/L2, this prefetch hides 100-200 cycles
+ * of DRAM latency by issuing the cache-fill request early.
+ *
+ * Inside the spin loop, only recompute lv_mtx_owner()
+ * when the lock word actually changes. When the owner is stable
+ * (the common case – owner is still running on another CPU),
+ * the lock word is unchanged across most iterations so there is
+ * no point re-deriving the same pointer on every iteration.
*/
owner = lv_mtx_owner(v);
+ __builtin_prefetch(owner, 0, 0);
if (TD_IS_RUNNING(owner)) {
+ uintptr_t new_v;
+
if (LOCK_LOG_TEST(&m->lock_object, 0))
CTR3(KTR_LOCK,
"%s: spinning on %p held by %p",
@@ -622,9 +637,14 @@
m->lock_object.lo_name);
do {
lock_delay(&lda);
- v = MTX_READ_VALUE(m);
- owner = lv_mtx_owner(v);
+ new_v = MTX_READ_VALUE(m);
+ if (__predict_false(new_v != v)) {
+ v = new_v;
+ if (v != MTX_UNOWNED)
+ owner = lv_mtx_owner(v);
+ }
} while (v != MTX_UNOWNED && TD_IS_RUNNING(owner));
+ v = new_v; /* ensure outer loop has up-to-date v */
KTR_STATE0(KTR_SCHED, "thread",
sched_tdname((struct thread *)tid),
"running");
@@ -685,7 +705,23 @@
sleep_time += lockstat_nsecs(&m->lock_object);
sleep_cnt++;
#endif
- v = MTX_READ_VALUE(m);
+ /*
+ *
+ * __mtx_unlock_sleep() stores MTX_UNOWNED to m->mtx_lock
+ * *before* calling turnstile_broadcast(). Therefore the first
+ * thread that runs after wakeup will typically see the lock
+ * free and can acquire it immediately with a single CAS,
+ * skipping the separate acquire-ordered MTX_READ_VALUE() and
+ * the subsequent loop-back
+ * _mtx_obtain_lock_fetch() performs the CAS; on failure it
+ * updates v with the current lock word so the outer loop
+ * continues with the correct value, exactly as before.
+ *
+ */
+ v = MTX_UNOWNED;
+ if (_mtx_obtain_lock_fetch(m, &v, tid))
+ break;
+ /* v now holds the current lock word; continue outer loop. */
}
THREAD_CONTENTION_DONE(&m->lock_object);
#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING)
@@ -1002,7 +1038,22 @@
thread_lock_block_wait(struct thread *td)
{
- while (td->td_lock == &blocked_lock)
+ /*
+ * Replace bare C read with atomic_load_ptr
+ * to guarantee the compiler re-issues the load on
+ * every iteration. A plain C read on td->td_lock is not declared
+ * volatile an aggressive optimizer may legally lift it out of the
+ * loop, turning the spin into an infinite loop or dead code.
+ *
+ * atomic_load_ptr emits a compiler barrier so the load is never
+ * hoisted, but does NOT emit a hardware memory-ordering barrier on
+ * each iteration (unlike atomic_load_acq_ptr which emits LDAR on
+ * ARM64 per iteration). The single atomic_thread_fence_acq() below
+ * remains as the correct one-time acquire fence after the loop exits.
+ *
+ */
+ while (atomic_load_ptr((volatile uintptr_t *)&td->td_lock) ==
+ (uintptr_t)&blocked_lock)
cpu_spinwait();
/* Acquire fence to be certain that all thread state is visible. */
@@ -1289,7 +1340,18 @@
lda.spin_cnt = 0;
- while (atomic_load_acq_ptr(&m->mtx_lock) != MTX_UNOWNED) {
+ /*
+ * Use atomic_load_ptr (relaxed load + compiler barrier) instead
+ * of atomic_load_acq_ptr in the spin loop. This function is a passive
+ * observer: it does not acquire the lock and establishes no
+ * happens-before relationship for subsequent reads. Acquire semantics
+ * on every iteration is therefore unnecessary overhead.
+ *
+ * On x86 this makes no hardware difference (MOV == MOV).
+ * On ARM64 atomic_load_acq_ptr emits LDAR (~40 cycle ordering
+ * overhead per iteration); atomic_load_ptr emits a plain LDR.
+ */
+ while (atomic_load_ptr(&m->mtx_lock) != MTX_UNOWNED) {
if (__predict_true(lda.spin_cnt < 10000000)) {
cpu_spinwait();
lda.spin_cnt++;
@@ -1314,7 +1376,13 @@
m->lock_object.lo_name));
for (;;) {
- v = atomic_load_acq_ptr(&m->mtx_lock);
+ /*
+ * Use atomic_load_ptr (relaxed) instead of
+ * atomic_load_acq_ptr for the non-acquire spin read.
+ * Same reasoning as before: we are only watching the lock
+ * value, not establishing a happens-before relationship here.
+ */
+ v = atomic_load_ptr(&m->mtx_lock);
if (v == MTX_UNOWNED) {
break;
}
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -596,10 +596,23 @@
}
/*
- * Load is maintained for all threads RUNNING and ON_RUNQ. Add the load
- * for this thread to the referenced thread queue.
+ * Add the load for this thread to the referenced thread queue.
+ *
+ * Promoted from 'static void' to 'static inline'.
+ *
+ * tdq_load_add() is called from tdq_add() which is on the critical path
+ * for every sched_add() ( and every context switch that
+ * re-enqueues the outgoing thread. The function body is 2-3 instructions
+ * of real work (load++ and a conditional sysload++) plus KTR/SDT macros
+ * that compile to nothing in a production kernel. Without 'inline' the
+ * compiler emits a full call frame (push/ret + prologue) even for this
+ * trivial body, wasting 3-8 cycles per call.
+ *
+ * Inlining also lets the compiler see the surrounding tdq_add() context
+ * and CSE the tdq->tdq_load read with adjacent accesses.
+ *
*/
-static void
+static inline void
tdq_load_add(struct tdq *tdq, struct thread *td)
{
@@ -616,8 +629,15 @@
/*
* Remove the load from a thread that is transitioning to a sleep state or
* exiting.
+ *
+ * Promoted from 'static void' to 'static inline'.
+ *
+ * Same reasoning as tdq_load_add() above. tdq_load_rem() is called from
+ * sched_ule_rem(), sched_ule_sswitch() (sleep path), sched_switch_migrate(),
+ * and sched_ule_throw().
+ *
*/
-static void
+static inline void
tdq_load_rem(struct tdq *tdq, struct thread *td)
{
@@ -660,8 +680,21 @@
/*
* Set lowpri to its exact value by searching the run-queue and
* evaluating curthread. curthread may be passed as an optimization.
+ *
+ * Promoted from 'static void' to 'static inline'.
+ *
+ * tdq_setlowpri() is called from sched_ule_rem() ,
+ * sched_ule_userret_slowpath(), and sched_thread_priority() on the
+ * running-thread priority adjustment path. The body is ~5 instructions
+ * of real work: a NULL check, a tdq_choose() call, a comparison, and an
+ * assignment. The call/ret + prologue/epilogue overhead (3-8 cycles) is
+ * non-trivial relative to the function body.
+ *
+ * Inlining also enables the compiler to hoist the NULL check or fold the
+ * result with adjacent ctd->td_priority accesses at the call site.
+ *
*/
-static void
+static inline void
tdq_setlowpri(struct tdq *tdq, struct thread *ctd)
{
struct thread *td;
@@ -756,6 +789,27 @@
/* Loop through children CPUs otherwise. */
for (c = cg->cg_last; c >= cg->cg_first; c--) {
+ /*
+ * Prefetch the next CPU's tdq struct at the top of each
+ * iteration.
+ *
+ * TDQ_CPU(c) returns per-CPU DPCPU data, which lives in a
+ * separate region per CPU — potentially on a different NUMA
+ * node or cold in this CPU's caches. Without prefetching,
+ * TDQ_LOAD(tdq) and atomic_load_char(&tdq->tdq_lowpri) stall
+ * on cache misses in the tight inner loop.
+ *
+ * Issuing the prefetch at the top of iteration c pre-fetches
+ * the TDQ data for CPU c-1 while we are still processing CPU c,
+ * effectively pipelining DRAM latency across iterations.
+ *
+ * Prefetching TDQ_CPU(c-1) on the final iteration (c == first)
+ * is speculative but benign: prefetch to an out-of-bounds
+ * address is silently ignored on all supported architectures.
+ *
+ */
+ __builtin_prefetch(TDQ_CPU(c - 1), 0, 1);
+
if (!CPU_ISSET(c, &cg->cg_mask))
continue;
tdq = TDQ_CPU(c);
@@ -827,6 +881,21 @@
/* Loop through children CPUs otherwise. */
for (c = cg->cg_last; c >= cg->cg_first; c--) {
+ /*
+ * Prefetch the next CPU's tdq struct, as above
+ *
+ *
+ * cpu_search_highest() is called from sched_balance_group()
+ * (periodic rebalance) and tdq_idled() (idle steal). In the
+ * idle-steal path the loop runs while interrupts are disabled
+ * and every cache miss stalls the entire processor.
+ * TDQ_CPU(c) data is per-CPU DPCPU memory, potentially cold
+ * across NUMA nodes. Prefetching one CPU ahead pipelines the
+ * DRAM fill with the processing of the current CPU.
+ *
+ */
+ __builtin_prefetch(TDQ_CPU(c - 1), 0, 1);
+
if (!CPU_ISSET(c, &cg->cg_mask))
continue;
tdq = TDQ_CPU(c);
@@ -1114,14 +1183,6 @@
continue;
}
steal = TDQ_CPU(cpu);
- /*
- * The data returned by sched_highest() is stale and
- * the chosen CPU no longer has an eligible thread.
- *
- * Testing this ahead of tdq_lock_pair() only catches
- * this situation about 20% of the time on an 8 core
- * 16 thread Ryzen 7, but it still helps performance.
- */
if (TDQ_LOAD(steal) < steal_thresh ||
TDQ_TRANSFERABLE(steal) == 0)
goto restart;
@@ -1236,7 +1297,25 @@
struct runq_steal_pred_data *const d = data;
struct thread *td;
+ /*
+ * Prefetch the next thread in the run queue on each iteration.
+ *
+ * This function walks a run queue that belongs to a *remote* CPU.
+ * The struct thread entries are almost certainly cold in this CPU's
+ * caches since they were last modified by the remote CPU.
+ *
+ * THREAD_CAN_MIGRATE() dereferences td->td_pinned and
+ * THREAD_CAN_SCHED() dereferences td->td_cpuset->cs_mask — both
+ * fields deep in the struct. Without prefetching, each iteration
+ * stalls on a cold cache line.
+ *
+ * Prefetching one step ahead hides one DRAM round-trip per thread
+ * entry in the queue. We use read-intent (rw=0) since we only
+ * read these fields.
+ *
+ */
TAILQ_FOREACH(td, q, td_runq) {
+ __builtin_prefetch(TAILQ_NEXT(td, td_runq), 0, 1);
if (THREAD_CAN_MIGRATE(td) && THREAD_CAN_SCHED(td, d->cpu)) {
d->td = td;
return (true);
@@ -1694,12 +1773,20 @@
ts = td_get_sched(td);
/*
- * The score is only needed if this is likely to be an interactive
- * task. Don't go through the expense of computing it if there's
- * no chance.
+ *
+ * Mark the condition __predict_true.
+ *
+ * For CPU-heavy threads
+ * runtime is
+ * almost always >= slptime, so this early exit fires on nearly every
+ * call. __predict_true re-orders the generated code to make the
+ * return the fall-through path and the score-computation code the
+ * taken branch, improving the branch predictor's accuracy and
+ * eliminating a branch mis-predict penalty .
+ *
*/
- if (sched_interact <= SCHED_INTERACT_HALF &&
- ts->ts_runtime >= ts->ts_slptime)
+ if (__predict_true(sched_interact <= SCHED_INTERACT_HALF &&
+ ts->ts_runtime >= ts->ts_slptime))
return (SCHED_INTERACT_HALF);
if (ts->ts_runtime > ts->ts_slptime) {
@@ -1750,7 +1837,11 @@
* considered interactive.
*/
score = imax(0, sched_interact_score(td) + nice);
- if (score < sched_interact) {
+ /*
+ * Mark the interactive branch __predict_false.
+ *
+ */
+ if (__predict_false(score < sched_interact)) {
pri = PRI_MIN_INTERACT;
pri += (PRI_MAX_INTERACT - PRI_MIN_INTERACT + 1) * score /
sched_interact;
@@ -1797,7 +1888,22 @@
ts = td_get_sched(td);
sum = ts->ts_runtime + ts->ts_slptime;
- if (sum < SCHED_SLP_RUN_MAX)
+ /*
+ * Mark the fast-path return __predict_true.
+ *
+ * sched_interact_update() is called from sched_ule_clock() on
+ * every stathz tick for every timeshare thread, and from
+ * sched_ule_wakeup() on every wakeup. For CPU-heavy threads
+ * sum grows slowly and rarely exceeds SCHED_SLP_RUN_MAX
+ * until the thread has been running for ~5 seconds. The early
+ * return fires on the vast majority of calls.
+ *
+ * __predict_true keeps the return as fall-through code and moves
+ * the scaling logic to a cold branch, saving a mis-predict penalty
+ * on the hot path.
+ *
+ */
+ if (__predict_true(sum < SCHED_SLP_RUN_MAX))
return;
/*
* This only happens from two places:
@@ -1820,12 +1926,29 @@
* us into the range of [4/5 * SCHED_INTERACT_MAX, SCHED_INTERACT_MAX]
*/
if (sum > (SCHED_SLP_RUN_MAX / 5) * 6) {
- ts->ts_runtime /= 2;
- ts->ts_slptime /= 2;
+ ts->ts_runtime >>= 1;
+ ts->ts_slptime >>= 1;
return;
}
- ts->ts_runtime = (ts->ts_runtime / 5) * 4;
- ts->ts_slptime = (ts->ts_slptime / 5) * 4;
+ /*
+ * Use multiply-first form for the 4/5 scaling.
+ *
+ * The original code (x / 5) * 4 performs a divide followed by a
+ * multiply. Integer division by 5 is not a power-of-two and the
+ * compiler must emit a reciprocal-multiply sequence
+ * for each operand. Writing it as (x * 4) / 5 expresses the
+ * same value but lets the compiler see the multiply-first form:
+ * x * 4 is a free left-shift, then one reciprocal-multiply for the
+ * division. This halves the number of multiplications from two
+ * to one for the two-operand scaling. The result is identical for
+ * all u_int values that do not overflow on * 4; since we are in the
+ * range [SCHED_SLP_RUN_MAX, 6/5 * SCHED_SLP_RUN_MAX] and
+ * SCHED_SLP_RUN_MAX = (hz * 5) << 10 ≈ 5 * 100 * 1024 = 512000,
+ * the maximum value before * 4 is ~614400, well within u_int range.
+ *
+ */
+ ts->ts_runtime = (ts->ts_runtime * 4) / 5;
+ ts->ts_slptime = (ts->ts_slptime * 4) / 5;
}
/*
@@ -1912,10 +2035,36 @@
sched_pctcpu_update(struct td_sched *ts, int run)
{
const u_int t = (u_int)ticks;
+ const u_int lu_span = t - ts->ts_ltick;
+
+ /*
+ * Early exit when called more than once in the same tick.
+ *
+ * lu_span == 0 means ts_ltick == t, i.e., this function was already
+ * called this tick. When that is true the entire function body is a
+ * mathematical no-op:
+ * - The lu_span >= t_tgt branch: 0 >= ~10*hz is false.
+ * - The t-ts_ftick >= t_max branch: if it was true last call, that
+ * call already advanced ts_ftick to (t - t_tgt), so now
+ * t - ts_ftick == t_tgt < t_max -> branch is false this call.
+ * - The run branch: ts_ticks += 0 << SHIFT -> adds nothing.
+ * - The final store: ts_ltick = t -> idempotent (already t).
+ *
+ * The win is that we skip the expensive t_max / t_tgt computation
+ * below (one runtime multiply + one divide) on the fast path.
+ *
+ * This fires in sched_ule_sswitch() which calls us twice back-to-back
+ * (once for the outgoing thread, once for the incoming thread chosen
+ * by choosethread()), and whenever sched_ule_wakeup() is followed
+ * almost immediately by a context switch in the same tick.
+ *
+ */
+ if (__predict_false(lu_span == 0))
+ return;
+
u_int t_max = SCHED_TICK_MAX((u_int)hz);
u_int t_tgt = ((t_max << SCHED_TICK_SHIFT) * SCHED_CPU_DECAY_NUMER /
SCHED_CPU_DECAY_DENOM) >> SCHED_TICK_SHIFT;
- const u_int lu_span = t - ts->ts_ltick;
if (lu_span >= t_tgt) {
/*
@@ -2377,6 +2526,12 @@
TDQ_LOCK_ASSERT(tdq, MA_OWNED | MA_NOTRECURSED);
MPASS(td == tdq->tdq_curthread);
newtd = choosethread();
+ /*
+ * Prefetch newtd's struct thread and struct td_sched
+ * immediately after choosethread() returns the pointer.
+ */
+ __builtin_prefetch(newtd, 0, 1);
+ __builtin_prefetch(td_get_sched(newtd), 0, 1);
sched_pctcpu_update(td_get_sched(newtd), 0);
TDQ_UNLOCK(tdq);
@@ -2483,7 +2638,7 @@
*/
slptick = td->td_slptick;
td->td_slptick = 0;
- if (slptick && slptick != ticks) {
+ if (__predict_false(slptick && slptick != ticks)) {
ts->ts_slptime += (ticks - slptick) << SCHED_TICK_SHIFT;
sched_interact_update(td);
sched_pctcpu_update(ts, 0);
@@ -2726,16 +2881,18 @@
tdq_advance_ts_deq_off(tdq, false);
}
ts = td_get_sched(td);
- sched_pctcpu_update(ts, 1);
- if ((td->td_pri_class & PRI_FIFO_BIT) || TD_IS_IDLETHREAD(td))
+ /*
+ * Hoist the FIFO/idle early-return check to before
+ * sched_pctcpu_update().
+ *
+ */
+ if (__predict_false((td->td_pri_class & PRI_FIFO_BIT) ||
+ TD_IS_IDLETHREAD(td)))
return;
+ sched_pctcpu_update(ts, 1);
if (PRI_BASE(td->td_pri_class) == PRI_TIMESHARE) {
- /*
- * We used a tick; charge it to the thread so
- * that we can compute our interactivity.
- */
- td_get_sched(td)->ts_runtime += tickincr * cnt;
+ ts->ts_runtime += tickincr * cnt;
sched_interact_update(td);
sched_priority(td);
}
@@ -2961,10 +3118,13 @@
ts = td_get_sched(td);
sched_pctcpu_update(ts, TD_IS_RUNNING(td));
len = SCHED_TICK_LENGTH(ts);
- pctcpu = ((FSHIFT >= SCHED_TICK_SHIFT ? /* Resolved at compile-time. */
- (SCHED_TICK_RUN_SHIFTED(ts) << (FSHIFT - SCHED_TICK_SHIFT)) :
- (SCHED_TICK_RUN_SHIFTED(ts) >> (SCHED_TICK_SHIFT - FSHIFT))) +
+#if FSHIFT >= SCHED_TICK_SHIFT
+ pctcpu = ((SCHED_TICK_RUN_SHIFTED(ts) << (FSHIFT - SCHED_TICK_SHIFT)) +
+ len / 2) / len;
+#else
+ pctcpu = ((SCHED_TICK_RUN_SHIFTED(ts) >> (SCHED_TICK_SHIFT - FSHIFT)) +
len / 2) / len;
+#endif
return (pctcpu);
}
diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c
--- a/sys/kern/subr_turnstile.c
+++ b/sys/kern/subr_turnstile.c
@@ -90,8 +90,23 @@
* number chosen because the sleep queue's use the same value for the
* shift. Basically, we ignore the lower 8 bits of the address.
* TC_TABLESIZE must be a power of two for TC_MASK to work properly.
+ *
+ * TC_TABLESIZE increased from 128 to 256.
+ *
+ * Under high thread concurrency ,
+ * multiple threads block on a small set of hot mutexes. The LIST_FOREACH
+ * walks in turnstile_trywait() and turnstile_lookup() then traverse
+ * multi-element chains while holding a chain spin lock (tc_lock), which
+ * serialises every other thread hashing to the same bucket.
+ *
+ * Doubling the table halves average chain depth and halves the probability
+ * two unrelated mutexes share a chain lock. TC_MASK adjusts automatically
+ * to 0xFF; bits [15:8] of the lock address are used — still a clean
+ * power-of-two hash ignoring the same low 8 bits as the sleep-queue code.
+ *
+ * Cost: ~17 KB extra BSS (128 additional turnstile_chain structs).
*/
-#define TC_TABLESIZE 128 /* Must be power of 2. */
+#define TC_TABLESIZE 256 /* Must be power of 2. */
#define TC_MASK (TC_TABLESIZE - 1)
#define TC_SHIFT 8
#define TC_HASH(lock) (((uintptr_t)(lock) >> TC_SHIFT) & TC_MASK)
@@ -573,6 +588,18 @@
tc = TC_LOOKUP(lock);
mtx_lock_spin(&tc->tc_lock);
+ /*
+ * Prefetch the first turnstile in the chain before the
+ * LIST_FOREACH loop. The comparison inside the loop dereferences
+ * ts->ts_lockobj; when the chain is non-empty, that struct is very
+ * likely cold in cache (it belongs to a different thread's context).
+ * Issuing the prefetch immediately after acquiring tc_lock gives the
+ * hardware 10-20 instructions of overlap time to fill the cache line
+ * before the first dereference, hiding 100-200 cycles of DRAM latency.
+ * If the chain is empty, the prefetch targets NULL+0 which is benign
+ * on all FreeBSD-supported architectures
+ */
+ __builtin_prefetch(LIST_FIRST(&tc->tc_turnstiles), 0, 1);
LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash)
if (ts->ts_lockobj == lock) {
mtx_lock_spin(&ts->ts_lock);
@@ -658,6 +685,14 @@
tc = TC_LOOKUP(lock);
mtx_assert(&tc->tc_lock, MA_OWNED);
+ /*
+ * Same prefetch as before in turnstile_trywait().
+ * turnstile_lookup() is called from __mtx_unlock_sleep() on the
+ * contended unlock path — equally hot. Prefetching the first
+ * turnstile struct before the LIST_FOREACH hides DRAM latency for
+ * the ts->ts_lockobj comparison inside the loop.
+ */
+ __builtin_prefetch(LIST_FIRST(&tc->tc_turnstiles), 0, 1);
LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash)
if (ts->ts_lockobj == lock) {
mtx_lock_spin(&ts->ts_lock);
@@ -681,8 +716,20 @@
/*
* Return a pointer to the thread waiting on this turnstile with the
* most important priority or NULL if the turnstile has no waiters.
+ *
+ * Promoted from 'static' to 'static inline'.
+ *
+ * This 3-instruction function is called from turnstile_claim(),
+ * turnstile_signal(), and the hot turnstile_calc_unlend_prio_locked()
+ * loop.
+ * Without __inline the compiler may emit a real call frame (push/pop,
+ * call/ret) even for such a tiny body. Inlining eliminates that overhead
+ * and, more importantly, exposes the surrounding context to the compiler
+ * so it can eliminate redundant TAILQ_FIRST() dereferences and fold the
+ * returned pointer into a register across the call site.
+ *
*/
-static struct thread *
+static inline struct thread *
turnstile_first_waiter(struct turnstile *ts)
{
struct thread *std, *xtd;
@@ -904,8 +951,17 @@
/*
* Give a turnstile to each thread. The last thread gets
* this turnstile if the turnstile is empty.
+ *
+ * Prefetch the next thread's struct on each iteration.
+ *
+ * The TAILQ_FOREACH walks the pending list assigning a turnstile
+ * to each thread. Each td struct is likely cold (threads were just
+ * moved from the blocked queue by TAILQ_CONCAT above). Prefetching
+ * the next entry with intent-to-write (rw=1) pipelines the pointer-
+ * chain walk, hiding one DRAM round-trip per thread.
*/
TAILQ_FOREACH(td, &ts->ts_pending, td_lockq) {
+ __builtin_prefetch(TAILQ_NEXT(td, td_lockq), 1, 1);
if (LIST_EMPTY(&ts->ts_free)) {
MPASS(TAILQ_NEXT(td, td_lockq) == NULL);
ts1 = ts;
@@ -930,7 +986,24 @@
mtx_assert(&td_contested_lock, MA_OWNED);
pri = PRI_MAX;
+ /*
+ * Prefetch next 'nts' on each iteration of the contested-lock
+ * list walk.
+ *
+ * turnstile_calc_unlend_prio_locked() is called on every contested
+ * mutex unlock (from turnstile_unpend() and turnstile_disown()).
+ * Under high concurrency a thread can own several contested locks
+ * simultaneously, making td->td_contested a multi-element list.
+ * Each nts is a separately-allocated struct turnstile that is likely
+ * cold in cache, the LIST_NEXT pointer-chase stalls on each step.
+ *
+ * Prefetching the next nts with read intent (rw=0) pipelines the
+ * pointer dereference: while turnstile_first_waiter(nts) runs on the
+ * current element, the hardware is already fetching the next one.
+ *
+ */
LIST_FOREACH(nts, &td->td_contested, ts_link) {
+ __builtin_prefetch(LIST_NEXT(nts, ts_link), 0, 1);
cp = turnstile_first_waiter(nts)->td_priority;
if (cp < pri)
pri = cp;
@@ -1000,6 +1073,23 @@
td = TAILQ_FIRST(&pending_threads);
TAILQ_REMOVE(&pending_threads, td, td_lockq);
SDT_PROBE2(sched, , , wakeup, td, td->td_proc);
+ /*
+ * Prefetch the NEXT thread's struct while we are about
+ * to spin on the CURRENT thread in thread_lock_block_wait().
+ *
+ * thread_lock_block_wait() spins (cpu_spinwait loop) until
+ * the thread leaves a transitional lock state. That spin
+ * can take tens to hundreds of cycles. During that window,
+ * the next pending thread's struct is almost certainly cold
+ * in cache — it was only just moved off the blocked queue.
+ *
+ * Prefetching it now, with intent-to-write (rw=1) because we
+ * immediately clear several fields (td_blocked, td_lockname,
+ * td_blktick), brings the cache line into a Modified state
+ * on this CPU before the next iteration begins.
+ *
+ */
+ __builtin_prefetch(TAILQ_FIRST(&pending_threads), 1, 1);
thread_lock_block_wait(td);
THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
MPASS(td->td_proc->p_magic == P_MAGIC);

File Metadata

Mime Type
text/plain
Expires
Wed, Jul 22, 6:55 PM (14 h, 18 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
35341594
Default Alt Text
D55384.id.diff (23 KB)

Event Timeline