Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/subr_lock.c
Show First 20 Lines • Show All 303 Lines • ▼ Show 20 Lines | lock_prof_reset_wait(void) | ||||
* Spin relinquishing our cpu so that quiesce_all_cpus may | * Spin relinquishing our cpu so that quiesce_all_cpus may | ||||
* complete. | * complete. | ||||
*/ | */ | ||||
while (lock_prof_resetting) | while (lock_prof_resetting) | ||||
sched_relinquish(curthread); | sched_relinquish(curthread); | ||||
} | } | ||||
static void | static void | ||||
lock_prof_reset_cpu_wait(int cpu) | |||||
{ | |||||
struct thread *td, *newtd; | |||||
struct pcpu *pcpu; | |||||
MPASS(td->td_critnest == 0); | |||||
kib: Don't you need this function to execute in critical section ? | |||||
Done Inline Actionsno. even if we migrate it's harmless since we will find ourselves not in the crit section in the loop below. in fact the check can be removed if it appears confusing. mjg: no. even if we migrate it's harmless since we will find ourselves not in the crit section in… | |||||
/* | |||||
Not Done Inline ActionsSeems weird to not cache the pcpu ptr. jeff: Seems weird to not cache the pcpu ptr. | |||||
Done Inline Actionssure, i can change that but wont be reuploading the patch if that's all mjg: sure, i can change that but wont be reuploading the patch if that's all | |||||
* Observe all CPUs not executing in critical section. | |||||
* We are not in one so the check for us is safe. If the found | |||||
* thread changes to something else we know the section was | |||||
* exited as well. | |||||
*/ | |||||
pcpu = cpuid_to_pcpu[cpu]; | |||||
td = pcpu->pc_curthread; | |||||
Not Done Inline ActionsAre both casts needed ? kib: Are both casts needed ? | |||||
Done Inline Actionsthe compiler complained about both cases. imo these macros should be reworked to do this interally but that's for another time, there are already users which cast mjg: the compiler complained about both cases. imo these macros should be reworked to do this… | |||||
for (;;) { | |||||
if (td->td_critnest == 0) | |||||
break; | |||||
cpu_spinwait(); | |||||
newtd = (struct thread *)atomic_load_acq_ptr((u_long *)pcpu->pc_curthread); | |||||
if (td != newtd) | |||||
break; | |||||
} | |||||
} | |||||
static void | |||||
lock_prof_reset(void) | lock_prof_reset(void) | ||||
{ | { | ||||
struct lock_prof_cpu *lpc; | struct lock_prof_cpu *lpc; | ||||
int enabled, i, cpu; | int enabled, i, cpu; | ||||
/* | /* | ||||
* We not only race with acquiring and releasing locks but also | * We not only race with acquiring and releasing locks but also | ||||
* thread exit. To be certain that threads exit without valid head | * thread exit. To be certain that threads exit without valid head | ||||
* pointers they must see resetting set before enabled is cleared. | * pointers they must see resetting set before enabled is cleared. | ||||
* Otherwise a lock may not be removed from a per-thread list due | * Otherwise a lock may not be removed from a per-thread list due | ||||
* to disabled being set but not wait for reset() to remove it below. | * to disabled being set but not wait for reset() to remove it below. | ||||
*/ | */ | ||||
atomic_store_rel_int(&lock_prof_resetting, 1); | atomic_store_rel_int(&lock_prof_resetting, 1); | ||||
enabled = lock_prof_enable; | enabled = lock_prof_enable; | ||||
lock_prof_enable = 0; | lock_prof_enable = 0; | ||||
quiesce_all_cpus("profreset", 0); | |||||
/* | /* | ||||
* This both publishes lock_prof_enable as disabled and makes sure | |||||
* everyone else reads it if they are not far enough. We wait for the | |||||
* rest down below. | |||||
*/ | |||||
cpus_fence_seq_cst(); | |||||
/* | |||||
* Some objects may have migrated between CPUs. Clear all links | * Some objects may have migrated between CPUs. Clear all links | ||||
* before we zero the structures. Some items may still be linked | * before we zero the structures. Some items may still be linked | ||||
* into per-thread lists as well. | * into per-thread lists as well. | ||||
*/ | */ | ||||
CPU_FOREACH(cpu) { | CPU_FOREACH(cpu) { | ||||
lpc = LP_CPU(cpu); | lpc = LP_CPU(cpu); | ||||
for (i = 0; i < LPROF_CACHE_SIZE; i++) { | for (i = 0; i < LPROF_CACHE_SIZE; i++) { | ||||
LIST_REMOVE(&lpc->lpc_types[0].lpt_objs[i], lpo_link); | LIST_REMOVE(&lpc->lpc_types[0].lpt_objs[i], lpo_link); | ||||
LIST_REMOVE(&lpc->lpc_types[1].lpt_objs[i], lpo_link); | LIST_REMOVE(&lpc->lpc_types[1].lpt_objs[i], lpo_link); | ||||
} | } | ||||
} | } | ||||
CPU_FOREACH(cpu) { | CPU_FOREACH(cpu) { | ||||
lpc = LP_CPU(cpu); | lpc = LP_CPU(cpu); | ||||
bzero(lpc, sizeof(*lpc)); | bzero(lpc, sizeof(*lpc)); | ||||
lock_prof_init_type(&lpc->lpc_types[0]); | lock_prof_init_type(&lpc->lpc_types[0]); | ||||
lock_prof_init_type(&lpc->lpc_types[1]); | lock_prof_init_type(&lpc->lpc_types[1]); | ||||
} | } | ||||
/* | |||||
* Paired with the fence from cpus_fence_seq_cst() | |||||
*/ | |||||
atomic_store_rel_int(&lock_prof_resetting, 0); | atomic_store_rel_int(&lock_prof_resetting, 0); | ||||
lock_prof_enable = enabled; | lock_prof_enable = enabled; | ||||
} | } | ||||
static void | static void | ||||
lock_prof_output(struct lock_prof *lp, struct sbuf *sb) | lock_prof_output(struct lock_prof *lp, struct sbuf *sb) | ||||
{ | { | ||||
const char *p; | const char *p; | ||||
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines | dump_lock_prof_stats(SYSCTL_HANDLER_ARGS) | ||||
error = sysctl_wire_old_buffer(req, 0); | error = sysctl_wire_old_buffer(req, 0); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
sb = sbuf_new_for_sysctl(NULL, NULL, LPROF_SBUF_SIZE, req); | sb = sbuf_new_for_sysctl(NULL, NULL, LPROF_SBUF_SIZE, req); | ||||
sbuf_printf(sb, "\n%8s %9s %11s %11s %11s %6s %6s %2s %6s %s\n", | sbuf_printf(sb, "\n%8s %9s %11s %11s %11s %6s %6s %2s %6s %s\n", | ||||
"max", "wait_max", "total", "wait_total", "count", "avg", "wait_avg", "cnt_hold", "cnt_lock", "name"); | "max", "wait_max", "total", "wait_total", "count", "avg", "wait_avg", "cnt_hold", "cnt_lock", "name"); | ||||
enabled = lock_prof_enable; | enabled = lock_prof_enable; | ||||
lock_prof_enable = 0; | lock_prof_enable = 0; | ||||
quiesce_all_cpus("profstat", 0); | /* | ||||
* See the comment in lock_prof_reset | |||||
*/ | |||||
cpus_fence_seq_cst(); | |||||
CPU_FOREACH(cpu) { | |||||
lock_prof_reset_cpu_wait(cpu); | |||||
} | |||||
t = ticks; | t = ticks; | ||||
CPU_FOREACH(cpu) { | CPU_FOREACH(cpu) { | ||||
lock_prof_type_stats(&LP_CPU(cpu)->lpc_types[0], sb, 0, t); | lock_prof_type_stats(&LP_CPU(cpu)->lpc_types[0], sb, 0, t); | ||||
lock_prof_type_stats(&LP_CPU(cpu)->lpc_types[1], sb, 1, t); | lock_prof_type_stats(&LP_CPU(cpu)->lpc_types[1], sb, 1, t); | ||||
} | } | ||||
atomic_thread_fence_rel(); | |||||
lock_prof_enable = enabled; | lock_prof_enable = enabled; | ||||
error = sbuf_finish(sb); | error = sbuf_finish(sb); | ||||
/* Output a trailing NUL. */ | /* Output a trailing NUL. */ | ||||
if (error == 0) | if (error == 0) | ||||
error = SYSCTL_OUT(req, "", 1); | error = SYSCTL_OUT(req, "", 1); | ||||
sbuf_delete(sb); | sbuf_delete(sb); | ||||
return (error); | return (error); | ||||
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines | if (++l->lpo_ref > 1) | ||||
goto out; | goto out; | ||||
l->lpo_contest_locking = contested; | l->lpo_contest_locking = contested; | ||||
l->lpo_acqtime = nanoseconds(); | l->lpo_acqtime = nanoseconds(); | ||||
if (waittime && (l->lpo_acqtime > waittime)) | if (waittime && (l->lpo_acqtime > waittime)) | ||||
l->lpo_waittime = l->lpo_acqtime - waittime; | l->lpo_waittime = l->lpo_acqtime - waittime; | ||||
else | else | ||||
l->lpo_waittime = 0; | l->lpo_waittime = 0; | ||||
out: | out: | ||||
/* | |||||
* Paired with cpus_fence_seq_cst(). | |||||
*/ | |||||
atomic_thread_fence_rel(); | |||||
critical_exit(); | critical_exit(); | ||||
} | } | ||||
void | void | ||||
lock_profile_thread_exit(struct thread *td) | lock_profile_thread_exit(struct thread *td) | ||||
{ | { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
struct lock_profile_object *l; | struct lock_profile_object *l; | ||||
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | lock_profile_release_lock(struct lock_object *lo) | ||||
lp->cnt_wait += l->lpo_waittime; | lp->cnt_wait += l->lpo_waittime; | ||||
lp->cnt_contest_locking += l->lpo_contest_locking; | lp->cnt_contest_locking += l->lpo_contest_locking; | ||||
lp->cnt_cur += l->lpo_cnt; | lp->cnt_cur += l->lpo_cnt; | ||||
release: | release: | ||||
LIST_REMOVE(l, lpo_link); | LIST_REMOVE(l, lpo_link); | ||||
type = &LP_CPU_SELF->lpc_types[spin]; | type = &LP_CPU_SELF->lpc_types[spin]; | ||||
LIST_INSERT_HEAD(&type->lpt_lpoalloc, l, lpo_link); | LIST_INSERT_HEAD(&type->lpt_lpoalloc, l, lpo_link); | ||||
out: | out: | ||||
/* | |||||
* Paired with cpus_fence_seq_cst(). | |||||
*/ | |||||
atomic_thread_fence_rel(); | |||||
critical_exit(); | critical_exit(); | ||||
} | } | ||||
static SYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL, | static SYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL, | ||||
"lock profiling"); | "lock profiling"); | ||||
SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipspin, CTLFLAG_RW, | SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipspin, CTLFLAG_RW, | ||||
&lock_prof_skipspin, 0, "Skip profiling on spinlocks."); | &lock_prof_skipspin, 0, "Skip profiling on spinlocks."); | ||||
SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipcount, CTLFLAG_RW, | SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipcount, CTLFLAG_RW, | ||||
Show All 11 Lines |
Don't you need this function to execute in critical section ?