Page MenuHomeFreeBSD
Files F177568650

D60567.1794762223.diff
No OneTemporary
Actions

Authored By
Unknown
Size
11 KB
Referenced Files
None
Subscribers
None

D60567.1794762223.diff
View Options

diff --git a/UPDATING b/UPDATING
--- a/UPDATING
+++ b/UPDATING
@@ -29,10 +29,12 @@
20261010:
The hwpmc(4) kernel/module interface version is now 0x0A040000:
- struct pmc_multipart grew to 32 payload words. Rebuild and install
- the kernel and hwpmc.ko together; a mismatched module refuses to
- load. The minimum kern.hwpmc.callchaindepth is now 34 (67 on 32-bit
- platforms).
+ struct pmc_mdep gained the pmd_lbr_csw and pmd_lbr_exec hooks,
+ struct pmc_multipart grew to 32 payload words and the scheduler
+ calls the context-switch hooks through the new pmc_ss_csw_count.
+ Rebuild and install the kernel and hwpmc.ko together; a mismatched
+ module refuses to load. The minimum kern.hwpmc.callchaindepth is
+ now 34 (67 on 32-bit platforms).
20260919:
less(1) has been updated to version 710, which removes the obsolete
diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c
--- a/sys/dev/hwpmc/hwpmc_mod.c
+++ b/sys/dev/hwpmc/hwpmc_mod.c
@@ -143,9 +143,18 @@
__LINE__)); \
} while (0)
+/*
+ * A running SS PMC with this property makes the scheduler call the context
+ * switch hooks for every thread, not only for threads of processes using
+ * PMCs (see PMC_SYSTEM_CSW_ACTIVE()). Only LBR needs this today.
+ */
+#define PMC_NEEDS_ALL_CSW(PM) \
+ (PMC_TO_MODE(PM) == PMC_MODE_SS && md->pmd_lbr_csw != NULL && \
+ ((PM)->pm_caps & PMC_CAP_LBR) != 0)
+
/* various event handlers */
static eventhandler_tag pmc_exit_tag, pmc_fork_tag, pmc_kld_load_tag,
- pmc_kld_unload_tag;
+ pmc_kld_unload_tag, pmc_exec_tag;
/* Module statistics */
struct pmc_driverstats pmc_stats;
@@ -1309,6 +1318,25 @@
return (0);
}
+/*
+ * process_exec event handler. It runs in exec_new_vmspace(), before the old
+ * address space is torn down, for every exec while hwpmc is loaded. Branch
+ * records are not tagged with an address space, so ask the MD layer to
+ * discard any record of the old image; the PMC_FN_PROCESS_EXEC hook runs
+ * much later and only when PMCs are in use.
+ */
+static void
+pmc_process_exec_early(void *arg __unused, struct proc *p __unused,
+ struct image_params *imgp __unused)
+{
+
+ if (md->pmd_lbr_exec == NULL)
+ return;
+ critical_enter();
+ md->pmd_lbr_exec(pmc_pcpu[curcpu]);
+ critical_exit();
+}
+
/*
* Handle events after an exec() for a process:
* - Inform log owners of the new exec() event
@@ -1480,8 +1508,19 @@
p = td->td_proc;
pt = NULL;
- if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
+ /*
+ * With PMC_SYSTEM_CSW_ACTIVE(), the scheduler calls this hook for
+ * every thread. Skip the process hash lookup for threads of
+ * processes that do not use PMCs, which is the common case.
+ */
+ if (!PMC_PROC_IS_USING_PMCS(p) ||
+ (pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL) {
+ if (md->pmd_lbr_csw != NULL) {
+ CRITICAL_ASSERT(curthread);
+ md->pmd_lbr_csw(pmc_pcpu[curcpu], true);
+ }
return;
+ }
KASSERT(pp->pp_proc == td->td_proc,
("[pmc,%d] not my thread state", __LINE__));
@@ -1616,6 +1655,10 @@
*/
(void)(*md->pmd_switch_in)(pc, pp);
+ /* Wipe the branch history before the incoming counters go live. */
+ if (md->pmd_lbr_csw != NULL)
+ md->pmd_lbr_csw(pc, true);
+
/* Commit all class PMC start updates at one boundary. */
pmc_process_csw_start_all(cpu);
@@ -1726,6 +1769,21 @@
* are currently running on hardware.
*/
p = td->td_proc;
+
+ /*
+ * A process without P_HWPMC has no process-virtual PMC loaded on
+ * any CPU (see pmc_detach_one_process()); this is the condition the
+ * scheduler used before PMC_SYSTEM_CSW_ACTIVE(). Only the MD
+ * branch-record state needs to see the switch.
+ */
+ if (!PMC_PROC_IS_USING_PMCS(p)) {
+ if (md->pmd_lbr_csw != NULL) {
+ CRITICAL_ASSERT(curthread);
+ md->pmd_lbr_csw(pmc_pcpu[curcpu], false);
+ }
+ return;
+ }
+
pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE);
critical_enter();
@@ -1740,6 +1798,10 @@
pc = pmc_pcpu[cpu];
+ /* Suspend branch recording before changing the address space. */
+ if (md->pmd_lbr_csw != NULL)
+ md->pmd_lbr_csw(pc, false);
+
/* Close shared class gates before any PMC stop or read. */
pmc_process_csw_stop_all(cpu);
@@ -1877,7 +1939,8 @@
/*
* Perform any other architecture/cpu dependent thread
- * switch out functions.
+ * switch out functions. The descriptor may be NULL after detach;
+ * these callbacks must still revoke access such as CR4.PCE.
*/
(void)(*md->pmd_switch_out)(pc, pp);
@@ -2868,6 +2931,8 @@
/* adjust the global and process count of SS mode PMCs */
if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
+ if (PMC_NEEDS_ALL_CSW(pm))
+ atomic_subtract_rel_int(&pmc_ss_csw_count, 1);
po = pm->pm_owner;
po->po_sscount--;
if (po->po_sscount == 0) {
@@ -3247,6 +3312,12 @@
return (EXTERROR(ENXIO, "PMC CPU %ju is not active for start",
(uintmax_t)cpu));
}
+ /*
+ * Make every context switch call the hooks before the hardware
+ * starts; the CPU migration below orders this store.
+ */
+ if (PMC_NEEDS_ALL_CSW(pm))
+ atomic_add_rel_int(&pmc_ss_csw_count, 1);
pmc_select_cpu(cpu);
/*
@@ -3313,6 +3384,8 @@
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
if (!pmc_cpu_is_active(cpu)) {
+ if (PMC_NEEDS_ALL_CSW(pm))
+ atomic_subtract_rel_int(&pmc_ss_csw_count, 1);
return (EXTERROR(ENXIO, "PMC CPU %ju is not active for stop",
(uintmax_t)cpu));
}
@@ -3331,6 +3404,10 @@
pmc_restore_cpu_binding(&pb);
+ /* Drop the all-switches reference once the hardware is stopped. */
+ if (PMC_NEEDS_ALL_CSW(pm))
+ atomic_subtract_rel_int(&pmc_ss_csw_count, 1);
+
/* Remove this owner from the global list of SS PMC owners. */
po = pm->pm_owner;
if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
@@ -5890,6 +5967,7 @@
CK_LIST_INIT(&pmc_ss_owners);
pmc_ss_count = 0;
+ pmc_ss_csw_count = 0;
/* allocate a pool of spin mutexes */
pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size,
@@ -5911,6 +5989,9 @@
pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY);
pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork,
pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY);
+ if (md->pmd_lbr_exec != NULL)
+ pmc_exec_tag = EVENTHANDLER_REGISTER(process_exec,
+ pmc_process_exec_early, NULL, EVENTHANDLER_PRI_FIRST);
/* register kld event handlers */
pmc_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, pmc_kld_load,
@@ -5977,6 +6058,10 @@
/* deregister event handlers */
EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag);
EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag);
+ if (pmc_exec_tag != NULL) {
+ EVENTHANDLER_DEREGISTER(process_exec, pmc_exec_tag);
+ pmc_exec_tag = NULL;
+ }
EVENTHANDLER_DEREGISTER(kld_load, pmc_kld_load_tag);
EVENTHANDLER_DEREGISTER(kld_unload, pmc_kld_unload_tag);
@@ -6037,6 +6122,8 @@
("[pmc,%d] Global SS owner list not empty", __LINE__));
KASSERT(pmc_ss_count == 0,
("[pmc,%d] Global SS count not empty", __LINE__));
+ KASSERT(pmc_ss_csw_count == 0,
+ ("[pmc,%d] Global SS context-switch count not empty", __LINE__));
/* do processor and pmc-class dependent cleanup */
maxcpu = pmc_cpu_max();
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -1226,7 +1226,7 @@
#ifdef HWPMC_HOOKS
/* First-run threads bypass the normal scheduler switch-in path. */
- if (PMC_PROC_IS_USING_PMCS(p))
+ if (PMC_PROC_IS_USING_PMCS(p) || PMC_SYSTEM_CSW_ACTIVE())
PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN);
#endif
diff --git a/sys/kern/kern_pmc.c b/sys/kern/kern_pmc.c
--- a/sys/kern/kern_pmc.c
+++ b/sys/kern/kern_pmc.c
@@ -84,6 +84,12 @@
*/
volatile int pmc_ss_count;
+/*
+ * A global count of running SS mode PMCs that need every context switch;
+ * see PMC_SYSTEM_CSW_ACTIVE().
+ */
+volatile int pmc_ss_csw_count;
+
/*
* Since PMC(4) may not be loaded in the current kernel, the
* convention followed is that a non-NULL value of 'pmc_hook' implies
diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c
--- a/sys/kern/kern_thread.c
+++ b/sys/kern/kern_thread.c
@@ -997,14 +997,14 @@
}
}
#ifdef HWPMC_HOOKS
- /*
- * If this thread is part of a process that is being tracked by hwpmc(4),
- * inform the module of the thread's impending exit.
- */
- if (PMC_PROC_IS_USING_PMCS(td->td_proc)) {
+ /* sched_throw() bypasses the normal scheduler switch-out path. */
+ if (PMC_PROC_IS_USING_PMCS(td->td_proc) || PMC_SYSTEM_CSW_ACTIVE())
PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
+
+ /* Inform owners of the thread's impending exit. */
+ if (PMC_PROC_IS_USING_PMCS(td->td_proc))
PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT, NULL);
- } else if (PMC_SYSTEM_SAMPLING_ACTIVE())
+ else if (PMC_SYSTEM_SAMPLING_ACTIVE())
PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_THR_EXIT_LOG, NULL);
#endif
diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c
--- a/sys/kern/sched_4bsd.c
+++ b/sys/kern/sched_4bsd.c
@@ -1076,7 +1076,8 @@
if (td != newtd) {
#ifdef HWPMC_HOOKS
- if (PMC_PROC_IS_USING_PMCS(td->td_proc))
+ if (PMC_PROC_IS_USING_PMCS(td->td_proc) ||
+ PMC_SYSTEM_CSW_ACTIVE())
PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
#endif
@@ -1121,7 +1122,8 @@
SDT_PROBE0(sched, , , on__cpu);
#ifdef HWPMC_HOOKS
- if (PMC_PROC_IS_USING_PMCS(td->td_proc))
+ if (PMC_PROC_IS_USING_PMCS(td->td_proc) ||
+ PMC_SYSTEM_CSW_ACTIVE())
PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN);
#endif
} else {
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -2415,7 +2415,8 @@
*/
if (td != newtd) {
#ifdef HWPMC_HOOKS
- if (PMC_PROC_IS_USING_PMCS(td->td_proc))
+ if (PMC_PROC_IS_USING_PMCS(td->td_proc) ||
+ PMC_SYSTEM_CSW_ACTIVE())
PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
#endif
SDT_PROBE2(sched, , , off__cpu, newtd, newtd->td_proc);
@@ -2441,7 +2442,8 @@
SDT_PROBE0(sched, , , on__cpu);
#ifdef HWPMC_HOOKS
- if (PMC_PROC_IS_USING_PMCS(td->td_proc))
+ if (PMC_PROC_IS_USING_PMCS(td->td_proc) ||
+ PMC_SYSTEM_CSW_ACTIVE())
PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN);
#endif
} else {
diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h
--- a/sys/sys/pmc.h
+++ b/sys/sys/pmc.h
@@ -1096,6 +1096,10 @@
/* handle a PMC interrupt */
int (*pmd_intr)(struct trapframe *_tf);
+ /* optional MD hook for branch-trace context hygiene */
+ void (*pmd_lbr_csw)(struct pmc_cpu *_p, bool _switch_in);
+ void (*pmd_lbr_exec)(struct pmc_cpu *_p);
+
/*
* PMC class dependent information.
*/
diff --git a/sys/sys/pmckern.h b/sys/sys/pmckern.h
--- a/sys/sys/pmckern.h
+++ b/sys/sys/pmckern.h
@@ -188,6 +188,13 @@
/* Count of system-wide sampling PMCs in existence */
extern volatile int pmc_ss_count;
+/*
+ * Count of running system-wide PMCs that need to observe every context
+ * switch, including those of processes that do not use PMCs (e.g. to reset
+ * branch-record state that is not tagged with an address space).
+ */
+extern volatile int pmc_ss_csw_count;
+
/* kernel version number */
extern const int pmc_kernel_version;
@@ -244,6 +251,10 @@
#define PMC_SYSTEM_SAMPLING_ACTIVE() (pmc_ss_count > 0)
+/* Check if the scheduler must call the context-switch hooks for everyone. */
+#define PMC_SYSTEM_CSW_ACTIVE() \
+ (__predict_false(pmc_ss_csw_count > 0))
+
/* Check if a CPU has recorded samples. */
#define PMC_CPU_HAS_SAMPLES(C) (__predict_false(DPCPU_ID_GET((C), pmc_sampled)))

File Metadata

Mime Type
text/plain
Expires
Sun, Nov 15, 5:03 PM (16 h, 59 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
41745661
Default Alt Text
D60567.1794762223.diff (11 KB)

Event Timeline