D57631.id185746.diff
No OneTemporary
Actions

Size

73 KB

Referenced Files

None

Subscribers

None

D57631.id185746.diff
View Options

	diff --git a/sys/dev/hwpmc/hwpmc_pmu.h b/sys/dev/hwpmc/hwpmc_pmu.h
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/hwpmc/hwpmc_pmu.h
	@@ -0,0 +1,306 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause
	+ *
	+ * Copyright (c) 2026 Advanced Micro Devices, Inc.
	+ *
	+ * This is the PMU event grouping and multiplexing interface.
	+ * It defers hardware row allocation until the group starts.
	+ */
	+
	+#ifndef _DEV_HWPMC_PMU_H_
	+#define _DEV_HWPMC_PMU_H_
	+
	+#include <sys/mutex.h>
	+#include <sys/pmc.h>
	+#include <sys/queue.h>
	+
	+#define PMC_ROW_IS_UNASSIGNED(P) (PMC_TO_ROWINDEX(P) == PMC_ROW_UNASSIGNED)
	+
	+#ifdef _KERNEL
	+
	+/*
	+ * These are the flags for process and thread descriptor lookups.
	+ */
	+enum pmc_flags {
	+ PMC_FLAG_NONE = 0x00, /* do nothing */
	+ PMC_FLAG_REMOVE = 0x01, /* atomically remove entry from hash */
	+ PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */
	+ PMC_FLAG_NOWAIT = 0x04, /* do not wait for mallocs */
	+};
	+
	+/*
	+ * Scheduling constraint from the class backend.
	+ * pc_allowed_rows is a bitmask of valid class row indices.
	+ */
	+#define PMC_SC_F_FIXED 0x0001 /* must use pc_fixed_row */
	+#define PMC_SC_F_SHARED 0x0004 /* tolerates sharing siblings */
	+
	+struct pmc_sched_constraint {
	+ uint32_t pc_allowed_rows; /* bitmask of legal rows */
	+ uint8_t pc_weight; /* popcount(pc_allowed_rows) */
	+ uint8_t pc_fixed_row; /* used iff PMC_SC_F_FIXED */
	+ uint16_t pc_flags; /* PMC_SC_F_* */
	+};
	+typedef struct pmc_sched_constraint pmc_sched_constraint_t;
	+
	+/* This struct wraps an event for group allocation state. */
	+struct pmu_event {
	+ TAILQ_ENTRY(pmu_event) pe_sibling;
	+ pmu_group_t *pe_group;
	+ struct pmc *pe_pmc;
	+ counter_u64_t pe_samples;
	+#ifdef HWPMC_DEBUG
	+ counter_u64_t pe_samples_emitted;
	+ counter_u64_t pe_samples_dropped;
	+ volatile u_int pe_test_sample_pid;
	+ volatile u_int pe_test_sample_tid;
	+#endif
	+ bool pe_is_leader;
	+ struct pmc_op_pmcallocate pe_alloc;
	+ pmc_sched_constraint_t pe_cons;
	+ /*
	+ * This field holds system-mode progress toward the next sample.
	+ * The value survives eviction.
	+ * For virtual mode, per-thread state is in pp_pmu_residuals instead.
	+ * A value of zero means: start from the reload count.
	+ */
	+ pmc_value_t pe_residual;
	+};
	+
	+/*
	+ * This is the explicit residual state.
	+ * A raw count of zero can mean overflow, or no saved progress. To
	+ * remove this ambiguity, track the state in a separate field.
	+ */
	+enum pmu_residual_state {
	+ PMU_RESIDUAL_UNINITIALIZED = 0, /* no saved progress; full reload */
	+ PMU_RESIDUAL_COUNT_VALID = 1, /* ptr_value is a partial count */
	+ PMU_RESIDUAL_OVERFLOW_PENDING = 2, /* overflow seen at schedule-out */
	+};
	+
	+/*
	+ * This struct holds saved sampling progress for one virtual thread.
	+ * Hardware rows are not permanent, so use the event and the TID as the
	+ * key. The process's pp_tdslock protects this data.
	+ */
	+struct pmu_thread_residual {
	+ LIST_ENTRY(pmu_thread_residual) ptr_next;
	+ pmu_event_t *ptr_event;
	+ lwpid_t ptr_tid;
	+ pmc_value_t ptr_value;
	+ uint8_t ptr_state; /* pmu_residual_state */
	+};
	+
	+/*
	+ * This struct holds one authoritative group-target edge.
	+ * The edge can be explicit or inherited.
	+ * This struct does not store hardware-row links.
	+ * To change this data, hold pmc_sx exclusive.
	+ * If you also need a process-tree snapshot, take the locks in this
	+ * order: pmc_sx, then proctree_lock, then pp_pmu_lock.
	+ */
	+struct pmu_group_target {
	+ LIST_ENTRY(pmu_group_target) pgt_group_next; /* pg_targets */
	+ LIST_ENTRY(pmu_group_target) pgt_process_next; /* pp_pmu_targets */
	+ pmu_group_t *pgt_group;
	+ struct pmc_process *pgt_pp;
	+ bool pgt_explicit;
	+};
	+
	+struct pmu_group_cpu_state {
	+ LIST_ENTRY(pmu_group_cpu_state) pgcs_next;
	+ pmu_group_t *pgcs_group;
	+ struct thread *pgcs_td;
	+ bool pgcs_counted;
	+ bool pgcs_placed;
	+ bool pgcs_transitioning;
	+};
	+
	+struct pmu_group_time_snapshot {
	+ uint64_t pgts_enabled;
	+ uint64_t pgts_running;
	+ uint64_t pgts_enabled_wall;
	+ uint64_t pgts_wall;
	+ bool pgts_system;
	+};
	+
	+/*
	+ * This is a PMU event group for atomic scheduling.
	+ * All sibling events are assigned together, or deferred together.
	+ */
	+struct pmu_group {
	+ LIST_ENTRY(pmu_group) pg_owner_next;
	+ LIST_ENTRY(pmu_group) pg_proc_next;
	+ uint32_t pg_id;
	+ struct pmc_owner *pg_owner;
	+ pmu_event_t *pg_leader;
	+ TAILQ_HEAD(, pmu_event) pg_events;
	+ u_int pg_nevents;
	+ bool pg_committed;
	+ bool pg_assigned; /* SCHEDULED if true */
	+ bool pg_running; /* between start/stop */
	+ bool pg_defer_ok; /* PMC_F_GROUP_MUX hint */
	+ struct pmu_group_target *pg_explicit_target;
	+ struct pmc_process pg_pp; / current scheduling anchor */
	+ /*
	+ * This is the authoritative target set.
	+ * If pg_explicit_target is not NULL, it holds the user-requested
	+ * edge from this list. pg_pp is only the current anchor.
	+ */
	+ LIST_HEAD(, pmu_group_target) pg_targets;
	+ /* Virtual enabled and running times use thread ticks; wall uses TSC. */
	+ uint64_t pg_time_enabled_ticks;
	+ uint64_t pg_time_running_ticks;
	+ uint64_t pg_enabled_wall_ticks;
	+ uint64_t pg_wall_start_ticks;
	+ uint64_t pg_wall_ticks;
	+ uint64_t pg_timestamp_ticks;
	+ uint64_t pg_tickrate;
	+ struct mtx pg_snapshot_lock;
	+ u_int pg_oncpu_threads;
	+ u_int pg_running_threads;
	+ struct pmu_group_cpu_state *pg_cpu_state;
	+ u_int pg_ncpu;
	+ bool pg_account_blocked;
	+ bool pg_account_placement_admit;
	+ bool pg_snapshot_pending;
	+ bool pg_snapshot_active;
	+ bool pg_releasing;
	+ /*
	+ * This is the system-mode (PMC_MODE_SC) state.
	+ * A system group binds to one CPU and rotates on that CPU.
	+ */
	+ bool pg_system; /* system-wide group */
	+ bool pg_sys_listed; /* on pmu_syscpu list */
	+ bool pg_sscounted; /* in po_sscount */
	+ int pg_cpu; /* bound CPU (system) */
	+};
	+
	+/*
	+ * These are the core hooks from hwpmc_mod.c.
	+ */
	+struct pmc_mdep *hwpmc_get_mdep(void);
	+struct pmc_classdep hwpmc_ri_to_classdep(int ri, int adjri);
	+bool hwpmc_can_allocate_row(int ri, enum pmc_mode mode);
	+bool hwpmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu);
	+void hwpmc_mark_row_thread(int ri);
	+void hwpmc_unmark_row_thread(int ri);
	+void hwpmc_mark_row_standalone(int ri);
	+void hwpmc_unmark_row_standalone(int ri);
	+bool hwpmc_row_is_unallocated(int cpu, int ri);
	+void hwpmc_unconfigure_row_all_cpus(struct pmc *pm, int ri);
	+
	+/*
	+ * Start and stop system-mode PMC hardware on the target CPU.
	+ * The values accumulate across multiplex windows.
	+ */
	+int hwpmc_pmu_sys_start_row(int cpu, struct pmc *pm);
	+void hwpmc_sscount_add(struct pmc_owner *po);
	+void hwpmc_sscount_sub(struct pmc_owner *po);
	+void hwpmc_pmu_sys_stop_row(int cpu, struct pmc *pm);
	+
	+/*
	+ * This is the hardware row assigner (hwpmc_assign.c).
	+ */
	+int pmu_assign_group(pmu_group_t pg, struct proc p, int cpu);
	+void pmu_unassign_group(pmu_group_t *pg, int cpu);
	+int pmu_group_can_fit(pmu_group_t *pg);
	+int pmu_group_can_place(pmu_group_t pg, struct proc p, int cpu);
	+int pmu_group_satisfiable(pmu_group_t pg, struct proc p, int cpu,
	+ uint64_t evictable_rows);
	+int pmu_validate_group(pmu_group_t *pg);
	+int pmu_event_get_constraint(pmu_event_t *pe,
	+ pmc_sched_constraint_t *cons);
	+bool pmu_class_supports_grouping(enum pmc_class class);
	+
	+/*
	+ * This is the group lifecycle (hwpmc_pmu.c).
	+ */
	+void pmu_group_create(struct pmc_owner po, uint32_t pg_id);
	+int pmu_group_add(pmu_group_t pg, struct pmc pm, bool leader);
	+int pmu_group_commit(pmu_group_t *pg);
	+pmu_group_t pmu_group_lookup(struct pmc_owner po, uint32_t pg_id);
	+u_int pmu_group_prepare_release(pmu_group_t pg, struct pmc *members,
	+ u_int capacity, struct pmc_process **released_pp);
	+void pmu_group_release(pmu_group_t *pg);
	+void pmu_group_accounting_initialize(void);
	+void pmu_group_accounting_finalize(void);
	+void pmu_group_time_snapshot_locked(pmu_group_t *pg,
	+ struct pmu_group_time_snapshot *snapshot, uint64_t now);
	+void pmu_event_destroy(pmu_event_t *pe);
	+pmu_event_t pmu_event_from_pmc(struct pmc pm);
	+pmu_group_t pmu_group_from_pmc(struct pmc pm);
	+u_int pmu_group_inherit(struct pmc_process ppold, struct pmc_process ppnew,
	+ struct pmc *missed_leaders, u_int missed_capacity, u_int nmissed);
	+bool pmu_group_follows_fork(pmu_group_t *pg);
	+void pmu_group_disinherit(struct pmc_process *pp);
	+struct pmu_group_target pmu_group_target_alloc(pmu_group_t pg,
	+ struct pmc_process *pp, bool explicit, int malloc_flags);
	+void pmu_group_target_publish(struct pmu_group_target *pgt);
	+void pmu_group_target_discard(struct pmu_group_target *pgt);
	+void pmu_group_target_remove(struct pmu_group_target *pgt);
	+void pmu_event_save_residual(struct pmc pm, struct pmc_process pp, int ri);
	+void pmu_event_set_residual(struct pmc *pm, pmc_value_t residual);
	+void pmu_event_restore_thread_residual(struct pmc *pm,
	+ struct pmc_process *pp, int ri);
	+pmc_value_t pmu_event_restore_residual(struct pmc *pm);
	+void pmu_thread_residual_remove(struct pmc_process *pp, lwpid_t tid);
	+#ifdef HWPMC_DEBUG
	+int pmu_event_get_thread_residual(struct pmc pm, struct pmc_process pp,
	+ lwpid_t tid, pmc_value_t value, uint8_t state);
	+int pmu_event_set_thread_residual(struct pmc pm, struct pmc_process pp,
	+ lwpid_t tid, pmc_value_t value, uint8_t state);
	+#endif
	+
	+int pmu_group_on_allocate(struct pmc pm, const struct pmc_op_pmcallocate pa);
	+int pmu_group_on_attach(struct pmc pm, struct proc p);
	+int pmu_group_on_start(struct pmc *pm);
	+void pmu_group_on_stop(struct pmc *pm);
	+void pmu_group_on_release(struct pmc *pm);
	+void pmu_group_csw_in(struct thread td, struct pmc_process pp);
	+void pmu_group_csw_in_complete(struct thread *td, int cpu);
	+bool pmu_group_csw_can_start(struct pmc pm, struct thread td, int cpu);
	+void pmu_group_csw_out(struct thread *td, int cpu);
	+void pmu_group_csw_out_complete(struct thread *td, int cpu);
	+
	+/*
	+ * These are the system-mode (PMC_MODE_SC) group lifecycle functions.
	+ */
	+int pmu_sys_group_on_start(struct pmc *pm);
	+void pmu_sys_group_on_stop(struct pmc *pm);
	+void pmu_sys_group_pre_release(struct pmc *pm);
	+
	+/*
	+ * These are the process descriptor PMU lifecycle functions.
	+ * Before you call pmu_pp_destroy(), hold pmc_sx exclusive.
	+ */
	+void pmu_pp_init(struct pmc_process *pp);
	+void pmu_pp_destroy(struct pmc_process *pp);
	+void pmu_group_detach_target(pmu_group_t pg, struct pmc_process pp);
	+void pmu_group_detach_inherited_target(pmu_group_t pg, struct pmc_process pp);
	+void pmu_pp_kick_after_exec(struct pmc_process *pp);
	+
	+/*
	+ * Find a process descriptor for the PMU layer.
	+ */
	+struct pmc_process pmc_find_process_descriptor_pmu(struct proc p,
	+ uint32_t mode);
	+
	+void hwpmc_pmu_force_context_switch(void);
	+
	+/*
	+ * These are the rotation helper functions (hwpmc_mod.c).
	+ */
	+void pmc_rotation_drain_set(struct pmc *pm, bool drain);
	+void pmc_rotation_drain(struct pmc *pm);
	+void pmc_rotation_drain_cpu(struct pmc *pm, int cpu);
	+void pmc_rotation_detach(struct pmc pm, struct pmc_process pp);
	+void pmc_rotation_attach(struct pmc pm, struct pmc_process pp);
	+struct pmc_target *pmc_target_object_alloc(int malloc_flags);
	+void pmc_target_object_free(struct pmc_target *pt);
	+void pmc_link_target_process_preallocated(struct pmc *pm,
	+ struct pmc_process pp, struct pmc_target pt);
	+
	+#endif /* _KERNEL */
	+
	+#endif /* _DEV_HWPMC_PMU_H_ */
	diff --git a/sys/dev/hwpmc/hwpmc_pmu.c b/sys/dev/hwpmc/hwpmc_pmu.c
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/hwpmc/hwpmc_pmu.c
	@@ -0,0 +1,2302 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause
	+ *
	+ * Copyright (c) 2026 Advanced Micro Devices, Inc.
	+ *
	+ * This file has PMU event grouping, time accounting, and multiplex
	+ * rotation.
	+ */
	+
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <sys/kernel.h>
	+#include <sys/kthread.h>
	+#include <sys/lock.h>
	+#include <sys/malloc.h>
	+#include <sys/mutex.h>
	+#include <sys/pcpu.h>
	+#include <sys/pmc.h>
	+#include <sys/pmckern.h>
	+#include <sys/pmclog.h>
	+#include <sys/prng.h>
	+#include <sys/proc.h>
	+#include <sys/sysctl.h>
	+#include <sys/sx.h>
	+#include <sys/taskqueue.h>
	+#include <sys/unistd.h>
	+
	+#include "hwpmc_pmu.h"
	+
	+static MALLOC_DEFINE(M_PMU, "pmu", "hwpmc PMU grouping");
	+
	+static uint32_t pmu_next_group_id = 1;
	+
	+extern struct mtx_pool *pmc_mtxpool;
	+
	+LIST_HEAD(pmu_group_cpu_list, pmu_group_cpu_state);
	+static struct pmu_group_cpu_list pmu_group_cpu_active[MAXCPU];
	+static LIST_HEAD(, pmu_thread_residual) pmu_residual_freelist;
	+static struct mtx pmu_residual_freelist_mtx;
	+static struct task pmu_residual_free_task;
	+
	+SYSCTL_DECL(_kern_hwpmc);
	+
	+/*
	+ * This is the multiplex rotation period, in milliseconds.
	+ */
	+static int pmu_mux_period_ms = 50;
	+SYSCTL_INT(_kern_hwpmc, OID_AUTO, mux_period_ms, CTLFLAG_RWTUN,
	+ &pmu_mux_period_ms, 0,
	+ "PMU multiplex rotation period floor in milliseconds");
	+
	+/* Return the rotation window in ticks, with random jitter. */
	+static int
	+pmu_rot_period_ticks(void)
	+{
	+ int period_ticks;
	+
	+ period_ticks = (pmu_mux_period_ms * hz) / 1000;
	+ if (period_ticks < 1)
	+ period_ticks = 1;
	+ return (period_ticks + prng32_bounded(period_ticks / 4 + 1));
	+}
	+
	+static void pmu_pp_rotate_thread(void *arg);
	+static int pmu_pp_schedule_in(struct pmc_process pp, pmu_group_t pg);
	+static void pmu_pp_schedule_out(struct pmc_process pp, pmu_group_t pg,
	+ bool drain_samples);
	+static void pmu_pp_kick_rotate(struct pmc_process *pp);
	+static void pmu_pp_unlink_group(pmu_group_t pg, struct pmc_process pp);
	+static void pmu_group_attach_siblings(pmu_group_t *pg,
	+ struct pmc_process *pp);
	+static void pmu_event_remove_target_residuals(pmu_event_t *pe,
	+ struct pmc_process *pp);
	+static int pmu_event_set_thread_residual_impl(struct pmc *pm,
	+ struct pmc_process *pp, lwpid_t tid, pmc_value_t value, uint8_t state);
	+static struct pmu_thread_residual *pmu_thread_residual_find_locked(
	+ struct pmc_process pp, pmu_event_t pe, lwpid_t tid);
	+static bool pmu_thread_exists_locked(struct pmc_process *pp, lwpid_t tid);
	+static void pmu_thread_residual_defer_free(struct pmu_thread_residual *ptr);
	+static void pmu_thread_residual_free_task_fn(void *arg, int pending);
	+
	+/*
	+ * This struct holds state for system-mode (PMC_MODE_SC) multiplexing.
	+ */
	+struct pmu_syscpu {
	+ struct pmu_group_list sc_groups; /* Groups bound to this CPU */
	+ pmu_group_t sc_cursor; / Rotation cursor */
	+ struct thread sc_td; / Rotation thread */
	+ u_int sc_quiesce;
	+ bool sc_running; /* Is the rotation thread active? */
	+ bool sc_needed; /* Is a deferred group waiting? */
	+};
	+static struct pmu_syscpu pmu_syscpu[MAXCPU];
	+
	+
	+void
	+pmu_group_accounting_initialize(void)
	+{
	+ u_int cpu;
	+
	+ for (cpu = 0; cpu < MAXCPU; cpu++)
	+ LIST_INIT(&pmu_group_cpu_active[cpu]);
	+ LIST_INIT(&pmu_residual_freelist);
	+ mtx_init(&pmu_residual_freelist_mtx, "pmu-residuals", "pmc-leaf",
	+ MTX_SPIN);
	+ TASK_INIT(&pmu_residual_free_task, 0, pmu_thread_residual_free_task_fn,
	+ NULL);
	+}
	+
	+void
	+pmu_group_accounting_finalize(void)
	+{
	+ u_int cpu;
	+
	+ for (cpu = 0; cpu < MAXCPU; cpu++)
	+ KASSERT(LIST_EMPTY(&pmu_group_cpu_active[cpu]),
	+ ("[pmu] active accounting markers on CPU %u", cpu));
	+ taskqueue_drain(taskqueue_fast, &pmu_residual_free_task);
	+ pmu_thread_residual_free_task_fn(NULL, 0);
	+ KASSERT(LIST_EMPTY(&pmu_residual_freelist),
	+ ("[pmu] deferred residual frees remain"));
	+ mtx_destroy(&pmu_residual_freelist_mtx);
	+}
	+
	+static void
	+pmu_group_time_update_locked(pmu_group_t *pg, uint64_t now)
	+{
	+ uint64_t delta;
	+
	+ if (pg->pg_timestamp_ticks == 0) {
	+ pg->pg_timestamp_ticks = now;
	+ return;
	+ }
	+ if (now <= pg->pg_timestamp_ticks)
	+ return;
	+
	+ delta = now - pg->pg_timestamp_ticks;
	+ if (pg->pg_running) {
	+ if (pg->pg_system) {
	+ if (pg->pg_cpu >= 0 && pmc_cpu_is_active(pg->pg_cpu)) {
	+ pg->pg_time_enabled_ticks += delta;
	+ pg->pg_enabled_wall_ticks += delta;
	+ if (pg->pg_account_placement_admit)
	+ pg->pg_time_running_ticks += delta;
	+ }
	+ } else {
	+ pg->pg_time_enabled_ticks +=
	+ (uint64_t)pg->pg_oncpu_threads * delta;
	+ pg->pg_time_running_ticks +=
	+ (uint64_t)pg->pg_running_threads * delta;
	+ if (pg->pg_oncpu_threads != 0)
	+ pg->pg_enabled_wall_ticks += delta;
	+ }
	+ }
	+ pg->pg_timestamp_ticks = now;
	+}
	+
	+static void
	+pmu_group_running_start_locked(pmu_group_t *pg, uint64_t now)
	+{
	+
	+ pmu_group_time_update_locked(pg, now);
	+ if (!pg->pg_running) {
	+ pg->pg_running = true;
	+ pg->pg_wall_start_ticks = now;
	+ }
	+}
	+
	+static void
	+pmu_group_running_stop_locked(pmu_group_t *pg, uint64_t now)
	+{
	+
	+ pmu_group_time_update_locked(pg, now);
	+ if (!pg->pg_running)
	+ return;
	+ if (now > pg->pg_wall_start_ticks)
	+ pg->pg_wall_ticks += now - pg->pg_wall_start_ticks;
	+ pg->pg_wall_start_ticks = 0;
	+ pg->pg_running = false;
	+}
	+
	+static uint64_t
	+pmu_group_ticks_to_ns(uint64_t ticks, uint64_t tickrate)
	+{
	+ uint64_t frac, quot, rem, secs;
	+
	+ if (tickrate == 0)
	+ return (0);
	+
	+ /*
	+ * Scale ticks to nanoseconds without 128-bit division:
	+ * (ticks / rate) * 1e9 + (ticks % rate) * 1e9 / rate.
	+ */
	+ quot = ticks / tickrate;
	+ rem = ticks % tickrate;
	+
	+ if (quot > UINT64_MAX / 1000000000)
	+ return (UINT64_MAX);
	+ secs = quot * 1000000000;
	+
	+ /* Prevent overflow for high tick rates. */
	+ if (rem <= UINT64_MAX / 1000000000)
	+ frac = rem * 1000000000 / tickrate;
	+ else
	+ frac = rem / (tickrate / 1000000000);
	+
	+ if (secs > UINT64_MAX - frac)
	+ return (UINT64_MAX);
	+ return (secs + frac);
	+}
	+
	+void
	+pmu_group_time_snapshot_locked(pmu_group_t *pg,
	+ struct pmu_group_time_snapshot *snapshot, uint64_t now)
	+{
	+ uint64_t enabled, enabled_wall, running, tickrate, wall;
	+
	+ KASSERT(pg != NULL && snapshot != NULL,
	+ ("[pmu] invalid group time snapshot"));
	+ pmu_group_time_update_locked(pg, now);
	+ enabled = pg->pg_time_enabled_ticks;
	+ running = MIN(pg->pg_time_running_ticks, enabled);
	+ enabled_wall = pg->pg_enabled_wall_ticks;
	+ wall = pg->pg_wall_ticks;
	+ if (pg->pg_running && now > pg->pg_wall_start_ticks)
	+ wall += now - pg->pg_wall_start_ticks;
	+ tickrate = pg->pg_tickrate;
	+ snapshot->pgts_system = pg->pg_system;
	+
	+ snapshot->pgts_enabled = pmu_group_ticks_to_ns(enabled, tickrate);
	+ snapshot->pgts_running = pmu_group_ticks_to_ns(running, tickrate);
	+ snapshot->pgts_enabled_wall = pmu_group_ticks_to_ns(enabled_wall,
	+ tickrate);
	+ snapshot->pgts_wall = pmu_group_ticks_to_ns(wall, tickrate);
	+ if (snapshot->pgts_running > snapshot->pgts_enabled)
	+ snapshot->pgts_running = snapshot->pgts_enabled;
	+}
	+
	+static void
	+pmu_group_accounting_block_locked(pmu_group_t *pg, uint64_t now)
	+{
	+
	+ pmu_group_time_update_locked(pg, now);
	+ pg->pg_account_blocked = true;
	+ pg->pg_account_placement_admit = false;
	+}
	+
	+static void
	+pmu_group_accounting_block(pmu_group_t *pg)
	+{
	+
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ pmu_group_accounting_block_locked(pg, cpu_ticks());
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+}
	+
	+static void
	+pmu_group_accounting_drain(pmu_group_t *pg, bool release)
	+{
	+ struct pmc_binding pb;
	+ u_int cpu;
	+#ifdef INVARIANTS
	+ int maxloop;
	+
	+ maxloop = 100 * MAX(1, pg->pg_ncpu);
	+#endif
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ KASSERT(pg->pg_account_blocked,
	+ ("[pmu] draining unblocked group %u", pg->pg_id));
	+
	+ if (pg->pg_ncpu != 0)
	+ pmc_save_cpu_binding(&pb);
	+ for (;;) {
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ for (cpu = 0; cpu < pg->pg_ncpu; cpu++) {
	+ if (pg->pg_cpu_state[cpu].pgcs_counted)
	+ break;
	+ }
	+ if (cpu == pg->pg_ncpu) {
	+ KASSERT(pg->pg_oncpu_threads == 0,
	+ ("[pmu] group %u has count without marker",
	+ pg->pg_id));
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ break;
	+ }
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+#ifdef INVARIANTS
	+ KASSERT(maxloop-- > 0,
	+ ("[pmu] group %u accounting drain stuck", pg->pg_id));
	+#endif
	+ KASSERT(pmc_cpu_is_active(cpu),
	+ ("[pmu] group %u marker on inactive CPU %u", pg->pg_id,
	+ cpu));
	+ pmc_select_cpu(cpu);
	+ hwpmc_pmu_force_context_switch();
	+ }
	+ if (pg->pg_ncpu != 0)
	+ pmc_restore_cpu_binding(&pb);
	+
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ pmu_group_running_stop_locked(pg, cpu_ticks());
	+ if (!release)
	+ pg->pg_account_blocked = false;
	+ KASSERT(pg->pg_running_threads == 0,
	+ ("[pmu] group %u has running accounting threads", pg->pg_id));
	+ for (cpu = 0; cpu < pg->pg_ncpu; cpu++) {
	+ KASSERT(!pg->pg_cpu_state[cpu].pgcs_counted &&
	+ !pg->pg_cpu_state[cpu].pgcs_placed &&
	+ !pg->pg_cpu_state[cpu].pgcs_transitioning &&
	+ pg->pg_cpu_state[cpu].pgcs_td == NULL,
	+ ("[pmu] group %u has active CPU %u marker", pg->pg_id,
	+ cpu));
	+ }
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+}
	+
	+static void
	+pmu_group_kick_placement(pmu_group_t *pg)
	+{
	+ struct pmc_binding pb;
	+ bool kick;
	+ u_int cpu;
	+
	+ KASSERT(!pg->pg_system && pg->pg_ncpu != 0,
	+ ("[pmu] kick_placement: bad group %u", pg->pg_id));
	+ pmc_save_cpu_binding(&pb);
	+ for (cpu = 0; cpu < pg->pg_ncpu; cpu++) {
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ kick = pg->pg_cpu_state[cpu].pgcs_counted;
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ if (!kick \|\| !pmc_cpu_is_active(cpu))
	+ continue;
	+ pmc_select_cpu(cpu);
	+ hwpmc_pmu_force_context_switch();
	+ }
	+ pmc_restore_cpu_binding(&pb);
	+}
	+
	+static void
	+pmu_group_accounting_prepare_release(pmu_group_t *pg)
	+{
	+ struct pmc_process *pp;
	+
	+ pp = pg->pg_pp;
	+ if (pp != NULL)
	+ mtx_lock_spin(&pp->pp_pmu_lock);
	+ pmu_group_accounting_block(pg);
	+ if (pp != NULL)
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+ pmu_group_accounting_drain(pg, true);
	+}
	+
	+static void
	+pmu_pp_stop_rotate(struct pmc_process pp, const char wmesg)
	+{
	+
	+ mtx_lock_spin(&pp->pp_pmu_lock);
	+ pp->pp_pmu_rot_quiesce++;
	+ pp->pp_pmu_rot_running = false;
	+ pp->pp_pmu_rot_needed = false;
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+ wakeup(&pp->pp_pmu_rot_needed);
	+ while (pp->pp_pmu_rot_td != NULL)
	+ (void)sx_sleep(&pp->pp_pmu_rot_td, &pmc_sx, 0, wmesg, 1);
	+ mtx_lock_spin(&pp->pp_pmu_lock);
	+ KASSERT(pp->pp_pmu_rot_quiesce > 0,
	+ ("[pmu] rotation quiesce underflow"));
	+ pp->pp_pmu_rot_quiesce--;
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+}
	+
	+/* Add the group to the process list if it is not already there. */
	+static void
	+pmu_pp_link_group(pmu_group_t pg, struct pmc_process pp)
	+{
	+
	+ if (pg->pg_pp != NULL)
	+ return;
	+ mtx_lock_spin(&pp->pp_pmu_lock);
	+ LIST_INSERT_HEAD(&pp->pp_pmu_groups, pg, pg_proc_next);
	+ pg->pg_pp = pp;
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+}
	+
	+/*
	+ * Does this group follow its targets through fork? This flag is
	+ * leader-only and governs the whole group. Commit already rejects
	+ * this flag on a system group.
	+ */
	+bool
	+pmu_group_follows_fork(pmu_group_t *pg)
	+{
	+
	+ return (pg->pg_committed && !pg->pg_releasing && !pg->pg_system &&
	+ pg->pg_leader != NULL &&
	+ (pg->pg_leader->pe_alloc.pm_flags & PMC_F_DESCENDANTS) != 0);
	+}
	+
	+/* Find the authoritative edge for pp, if this group follows it already. */
	+static struct pmu_group_target *
	+pmu_group_find_target(pmu_group_t pg, struct pmc_process pp)
	+{
	+ struct pmu_group_target *pgt;
	+
	+ LIST_FOREACH(pgt, &pg->pg_targets, pgt_group_next) {
	+ if (pgt->pgt_pp == pp)
	+ return (pgt);
	+ }
	+ return (NULL);
	+}
	+
	+/*
	+ * This function selects a surviving target for a scheduling-anchor handoff.
	+ * Prefer the explicit edge if it exists. If not, use any inherited edge.
	+ * To call this function, hold pmc_sx exclusive.
	+ */
	+static struct pmu_group_target *
	+pmu_group_replacement_target(pmu_group_t pg, struct pmc_process old_pp)
	+{
	+ struct pmu_group_target pgt, replacement;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ replacement = NULL;
	+ LIST_FOREACH(pgt, &pg->pg_targets, pgt_group_next) {
	+ if (pgt->pgt_pp == old_pp)
	+ continue;
	+ if (pgt->pgt_explicit)
	+ return (pgt);
	+ if (replacement == NULL)
	+ replacement = pgt;
	+ }
	+ return (replacement);
	+}
	+
	+static bool
	+pmu_group_has_target(pmu_group_t pg, struct pmc_process pp)
	+{
	+
	+ return (pmu_group_find_target(pg, pp) != NULL);
	+}
	+
	+/*
	+ * This function allocates a target edge. It does not publish it.
	+ * Publication is a separate step. This lets a multi-process attach
	+ * finish every fallible check before any edge becomes visible.
	+ */
	+struct pmu_group_target *
	+pmu_group_target_alloc(pmu_group_t pg, struct pmc_process pp, bool explicit,
	+ int malloc_flags)
	+{
	+ struct pmu_group_target *pgt;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ KASSERT(pg != NULL && pp != NULL,
	+ ("[pmu] invalid target allocation pg=%p pp=%p", pg, pp));
	+ pgt = malloc(sizeof(*pgt), M_PMU, malloc_flags \| M_ZERO);
	+ if (pgt == NULL)
	+ return (NULL);
	+ pgt->pgt_pp = pp;
	+ pgt->pgt_group = pg;
	+ pgt->pgt_explicit = explicit;
	+ return (pgt);
	+}
	+
	+void
	+pmu_group_target_publish(struct pmu_group_target *pgt)
	+{
	+ pmu_group_t *pg;
	+ struct pmc_process *pp;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ KASSERT(pgt != NULL && pgt->pgt_group != NULL && pgt->pgt_pp != NULL,
	+ ("[pmu] invalid target publication pgt=%p", pgt));
	+ pg = pgt->pgt_group;
	+ pp = pgt->pgt_pp;
	+ KASSERT(!pmu_group_has_target(pg, pp),
	+ ("[pmu] duplicate group %u target pp=%p", pg->pg_id, pp));
	+
	+ LIST_INSERT_HEAD(&pg->pg_targets, pgt, pgt_group_next);
	+ mtx_lock_spin(&pp->pp_pmu_lock);
	+ LIST_INSERT_HEAD(&pp->pp_pmu_targets, pgt, pgt_process_next);
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+ if (pgt->pgt_explicit) {
	+ KASSERT(pg->pg_explicit_target == NULL && pg->pg_pp == NULL,
	+ ("[pmu] group %u already has an explicit target", pg->pg_id));
	+ pg->pg_explicit_target = pgt;
	+ pmu_pp_link_group(pg, pp);
	+ }
	+}
	+
	+void
	+pmu_group_target_discard(struct pmu_group_target *pgt)
	+{
	+
	+ if (pgt == NULL)
	+ return;
	+ free(pgt, M_PMU);
	+}
	+
	+void
	+pmu_group_target_remove(struct pmu_group_target *pgt)
	+{
	+ pmu_event_t *pe;
	+ pmu_group_t *pg;
	+ struct pmc_process *pp;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ KASSERT(pgt != NULL && pgt->pgt_group != NULL && pgt->pgt_pp != NULL,
	+ ("[pmu] invalid target removal pgt=%p", pgt));
	+ pg = pgt->pgt_group;
	+ pp = pgt->pgt_pp;
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling)
	+ pmu_event_remove_target_residuals(pe, pp);
	+ if (pg->pg_pp == pp)
	+ pmu_pp_unlink_group(pg, pp);
	+ if (pg->pg_explicit_target == pgt)
	+ pg->pg_explicit_target = NULL;
	+ mtx_lock_spin(&pp->pp_pmu_lock);
	+ LIST_REMOVE(pgt, pgt_process_next);
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+ LIST_REMOVE(pgt, pgt_group_next);
	+ pmu_group_target_discard(pgt);
	+}
	+
	+/*
	+ * This function records a target reached through fork.
	+ * The allocation must not sleep. A fork must not fail.
	+ * If the allocation fails, the group does not monitor the child.
	+ * The caller then records the miss.
	+ */
	+static bool
	+pmu_group_add_inherited(pmu_group_t pg, struct pmc_process pp)
	+{
	+ pmu_event_t *pe;
	+ struct pmc_target *links[PMC_GROUP_MAX_MEMBERS];
	+ struct pmu_group_target *pgt;
	+ struct pmc *pm;
	+ u_int i, nlinks;
	+ int ri;
	+
	+ if (pmu_group_has_target(pg, pp))
	+ return (true);
	+ KASSERT(pg->pg_nevents <= PMC_GROUP_MAX_MEMBERS,
	+ ("[pmu] group %u has too many members", pg->pg_id));
	+ memset(links, 0, sizeof(links));
	+ nlinks = 0;
	+ pgt = pmu_group_target_alloc(pg, pp, false, M_NOWAIT);
	+ if (pgt == NULL)
	+ return (false);
	+
	+ if (pg->pg_assigned) {
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling) {
	+ pm = pe->pe_pmc;
	+ KASSERT(!PMC_ROW_IS_UNASSIGNED(pm),
	+ ("[pmu] assigned group %u has unassigned member",
	+ pg->pg_id));
	+ ri = PMC_TO_ROWINDEX(pm);
	+ if (pp->pp_pmcs[ri].pp_pmc != NULL)
	+ goto fail;
	+ links[nlinks] =
	+ pmc_target_object_alloc(M_NOWAIT);
	+ if (links[nlinks] == NULL)
	+ goto fail;
	+ nlinks++;
	+ }
	+
	+ i = 0;
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling) {
	+ pmc_link_target_process_preallocated(pe->pe_pmc, pp,
	+ links[i]);
	+ links[i++] = NULL;
	+ }
	+ }
	+
	+ /*
	+ * This is the linearization point. Every fallible allocation and
	+ * every current row link is complete before the authoritative
	+ * target becomes visible.
	+ */
	+ pmu_group_target_publish(pgt);
	+ return (true);
	+
	+fail:
	+ for (i = 0; i < nlinks; i++)
	+ pmc_target_object_free(links[i]);
	+ pmu_group_target_discard(pgt);
	+ return (false);
	+}
	+
	+/*
	+ * Attach every group the parent is a target of to the child (spec §3.9).
	+ * A group is taken all at once, or not at all, so that §2.4 holds in the
	+ * child. If a group cannot be recorded, skip it whole and count it.
	+ * The caller reports how many were missed.
	+ *
	+ * Only groups whose leader carries PMC_F_DESCENDANTS follow a fork, and
	+ * only virtual ones. A system group is bound to a CPU and has no process
	+ * target, so commit already rejects it.
	+ */
	+u_int
	+pmu_group_inherit(struct pmc_process ppold, struct pmc_process ppnew,
	+ struct pmc *missed_leaders, u_int missed_capacity, u_int nmissed)
	+{
	+ pmu_group_t *pg;
	+ struct pmu_group_target *pgt;
	+ u_int ninherited, nmiss;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ ninherited = 0;
	+ nmiss = 0;
	+
	+ /* Direct parents and inherited parents share one target list. */
	+ LIST_FOREACH(pgt, &ppold->pp_pmu_targets, pgt_process_next) {
	+ pg = pgt->pgt_group;
	+ if (!pmu_group_follows_fork(pg))
	+ continue;
	+ if (pmu_group_add_inherited(pg, ppnew)) {
	+ ninherited++;
	+ continue;
	+ }
	+ /* Record the missed group by its leader handle, for per-group logs. */
	+ if (missed_leaders != NULL && nmiss < missed_capacity &&
	+ pg->pg_leader != NULL)
	+ missed_leaders[nmiss] = pg->pg_leader->pe_pmc;
	+ nmiss++;
	+ }
	+
	+ if (nmissed != NULL)
	+ *nmissed = nmiss;
	+ return (ninherited);
	+}
	+
	+/* Remove every group-target edge held by pp. This includes both explicit and inherited edges. */
	+void
	+pmu_group_disinherit(struct pmc_process *pp)
	+{
	+ struct pmu_group_target *pgt;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ while ((pgt = LIST_FIRST(&pp->pp_pmu_targets)) != NULL)
	+ pmu_group_target_remove(pgt);
	+}
	+
	+/* Remove the group from the process list and update the cursor. */
	+static void
	+pmu_pp_unlink_group(pmu_group_t pg, struct pmc_process pp)
	+{
	+
	+ mtx_lock_spin(&pp->pp_pmu_lock);
	+ if (pp->pp_pmu_rot_cursor == pg)
	+ pp->pp_pmu_rot_cursor = LIST_NEXT(pg, pg_proc_next);
	+ LIST_REMOVE(pg, pg_proc_next);
	+ pg->pg_pp = NULL;
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+}
	+
	+/* Assign deferred groups and restart rotation. */
	+static void
	+pmu_pp_backfill(struct pmc_process *pp)
	+{
	+ pmu_group_t *pg;
	+
	+ if (pp->pp_pmu_unhashed)
	+ return;
	+ LIST_FOREACH(pg, &pp->pp_pmu_groups, pg_proc_next) {
	+ if (pg->pg_running && !pg->pg_assigned)
	+ (void)pmu_pp_schedule_in(pp, pg);
	+ }
	+ pmu_pp_kick_rotate(pp);
	+}
	+
	+pmu_event_t *
	+pmu_event_from_pmc(struct pmc *pm)
	+{
	+ return (pm != NULL ? pm->pm_pmu : NULL);
	+}
	+
	+/* Return the group for the PMC, or NULL. */
	+pmu_group_t *
	+pmu_group_from_pmc(struct pmc *pm)
	+{
	+ pmu_event_t *pe;
	+
	+ pe = pmu_event_from_pmc(pm);
	+ return (pe != NULL ? pe->pe_group : NULL);
	+}
	+
	+void
	+pmu_event_destroy(pmu_event_t *pe)
	+{
	+ if (pe == NULL)
	+ return;
	+ counter_u64_free(pe->pe_samples);
	+#ifdef HWPMC_DEBUG
	+ counter_u64_free(pe->pe_samples_emitted);
	+ counter_u64_free(pe->pe_samples_dropped);
	+#endif
	+ free(pe, M_PMU);
	+}
	+
	+void
	+pmu_group_create(struct pmc_owner po, uint32_t pg_id)
	+{
	+ pmu_group_t *pg;
	+
	+ KASSERT(po != NULL && pg_id != NULL,
	+ ("[pmu] pmu_group_create: null po=%p pg_id=%p", po, pg_id));
	+
	+ pg = malloc(sizeof(*pg), M_PMU, M_WAITOK \| M_ZERO);
	+ mtx_init(&pg->pg_snapshot_lock, "pmu group snapshot", NULL, MTX_SPIN);
	+ pg->pg_id = pmu_next_group_id++;
	+ pg->pg_owner = po;
	+ TAILQ_INIT(&pg->pg_events);
	+ LIST_INIT(&pg->pg_targets);
	+ LIST_INSERT_HEAD(&po->po_groups, pg, pg_owner_next);
	+ *pg_id = pg->pg_id;
	+}
	+
	+pmu_group_t *
	+pmu_group_lookup(struct pmc_owner *po, uint32_t pg_id)
	+{
	+ pmu_group_t *pg;
	+
	+ LIST_FOREACH(pg, &po->po_groups, pg_owner_next) {
	+ if (pg->pg_id == pg_id && !pg->pg_releasing)
	+ return (pg);
	+ }
	+ return (NULL);
	+}
	+
	+static void
	+pmu_group_accounting_init(pmu_group_t *pg)
	+{
	+ u_int cpu;
	+
	+ if (pg->pg_tickrate != 0)
	+ return;
	+ pg->pg_tickrate = cpu_tickrate();
	+ KASSERT(pg->pg_tickrate != 0, ("[pmu] zero CPU tick rate"));
	+ if (PMC_IS_SYSTEM_MODE(pg->pg_leader->pe_alloc.pm_mode))
	+ return;
	+ pg->pg_ncpu = pmc_cpu_max();
	+ pg->pg_cpu_state = malloc(sizeof(pg->pg_cpu_state) pg->pg_ncpu,
	+ M_PMU, M_WAITOK \| M_ZERO);
	+ for (cpu = 0; cpu < pg->pg_ncpu; cpu++)
	+ pg->pg_cpu_state[cpu].pgcs_group = pg;
	+}
	+
	+int
	+pmu_group_add(pmu_group_t pg, struct pmc pm, bool leader)
	+{
	+ pmu_event_t *pe;
	+
	+ KASSERT(pg != NULL && pm != NULL,
	+ ("[pmu] pmu_group_add: null pg=%p pm=%p", pg, pm));
	+
	+ if (pg->pg_committed) {
	+ PMCDBG1(PMC, OPS, 1, "group_add: gid=%u already committed",
	+ pg->pg_id);
	+ return (EBUSY);
	+ }
	+
	+ pe = pmu_event_from_pmc(pm);
	+ if (pe == NULL) {
	+ PMCDBG2(PMC, OPS, 1,
	+ "group_add: pm_pmu==NULL pm=%p flags=0x%x",
	+ pm, pm->pm_flags);
	+ return (EINVAL);
	+ }
	+ if (!PMC_ROW_IS_UNASSIGNED(pe->pe_pmc)) {
	+ PMCDBG2(PMC, OPS, 1,
	+ "group_add: already assigned pm=%p id=0x%jx",
	+ pm, (uintmax_t)pm->pm_id);
	+ return (EBUSY);
	+ }
	+ if (pe->pe_group == pg)
	+ return (leader ? EINVAL : 0);
	+ if (pe->pe_group != NULL) {
	+ PMCDBG3(PMC, OPS, 1,
	+ "group_add: pe %p in gid=%u, requested gid=%u",
	+ pe, pe->pe_group->pg_id, pg->pg_id);
	+ return (EBUSY);
	+ }
	+ if (pg->pg_nevents >= PMC_GROUP_MAX_MEMBERS)
	+ return (E2BIG);
	+ if (leader && pg->pg_leader != NULL)
	+ return (EINVAL);
	+
	+ pe->pe_group = pg;
	+ pe->pe_is_leader = leader;
	+ TAILQ_INSERT_TAIL(&pg->pg_events, pe, pe_sibling);
	+ pg->pg_nevents++;
	+
	+ if (leader)
	+ pg->pg_leader = pe;
	+
	+ return (0);
	+}
	+
	+int
	+pmu_group_commit(pmu_group_t *pg)
	+{
	+ pmu_event_t *pe;
	+ struct pmc_mdep *mdep;
	+ struct pmc_owner *po;
	+ struct proc *p;
	+ int error;
	+
	+ if (pg == NULL)
	+ return (EINVAL);
	+ if (pg->pg_committed)
	+ return (EBUSY);
	+ mdep = hwpmc_get_mdep();
	+ if (mdep == NULL)
	+ return (EOPNOTSUPP);
	+ if (pg->pg_nevents > (u_int)mdep->pmd_npmc)
	+ return (ENOSPC);
	+
	+ error = pmu_validate_group(pg);
	+ if (error != 0) {
	+ PMCDBG3(PMC, OPS, 1,
	+ "group_commit: validate gid=%u err=%d nevents=%u",
	+ pg->pg_id, error, pg->pg_nevents);
	+ return (error);
	+ }
	+
	+ /* Cache the event constraints, for sorting. */
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling) {
	+ error = pmu_event_get_constraint(pe, &pe->pe_cons);
	+ if (error != 0) {
	+ PMCDBG2(PMC, OPS, 1,
	+ "group_commit: constraint err=%d class=%d",
	+ error, (int)pe->pe_alloc.pm_class);
	+ return (error);
	+ }
	+ }
	+
	+ pg->pg_defer_ok = (pg->pg_leader->pe_alloc.pm_flags &
	+ PMC_F_GROUP_MUX) != 0;
	+
	+ error = pmu_group_can_fit(pg);
	+ if (error != 0)
	+ return (error);
	+ pmu_group_accounting_init(pg);
	+
	+ po = pg->pg_owner;
	+ p = po != NULL ? po->po_owner : NULL;
	+ if (p == NULL)
	+ return (EINVAL);
	+
	+ /*
	+ * A system group binds to one CPU.
	+ * Hardware assignment happens when the group starts.
	+ */
	+ if (PMC_IS_SYSTEM_MODE(pg->pg_leader->pe_alloc.pm_mode)) {
	+ pg->pg_system = true;
	+ pg->pg_cpu = pg->pg_leader->pe_alloc.pm_cpu;
	+
	+ /* The commit succeeds if the group can fit on the PMU. */
	+ error = pmu_group_can_place(pg, p, pg->pg_cpu);
	+ PMCDBG5(PMC, OPS, 1,
	+ "group_commit: SYS gid=%u cpu=%d nevents=%u "
	+ "place=%d defer_ok=%d", pg->pg_id, pg->pg_cpu,
	+ pg->pg_nevents, error, (int)pg->pg_defer_ok);
	+ if (error != 0 && error != ENOSPC)
	+ return (error);
	+
	+ pg->pg_committed = true;
	+ return (0);
	+ }
	+
	+ /*
	+ * Assign hardware rows now, if rows are free.
	+ * If rows are busy and PMC_F_GROUP_MUX is set, defer the assignment.
	+ */
	+ error = pmu_group_can_place(pg, p, PMC_CPU_ANY);
	+ if (error == 0)
	+ error = pmu_assign_group(pg, p, PMC_CPU_ANY);
	+ if (error == 0) {
	+ PMCDBG1(PMC, OPS, 1,
	+ "group_commit: gid=%u BOUND", pg->pg_id);
	+ } else if (error != EBUSY && error != ENOSPC) {
	+ return (error);
	+ } else if (!pg->pg_defer_ok) {
	+ return (ENOSPC);
	+ } else {
	+ PMCDBG1(PMC, OPS, 1,
	+ "group_commit: gid=%u DEFERRED", pg->pg_id);
	+ }
	+
	+ pg->pg_committed = true;
	+ return (0);
	+}
	+
	+u_int
	+pmu_group_prepare_release(pmu_group_t pg, struct pmc *members,
	+ u_int capacity, struct pmc_process **released_pp)
	+{
	+ pmu_event_t pe, tmp;
	+ struct pmu_group_target *pgt;
	+ struct pmc_process held_pp, pp;
	+ bool pp_unhashed;
	+ u_int n;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ KASSERT(pg != NULL && members != NULL,
	+ ("[pmu] invalid group release preparation"));
	+ KASSERT(capacity >= pg->pg_nevents,
	+ ("[pmu] group release capacity %u < %u", capacity,
	+ pg->pg_nevents));
	+ KASSERT(!pg->pg_releasing,
	+ ("[pmu] group %u release already active", pg->pg_id));
	+
	+ pg->pg_releasing = true;
	+ if (released_pp != NULL)
	+ *released_pp = NULL;
	+
	+ if (pg->pg_system) {
	+ if (pg->pg_committed && !pg->pg_account_blocked)
	+ pmu_group_accounting_prepare_release(pg);
	+ } else {
	+ if (pg->pg_committed && !pg->pg_account_blocked)
	+ pmu_group_accounting_prepare_release(pg);
	+ held_pp = pg->pg_pp;
	+ if (held_pp != NULL) {
	+ mtx_lock_spin(&held_pp->pp_pmu_lock);
	+ held_pp->pp_pmu_refs++;
	+ mtx_unlock_spin(&held_pp->pp_pmu_lock);
	+ pmu_pp_stop_rotate(held_pp, "muxrel");
	+ }
	+
	+ pp = pg->pg_pp;
	+ if (pg->pg_assigned) {
	+ if (pp != NULL)
	+ pmu_pp_schedule_out(pp, pg, false);
	+ else
	+ pmu_unassign_group(pg, 0);
	+ }
	+ if (pg->pg_pp != NULL) {
	+ pp = pg->pg_pp;
	+ pmu_pp_unlink_group(pg, pp);
	+ pmu_pp_backfill(pp);
	+ }
	+ if (held_pp != NULL) {
	+ mtx_lock_spin(&held_pp->pp_pmu_lock);
	+ KASSERT(held_pp->pp_pmu_refs > 0,
	+ ("[pmu] process reference underflow"));
	+ held_pp->pp_pmu_refs--;
	+ pp_unhashed = held_pp->pp_pmu_unhashed;
	+ mtx_unlock_spin(&held_pp->pp_pmu_lock);
	+ wakeup(&held_pp->pp_pmu_refs);
	+ if (released_pp != NULL && !pp_unhashed && pp == held_pp)
	+ *released_pp = held_pp;
	+ }
	+ }
	+
	+ KASSERT(!pg->pg_assigned && pg->pg_pp == NULL &&
	+ !pg->pg_sys_listed,
	+ ("[pmu] group %u still scheduled during release", pg->pg_id));
	+ n = 0;
	+ TAILQ_FOREACH_SAFE(pe, &pg->pg_events, pe_sibling, tmp) {
	+ KASSERT(n < capacity,
	+ ("[pmu] group %u release member overflow", pg->pg_id));
	+ KASSERT(PMC_ROW_IS_UNASSIGNED(pe->pe_pmc),
	+ ("[pmu] group %u member still assigned", pg->pg_id));
	+ LIST_FOREACH(pgt, &pg->pg_targets, pgt_group_next)
	+ pmu_event_remove_target_residuals(pe, pgt->pgt_pp);
	+ members[n++] = pe->pe_pmc;
	+ TAILQ_REMOVE(&pg->pg_events, pe, pe_sibling);
	+ pe->pe_group = NULL;
	+ pe->pe_is_leader = false;
	+ }
	+ pg->pg_leader = NULL;
	+ pg->pg_nevents = 0;
	+ if (pg->pg_owner != NULL) {
	+ LIST_REMOVE(pg, pg_owner_next);
	+ pg->pg_owner = NULL;
	+ }
	+ return (n);
	+}
	+
	+void
	+pmu_group_release(pmu_group_t *pg)
	+{
	+ u_int cpu;
	+
	+ if (pg == NULL)
	+ return;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ KASSERT(LIST_EMPTY(&pg->pg_targets),
	+ ("[pmu] freeing group %u with live targets", pg->pg_id));
	+ KASSERT(pg->pg_explicit_target == NULL,
	+ ("[pmu] freeing group %u with explicit target", pg->pg_id));
	+ KASSERT(TAILQ_EMPTY(&pg->pg_events) && pg->pg_nevents == 0 &&
	+ pg->pg_leader == NULL,
	+ ("[pmu] freeing non-empty group %u", pg->pg_id));
	+ KASSERT(!pg->pg_assigned && pg->pg_pp == NULL &&
	+ !pg->pg_sys_listed && !pg->pg_sscounted,
	+ ("[pmu] freeing scheduled group %u", pg->pg_id));
	+ if (pg->pg_owner != NULL) {
	+ LIST_REMOVE(pg, pg_owner_next);
	+ pg->pg_owner = NULL;
	+ }
	+ KASSERT(pg->pg_oncpu_threads == 0 && pg->pg_running_threads == 0,
	+ ("[pmu] freeing group with active accounting"));
	+ KASSERT(!pg->pg_snapshot_pending && !pg->pg_snapshot_active,
	+ ("[pmu] freeing group with active snapshot"));
	+ for (cpu = 0; cpu < pg->pg_ncpu; cpu++) {
	+ KASSERT(!pg->pg_cpu_state[cpu].pgcs_counted &&
	+ !pg->pg_cpu_state[cpu].pgcs_placed &&
	+ !pg->pg_cpu_state[cpu].pgcs_transitioning &&
	+ pg->pg_cpu_state[cpu].pgcs_td == NULL,
	+ ("[pmu] freeing group with active CPU marker"));
	+ }
	+ free(pg->pg_cpu_state, M_PMU);
	+ mtx_destroy(&pg->pg_snapshot_lock);
	+ free(pg, M_PMU);
	+}
	+
	+int
	+pmu_group_on_allocate(struct pmc pm, const struct pmc_op_pmcallocate pa)
	+{
	+ pmu_event_t *pe;
	+
	+ pe = malloc(sizeof(*pe), M_PMU, M_WAITOK \| M_ZERO);
	+ pe->pe_pmc = pm;
	+ pe->pe_samples = counter_u64_alloc(M_WAITOK);
	+#ifdef HWPMC_DEBUG
	+ pe->pe_samples_emitted = counter_u64_alloc(M_WAITOK);
	+ pe->pe_samples_dropped = counter_u64_alloc(M_WAITOK);
	+#endif
	+ pe->pe_alloc = *pa;
	+ pm->pm_pmu = pe;
	+ return (0);
	+}
	+
	+int
	+pmu_group_on_attach(struct pmc pm, struct proc p)
	+{
	+ pmu_event_t *pe;
	+ pmu_group_t *pg;
	+ struct pmu_group_target *pgt;
	+ struct pmc_process *pp;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ pe = pmu_event_from_pmc(pm);
	+ if (pe == NULL \|\| pe->pe_group == NULL)
	+ return (0);
	+ pg = pe->pe_group;
	+ if (pg->pg_explicit_target != NULL &&
	+ pg->pg_explicit_target->pgt_pp->pp_proc != p)
	+ return (EBUSY);
	+ pp = pmc_find_process_descriptor_pmu(p, PMC_FLAG_ALLOCATE);
	+ if (pp == NULL)
	+ return (ENOMEM);
	+ if (pg->pg_pp != NULL && pg->pg_pp != pp)
	+ return (EBUSY);
	+ if (pmu_group_has_target(pg, pp))
	+ return (0);
	+ pgt = pmu_group_target_alloc(pg, pp, true, M_WAITOK);
	+ if (pgt == NULL)
	+ return (ENOMEM);
	+ pmu_group_target_publish(pgt);
	+ return (0);
	+}
	+
	+/*
	+ * Find the target process for a group.
	+ */
	+static struct proc *
	+pmu_group_target_proc(pmu_group_t *pg)
	+{
	+ if (pg == NULL \|\| pg->pg_pp == NULL)
	+ return (NULL);
	+ return (pg->pg_pp->pp_proc);
	+}
	+
	+int
	+pmu_group_on_start(struct pmc *pm)
	+{
	+ pmu_event_t *pe;
	+ pmu_group_t *pg;
	+ struct pmc_process *pp;
	+ struct proc *p;
	+ int error;
	+
	+ pe = pmu_event_from_pmc(pm);
	+ if (pe == NULL \|\| pe->pe_group == NULL)
	+ return (0);
	+
	+ pg = pe->pe_group;
	+ if (!pg->pg_committed)
	+ return (EDOOFUS);
	+
	+ p = pmu_group_target_proc(pg);
	+ if (p == NULL) {
	+ PMCDBG2(PMC, OPS, 1,
	+ "on_start: gid=%u pm=%p no target proc -> EINVAL",
	+ pg->pg_id, pm);
	+ return (EINVAL);
	+ }
	+
	+ pp = pg->pg_pp;
	+ if (pp != NULL && pp->pp_pmu_unhashed)
	+ return (ESRCH);
	+ if (pp == NULL)
	+ pp = pmc_find_process_descriptor_pmu(p, PMC_FLAG_ALLOCATE);
	+ if (pp == NULL)
	+ return (ENOMEM);
	+
	+ pmu_pp_link_group(pg, pp);
	+
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ if (pg->pg_account_blocked) {
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ return (EBUSY);
	+ }
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+
	+ if (pg->pg_assigned) {
	+ PMCDBG3(PMC, OPS, 2,
	+ "on_start: gid=%u pm=%p ASSIGNED, attaching siblings "
	+ "on pp=%p", pg->pg_id, pm, pp);
	+ pmu_group_attach_siblings(pg, pp);
	+ } else {
	+ error = pmu_pp_schedule_in(pp, pg);
	+ PMCDBG4(PMC, OPS, 2,
	+ "on_start: gid=%u pm=%p DEFERRED schedule_in=%d pp=%p",
	+ pg->pg_id, pm, error, pp);
	+ if (error != 0 && error != ENOSPC)
	+ return (error);
	+ }
	+
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling) {
	+ pe->pe_pmc->pm_state = PMC_STATE_RUNNING;
	+ PMCDBG3(PMC, OPS, 3,
	+ "on_start: gid=%u pm=%p ri=%d -> RUNNING",
	+ pg->pg_id, pe->pe_pmc, PMC_TO_ROWINDEX(pe->pe_pmc));
	+ }
	+
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ pmu_group_running_start_locked(pg, cpu_ticks());
	+ pg->pg_account_placement_admit = pg->pg_assigned;
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+
	+ pmu_pp_kick_rotate(pp);
	+ return (0);
	+}
	+
	+/*
	+ * Mark the group stopped. Assigned rows stay reserved until release.
	+ */
	+void
	+pmu_group_on_stop(struct pmc *pm)
	+{
	+ pmu_event_t *pe;
	+ pmu_group_t *pg;
	+ struct pmc_process *pp;
	+
	+ pe = pmu_event_from_pmc(pm);
	+ if (pe == NULL \|\| pe->pe_group == NULL)
	+ return;
	+ /* Require exclusive lock ownership for grouped stops. */
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ pg = pe->pe_group;
	+ pp = pg->pg_pp;
	+ if (pp != NULL)
	+ mtx_lock_spin(&pp->pp_pmu_lock);
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ if (!pg->pg_running) {
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ if (pp != NULL)
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+ return;
	+ }
	+ pmu_group_accounting_block_locked(pg, cpu_ticks());
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ if (pp != NULL)
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling)
	+ pe->pe_pmc->pm_state = PMC_STATE_STOPPED;
	+ pmu_group_accounting_drain(pg, false);
	+}
	+
	+/* Mark one group on-CPU for the thread that is switching in. */
	+static void
	+pmu_group_csw_in_one(pmu_group_t pg, struct thread td, int cpu,
	+ uint64_t now)
	+{
	+ struct pmu_group_cpu_state *pgcs;
	+
	+ if (pg->pg_system \|\| cpu < 0 \|\| (u_int)cpu >= pg->pg_ncpu)
	+ return;
	+ pgcs = &pg->pg_cpu_state[cpu];
	+ /* Do not wait for a snapshot during a context switch. */
	+ mtx_lock_spin(&pg->pg_snapshot_lock);
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ if (!pg->pg_committed \|\| !pg->pg_running \|\| pg->pg_account_blocked) {
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ mtx_unlock_spin(&pg->pg_snapshot_lock);
	+ return;
	+ }
	+ KASSERT(!pgcs->pgcs_counted,
	+ ("[pmu] duplicate CPU %d marker for group %u", cpu, pg->pg_id));
	+ pmu_group_time_update_locked(pg, now);
	+ pgcs->pgcs_td = td;
	+ pgcs->pgcs_counted = true;
	+ pgcs->pgcs_placed = false;
	+ pgcs->pgcs_transitioning = true;
	+ pg->pg_oncpu_threads++;
	+ LIST_INSERT_HEAD(&pmu_group_cpu_active[cpu], pgcs, pgcs_next);
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ mtx_unlock_spin(&pg->pg_snapshot_lock);
	+}
	+
	+void
	+pmu_group_csw_in(struct thread td, struct pmc_process pp)
	+{
	+ struct pmu_group_target *pgt;
	+ uint64_t now;
	+ int cpu;
	+
	+ /* Check the lists without a lock. */
	+ if (LIST_EMPTY(&pp->pp_pmu_targets))
	+ return;
	+ cpu = PCPU_GET(cpuid);
	+ now = cpu_ticks();
	+ mtx_lock_spin(&pp->pp_pmu_lock);
	+ LIST_FOREACH(pgt, &pp->pp_pmu_targets, pgt_process_next)
	+ pmu_group_csw_in_one(pgt->pgt_group, td, cpu, now);
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+}
	+
	+void
	+pmu_group_csw_in_complete(struct thread *td, int cpu)
	+{
	+ struct pmu_group_cpu_state *pgcs;
	+ pmu_group_t *pg;
	+
	+ KASSERT(cpu >= 0 && cpu < MAXCPU,
	+ ("[pmu] invalid csw-in completion CPU %d", cpu));
	+ LIST_FOREACH(pgcs, &pmu_group_cpu_active[cpu], pgcs_next) {
	+ if (pgcs->pgcs_td != td)
	+ continue;
	+ pg = pgcs->pgcs_group;
	+ mtx_lock_spin(&pg->pg_snapshot_lock);
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ if (pgcs->pgcs_counted && pgcs->pgcs_td == td &&
	+ pgcs->pgcs_transitioning)
	+ pgcs->pgcs_transitioning = false;
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ mtx_unlock_spin(&pg->pg_snapshot_lock);
	+ }
	+}
	+
	+bool
	+pmu_group_csw_can_start(struct pmc pm, struct thread td, int cpu)
	+{
	+ struct pmu_group_cpu_state *pgcs;
	+ pmu_event_t *pe;
	+ pmu_group_t *pg;
	+ bool can_start;
	+
	+ pe = pmu_event_from_pmc(pm);
	+ if (pe == NULL \|\| pe->pe_group == NULL)
	+ return (true);
	+ pg = pe->pe_group;
	+ if (pg->pg_system \|\| cpu < 0 \|\| (u_int)cpu >= pg->pg_ncpu)
	+ return (false);
	+
	+ pgcs = &pg->pg_cpu_state[cpu];
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ can_start = pg->pg_committed && pg->pg_running &&
	+ !pg->pg_account_blocked && pg->pg_account_placement_admit &&
	+ pgcs->pgcs_counted && pgcs->pgcs_td == td;
	+ if (can_start && !pgcs->pgcs_placed) {
	+ pmu_group_time_update_locked(pg, cpu_ticks());
	+ pgcs->pgcs_placed = true;
	+ pg->pg_running_threads++;
	+ KASSERT(pg->pg_running_threads <= pg->pg_oncpu_threads,
	+ ("[pmu] group %u running threads exceed on-CPU threads",
	+ pg->pg_id));
	+ }
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ return (can_start);
	+}
	+
	+void
	+pmu_group_csw_out(struct thread *td, int cpu)
	+{
	+ struct pmu_group_cpu_state *pgcs;
	+ pmu_group_t *pg;
	+
	+ KASSERT(cpu >= 0 && cpu < MAXCPU,
	+ ("[pmu] invalid csw-out CPU %d", cpu));
	+ LIST_FOREACH(pgcs, &pmu_group_cpu_active[cpu], pgcs_next) {
	+ if (pgcs->pgcs_td != td)
	+ continue;
	+ pg = pgcs->pgcs_group;
	+ /* Do not wait for a snapshot during a context switch. */
	+ mtx_lock_spin(&pg->pg_snapshot_lock);
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ if (pgcs->pgcs_counted && pgcs->pgcs_td == td &&
	+ !pgcs->pgcs_transitioning)
	+ pgcs->pgcs_transitioning = true;
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ mtx_unlock_spin(&pg->pg_snapshot_lock);
	+ }
	+}
	+
	+void
	+pmu_group_csw_out_complete(struct thread *td, int cpu)
	+{
	+ struct pmu_group_cpu_state pgcs, next;
	+ pmu_group_t *pg;
	+ uint64_t now;
	+
	+ KASSERT(cpu >= 0 && cpu < MAXCPU,
	+ ("[pmu] invalid csw-out completion CPU %d", cpu));
	+ now = cpu_ticks();
	+ LIST_FOREACH_SAFE(pgcs, &pmu_group_cpu_active[cpu], pgcs_next,
	+ next) {
	+ if (pgcs->pgcs_td != td)
	+ continue;
	+ pg = pgcs->pgcs_group;
	+ mtx_lock_spin(&pg->pg_snapshot_lock);
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ if (!pgcs->pgcs_counted \|\| pgcs->pgcs_td != td) {
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ mtx_unlock_spin(&pg->pg_snapshot_lock);
	+ continue;
	+ }
	+ pmu_group_time_update_locked(pg, now);
	+ KASSERT(pg->pg_oncpu_threads > 0,
	+ ("[pmu] group %u on-CPU thread underflow", pg->pg_id));
	+ pg->pg_oncpu_threads--;
	+ if (pgcs->pgcs_placed) {
	+ KASSERT(pg->pg_running_threads > 0,
	+ ("[pmu] group %u running thread underflow",
	+ pg->pg_id));
	+ pg->pg_running_threads--;
	+ }
	+ LIST_REMOVE(pgcs, pgcs_next);
	+ pgcs->pgcs_td = NULL;
	+ pgcs->pgcs_counted = false;
	+ pgcs->pgcs_placed = false;
	+ pgcs->pgcs_transitioning = false;
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ mtx_unlock_spin(&pg->pg_snapshot_lock);
	+ }
	+}
	+
	+void
	+pmu_group_on_release(struct pmc *pm)
	+{
	+ pmu_event_t *pe;
	+ pmu_group_t *pg;
	+
	+ pe = pmu_event_from_pmc(pm);
	+ if (pe == NULL)
	+ return;
	+ pg = pe->pe_group;
	+ if (pg != NULL) {
	+ KASSERT(!pg->pg_committed,
	+ ("[pmu] committed group %u released by member", pg->pg_id));
	+ TAILQ_REMOVE(&pg->pg_events, pe, pe_sibling);
	+ if (pg->pg_leader == pe)
	+ pg->pg_leader = NULL;
	+ KASSERT(pg->pg_nevents > 0,
	+ ("[pmu] empty group %u member release", pg->pg_id));
	+ pg->pg_nevents--;
	+ pe->pe_group = NULL;
	+ pe->pe_is_leader = false;
	+ }
	+ pm->pm_pmu = NULL;
	+ pmu_event_destroy(pe);
	+ if (pg != NULL && pg->pg_nevents == 0)
	+ pmu_group_release(pg);
	+}
	+
	+/*
	+ * This function finds saved sampling progress.
	+ * pp_tdslock protects both the thread list and the residual list.
	+ */
	+static struct pmu_thread_residual *
	+pmu_thread_residual_find_locked(struct pmc_process pp, pmu_event_t pe,
	+ lwpid_t tid)
	+{
	+ struct pmu_thread_residual *ptr;
	+
	+ mtx_assert(pp->pp_tdslock, MA_OWNED);
	+ LIST_FOREACH(ptr, &pp->pp_pmu_residuals, ptr_next) {
	+ if (ptr->ptr_event == pe && ptr->ptr_tid == tid)
	+ return (ptr);
	+ }
	+ return (NULL);
	+}
	+
	+static bool
	+pmu_thread_exists_locked(struct pmc_process *pp, lwpid_t tid)
	+{
	+ struct pmc_thread *pt;
	+
	+ mtx_assert(pp->pp_tdslock, MA_OWNED);
	+ LIST_FOREACH(pt, &pp->pp_tds, pt_next) {
	+ if (pt->pt_td->td_tid == tid)
	+ return (true);
	+ }
	+ return (false);
	+}
	+
	+/*
	+ * This function publishes or replaces one saved value.
	+ * pp_tdslock is a spin lock. Allocate memory before you take this lock.
	+ * At thread exit, remove the thread descriptor first. This stops a
	+ * concurrent save from creating stale state again.
	+ */
	+static int
	+pmu_event_set_thread_residual_impl(struct pmc pm, struct pmc_process pp,
	+ lwpid_t tid, pmc_value_t value, uint8_t state)
	+{
	+ struct pmu_thread_residual discard, ptr;
	+ pmu_event_t *pe;
	+ bool keep;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ pe = pmu_event_from_pmc(pm);
	+ if (pe == NULL \|\| value > pm->pm_sc.pm_reloadcount)
	+ return (EINVAL);
	+
	+ /*
	+ * Keep an entry only for a partial count, or for a pending overflow.
	+ * The UNINITIALIZED state removes any existing entry.
	+ * Check the value and the state together. This stops a zero value
	+ * from looking like an overflow, and stops the opposite case too.
	+ */
	+ switch (state) {
	+ case PMU_RESIDUAL_UNINITIALIZED:
	+ if (value != 0)
	+ return (EINVAL);
	+ keep = false;
	+ break;
	+ case PMU_RESIDUAL_COUNT_VALID:
	+ if (value == 0)
	+ return (EINVAL);
	+ keep = true;
	+ break;
	+ case PMU_RESIDUAL_OVERFLOW_PENDING:
	+ if (value != 0)
	+ return (EINVAL);
	+ keep = true;
	+ break;
	+ default:
	+ return (EINVAL);
	+ }
	+
	+ ptr = keep ? malloc(sizeof(*ptr), M_PMU, M_WAITOK \| M_ZERO) : NULL;
	+ discard = NULL;
	+ mtx_lock_spin(pp->pp_tdslock);
	+ if (!pmu_thread_exists_locked(pp, tid)) {
	+ mtx_unlock_spin(pp->pp_tdslock);
	+ free(ptr, M_PMU);
	+ return (ESRCH);
	+ }
	+
	+ discard = pmu_thread_residual_find_locked(pp, pe, tid);
	+ if (!keep) {
	+ if (discard != NULL) {
	+ LIST_REMOVE(discard, ptr_next);
	+ }
	+ } else if (discard != NULL) {
	+ discard->ptr_value = value;
	+ discard->ptr_state = state;
	+ discard = ptr;
	+ ptr = NULL;
	+ } else {
	+ ptr->ptr_event = pe;
	+ ptr->ptr_tid = tid;
	+ ptr->ptr_value = value;
	+ ptr->ptr_state = state;
	+ LIST_INSERT_HEAD(&pp->pp_pmu_residuals, ptr, ptr_next);
	+ ptr = NULL;
	+ }
	+ mtx_unlock_spin(pp->pp_tdslock);
	+
	+ if (discard != NULL)
	+ free(discard, M_PMU);
	+ return (0);
	+}
	+
	+/*
	+ * This function saves the progress of each target thread.
	+ * First, take a snapshot of the row-indexed values under pp_tdslock.
	+ * Then release the lock and allocate the stable entries.
	+ */
	+void
	+pmu_event_save_residual(struct pmc pm, struct pmc_process pp, int ri)
	+{
	+ struct pmu_thread_residual_snapshot {
	+ lwpid_t tid;
	+ pmc_value_t value;
	+ } *snapshot;
	+ pmu_event_t *pe;
	+ struct pmc_thread *pt;
	+ u_int capacity, i, nthreads;
	+ bool retry;
	+ int error __diagused;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ pe = pmu_event_from_pmc(pm);
	+ if (pe == NULL)
	+ return;
	+
	+ for (;;) {
	+ capacity = 0;
	+ mtx_lock_spin(pp->pp_tdslock);
	+ LIST_FOREACH(pt, &pp->pp_tds, pt_next)
	+ capacity++;
	+ mtx_unlock_spin(pp->pp_tdslock);
	+ if (capacity == 0)
	+ return;
	+
	+ snapshot = mallocarray(capacity, sizeof(*snapshot), M_PMU,
	+ M_WAITOK);
	+ nthreads = 0;
	+ retry = false;
	+ mtx_lock_spin(pp->pp_tdslock);
	+ LIST_FOREACH(pt, &pp->pp_tds, pt_next) {
	+ if (nthreads == capacity) {
	+ retry = true;
	+ break;
	+ }
	+ snapshot[nthreads].tid = pt->pt_td->td_tid;
	+ snapshot[nthreads].value = pt->pt_pmcs[ri].pt_pmcval;
	+ nthreads++;
	+ }
	+ mtx_unlock_spin(pp->pp_tdslock);
	+ if (!retry)
	+ break;
	+ free(snapshot, M_PMU);
	+ }
	+
	+ for (i = 0; i < nthreads; i++) {
	+ if (snapshot[i].value == 0 \|\|
	+ snapshot[i].value > pm->pm_sc.pm_reloadcount)
	+ continue;
	+ error = pmu_event_set_thread_residual_impl(pm, pp,
	+ snapshot[i].tid, snapshot[i].value,
	+ PMU_RESIDUAL_COUNT_VALID);
	+ KASSERT(error == 0 \|\| error == ESRCH,
	+ ("[pmu] residual save failed for tid %d: %d",
	+ snapshot[i].tid, error));
	+ }
	+ free(snapshot, M_PMU);
	+}
	+
	+/*
	+ * Save a system-mode sampling member's progress. Read it straight off
	+ * the hardware at eviction. A value outside (0, reloadcount] means
	+ * the counter overflowed, or was never seeded, so the next placement
	+ * should start from a full period.
	+ */
	+void
	+pmu_event_set_residual(struct pmc *pm, pmc_value_t residual)
	+{
	+ pmu_event_t *pe;
	+
	+ pe = pmu_event_from_pmc(pm);
	+ if (pe == NULL)
	+ return;
	+ pe->pe_residual = (residual > 0 &&
	+ residual <= pm->pm_sc.pm_reloadcount) ? residual : 0;
	+}
	+
	+/*
	+ * Seed the per-thread counts that a sampling member resumes from. A
	+ * context switch reads pt_pmcval, and falls back to the per-process
	+ * value only when the thread has no descriptor. Restoring only the
	+ * per-process value would be ignored for every real thread.
	+ */
	+void
	+pmu_event_restore_thread_residual(struct pmc pm, struct pmc_process pp,
	+ int ri)
	+{
	+ struct pmu_thread_residual *ptr;
	+ pmu_event_t *pe;
	+ struct pmc_thread *pt;
	+
	+ pe = pmu_event_from_pmc(pm);
	+ if (pe == NULL)
	+ return;
	+ mtx_lock_spin(pp->pp_tdslock);
	+ LIST_FOREACH(pt, &pp->pp_tds, pt_next) {
	+ ptr = pmu_thread_residual_find_locked(pp, pe,
	+ pt->pt_td->td_tid);
	+ /*
	+ * A partial count resumes from the saved point.
	+ * Both the overflow-pending state and the no-saved-progress
	+ * state reload a full period.
	+ */
	+ pt->pt_pmcs[ri].pt_pmcval =
	+ (ptr != NULL && ptr->ptr_state == PMU_RESIDUAL_COUNT_VALID) ?
	+ ptr->ptr_value : pm->pm_sc.pm_reloadcount;
	+ }
	+ mtx_unlock_spin(pp->pp_tdslock);
	+}
	+
	+/*
	+ * The count a sampling member resumes from at placement: its saved
	+ * progress, or a full period if it has none yet.
	+ */
	+pmc_value_t
	+pmu_event_restore_residual(struct pmc *pm)
	+{
	+ pmu_event_t *pe;
	+
	+ pe = pmu_event_from_pmc(pm);
	+ if (pe == NULL \|\| pe->pe_residual == 0 \|\|
	+ pe->pe_residual > pm->pm_sc.pm_reloadcount)
	+ return (pm->pm_sc.pm_reloadcount);
	+ return (pe->pe_residual);
	+}
	+
	+#ifdef HWPMC_DEBUG
	+int
	+pmu_event_get_thread_residual(struct pmc pm, struct pmc_process pp,
	+ lwpid_t tid, pmc_value_t value, uint8_t state)
	+{
	+ struct pmu_thread_residual *ptr;
	+ pmu_event_t *pe;
	+ int error;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ pe = pmu_event_from_pmc(pm);
	+ if (pe == NULL \|\| value == NULL \|\| state == NULL)
	+ return (EINVAL);
	+
	+ error = 0;
	+ mtx_lock_spin(pp->pp_tdslock);
	+ if (!pmu_thread_exists_locked(pp, tid))
	+ error = ESRCH;
	+ else {
	+ ptr = pmu_thread_residual_find_locked(pp, pe, tid);
	+ if (ptr != NULL) {
	+ *value = ptr->ptr_value;
	+ *state = ptr->ptr_state;
	+ } else {
	+ *value = 0;
	+ *state = PMU_RESIDUAL_UNINITIALIZED;
	+ }
	+ }
	+ mtx_unlock_spin(pp->pp_tdslock);
	+ return (error);
	+}
	+
	+int
	+pmu_event_set_thread_residual(struct pmc pm, struct pmc_process pp,
	+ lwpid_t tid, pmc_value_t value, uint8_t state)
	+{
	+
	+ return (pmu_event_set_thread_residual_impl(pm, pp, tid, value, state));
	+}
	+#endif
	+
	+static void
	+pmu_thread_residual_free_task_fn(void *arg __unused, int pending __unused)
	+{
	+ struct pmu_thread_residual *ptr;
	+ LIST_HEAD(, pmu_thread_residual) tmplist;
	+
	+ LIST_INIT(&tmplist);
	+ mtx_lock_spin(&pmu_residual_freelist_mtx);
	+ while ((ptr = LIST_FIRST(&pmu_residual_freelist)) != NULL) {
	+ LIST_REMOVE(ptr, ptr_next);
	+ LIST_INSERT_HEAD(&tmplist, ptr, ptr_next);
	+ }
	+ mtx_unlock_spin(&pmu_residual_freelist_mtx);
	+
	+ while ((ptr = LIST_FIRST(&tmplist)) != NULL) {
	+ LIST_REMOVE(ptr, ptr_next);
	+ free(ptr, M_PMU);
	+ }
	+}
	+
	+static void
	+pmu_thread_residual_defer_free(struct pmu_thread_residual *ptr)
	+{
	+
	+ mtx_lock_spin(&pmu_residual_freelist_mtx);
	+ LIST_INSERT_HEAD(&pmu_residual_freelist, ptr, ptr_next);
	+ taskqueue_enqueue(taskqueue_fast, &pmu_residual_free_task);
	+ mtx_unlock_spin(&pmu_residual_freelist_mtx);
	+}
	+
	+void
	+pmu_thread_residual_remove(struct pmc_process *pp, lwpid_t tid)
	+{
	+ struct pmu_thread_residual *ptr;
	+
	+ for (;;) {
	+ mtx_lock_spin(pp->pp_tdslock);
	+ LIST_FOREACH(ptr, &pp->pp_pmu_residuals, ptr_next) {
	+ if (ptr->ptr_tid == tid)
	+ break;
	+ }
	+ if (ptr != NULL) {
	+ LIST_REMOVE(ptr, ptr_next);
	+ }
	+ mtx_unlock_spin(pp->pp_tdslock);
	+ if (ptr == NULL)
	+ break;
	+ pmu_thread_residual_defer_free(ptr);
	+ }
	+}
	+
	+static void
	+pmu_event_remove_target_residuals(pmu_event_t pe, struct pmc_process pp)
	+{
	+ struct pmu_thread_residual *ptr;
	+
	+ for (;;) {
	+ mtx_lock_spin(pp->pp_tdslock);
	+ LIST_FOREACH(ptr, &pp->pp_pmu_residuals, ptr_next) {
	+ if (ptr->ptr_event == pe)
	+ break;
	+ }
	+ if (ptr != NULL) {
	+ LIST_REMOVE(ptr, ptr_next);
	+ }
	+ mtx_unlock_spin(pp->pp_tdslock);
	+ if (ptr == NULL)
	+ break;
	+ free(ptr, M_PMU);
	+ }
	+}
	+
	+/*
	+ * Attach all sibling events of a group to the process descriptor.
	+ */
	+static void
	+pmu_group_attach_siblings(pmu_group_t pg, struct pmc_process anchor_pp)
	+{
	+ pmu_event_t *pe;
	+ struct pmc *pm;
	+ struct pmc_process *pp;
	+ struct pmu_group_target *pgt;
	+
	+ KASSERT(anchor_pp == pg->pg_pp,
	+ ("[pmu] group %u anchor mismatch", pg->pg_id));
	+ LIST_FOREACH(pgt, &pg->pg_targets, pgt_group_next) {
	+ pp = pgt->pgt_pp;
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling) {
	+ pm = pe->pe_pmc;
	+ KASSERT(!PMC_ROW_IS_UNASSIGNED(pm),
	+ ("[pmu] attach_siblings: gid=%u pm_id=0x%jx "
	+ "unassigned", pg->pg_id, (uintmax_t)pm->pm_id));
	+ if (pp->pp_pmcs[PMC_TO_ROWINDEX(pm)].pp_pmc == pm)
	+ continue; /* Already attached at this placement. */
	+ KASSERT(pp->pp_pmcs[PMC_TO_ROWINDEX(pm)].pp_pmc == NULL,
	+ ("[pmu] group %u target pp=%p row=%d busy",
	+ pg->pg_id, pp, PMC_TO_ROWINDEX(pm)));
	+ pmc_rotation_attach(pm, pp);
	+ }
	+ }
	+}
	+
	+/*
	+ * Schedule a group in by assigning hardware rows for all siblings.
	+ */
	+static int
	+pmu_pp_schedule_in(struct pmc_process pp, pmu_group_t pg)
	+{
	+ pmu_event_t *pe;
	+ struct proc *p;
	+ int error;
	+
	+ if (pg == NULL \|\| pg->pg_assigned)
	+ return (0);
	+ /* Use the target process only. */
	+ p = pmu_group_target_proc(pg);
	+ if (p == NULL)
	+ return (EINVAL);
	+ KASSERT(pp != NULL && pp->pp_proc == p,
	+ ("[pmu] schedule_in: pp/p mismatch pp=%p pp->proc=%p p=%p",
	+ pp, pp != NULL ? pp->pp_proc : NULL, p));
	+
	+ error = pmu_group_can_place(pg, p, PMC_CPU_ANY);
	+ if (error != 0)
	+ return (error);
	+
	+ error = pmu_assign_group(pg, p, PMC_CPU_ANY);
	+ if (error == EBUSY)
	+ error = ENOSPC;
	+ if (error != 0) {
	+ PMCDBG2(PMC, OPS, 1,
	+ "pp_schedule_in: assign gid=%u err=%d",
	+ pg->pg_id, error);
	+ return (error);
	+ }
	+
	+ /* Set the running state before you publish the rows. */
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling)
	+ pe->pe_pmc->pm_state = PMC_STATE_RUNNING;
	+
	+ pmu_group_attach_siblings(pg, pp);
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ pg->pg_account_placement_admit = true;
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ pmu_group_kick_placement(pg);
	+ return (0);
	+}
	+
	+/*
	+ * Schedule a group out and release assigned hardware rows.
	+ */
	+static void
	+pmu_pp_schedule_out(struct pmc_process pp, pmu_group_t pg,
	+ bool drain_samples)
	+{
	+ pmu_event_t *pe;
	+ struct pmc_process *target_pp;
	+ struct pmu_group_target *pgt;
	+
	+ if (pg == NULL \|\| !pg->pg_assigned)
	+ return;
	+
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ pmu_group_time_update_locked(pg, cpu_ticks());
	+ pg->pg_account_placement_admit = false;
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+
	+ /* Phase 1: stop the sibling events. */
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling) {
	+ if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pe->pe_pmc))) {
	+ pmc_rotation_drain_set(pe->pe_pmc, drain_samples);
	+ wmb();
	+ }
	+ pe->pe_pmc->pm_state = PMC_STATE_STOPPED;
	+ }
	+
	+ /* Phase 2: flush the log and drain the context switches. */
	+ if (pg->pg_owner != NULL &&
	+ (pg->pg_owner->po_flags & PMC_PO_OWNS_LOGFILE) != 0)
	+ (void)pmclog_flush(pg->pg_owner, 1);
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling)
	+ pmc_rotation_drain(pe->pe_pmc);
	+
	+ /* Phase 3: remove every transient row link from every stable target. */
	+ KASSERT(pp == pg->pg_pp,
	+ ("[pmu] group %u schedule-out anchor mismatch", pg->pg_id));
	+ LIST_FOREACH(pgt, &pg->pg_targets, pgt_group_next) {
	+ target_pp = pgt->pgt_pp;
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling) {
	+ struct pmc *pm = pe->pe_pmc;
	+
	+ KASSERT(!PMC_ROW_IS_UNASSIGNED(pm),
	+ ("[pmu] schedule_out: gid=%u pm_id=0x%jx "
	+ "unassigned", pg->pg_id, (uintmax_t)pm->pm_id));
	+ if (target_pp->pp_pmcs[PMC_TO_ROWINDEX(pm)].pp_pmc == pm)
	+ pmc_rotation_detach(pm, target_pp);
	+ }
	+ }
	+
	+ /* Phase 4: release the hardware rows. */
	+ pmu_unassign_group(pg, 0);
	+}
	+
	+/*
	+ * Remove all groups from the process and stop the rotation thread.
	+ */
	+static void
	+pmu_pp_release_all(struct pmc_process *pp)
	+{
	+ pmu_event_t *pe;
	+ pmu_group_t *pg;
	+ struct pmc_process *replacement_pp;
	+ struct pmu_group_target departing, replacement;
	+ bool was_running;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+
	+ if (pp == NULL)
	+ return;
	+
	+ mtx_lock_spin(&pp->pp_pmu_lock);
	+ LIST_FOREACH(pg, &pp->pp_pmu_groups, pg_proc_next)
	+ pmu_group_accounting_block(pg);
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+
	+ /* Stop the rotation thread before freeing the process descriptor. */
	+ pmu_pp_stop_rotate(pp, "muxpurge");
	+ while (pp->pp_pmu_refs != 0)
	+ (void)sx_sleep(&pp->pp_pmu_refs, &pmc_sx, 0, "muxrefs", 1);
	+
	+ /* Schedule out assigned groups and unhook every group. */
	+ for (;;) {
	+ mtx_lock_spin(&pp->pp_pmu_lock);
	+ pg = LIST_FIRST(&pp->pp_pmu_groups);
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+ if (pg == NULL)
	+ break;
	+
	+ departing = pmu_group_find_target(pg, pp);
	+ KASSERT(departing != NULL,
	+ ("[pmu] group %u anchor has no target edge", pg->pg_id));
	+ replacement = pmu_group_replacement_target(pg, pp);
	+ replacement_pp = replacement != NULL ? replacement->pgt_pp : NULL;
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ was_running = pg->pg_running;
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ pmu_group_accounting_drain(pg, true);
	+ if (pg->pg_assigned)
	+ pmu_pp_schedule_out(pp, pg, false);
	+ pmu_pp_unlink_group(pg, pp);
	+ pmu_group_target_remove(departing);
	+ if (replacement_pp == NULL)
	+ continue;
	+
	+ /*
	+ * pmc_sx serializes target-list changes and process teardown.
	+ * Change the old and new pp_pmu_groups lists one at a time, so
	+ * their spin locks are never held together.
	+ */
	+ pmu_pp_link_group(pg, replacement_pp);
	+ if (was_running) {
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling)
	+ pe->pe_pmc->pm_state = PMC_STATE_RUNNING;
	+ }
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ if (was_running)
	+ pmu_group_running_start_locked(pg, cpu_ticks());
	+ pg->pg_account_placement_admit = pg->pg_assigned;
	+ pg->pg_account_blocked = false;
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ pmu_pp_backfill(replacement_pp);
	+ }
	+}
	+
	+/* Initialize the PMU fields in the process descriptor. */
	+void
	+pmu_pp_init(struct pmc_process *pp)
	+{
	+
	+ mtx_init(&pp->pp_pmu_lock, "pmc-pmu-groups", "pmc-pmu", MTX_SPIN);
	+ LIST_INIT(&pp->pp_pmu_groups);
	+ LIST_INIT(&pp->pp_pmu_targets);
	+ LIST_INIT(&pp->pp_pmu_residuals);
	+}
	+
	+/* Destroy the PMU state in the process descriptor. */
	+void
	+pmu_pp_destroy(struct pmc_process *pp)
	+{
	+
	+ pmu_pp_release_all(pp);
	+ /* Remove every authoritative edge after the anchored groups stop. */
	+ pmu_group_disinherit(pp);
	+ KASSERT(LIST_EMPTY(&pp->pp_pmu_residuals),
	+ ("[pmu] pp %p destroyed with saved residuals", pp));
	+ KASSERT(pp->pp_pmu_refs == 0 && pp->pp_pmu_rot_quiesce == 0 &&
	+ pp->pp_pmu_rot_td == NULL,
	+ ("[pmu] pp %p destroyed while referenced", pp));
	+ mtx_destroy(&pp->pp_pmu_lock);
	+}
	+
	+void
	+pmu_group_detach_target(pmu_group_t pg, struct pmc_process pp)
	+{
	+ pmu_event_t *pe;
	+ struct pmu_group_target *pgt;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ KASSERT(pg != NULL && pp != NULL && pg->pg_pp == pp,
	+ ("[pmu] invalid group target detach"));
	+ KASSERT(!pg->pg_releasing,
	+ ("[pmu] group %u detach already active", pg->pg_id));
	+ pgt = pmu_group_find_target(pg, pp);
	+ KASSERT(pgt != NULL && pgt->pgt_explicit,
	+ ("[pmu] group %u explicit target edge missing", pg->pg_id));
	+
	+ pg->pg_releasing = true;
	+ mtx_lock_spin(&pp->pp_pmu_lock);
	+ pmu_group_accounting_block(pg);
	+ mtx_unlock_spin(&pp->pp_pmu_lock);
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling)
	+ pe->pe_pmc->pm_state = PMC_STATE_STOPPED;
	+ pmu_group_accounting_drain(pg, false);
	+ pmu_pp_stop_rotate(pp, "muxdetach");
	+ if (pg->pg_assigned)
	+ pmu_pp_schedule_out(pp, pg, false);
	+
	+ pmu_group_target_remove(pgt);
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling) {
	+ pe->pe_pmc->pm_flags &= ~(PMC_F_ATTACH_DONE \|
	+ PMC_F_ATTACHED_TO_OWNER \| PMC_F_NEEDS_LOGFILE);
	+ }
	+ pg->pg_releasing = false;
	+ wakeup(&pg->pg_releasing);
	+
	+ pmu_pp_backfill(pp);
	+}
	+
	+/*
	+ * This function detaches one inherited target from a group. This target
	+ * is not the anchor. The group stays alive for its other targets.
	+ * For example, use this function when a credential-changing exec revokes
	+ * one descendant's authorization. Only this process's edge and row links
	+ * are removed.
	+ *
	+ * The function briefly schedules the group out at its real anchor. This
	+ * drops every target's row links. The function then removes this edge.
	+ * Then the function puts the group back at the surviving anchor.
	+ */
	+void
	+pmu_group_detach_inherited_target(pmu_group_t pg, struct pmc_process pp)
	+{
	+ pmu_event_t *pe;
	+ struct pmc_process *anchor_pp;
	+ struct pmu_group_target *pgt;
	+ bool was_running, was_assigned;
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ KASSERT(pg != NULL && pp != NULL && pg->pg_pp != pp,
	+ ("[pmu] inherited detach on anchor pp=%p", pp));
	+ KASSERT(!pg->pg_releasing,
	+ ("[pmu] group %u detach already active", pg->pg_id));
	+ pgt = pmu_group_find_target(pg, pp);
	+ KASSERT(pgt != NULL && !pgt->pgt_explicit,
	+ ("[pmu] group %u inherited target edge missing", pg->pg_id));
	+
	+ anchor_pp = pg->pg_pp;
	+ pg->pg_releasing = true;
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ was_running = pg->pg_running;
	+ was_assigned = pg->pg_assigned;
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+
	+ /* First, quiesce the group at the anchor. Then drop every target's row links. */
	+ if (anchor_pp != NULL) {
	+ pmu_group_accounting_block(pg);
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling)
	+ pe->pe_pmc->pm_state = PMC_STATE_STOPPED;
	+ pmu_group_accounting_drain(pg, false);
	+ pmu_pp_stop_rotate(anchor_pp, "muxreauth");
	+ if (was_assigned)
	+ pmu_pp_schedule_out(anchor_pp, pg, false);
	+ }
	+
	+ pmu_group_target_remove(pgt);
	+
	+ /*
	+ * Restore the group for the remaining targets.
	+ * This edge was never the anchor, so pg_pp does not change.
	+ */
	+ if (was_running) {
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling)
	+ pe->pe_pmc->pm_state = PMC_STATE_RUNNING;
	+ }
	+ mtx_pool_lock_spin(pmc_mtxpool, pg);
	+ if (was_running)
	+ pmu_group_running_start_locked(pg, cpu_ticks());
	+ pg->pg_account_placement_admit = pg->pg_assigned;
	+ pg->pg_account_blocked = false;
	+ mtx_pool_unlock_spin(pmc_mtxpool, pg);
	+ pg->pg_releasing = false;
	+ wakeup(&pg->pg_releasing);
	+ if (anchor_pp != NULL)
	+ pmu_pp_backfill(anchor_pp);
	+}
	+
	+/* Return true if a running MUX group is unassigned. */
	+static bool
	+pmu_list_has_deferred(struct pmu_group_list *gl)
	+{
	+ pmu_group_t *pg;
	+
	+ LIST_FOREACH(pg, gl, pg_proc_next) {
	+ if (pg->pg_running && !pg->pg_assigned && pg->pg_defer_ok)
	+ return (true);
	+ }
	+ return (false);
	+}
	+
	+/*
	+ * Start or wake the process rotation thread.
	+ */
	+static void
	+pmu_pp_kick_rotate(struct pmc_process *pp)
	+{
	+ int error;
	+
	+ if (pp == NULL \|\| pp->pp_pmu_rot_quiesce != 0 \|\| pp->pp_pmu_unhashed)
	+ return;
	+ if (!pmu_list_has_deferred(&pp->pp_pmu_groups))
	+ return;
	+
	+ pp->pp_pmu_rot_needed = true;
	+ if (pp->pp_pmu_rot_running \|\| pp->pp_pmu_rot_td != NULL) {
	+ wakeup(&pp->pp_pmu_rot_needed);
	+ return;
	+ }
	+
	+ pp->pp_pmu_rot_running = true;
	+ error = kthread_add(pmu_pp_rotate_thread, pp, NULL,
	+ &pp->pp_pmu_rot_td, 0, 0, "pmu_rot_pp_%d",
	+ pp->pp_proc != NULL ? pp->pp_proc->p_pid : -1);
	+ if (error != 0) {
	+ pp->pp_pmu_rot_running = false;
	+ pp->pp_pmu_rot_td = NULL;
	+ PMCDBG1(PMC, OPS, 1,
	+ "pp_kick_rotate: kthread_add err=%d", error);
	+ }
	+}
	+
	+void
	+pmu_pp_kick_after_exec(struct pmc_process *pp)
	+{
	+
	+ sx_assert(&pmc_sx, SX_XLOCKED);
	+ pmu_pp_kick_rotate(pp);
	+}
	+
	+/*
	+ * Return the bitmask of rows held by assigned MUX groups.
	+ */
	+static uint64_t
	+pmu_list_evictable_rows(struct pmu_group_list *gl)
	+{
	+ pmu_event_t *pe;
	+ pmu_group_t *pg;
	+ uint64_t evictable;
	+ u_int n;
	+
	+ evictable = 0;
	+ LIST_FOREACH(pg, gl, pg_proc_next) {
	+ if (!pg->pg_assigned \|\| !pg->pg_defer_ok)
	+ continue;
	+ TAILQ_FOREACH(pe, &pg->pg_events, pe_sibling) {
	+ n = PMC_TO_ROWINDEX(pe->pe_pmc);
	+ if (n < 64)
	+ evictable \|= 1ULL << n;
	+ }
	+ }
	+ return (evictable);
	+}
	+
	+/*
	+ * Find the next assigned MUX group to evict.
	+ */
	+static pmu_group_t *
	+pmu_list_next_victim(struct pmu_group_list gl, pmu_group_t *vpg,
	+ u_int *vseen, u_int ngroups)
	+{
	+ pmu_group_t *pg;
	+
	+ while (*vseen < ngroups) {
	+ pg = *vpg;
	+ if (pg == NULL)
	+ pg = LIST_FIRST(gl);
	+ (*vseen)++;
	+ *vpg = LIST_NEXT(pg, pg_proc_next);
	+ if (pg->pg_assigned && pg->pg_defer_ok)
	+ return (pg);
	+ }
	+ return (NULL);
	+}
	+
	+/*
	+ * Run one multiplex rotation tick for a process.
	+ * Evicts assigned MUX groups and assigns deferred MUX groups.
	+ */
	+static void
	+pmu_pp_rotate_one(struct pmc_process *pp)
	+{
	+ pmu_group_t cursor, pg, victim, vpg;
	+ struct proc *p;
	+ uint64_t evictable;
	+ u_int ngroups, seen, vseen;
	+ int sin_err;
	+ bool found_cursor, placed_any, satisfiable;
	+
	+ if (pp == NULL)
	+ return;
	+ if (LIST_EMPTY(&pp->pp_pmu_groups)) {
	+ pp->pp_pmu_rot_needed = false;
	+ return;
	+ }
	+
	+ /* Reset the cursor to the head if it is invalid. */
	+ ngroups = 0;
	+ cursor = pp->pp_pmu_rot_cursor;
	+ found_cursor = false;
	+ LIST_FOREACH(pg, &pp->pp_pmu_groups, pg_proc_next) {
	+ if (pg == cursor)
	+ found_cursor = true;
	+ ngroups++;
	+ }
	+ if (!found_cursor)
	+ cursor = LIST_FIRST(&pp->pp_pmu_groups);
	+
	+ /* Do not rotate if no deferred group can run. */
	+ evictable = pmu_list_evictable_rows(&pp->pp_pmu_groups);
	+ satisfiable = false;
	+ LIST_FOREACH(pg, &pp->pp_pmu_groups, pg_proc_next) {
	+ if (!pg->pg_running \|\| pg->pg_assigned \|\| !pg->pg_defer_ok)
	+ continue;
	+ p = pmu_group_target_proc(pg);
	+ if (p != NULL && pmu_group_satisfiable(pg, p, PMC_CPU_ANY,
	+ evictable) == 0) {
	+ satisfiable = true;
	+ break;
	+ }
	+ }
	+ if (!satisfiable)
	+ goto out;
	+
	+ /* Evict group at cursor and advance cursor. */
	+ vpg = cursor;
	+ vseen = 0;
	+ victim = pmu_list_next_victim(&pp->pp_pmu_groups, &vpg, &vseen,
	+ ngroups);
	+ if (victim != NULL) {
	+ PMCDBG3(PMC, OPS, 4, "rotate: pp=%p evict gid=%u nevents=%u",
	+ pp, victim->pg_id, victim->pg_nevents);
	+ pmu_pp_schedule_out(pp, victim, true);
	+ cursor = LIST_NEXT(victim, pg_proc_next);
	+ if (cursor == NULL)
	+ cursor = LIST_FIRST(&pp->pp_pmu_groups);
	+ }
	+
	+ /* Assign deferred MUX groups in order. */
	+ placed_any = false;
	+ pg = cursor;
	+ seen = 0;
	+ while (seen < ngroups) {
	+ if (pg == NULL)
	+ pg = LIST_FIRST(&pp->pp_pmu_groups);
	+ seen++;
	+ if (!pg->pg_running \|\| pg->pg_assigned \|\| !pg->pg_defer_ok)
	+ goto next;
	+ p = pmu_group_target_proc(pg);
	+ if (p == NULL \|\| pmu_group_satisfiable(pg, p, PMC_CPU_ANY,
	+ evictable) != 0)
	+ goto next;
	+ sin_err = pmu_pp_schedule_in(pp, pg);
	+ while (sin_err == ENOSPC && !placed_any) {
	+ victim = pmu_list_next_victim(&pp->pp_pmu_groups,
	+ &vpg, &vseen, ngroups);
	+ if (victim == NULL)
	+ break;
	+ PMCDBG2(PMC, OPS, 4,
	+ "rotate: pp=%p escalate evict gid=%u",
	+ pp, victim->pg_id);
	+ pmu_pp_schedule_out(pp, victim, true);
	+ sin_err = pmu_pp_schedule_in(pp, pg);
	+ }
	+ PMCDBG4(PMC, OPS, 4,
	+ "rotate: pp=%p schedule_in gid=%u nevents=%u -> %d",
	+ pp, pg->pg_id, pg->pg_nevents, sin_err);
	+ if (sin_err == 0) {
	+ placed_any = true;
	+ } else if (sin_err == ENOSPC) {
	+ pp->pp_pmu_rot_cursor = pg;
	+ goto out;
	+ }
	+ /* Skip group on hard error. */
	+next:
	+ pg = LIST_NEXT(pg, pg_proc_next);
	+ }
	+ pp->pp_pmu_rot_cursor = cursor;
	+out:
	+ /* Stop thread if all groups are assigned. */
	+ pp->pp_pmu_rot_needed = pmu_list_has_deferred(&pp->pp_pmu_groups);
	+}
	+
	+/*
	+ * Rotation thread for process-mode groups.
	+ */
	+static void
	+pmu_pp_rotate_thread(void *arg)
	+{
	+ struct pmc_process *pp = arg;
	+
	+ sx_xlock(&pmc_sx);
	+ while (pp->pp_pmu_rot_running) {
	+ (void)sx_sleep(&pp->pp_pmu_rot_needed, &pmc_sx, 0, "muxrot",
	+ pmu_rot_period_ticks());
	+ if (!pp->pp_pmu_rot_running)
	+ break;
	+ pmu_pp_rotate_one(pp);
	+ if (!pp->pp_pmu_rot_needed)
	+ break;
	+ }
	+ pp->pp_pmu_rot_running = false;
	+ pp->pp_pmu_rot_td = NULL;
	+ wakeup(&pp->pp_pmu_rot_td);
	+ sx_xunlock(&pmc_sx);
	+ kthread_exit();
	+}

File Metadata

Mime Type: text/plain
Expires: Tue, Oct 20, 6:33 AM (10 h, 38 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 38586955
Default Alt Text: D57631.id185746.diff (73 KB)

D57631.id185746.diffNo OneTemporaryActions

D57631.id185746.diffView Options

File Metadata

Event Timeline

D57631.id185746.diff
No OneTemporary
Actions

D57631.id185746.diff
View Options