D28923.diff
No OneTemporary
Actions

Size

23 KB

Referenced Files

None

Subscribers

None

D28923.diff
View Options

	diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
	--- a/sys/amd64/amd64/genassym.c
	+++ b/sys/amd64/amd64/genassym.c
	@@ -227,7 +227,6 @@
	ASSYM(PC_LDT, offsetof(struct pcpu, pc_ldt));
	ASSYM(PC_COMMONTSS, offsetof(struct pcpu, pc_common_tss));
	ASSYM(PC_TSS, offsetof(struct pcpu, pc_tss));
	-ASSYM(PC_PM_SAVE_CNT, offsetof(struct pcpu, pc_pm_save_cnt));
	ASSYM(PC_KCR3, offsetof(struct pcpu, pc_kcr3));
	ASSYM(PC_UCR3, offsetof(struct pcpu, pc_ucr3));
	ASSYM(PC_UCR3_LOAD_MASK, offsetof(struct pcpu, pc_ucr3_load_mask));
	diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
	--- a/sys/amd64/amd64/pmap.c
	+++ b/sys/amd64/amd64/pmap.c
	@@ -535,21 +535,9 @@
	static int pmap_pkru_deassign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);
	static void pmap_pkru_deassign_all(pmap_t pmap);

	-static int
	-pmap_pcid_save_cnt_proc(SYSCTL_HANDLER_ARGS)
	-{
	- int i;
	- uint64_t res;
	-
	- res = 0;
	- CPU_FOREACH(i) {
	- res += cpuid_to_pcpu[i]->pc_pm_save_cnt;
	- }
	- return (sysctl_handle_64(oidp, &res, 0, req));
	-}
	-SYSCTL_PROC(_vm_pmap, OID_AUTO, pcid_save_cnt, CTLTYPE_U64 \| CTLFLAG_RD \|
	- CTLFLAG_MPSAFE, NULL, 0, pmap_pcid_save_cnt_proc, "QU",
	- "Count of saved TLB context on switch");
	+static COUNTER_U64_DEFINE_EARLY(pcid_save_cnt);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pcid_save_cnt, CTLFLAG_RD,
	+ &pcid_save_cnt, "Count of saved TLB context on switch");

	static LIST_HEAD(, pmap_invl_gen) pmap_invl_gen_tracker =
	LIST_HEAD_INITIALIZER(&pmap_invl_gen_tracker);
	@@ -769,19 +757,30 @@
	return (res);
	}

	+static COUNTER_U64_DEFINE_EARLY(pv_page_count);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_page_count, CTLFLAG_RD,
	+ &pv_page_count, "Current number of allocated pv pages");
	+
	+static COUNTER_U64_DEFINE_EARLY(pt_page_count);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pt_page_count, CTLFLAG_RD,
	+ &pt_page_count, "Current number of allocated page table pages");
	+
	#ifdef PV_STATS
	-static long invl_start_restart;
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_start_restart, CTLFLAG_RD,
	- &invl_start_restart, 0,
	- "");
	-static long invl_finish_restart;
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_finish_restart, CTLFLAG_RD,
	- &invl_finish_restart, 0,
	- "");
	+
	+static COUNTER_U64_DEFINE_EARLY(invl_start_restart);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_start_restart,
	+ CTLFLAG_RD, &invl_start_restart,
	+ "Number of delayed TLB invalidation request restarts");
	+
	+static COUNTER_U64_DEFINE_EARLY(invl_finish_restart);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_finish_restart, CTLFLAG_RD,
	+ &invl_finish_restart,
	+ "Number of delayed TLB invalidation completion restarts");
	+
	static int invl_max_qlen;
	SYSCTL_INT(_vm_pmap, OID_AUTO, invl_max_qlen, CTLFLAG_RD,
	&invl_max_qlen, 0,
	- "");
	+ "Maximum delayed TLB invalidation request queue length");
	#endif

	#define di_delay locks_delay
	@@ -819,7 +818,7 @@
	PV_STAT(i++);
	prevl = (uintptr_t)atomic_load_ptr(&p->next);
	if ((prevl & PMAP_INVL_GEN_NEXT_INVALID) != 0) {
	- PV_STAT(atomic_add_long(&invl_start_restart, 1));
	+ PV_STAT(counter_u64_add(invl_start_restart, 1));
	lock_delay(&lda);
	goto again;
	}
	@@ -833,7 +832,7 @@
	#endif

	if (!pmap_di_load_invl(p, &prev) \|\| prev.next != NULL) {
	- PV_STAT(atomic_add_long(&invl_start_restart, 1));
	+ PV_STAT(counter_u64_add(invl_start_restart, 1));
	lock_delay(&lda);
	goto again;
	}
	@@ -862,7 +861,7 @@
	*/
	if (!pmap_di_store_invl(p, &prev, &new_prev)) {
	critical_exit();
	- PV_STAT(atomic_add_long(&invl_start_restart, 1));
	+ PV_STAT(counter_u64_add(invl_start_restart, 1));
	lock_delay(&lda);
	goto again;
	}
	@@ -926,7 +925,7 @@
	for (p = &pmap_invl_gen_head; p != NULL; p = (void *)prevl) {
	prevl = (uintptr_t)atomic_load_ptr(&p->next);
	if ((prevl & PMAP_INVL_GEN_NEXT_INVALID) != 0) {
	- PV_STAT(atomic_add_long(&invl_finish_restart, 1));
	+ PV_STAT(counter_u64_add(invl_finish_restart, 1));
	lock_delay(&lda);
	goto again;
	}
	@@ -939,7 +938,7 @@
	* thread before us finished its DI and started it again.
	*/
	if (__predict_false(p == NULL)) {
	- PV_STAT(atomic_add_long(&invl_finish_restart, 1));
	+ PV_STAT(counter_u64_add(invl_finish_restart, 1));
	lock_delay(&lda);
	goto again;
	}
	@@ -951,7 +950,7 @@
	atomic_clear_ptr((uintptr_t *)&invl_gen->next,
	PMAP_INVL_GEN_NEXT_INVALID);
	critical_exit();
	- PV_STAT(atomic_add_long(&invl_finish_restart, 1));
	+ PV_STAT(counter_u64_add(invl_finish_restart, 1));
	lock_delay(&lda);
	goto again;
	}
	@@ -987,12 +986,15 @@
	#endif

	#ifdef PV_STATS
	-static long invl_wait;
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_wait, CTLFLAG_RD, &invl_wait, 0,
	+static COUNTER_U64_DEFINE_EARLY(invl_wait);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_wait,
	+ CTLFLAG_RD, &invl_wait,
	"Number of times DI invalidation blocked pmap_remove_all/write");
	-static long invl_wait_slow;
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_wait_slow, CTLFLAG_RD, &invl_wait_slow, 0,
	- "Number of slow invalidation waits for lockless DI");
	+
	+static COUNTER_U64_DEFINE_EARLY(invl_wait_slow);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_wait_slow, CTLFLAG_RD,
	+ &invl_wait_slow, "Number of slow invalidation waits for lockless DI");
	+
	#endif

	#ifdef NUMA
	@@ -1066,7 +1068,7 @@
	while (*m_gen > pmap_invl_gen) {
	#ifdef PV_STATS
	if (!accounted) {
	- atomic_add_long(&invl_wait, 1);
	+ counter_u64_add(invl_wait, 1);
	accounted = true;
	}
	#endif
	@@ -1086,7 +1088,7 @@
	lock_delay_arg_init(&lda, &di_delay);
	while (*m_gen > atomic_load_long(&pmap_invl_gen_head.gen)) {
	if (fast \|\| !pmap_invl_callout_inited) {
	- PV_STAT(atomic_add_long(&invl_wait, 1));
	+ PV_STAT(counter_u64_add(invl_wait, 1));
	lock_delay(&lda);
	fast = false;
	} else {
	@@ -1119,7 +1121,7 @@
	atomic_load_long(&pmap_invl_gen_head.gen)) {
	callout_reset(&pmap_invl_callout, 1,
	pmap_delayed_invl_callout_func, NULL);
	- PV_STAT(atomic_add_long(&invl_wait_slow, 1));
	+ PV_STAT(counter_u64_add(invl_wait_slow, 1));
	pmap_delayed_invl_wait_block(m_gen,
	&pmap_invl_gen_head.gen);
	}
	@@ -2459,28 +2461,28 @@
	static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"2MB page mapping counters");

	-static u_long pmap_pde_demotions;
	-SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD,
	- &pmap_pde_demotions, 0, "2MB page demotions");
	+static COUNTER_U64_DEFINE_EARLY(pmap_pde_demotions);
	+SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, demotions,
	+ CTLFLAG_RD, &pmap_pde_demotions, "2MB page demotions");

	-static u_long pmap_pde_mappings;
	-SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
	- &pmap_pde_mappings, 0, "2MB page mappings");
	+static COUNTER_U64_DEFINE_EARLY(pmap_pde_mappings);
	+SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
	+ &pmap_pde_mappings, "2MB page mappings");

	-static u_long pmap_pde_p_failures;
	-SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
	- &pmap_pde_p_failures, 0, "2MB page promotion failures");
	+static COUNTER_U64_DEFINE_EARLY(pmap_pde_p_failures);
	+SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
	+ &pmap_pde_p_failures, "2MB page promotion failures");

	-static u_long pmap_pde_promotions;
	-SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
	- &pmap_pde_promotions, 0, "2MB page promotions");
	+static COUNTER_U64_DEFINE_EARLY(pmap_pde_promotions);
	+SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
	+ &pmap_pde_promotions, "2MB page promotions");

	static SYSCTL_NODE(_vm_pmap, OID_AUTO, pdpe, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"1GB page mapping counters");

	-static u_long pmap_pdpe_demotions;
	-SYSCTL_ULONG(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD,
	- &pmap_pdpe_demotions, 0, "1GB page demotions");
	+static COUNTER_U64_DEFINE_EARLY(pmap_pdpe_demotions);
	+SYSCTL_COUNTER_U64(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD,
	+ &pmap_pdpe_demotions, "1GB page demotions");

	/***************************************************
	* Low level helper routines.....
	@@ -4016,6 +4018,8 @@
	pmap_unwire_ptp(pmap, va, pml4pg, free);
	}

	+ counter_u64_add(pt_page_count, -1);
	+
	/*
	* Put page on a list so that it is released after
	* ALL TLB shootdown is done
	@@ -4196,6 +4200,8 @@
	pmltop_pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL \| VM_ALLOC_NOOBJ \|
	VM_ALLOC_WIRED \| VM_ALLOC_ZERO \| VM_ALLOC_WAITOK);

	+ counter_u64_add(pt_page_count, 1);
	+
	pmltop_phys = VM_PAGE_TO_PHYS(pmltop_pg);
	pmap->pm_pmltop = (pml5_entry_t *)PHYS_TO_DMAP(pmltop_phys);

	@@ -4227,6 +4233,7 @@
	if ((curproc->p_md.md_flags & P_MD_KPTI) != 0) {
	pmltop_pgu = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL \|
	VM_ALLOC_NOOBJ \| VM_ALLOC_WIRED \| VM_ALLOC_WAITOK);
	+ counter_u64_add(pt_page_count, 1);
	pmap->pm_pmltopu = (pml4_entry_t *)PHYS_TO_DMAP(
	VM_PAGE_TO_PHYS(pmltop_pgu));
	if (pmap_is_la57(pmap))
	@@ -4414,6 +4421,7 @@
	if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ \|
	VM_ALLOC_WIRED \| VM_ALLOC_ZERO)) == NULL)
	return (NULL);
	+
	if ((m->flags & PG_ZERO) == 0)
	pmap_zero_page(m);

	@@ -4511,6 +4519,8 @@
	}

	pmap_resident_count_inc(pmap, 1);
	+ counter_u64_add(pt_page_count, 1);
	+
	return (m);
	}

	@@ -4673,12 +4683,14 @@

	vm_page_unwire_noq(m);
	vm_page_free_zero(m);
	+ counter_u64_add(pt_page_count, -1);

	if (pmap->pm_pmltopu != NULL) {
	m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->
	pm_pmltopu));
	vm_page_unwire_noq(m);
	vm_page_free(m);
	+ counter_u64_add(pt_page_count, -1);
	}
	if (pmap->pm_type == PT_X86 &&
	(cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0)
	@@ -4794,6 +4806,7 @@
	panic("pmap_growkernel: no memory to grow kernel");
	if ((nkpg->flags & PG_ZERO) == 0)
	pmap_zero_page(nkpg);
	+ counter_u64_add(pt_page_count, 1);
	paddr = VM_PAGE_TO_PHYS(nkpg);
	*pdpe = (pdp_entry_t)(paddr \| X86_PG_V \| X86_PG_RW \|
	X86_PG_A \| X86_PG_M);
	@@ -4816,6 +4829,7 @@
	panic("pmap_growkernel: no memory to grow kernel");
	if ((nkpg->flags & PG_ZERO) == 0)
	pmap_zero_page(nkpg);
	+ counter_u64_add(pt_page_count, 1);
	paddr = VM_PAGE_TO_PHYS(nkpg);
	newpdir = paddr \| X86_PG_V \| X86_PG_RW \| X86_PG_A \| X86_PG_M;
	pde_store(pde, newpdir);
	@@ -4852,28 +4866,39 @@
	static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 };

	#ifdef PV_STATS
	-static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
	-
	-SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
	- "Current number of pv entry chunks");
	-SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
	- "Current number of pv entry chunks allocated");
	-SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
	- "Current number of pv entry chunks frees");
	-SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
	- "Number of times tried to get a chunk page but failed.");
	-
	-static long pv_entry_frees, pv_entry_allocs, pv_entry_count;
	-static int pv_entry_spare;
	-
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
	- "Current number of pv entry frees");
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
	- "Current number of pv entry allocs");
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
	- "Current number of pv entries");
	-SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
	- "Current number of spare pv entries");
	+
	+static COUNTER_U64_DEFINE_EARLY(pc_chunk_count);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD,
	+ &pc_chunk_count, "Current number of pv entry cnunks");
	+
	+static COUNTER_U64_DEFINE_EARLY(pc_chunk_allocs);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD,
	+ &pc_chunk_allocs, "Total number of pv entry chunks allocated");
	+
	+static COUNTER_U64_DEFINE_EARLY(pc_chunk_frees);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD,
	+ &pc_chunk_frees, "Total number of pv entry chunks freed");
	+
	+static COUNTER_U64_DEFINE_EARLY(pc_chunk_tryfail);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD,
	+ &pc_chunk_tryfail,
	+ "Number of failed attempts to get a pv entry chunk page");
	+
	+static COUNTER_U64_DEFINE_EARLY(pv_entry_frees);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD,
	+ &pv_entry_frees, "Total number of pv entries freed");
	+
	+static COUNTER_U64_DEFINE_EARLY(pv_entry_allocs);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD,
	+ &pv_entry_allocs, "Total number of pv entries allocated");
	+
	+static COUNTER_U64_DEFINE_EARLY(pv_entry_count);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD,
	+ &pv_entry_count, "Current number of pv entries");
	+
	+static COUNTER_U64_DEFINE_EARLY(pv_entry_spare);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD,
	+ &pv_entry_spare, "Current number of spare pv entries");
	#endif

	static void
	@@ -5046,15 +5071,15 @@
	}
	/* Every freed mapping is for a 4 KB page. */
	pmap_resident_count_dec(pmap, freed);
	- PV_STAT(atomic_add_long(&pv_entry_frees, freed));
	- PV_STAT(atomic_add_int(&pv_entry_spare, freed));
	- PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
	+ PV_STAT(counter_u64_add(pv_entry_frees, freed));
	+ PV_STAT(counter_u64_add(pv_entry_spare, freed));
	+ PV_STAT(counter_u64_add(pv_entry_count, -freed));
	TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
	if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1 &&
	pc->pc_map[2] == PC_FREE2) {
	- PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
	- PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
	- PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
	+ PV_STAT(counter_u64_add(pv_entry_spare, -_NPCPV));
	+ PV_STAT(counter_u64_add(pc_chunk_count, -1));
	+ PV_STAT(counter_u64_add(pc_chunk_frees, 1));
	/* Entire chunk is free; return it. */
	m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
	dump_drop_page(m_pc->phys_addr);
	@@ -5127,9 +5152,9 @@
	int idx, field, bit;

	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
	- PV_STAT(atomic_add_long(&pv_entry_frees, 1));
	- PV_STAT(atomic_add_int(&pv_entry_spare, 1));
	- PV_STAT(atomic_subtract_long(&pv_entry_count, 1));
	+ PV_STAT(counter_u64_add(pv_entry_frees, 1));
	+ PV_STAT(counter_u64_add(pv_entry_spare, 1));
	+ PV_STAT(counter_u64_add(pv_entry_count, -1));
	pc = pv_to_chunk(pv);
	idx = pv - &pc->pc_pventry[0];
	field = idx / 64;
	@@ -5153,9 +5178,10 @@
	{
	vm_page_t m;

	- PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
	- PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
	- PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
	+ PV_STAT(counter_u64_add(pv_entry_spare, -_NPCPV));
	+ PV_STAT(counter_u64_add(pc_chunk_count, -1));
	+ PV_STAT(counter_u64_add(pc_chunk_frees, 1));
	+ counter_u64_add(pv_page_count, -1);
	/* entire chunk is free, return it */
	m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
	dump_drop_page(m->phys_addr);
	@@ -5218,7 +5244,7 @@
	vm_page_t m;

	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
	- PV_STAT(atomic_add_long(&pv_entry_allocs, 1));
	+ PV_STAT(counter_u64_add(pv_entry_allocs, 1));
	retry:
	pc = TAILQ_FIRST(&pmap->pm_pvchunk);
	if (pc != NULL) {
	@@ -5238,8 +5264,8 @@
	TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
	pc_list);
	}
	- PV_STAT(atomic_add_long(&pv_entry_count, 1));
	- PV_STAT(atomic_subtract_int(&pv_entry_spare, 1));
	+ PV_STAT(counter_u64_add(pv_entry_count, 1));
	+ PV_STAT(counter_u64_add(pv_entry_spare, -1));
	return (pv);
	}
	}
	@@ -5248,15 +5274,16 @@
	VM_ALLOC_WIRED);
	if (m == NULL) {
	if (lockp == NULL) {
	- PV_STAT(pc_chunk_tryfail++);
	+ PV_STAT(counter_u64_add(pc_chunk_tryfail, 1));
	return (NULL);
	}
	m = reclaim_pv_chunk(pmap, lockp);
	if (m == NULL)
	goto retry;
	- }
	- PV_STAT(atomic_add_int(&pc_chunk_count, 1));
	- PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
	+ } else
	+ counter_u64_add(pv_page_count, 1);
	+ PV_STAT(counter_u64_add(pc_chunk_count, 1));
	+ PV_STAT(counter_u64_add(pc_chunk_allocs, 1));
	dump_add_page(m->phys_addr);
	pc = (void *)PHYS_TO_DMAP(m->phys_addr);
	pc->pc_pmap = pmap;
	@@ -5269,8 +5296,8 @@
	mtx_unlock(&pvc->pvc_lock);
	pv = &pc->pc_pventry[0];
	TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
	- PV_STAT(atomic_add_long(&pv_entry_count, 1));
	- PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
	+ PV_STAT(counter_u64_add(pv_entry_count, 1));
	+ PV_STAT(counter_u64_add(pv_entry_spare, _NPCPV - 1));
	return (pv);
	}

	@@ -5354,9 +5381,10 @@
	if (m == NULL)
	goto retry;
	reclaimed = true;
	- }
	- PV_STAT(atomic_add_int(&pc_chunk_count, 1));
	- PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
	+ } else
	+ counter_u64_add(pv_page_count, 1);
	+ PV_STAT(counter_u64_add(pc_chunk_count, 1));
	+ PV_STAT(counter_u64_add(pc_chunk_allocs, 1));
	dump_add_page(m->phys_addr);
	pc = (void *)PHYS_TO_DMAP(m->phys_addr);
	pc->pc_pmap = pmap;
	@@ -5365,7 +5393,7 @@
	pc->pc_map[2] = PC_FREE2;
	TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
	TAILQ_INSERT_TAIL(&new_tail[vm_page_domain(m)], pc, pc_lru);
	- PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV));
	+ PV_STAT(counter_u64_add(pv_entry_spare, _NPCPV));

	/*
	* The reclaim might have freed a chunk from the current pmap.
	@@ -5440,7 +5468,7 @@
	TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
	m->md.pv_gen++;
	/* Instantiate the remaining NPTEPG - 1 pv entries. */
	- PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
	+ PV_STAT(counter_u64_add(pv_entry_allocs, NPTEPG - 1));
	va_last = va + NBPDR - PAGE_SIZE;
	for (;;) {
	pc = TAILQ_FIRST(&pmap->pm_pvchunk);
	@@ -5470,8 +5498,8 @@
	TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
	TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
	}
	- PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1));
	- PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1));
	+ PV_STAT(counter_u64_add(pv_entry_count, NPTEPG - 1));
	+ PV_STAT(counter_u64_add(pv_entry_spare, -(NPTEPG - 1)));
	}

	#if VM_NRESERVLEVEL > 0
	@@ -5731,6 +5759,8 @@
	return (FALSE);
	}

	+ counter_u64_add(pt_page_count, 1);
	+
	if (!in_kernel) {
	mpte->ref_count = NPTEPG;
	pmap_resident_count_inc(pmap, 1);
	@@ -5795,7 +5825,7 @@
	if ((oldpde & PG_MANAGED) != 0)
	pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME, lockp);

	- atomic_add_long(&pmap_pde_demotions, 1);
	+ counter_u64_add(pmap_pde_demotions, 1);
	CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx in pmap %p",
	va, pmap);
	return (TRUE);
	@@ -6517,7 +6547,7 @@
	if ((newpde & ((PG_FRAME & PDRMASK) \| PG_A \| PG_V)) != (PG_A \| PG_V) \|\|
	!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap,
	newpde))) {
	- atomic_add_long(&pmap_pde_p_failures, 1);
	+ counter_u64_add(pmap_pde_p_failures, 1);
	CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
	" in pmap %p", va, pmap);
	return;
	@@ -6542,7 +6572,7 @@
	setpte:
	oldpte = *pte;
	if ((oldpte & (PG_FRAME \| PG_A \| PG_V)) != pa) {
	- atomic_add_long(&pmap_pde_p_failures, 1);
	+ counter_u64_add(pmap_pde_p_failures, 1);
	CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
	" in pmap %p", va, pmap);
	return;
	@@ -6560,7 +6590,7 @@
	(va & ~PDRMASK), pmap);
	}
	if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
	- atomic_add_long(&pmap_pde_p_failures, 1);
	+ counter_u64_add(pmap_pde_p_failures, 1);
	CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
	" in pmap %p", va, pmap);
	return;
	@@ -6580,7 +6610,7 @@
	KASSERT(mpte->pindex == pmap_pde_pindex(va),
	("pmap_promote_pde: page table page's pindex is wrong"));
	if (pmap_insert_pt_page(pmap, mpte, true)) {
	- atomic_add_long(&pmap_pde_p_failures, 1);
	+ counter_u64_add(pmap_pde_p_failures, 1);
	CTR2(KTR_PMAP,
	"pmap_promote_pde: failure for va %#lx in pmap %p", va,
	pmap);
	@@ -6606,7 +6636,7 @@
	else
	pde_store(pde, PG_PROMOTED \| PG_PS \| newpde);

	- atomic_add_long(&pmap_pde_promotions, 1);
	+ counter_u64_add(pmap_pde_promotions, 1);
	CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx"
	" in pmap %p", va, pmap);
	}
	@@ -7182,7 +7212,7 @@
	*/
	pde_store(pde, newpde);

	- atomic_add_long(&pmap_pde_mappings, 1);
	+ counter_u64_add(pmap_pde_mappings, 1);
	CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx in pmap %p",
	va, pmap);
	return (KERN_SUCCESS);
	@@ -7444,7 +7474,7 @@
	pde_store(pde, pa \| PG_PS \| PG_M \| PG_A \|
	PG_U \| PG_RW \| PG_V);
	pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE);
	- atomic_add_long(&pmap_pde_mappings, 1);
	+ counter_u64_add(pmap_pde_mappings, 1);
	} else {
	/* Continue on if the PDE is already valid. */
	pdpg->ref_count--;
	@@ -7672,7 +7702,7 @@
	*pde = srcptepaddr & ~PG_W;
	pmap_resident_count_inc(dst_pmap, NBPDR /
	PAGE_SIZE);
	- atomic_add_long(&pmap_pde_mappings, 1);
	+ counter_u64_add(pmap_pde_mappings, 1);
	} else
	pmap_abort_ptp(dst_pmap, addr, dst_pdpg);
	continue;
	@@ -8157,9 +8187,9 @@
	freed++;
	}
	}
	- PV_STAT(atomic_add_long(&pv_entry_frees, freed));
	- PV_STAT(atomic_add_int(&pv_entry_spare, freed));
	- PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
	+ PV_STAT(counter_u64_add(pv_entry_frees, freed));
	+ PV_STAT(counter_u64_add(pv_entry_spare, freed));
	+ PV_STAT(counter_u64_add(pv_entry_count, -freed));
	if (allfree) {
	TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
	TAILQ_INSERT_TAIL(&free_chunks[pc_to_domain(pc)], pc, pc_list);
	@@ -9073,6 +9103,7 @@
	" in pmap %p", va, pmap);
	return (FALSE);
	}
	+ counter_u64_add(pt_page_count, 1);
	pdpgpa = VM_PAGE_TO_PHYS(pdpg);
	firstpde = (pd_entry_t *)PHYS_TO_DMAP(pdpgpa);
	newpdpe = pdpgpa \| PG_M \| PG_A \| (oldpdpe & PG_U) \| PG_RW \| PG_V;
	@@ -9100,7 +9131,7 @@
	*/
	pmap_invalidate_page(pmap, (vm_offset_t)vtopde(va));

	- pmap_pdpe_demotions++;
	+ counter_u64_add(pmap_pdpe_demotions, 1);
	CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx"
	" in pmap %p", va, pmap);
	return (TRUE);
	@@ -9622,7 +9653,7 @@
	PCPU_SET(kcr3, kcr3 \| CR3_PCID_SAVE);
	PCPU_SET(ucr3, ucr3 \| CR3_PCID_SAVE);
	if (cached)
	- PCPU_INC(pm_save_cnt);
	+ counter_u64_add(pcid_save_cnt, 1);

	pmap_activate_sw_pti_post(td, pmap);
	}
	@@ -9643,7 +9674,7 @@
	cached);
	PCPU_SET(curpmap, pmap);
	if (cached)
	- PCPU_INC(pm_save_cnt);
	+ counter_u64_add(pcid_save_cnt, 1);
	}

	static void
	@@ -10088,8 +10119,11 @@

	m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL \| VM_ALLOC_NOOBJ \|
	VM_ALLOC_ZERO);
	- if (m != NULL && (m->flags & PG_ZERO) == 0)
	- pmap_zero_page(m);
	+ if (m != NULL) {
	+ if ((m->flags & PG_ZERO) == 0)
	+ pmap_zero_page(m);
	+ counter_u64_add(pt_page_count, 1);
	+ }
	return (m);
	}

	diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
	--- a/sys/amd64/include/pcpu.h
	+++ b/sys/amd64/include/pcpu.h
	@@ -76,7 +76,6 @@
	struct system_segment_descriptor *pc_ldt; \
	/* Pointer to the CPU TSS descriptor */ \
	struct system_segment_descriptor *pc_tss; \
	- uint64_t pc_pm_save_cnt; \
	u_int pc_cmci_mask; /* MCx banks for CMCI */ \
	uint64_t pc_dbreg[16]; /* ddb debugging regs */ \
	uint64_t pc_pti_stack[PC_PTI_STACK_SZ]; \
	@@ -89,7 +88,7 @@
	uint32_t pc_ibpb_set; \
	void *pc_mds_buf; \
	void *pc_mds_buf64; \
	- uint32_t pc_pad[2]; \
	+ uint32_t pc_pad[4]; \
	uint8_t pc_mds_tmp[64]; \
	u_int pc_ipi_bitmap; \
	struct amd64tss pc_common_tss; \

File Metadata

Mime Type: text/plain
Expires: Tue, Feb 11, 12:10 AM (9 h, 48 s)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 16585782
Default Alt Text: D28923.diff (23 KB)

D28923.diffNo OneTemporaryActions

D28923.diffView Options

File Metadata

Event Timeline

D28923.diff
No OneTemporary
Actions

D28923.diff
View Options