D28923.id84908.diff
No OneTemporary
Actions

Size

27 KB

Referenced Files

None

Subscribers

None

D28923.id84908.diff
View Options

	Index: sys/amd64/amd64/genassym.c
	===================================================================
	--- sys/amd64/amd64/genassym.c
	+++ sys/amd64/amd64/genassym.c
	@@ -227,7 +227,6 @@
	ASSYM(PC_LDT, offsetof(struct pcpu, pc_ldt));
	ASSYM(PC_COMMONTSS, offsetof(struct pcpu, pc_common_tss));
	ASSYM(PC_TSS, offsetof(struct pcpu, pc_tss));
	-ASSYM(PC_PM_SAVE_CNT, offsetof(struct pcpu, pc_pm_save_cnt));
	ASSYM(PC_KCR3, offsetof(struct pcpu, pc_kcr3));
	ASSYM(PC_UCR3, offsetof(struct pcpu, pc_ucr3));
	ASSYM(PC_UCR3_LOAD_MASK, offsetof(struct pcpu, pc_ucr3_load_mask));
	Index: sys/amd64/amd64/pmap.c
	===================================================================
	--- sys/amd64/amd64/pmap.c
	+++ sys/amd64/amd64/pmap.c
	@@ -114,7 +114,9 @@
	#include <sys/param.h>
	#include <sys/bitstring.h>
	#include <sys/bus.h>
	+#include <sys/types.h>
	#include <sys/systm.h>
	+#include <sys/counter.h>
	#include <sys/kernel.h>
	#include <sys/ktr.h>
	#include <sys/lock.h>
	@@ -535,22 +537,6 @@
	static int pmap_pkru_deassign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);
	static void pmap_pkru_deassign_all(pmap_t pmap);

	-static int
	-pmap_pcid_save_cnt_proc(SYSCTL_HANDLER_ARGS)
	-{
	- int i;
	- uint64_t res;
	-
	- res = 0;
	- CPU_FOREACH(i) {
	- res += cpuid_to_pcpu[i]->pc_pm_save_cnt;
	- }
	- return (sysctl_handle_64(oidp, &res, 0, req));
	-}
	-SYSCTL_PROC(_vm_pmap, OID_AUTO, pcid_save_cnt, CTLTYPE_U64 \| CTLFLAG_RD \|
	- CTLFLAG_MPSAFE, NULL, 0, pmap_pcid_save_cnt_proc, "QU",
	- "Count of saved TLB context on switch");
	-
	static LIST_HEAD(, pmap_invl_gen) pmap_invl_gen_tracker =
	LIST_HEAD_INITIALIZER(&pmap_invl_gen_tracker);
	static struct mtx invl_gen_mtx;
	@@ -769,19 +755,30 @@
	return (res);
	}

	+static COUNTER_U64_DEFINE_EARLY(pv_page_count);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_page_count, CTLFLAG_RD,
	+ &pv_page_count, "Current number of allocated pv pages");
	+
	+static COUNTER_U64_DEFINE_EARLY(pt_page_count);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pt_page_count, CTLFLAG_RD,
	+ &pt_page_count, "Current number of allocated page table pages");
	+
	#ifdef PV_STATS
	-static long invl_start_restart;
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_start_restart, CTLFLAG_RD,
	- &invl_start_restart, 0,
	- "");
	-static long invl_finish_restart;
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_finish_restart, CTLFLAG_RD,
	- &invl_finish_restart, 0,
	- "");
	+
	+static COUNTER_U64_DEFINE_EARLY(invl_start_restart);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_start_restart,
	+ CTLFLAG_RD, &invl_start_restart,
	+ "Number of delayed TLB invalidation request restarts");
	+
	+static COUNTER_U64_DEFINE_EARLY(invl_finish_restart);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_finish_restart, CTLFLAG_RD,
	+ &invl_finish_restart,
	+ "Number of delayed TLB invalidation completion restarts");
	+
	static int invl_max_qlen;
	SYSCTL_INT(_vm_pmap, OID_AUTO, invl_max_qlen, CTLFLAG_RD,
	&invl_max_qlen, 0,
	- "");
	+ "Maximum delayed TLB invalidation request queue length");
	#endif

	#define di_delay locks_delay
	@@ -819,7 +816,7 @@
	PV_STAT(i++);
	prevl = (uintptr_t)atomic_load_ptr(&p->next);
	if ((prevl & PMAP_INVL_GEN_NEXT_INVALID) != 0) {
	- PV_STAT(atomic_add_long(&invl_start_restart, 1));
	+ PV_STAT(counter_u64_add(invl_start_restart, 1));
	lock_delay(&lda);
	goto again;
	}
	@@ -833,7 +830,7 @@
	#endif

	if (!pmap_di_load_invl(p, &prev) \|\| prev.next != NULL) {
	- PV_STAT(atomic_add_long(&invl_start_restart, 1));
	+ PV_STAT(counter_u64_add(invl_start_restart, 1));
	lock_delay(&lda);
	goto again;
	}
	@@ -862,7 +859,7 @@
	*/
	if (!pmap_di_store_invl(p, &prev, &new_prev)) {
	critical_exit();
	- PV_STAT(atomic_add_long(&invl_start_restart, 1));
	+ PV_STAT(counter_u64_add(invl_start_restart, 1));
	lock_delay(&lda);
	goto again;
	}
	@@ -926,7 +923,7 @@
	for (p = &pmap_invl_gen_head; p != NULL; p = (void *)prevl) {
	prevl = (uintptr_t)atomic_load_ptr(&p->next);
	if ((prevl & PMAP_INVL_GEN_NEXT_INVALID) != 0) {
	- PV_STAT(atomic_add_long(&invl_finish_restart, 1));
	+ PV_STAT(counter_u64_add(invl_finish_restart, 1));
	lock_delay(&lda);
	goto again;
	}
	@@ -939,7 +936,7 @@
	* thread before us finished its DI and started it again.
	*/
	if (__predict_false(p == NULL)) {
	- PV_STAT(atomic_add_long(&invl_finish_restart, 1));
	+ PV_STAT(counter_u64_add(invl_finish_restart, 1));
	lock_delay(&lda);
	goto again;
	}
	@@ -951,7 +948,7 @@
	atomic_clear_ptr((uintptr_t *)&invl_gen->next,
	PMAP_INVL_GEN_NEXT_INVALID);
	critical_exit();
	- PV_STAT(atomic_add_long(&invl_finish_restart, 1));
	+ PV_STAT(counter_u64_add(invl_finish_restart, 1));
	lock_delay(&lda);
	goto again;
	}
	@@ -987,12 +984,15 @@
	#endif

	#ifdef PV_STATS
	-static long invl_wait;
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_wait, CTLFLAG_RD, &invl_wait, 0,
	+static COUNTER_U64_DEFINE_EARLY(invl_wait);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_wait,
	+ CTLFLAG_RD, &invl_wait,
	"Number of times DI invalidation blocked pmap_remove_all/write");
	-static long invl_wait_slow;
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_wait_slow, CTLFLAG_RD, &invl_wait_slow, 0,
	- "Number of slow invalidation waits for lockless DI");
	+
	+static COUNTER_U64_DEFINE_EARLY(invl_wait_slow);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_wait_slow, CTLFLAG_RD,
	+ &invl_wait_slow, "Number of slow invalidation waits for lockless DI");
	+
	#endif

	#ifdef NUMA
	@@ -1066,7 +1066,7 @@
	while (*m_gen > pmap_invl_gen) {
	#ifdef PV_STATS
	if (!accounted) {
	- atomic_add_long(&invl_wait, 1);
	+ counter_u64_add(invl_wait, 1);
	accounted = true;
	}
	#endif
	@@ -1086,7 +1086,7 @@
	lock_delay_arg_init(&lda, &di_delay);
	while (*m_gen > atomic_load_long(&pmap_invl_gen_head.gen)) {
	if (fast \|\| !pmap_invl_callout_inited) {
	- PV_STAT(atomic_add_long(&invl_wait, 1));
	+ PV_STAT(counter_u64_add(invl_wait, 1));
	lock_delay(&lda);
	fast = false;
	} else {
	@@ -1119,7 +1119,7 @@
	atomic_load_long(&pmap_invl_gen_head.gen)) {
	callout_reset(&pmap_invl_callout, 1,
	pmap_delayed_invl_callout_func, NULL);
	- PV_STAT(atomic_add_long(&invl_wait_slow, 1));
	+ PV_STAT(counter_u64_add(invl_wait_slow, 1));
	pmap_delayed_invl_wait_block(m_gen,
	&pmap_invl_gen_head.gen);
	}
	@@ -1288,6 +1288,9 @@
	struct spglist *free);
	static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *);

	+static vm_page_t pmap_alloc_pt_page(vm_pindex_t, int);
	+static void pmap_free_pt_page(vm_page_t, bool);
	+
	/********************/
	/* Inline functions */
	/********************/
	@@ -2459,28 +2462,28 @@
	static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"2MB page mapping counters");

	-static u_long pmap_pde_demotions;
	-SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD,
	- &pmap_pde_demotions, 0, "2MB page demotions");
	+static COUNTER_U64_DEFINE_EARLY(pmap_pde_demotions);
	+SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, demotions,
	+ CTLFLAG_RD, &pmap_pde_demotions, "2MB page demotions");

	-static u_long pmap_pde_mappings;
	-SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
	- &pmap_pde_mappings, 0, "2MB page mappings");
	+static COUNTER_U64_DEFINE_EARLY(pmap_pde_mappings);
	+SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
	+ &pmap_pde_mappings, "2MB page mappings");

	-static u_long pmap_pde_p_failures;
	-SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
	- &pmap_pde_p_failures, 0, "2MB page promotion failures");
	+static COUNTER_U64_DEFINE_EARLY(pmap_pde_p_failures);
	+SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
	+ &pmap_pde_p_failures, "2MB page promotion failures");

	-static u_long pmap_pde_promotions;
	-SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
	- &pmap_pde_promotions, 0, "2MB page promotions");
	+static COUNTER_U64_DEFINE_EARLY(pmap_pde_promotions);
	+SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
	+ &pmap_pde_promotions, "2MB page promotions");

	static SYSCTL_NODE(_vm_pmap, OID_AUTO, pdpe, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"1GB page mapping counters");

	-static u_long pmap_pdpe_demotions;
	-SYSCTL_ULONG(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD,
	- &pmap_pdpe_demotions, 0, "1GB page demotions");
	+static COUNTER_U64_DEFINE_EARLY(pmap_pdpe_demotions);
	+SYSCTL_COUNTER_U64(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD,
	+ &pmap_pdpe_demotions, "1GB page demotions");

	/***************************************************
	* Low level helper routines.....
	@@ -4016,6 +4019,8 @@
	pmap_unwire_ptp(pmap, va, pml4pg, free);
	}

	+ counter_u64_add(pt_page_count, -1);
	+
	/*
	* Put page on a list so that it is released after
	* ALL TLB shootdown is done
	@@ -4179,6 +4184,42 @@
	pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, FALSE);
	}

	+/* Allocate a page table page and do related bookkeeping */
	+static vm_page_t
	+pmap_alloc_pt_page(vm_pindex_t pindex, int flags)
	+{
	+ vm_page_t m;
	+
	+ m = vm_page_alloc(NULL, pindex, flags \| VM_ALLOC_NOOBJ);
	+ if (__predict_false(m == NULL))
	+ return (NULL);
	+
	+ counter_u64_add(pt_page_count, 1);
	+ if ((flags & VM_ALLOC_ZERO) != 0 && (m->flags & PG_ZERO) == 0)
	+ pmap_zero_page(m);
	+
	+ return (m);
	+}
	+
	+static void
	+pmap_free_pt_page(vm_page_t m, bool zerofilled)
	+{
	+ /*
	+ * This function assumes the page will need to be unwired,
	+ * even though the counterpart allocation in pmap_alloc_pt_page()
	+ * doesn't enforce VM_ALLOC_WIRED. However, all current uses
	+ * of pmap_free_pt_page() require unwiring. The case in which
	+ * a PT page doesn't require unwiring because its ref_count has
	+ * naturally reached 0 is handled through _pmap_unwire_ptp().
	+ */
	+ vm_page_unwire_noq(m);
	+ if (zerofilled)
	+ vm_page_free_zero(m);
	+ else
	+ vm_page_free(m);
	+ counter_u64_add(pt_page_count, -1);
	+}
	+
	/*
	* Initialize a preallocated and zeroed pmap structure,
	* such as one in a vmspace structure.
	@@ -4193,8 +4234,8 @@
	/*
	* allocate the page directory page
	*/
	- pmltop_pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL \| VM_ALLOC_NOOBJ \|
	- VM_ALLOC_WIRED \| VM_ALLOC_ZERO \| VM_ALLOC_WAITOK);
	+ pmltop_pg = pmap_alloc_pt_page(0, VM_ALLOC_NORMAL \| VM_ALLOC_WIRED \|
	+ VM_ALLOC_ZERO \| VM_ALLOC_WAITOK);

	pmltop_phys = VM_PAGE_TO_PHYS(pmltop_pg);
	pmap->pm_pmltop = (pml5_entry_t *)PHYS_TO_DMAP(pmltop_phys);
	@@ -4208,8 +4249,6 @@
	pmap->pm_pmltopu = NULL;

	pmap->pm_type = pm_type;
	- if ((pmltop_pg->flags & PG_ZERO) == 0)
	- pagezero(pmap->pm_pmltop);

	/*
	* Do not install the host kernel mappings in the nested page
	@@ -4225,8 +4264,8 @@
	else
	pmap_pinit_pml4(pmltop_pg);
	if ((curproc->p_md.md_flags & P_MD_KPTI) != 0) {
	- pmltop_pgu = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL \|
	- VM_ALLOC_NOOBJ \| VM_ALLOC_WIRED \| VM_ALLOC_WAITOK);
	+ pmltop_pgu = pmap_alloc_pt_page(0, VM_ALLOC_WIRED \|
	+ VM_ALLOC_NORMAL \| VM_ALLOC_WAITOK);
	pmap->pm_pmltopu = (pml4_entry_t *)PHYS_TO_DMAP(
	VM_PAGE_TO_PHYS(pmltop_pgu));
	if (pmap_is_la57(pmap))
	@@ -4411,11 +4450,9 @@
	/*
	* Allocate a page table page.
	*/
	- if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ \|
	- VM_ALLOC_WIRED \| VM_ALLOC_ZERO)) == NULL)
	+ m = pmap_alloc_pt_page(ptepindex, VM_ALLOC_WIRED \| VM_ALLOC_ZERO);
	+ if (m == NULL)
	return (NULL);
	- if ((m->flags & PG_ZERO) == 0)
	- pmap_zero_page(m);

	/*
	* Map the pagetable page into the process address space, if
	@@ -4443,8 +4480,7 @@
	/* Wire up a new PDPE page */
	pml4 = pmap_allocpte_getpml4(pmap, lockp, va, true);
	if (pml4 == NULL) {
	- vm_page_unwire_noq(m);
	- vm_page_free_zero(m);
	+ pmap_free_pt_page(m, true);
	return (NULL);
	}
	KASSERT((*pml4 & PG_V) == 0,
	@@ -4471,8 +4507,7 @@
	/* Wire up a new PDE page */
	pdp = pmap_allocpte_getpdp(pmap, lockp, va, true);
	if (pdp == NULL) {
	- vm_page_unwire_noq(m);
	- vm_page_free_zero(m);
	+ pmap_free_pt_page(m, true);
	return (NULL);
	}
	KASSERT((*pdp & PG_V) == 0,
	@@ -4482,8 +4517,7 @@
	/* Wire up a new PTE page */
	pdp = pmap_allocpte_getpdp(pmap, lockp, va, false);
	if (pdp == NULL) {
	- vm_page_unwire_noq(m);
	- vm_page_free_zero(m);
	+ pmap_free_pt_page(m, true);
	return (NULL);
	}
	if ((*pdp & PG_V) == 0) {
	@@ -4492,8 +4526,7 @@
	lockp, va) == NULL) {
	pmap_allocpte_free_unref(pmap, va,
	pmap_pml4e(pmap, va));
	- vm_page_unwire_noq(m);
	- vm_page_free_zero(m);
	+ pmap_free_pt_page(m, true);
	return (NULL);
	}
	} else {
	@@ -4511,6 +4544,7 @@
	}

	pmap_resident_count_inc(pmap, 1);
	+
	return (m);
	}

	@@ -4671,14 +4705,12 @@
	pmap->pm_pmltop[LMSPML4I + i] = 0;
	}

	- vm_page_unwire_noq(m);
	- vm_page_free_zero(m);
	+ pmap_free_pt_page(m, true);

	if (pmap->pm_pmltopu != NULL) {
	m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->
	pm_pmltopu));
	- vm_page_unwire_noq(m);
	- vm_page_free(m);
	+ pmap_free_pt_page(m, false);
	}
	if (pmap->pm_type == PT_X86 &&
	(cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0)
	@@ -4787,13 +4819,11 @@
	pdpe = pmap_pdpe(kernel_pmap, kernel_vm_end);
	if ((*pdpe & X86_PG_V) == 0) {
	/* We need a new PDP entry */
	- nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDPSHIFT,
	- VM_ALLOC_INTERRUPT \| VM_ALLOC_NOOBJ \|
	- VM_ALLOC_WIRED \| VM_ALLOC_ZERO);
	+ nkpg = pmap_alloc_pt_page(kernel_vm_end >> PDPSHIFT,
	+ VM_ALLOC_WIRED \| VM_ALLOC_INTERRUPT \|
	+ VM_ALLOC_ZERO);
	if (nkpg == NULL)
	panic("pmap_growkernel: no memory to grow kernel");
	- if ((nkpg->flags & PG_ZERO) == 0)
	- pmap_zero_page(nkpg);
	paddr = VM_PAGE_TO_PHYS(nkpg);
	*pdpe = (pdp_entry_t)(paddr \| X86_PG_V \| X86_PG_RW \|
	X86_PG_A \| X86_PG_M);
	@@ -4809,13 +4839,10 @@
	continue;
	}

	- nkpg = vm_page_alloc(NULL, pmap_pde_pindex(kernel_vm_end),
	- VM_ALLOC_INTERRUPT \| VM_ALLOC_NOOBJ \| VM_ALLOC_WIRED \|
	- VM_ALLOC_ZERO);
	+ nkpg = pmap_alloc_pt_page(pmap_pde_pindex(kernel_vm_end),
	+ VM_ALLOC_WIRED \| VM_ALLOC_INTERRUPT \| VM_ALLOC_ZERO);
	if (nkpg == NULL)
	panic("pmap_growkernel: no memory to grow kernel");
	- if ((nkpg->flags & PG_ZERO) == 0)
	- pmap_zero_page(nkpg);
	paddr = VM_PAGE_TO_PHYS(nkpg);
	newpdir = paddr \| X86_PG_V \| X86_PG_RW \| X86_PG_A \| X86_PG_M;
	pde_store(pde, newpdir);
	@@ -4852,28 +4879,39 @@
	static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 };

	#ifdef PV_STATS
	-static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
	-
	-SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
	- "Current number of pv entry chunks");
	-SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
	- "Current number of pv entry chunks allocated");
	-SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
	- "Current number of pv entry chunks frees");
	-SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
	- "Number of times tried to get a chunk page but failed.");
	-
	-static long pv_entry_frees, pv_entry_allocs, pv_entry_count;
	-static int pv_entry_spare;
	-
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
	- "Current number of pv entry frees");
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
	- "Current number of pv entry allocs");
	-SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
	- "Current number of pv entries");
	-SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
	- "Current number of spare pv entries");
	+
	+static COUNTER_U64_DEFINE_EARLY(pc_chunk_count);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD,
	+ &pc_chunk_count, "Current number of pv entry cnunks");
	+
	+static COUNTER_U64_DEFINE_EARLY(pc_chunk_allocs);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD,
	+ &pc_chunk_allocs, "Total number of pv entry chunks allocated");
	+
	+static COUNTER_U64_DEFINE_EARLY(pc_chunk_frees);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD,
	+ &pc_chunk_frees, "Total number of pv entry chunks freed");
	+
	+static COUNTER_U64_DEFINE_EARLY(pc_chunk_tryfail);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD,
	+ &pc_chunk_tryfail,
	+ "Number of failed attempts to get a pv entry chunk page");
	+
	+static COUNTER_U64_DEFINE_EARLY(pv_entry_frees);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD,
	+ &pv_entry_frees, "Total number of pv entries freed");
	+
	+static COUNTER_U64_DEFINE_EARLY(pv_entry_allocs);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD,
	+ &pv_entry_allocs, "Total number of pv entries allocated");
	+
	+static COUNTER_U64_DEFINE_EARLY(pv_entry_count);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD,
	+ &pv_entry_count, "Current number of pv entries");
	+
	+static COUNTER_U64_DEFINE_EARLY(pv_entry_spare);
	+SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD,
	+ &pv_entry_spare, "Current number of spare pv entries");
	#endif

	static void
	@@ -5046,15 +5084,15 @@
	}
	/* Every freed mapping is for a 4 KB page. */
	pmap_resident_count_dec(pmap, freed);
	- PV_STAT(atomic_add_long(&pv_entry_frees, freed));
	- PV_STAT(atomic_add_int(&pv_entry_spare, freed));
	- PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
	+ PV_STAT(counter_u64_add(pv_entry_frees, freed));
	+ PV_STAT(counter_u64_add(pv_entry_spare, freed));
	+ PV_STAT(counter_u64_add(pv_entry_count, -freed));
	TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
	if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1 &&
	pc->pc_map[2] == PC_FREE2) {
	- PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
	- PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
	- PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
	+ PV_STAT(counter_u64_add(pv_entry_spare, -_NPCPV));
	+ PV_STAT(counter_u64_add(pc_chunk_count, -1));
	+ PV_STAT(counter_u64_add(pc_chunk_frees, 1));
	/* Entire chunk is free; return it. */
	m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
	dump_drop_page(m_pc->phys_addr);
	@@ -5127,9 +5165,9 @@
	int idx, field, bit;

	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
	- PV_STAT(atomic_add_long(&pv_entry_frees, 1));
	- PV_STAT(atomic_add_int(&pv_entry_spare, 1));
	- PV_STAT(atomic_subtract_long(&pv_entry_count, 1));
	+ PV_STAT(counter_u64_add(pv_entry_frees, 1));
	+ PV_STAT(counter_u64_add(pv_entry_spare, 1));
	+ PV_STAT(counter_u64_add(pv_entry_count, -1));
	pc = pv_to_chunk(pv);
	idx = pv - &pc->pc_pventry[0];
	field = idx / 64;
	@@ -5153,9 +5191,10 @@
	{
	vm_page_t m;

	- PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
	- PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
	- PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
	+ PV_STAT(counter_u64_add(pv_entry_spare, -_NPCPV));
	+ PV_STAT(counter_u64_add(pc_chunk_count, -1));
	+ PV_STAT(counter_u64_add(pc_chunk_frees, 1));
	+ counter_u64_add(pv_page_count, -1);
	/* entire chunk is free, return it */
	m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
	dump_drop_page(m->phys_addr);
	@@ -5218,7 +5257,7 @@
	vm_page_t m;

	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
	- PV_STAT(atomic_add_long(&pv_entry_allocs, 1));
	+ PV_STAT(counter_u64_add(pv_entry_allocs, 1));
	retry:
	pc = TAILQ_FIRST(&pmap->pm_pvchunk);
	if (pc != NULL) {
	@@ -5238,8 +5277,8 @@
	TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
	pc_list);
	}
	- PV_STAT(atomic_add_long(&pv_entry_count, 1));
	- PV_STAT(atomic_subtract_int(&pv_entry_spare, 1));
	+ PV_STAT(counter_u64_add(pv_entry_count, 1));
	+ PV_STAT(counter_u64_add(pv_entry_spare, -1));
	return (pv);
	}
	}
	@@ -5248,15 +5287,16 @@
	VM_ALLOC_WIRED);
	if (m == NULL) {
	if (lockp == NULL) {
	- PV_STAT(pc_chunk_tryfail++);
	+ PV_STAT(counter_u64_add(pc_chunk_tryfail, 1));
	return (NULL);
	}
	m = reclaim_pv_chunk(pmap, lockp);
	if (m == NULL)
	goto retry;
	- }
	- PV_STAT(atomic_add_int(&pc_chunk_count, 1));
	- PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
	+ } else
	+ counter_u64_add(pv_page_count, 1);
	+ PV_STAT(counter_u64_add(pc_chunk_count, 1));
	+ PV_STAT(counter_u64_add(pc_chunk_allocs, 1));
	dump_add_page(m->phys_addr);
	pc = (void *)PHYS_TO_DMAP(m->phys_addr);
	pc->pc_pmap = pmap;
	@@ -5269,8 +5309,8 @@
	mtx_unlock(&pvc->pvc_lock);
	pv = &pc->pc_pventry[0];
	TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
	- PV_STAT(atomic_add_long(&pv_entry_count, 1));
	- PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
	+ PV_STAT(counter_u64_add(pv_entry_count, 1));
	+ PV_STAT(counter_u64_add(pv_entry_spare, _NPCPV - 1));
	return (pv);
	}

	@@ -5354,9 +5394,10 @@
	if (m == NULL)
	goto retry;
	reclaimed = true;
	- }
	- PV_STAT(atomic_add_int(&pc_chunk_count, 1));
	- PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
	+ } else
	+ counter_u64_add(pv_page_count, 1);
	+ PV_STAT(counter_u64_add(pc_chunk_count, 1));
	+ PV_STAT(counter_u64_add(pc_chunk_allocs, 1));
	dump_add_page(m->phys_addr);
	pc = (void *)PHYS_TO_DMAP(m->phys_addr);
	pc->pc_pmap = pmap;
	@@ -5365,7 +5406,7 @@
	pc->pc_map[2] = PC_FREE2;
	TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
	TAILQ_INSERT_TAIL(&new_tail[vm_page_domain(m)], pc, pc_lru);
	- PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV));
	+ PV_STAT(counter_u64_add(pv_entry_spare, _NPCPV));

	/*
	* The reclaim might have freed a chunk from the current pmap.
	@@ -5440,7 +5481,7 @@
	TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
	m->md.pv_gen++;
	/* Instantiate the remaining NPTEPG - 1 pv entries. */
	- PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
	+ PV_STAT(counter_u64_add(pv_entry_allocs, NPTEPG - 1));
	va_last = va + NBPDR - PAGE_SIZE;
	for (;;) {
	pc = TAILQ_FIRST(&pmap->pm_pvchunk);
	@@ -5470,8 +5511,8 @@
	TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
	TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
	}
	- PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1));
	- PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1));
	+ PV_STAT(counter_u64_add(pv_entry_count, NPTEPG - 1));
	+ PV_STAT(counter_u64_add(pv_entry_spare, -(NPTEPG - 1)));
	}

	#if VM_NRESERVLEVEL > 0
	@@ -5718,9 +5759,8 @@
	* priority (VM_ALLOC_INTERRUPT). Otherwise, the
	* priority is normal.
	*/
	- mpte = vm_page_alloc(NULL, pmap_pde_pindex(va),
	- (in_kernel ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) \|
	- VM_ALLOC_NOOBJ \| VM_ALLOC_WIRED);
	+ mpte = pmap_alloc_pt_page(pmap_pde_pindex(va), VM_ALLOC_WIRED \|
	+ (in_kernel ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL));

	/*
	* If the allocation of the new page table page fails,
	@@ -5795,7 +5835,7 @@
	if ((oldpde & PG_MANAGED) != 0)
	pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME, lockp);

	- atomic_add_long(&pmap_pde_demotions, 1);
	+ counter_u64_add(pmap_pde_demotions, 1);
	CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx in pmap %p",
	va, pmap);
	return (TRUE);
	@@ -6517,7 +6557,7 @@
	if ((newpde & ((PG_FRAME & PDRMASK) \| PG_A \| PG_V)) != (PG_A \| PG_V) \|\|
	!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap,
	newpde))) {
	- atomic_add_long(&pmap_pde_p_failures, 1);
	+ counter_u64_add(pmap_pde_p_failures, 1);
	CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
	" in pmap %p", va, pmap);
	return;
	@@ -6542,7 +6582,7 @@
	setpte:
	oldpte = *pte;
	if ((oldpte & (PG_FRAME \| PG_A \| PG_V)) != pa) {
	- atomic_add_long(&pmap_pde_p_failures, 1);
	+ counter_u64_add(pmap_pde_p_failures, 1);
	CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
	" in pmap %p", va, pmap);
	return;
	@@ -6560,7 +6600,7 @@
	(va & ~PDRMASK), pmap);
	}
	if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
	- atomic_add_long(&pmap_pde_p_failures, 1);
	+ counter_u64_add(pmap_pde_p_failures, 1);
	CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
	" in pmap %p", va, pmap);
	return;
	@@ -6580,7 +6620,7 @@
	KASSERT(mpte->pindex == pmap_pde_pindex(va),
	("pmap_promote_pde: page table page's pindex is wrong"));
	if (pmap_insert_pt_page(pmap, mpte, true)) {
	- atomic_add_long(&pmap_pde_p_failures, 1);
	+ counter_u64_add(pmap_pde_p_failures, 1);
	CTR2(KTR_PMAP,
	"pmap_promote_pde: failure for va %#lx in pmap %p", va,
	pmap);
	@@ -6606,7 +6646,7 @@
	else
	pde_store(pde, PG_PROMOTED \| PG_PS \| newpde);

	- atomic_add_long(&pmap_pde_promotions, 1);
	+ counter_u64_add(pmap_pde_promotions, 1);
	CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx"
	" in pmap %p", va, pmap);
	}
	@@ -7182,7 +7222,7 @@
	*/
	pde_store(pde, newpde);

	- atomic_add_long(&pmap_pde_mappings, 1);
	+ counter_u64_add(pmap_pde_mappings, 1);
	CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx in pmap %p",
	va, pmap);
	return (KERN_SUCCESS);
	@@ -7444,7 +7484,7 @@
	pde_store(pde, pa \| PG_PS \| PG_M \| PG_A \|
	PG_U \| PG_RW \| PG_V);
	pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE);
	- atomic_add_long(&pmap_pde_mappings, 1);
	+ counter_u64_add(pmap_pde_mappings, 1);
	} else {
	/* Continue on if the PDE is already valid. */
	pdpg->ref_count--;
	@@ -7672,7 +7712,7 @@
	*pde = srcptepaddr & ~PG_W;
	pmap_resident_count_inc(dst_pmap, NBPDR /
	PAGE_SIZE);
	- atomic_add_long(&pmap_pde_mappings, 1);
	+ counter_u64_add(pmap_pde_mappings, 1);
	} else
	pmap_abort_ptp(dst_pmap, addr, dst_pdpg);
	continue;
	@@ -8157,9 +8197,9 @@
	freed++;
	}
	}
	- PV_STAT(atomic_add_long(&pv_entry_frees, freed));
	- PV_STAT(atomic_add_int(&pv_entry_spare, freed));
	- PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
	+ PV_STAT(counter_u64_add(pv_entry_frees, freed));
	+ PV_STAT(counter_u64_add(pv_entry_spare, freed));
	+ PV_STAT(counter_u64_add(pv_entry_count, -freed));
	if (allfree) {
	TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
	TAILQ_INSERT_TAIL(&free_chunks[pc_to_domain(pc)], pc, pc_list);
	@@ -9067,8 +9107,9 @@
	oldpdpe = *pdpe;
	KASSERT((oldpdpe & (PG_PS \| PG_V)) == (PG_PS \| PG_V),
	("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V"));
	- if ((pdpg = vm_page_alloc(NULL, va >> PDPSHIFT, VM_ALLOC_INTERRUPT \|
	- VM_ALLOC_NOOBJ \| VM_ALLOC_WIRED)) == NULL) {
	+ pdpg = pmap_alloc_pt_page(va >> PDPSHIFT,
	+ VM_ALLOC_WIRED \| VM_ALLOC_INTERRUPT);
	+ if (pdpg == NULL) {
	CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx"
	" in pmap %p", va, pmap);
	return (FALSE);
	@@ -9100,7 +9141,7 @@
	*/
	pmap_invalidate_page(pmap, (vm_offset_t)vtopde(va));

	- pmap_pdpe_demotions++;
	+ counter_u64_add(pmap_pdpe_demotions, 1);
	CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx"
	" in pmap %p", va, pmap);
	return (TRUE);
	@@ -9621,8 +9662,6 @@

	PCPU_SET(kcr3, kcr3 \| CR3_PCID_SAVE);
	PCPU_SET(ucr3, ucr3 \| CR3_PCID_SAVE);
	- if (cached)
	- PCPU_INC(pm_save_cnt);

	pmap_activate_sw_pti_post(td, pmap);
	}
	@@ -9642,8 +9681,6 @@
	load_cr3(pmap->pm_cr3 \| pmap->pm_pcids[cpuid].pm_pcid \|
	cached);
	PCPU_SET(curpmap, pmap);
	- if (cached)
	- PCPU_INC(pm_save_cnt);
	}

	static void
	@@ -10084,13 +10121,7 @@
	static vm_page_t
	pmap_large_map_getptp_unlocked(void)
	{
	- vm_page_t m;
	-
	- m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL \| VM_ALLOC_NOOBJ \|
	- VM_ALLOC_ZERO);
	- if (m != NULL && (m->flags & PG_ZERO) == 0)
	- pmap_zero_page(m);
	- return (m);
	+ return (pmap_alloc_pt_page(0, VM_ALLOC_NORMAL \| VM_ALLOC_ZERO));
	}

	static vm_page_t
	Index: sys/amd64/include/pcpu.h
	===================================================================
	--- sys/amd64/include/pcpu.h
	+++ sys/amd64/include/pcpu.h
	@@ -76,7 +76,6 @@
	struct system_segment_descriptor *pc_ldt; \
	/* Pointer to the CPU TSS descriptor */ \
	struct system_segment_descriptor *pc_tss; \
	- uint64_t pc_pm_save_cnt; \
	u_int pc_cmci_mask; /* MCx banks for CMCI */ \
	uint64_t pc_dbreg[16]; /* ddb debugging regs */ \
	uint64_t pc_pti_stack[PC_PTI_STACK_SZ]; \
	@@ -89,7 +88,7 @@
	uint32_t pc_ibpb_set; \
	void *pc_mds_buf; \
	void *pc_mds_buf64; \
	- uint32_t pc_pad[2]; \
	+ uint32_t pc_pad[4]; \
	uint8_t pc_mds_tmp[64]; \
	u_int pc_ipi_bitmap; \
	struct amd64tss pc_common_tss; \

File Metadata

Mime Type: text/plain
Expires: Mon, Oct 12, 7:28 AM (10 h, 12 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 40660326
Default Alt Text: D28923.id84908.diff (27 KB)

D28923.id84908.diffNo OneTemporaryActions

D28923.id84908.diffView Options

File Metadata

Event Timeline

D28923.id84908.diff
No OneTemporary
Actions

D28923.id84908.diff
View Options