Index: sys/amd64/amd64/genassym.c =================================================================== --- sys/amd64/amd64/genassym.c +++ sys/amd64/amd64/genassym.c @@ -227,7 +227,6 @@ ASSYM(PC_LDT, offsetof(struct pcpu, pc_ldt)); ASSYM(PC_COMMONTSS, offsetof(struct pcpu, pc_common_tss)); ASSYM(PC_TSS, offsetof(struct pcpu, pc_tss)); -ASSYM(PC_PM_SAVE_CNT, offsetof(struct pcpu, pc_pm_save_cnt)); ASSYM(PC_KCR3, offsetof(struct pcpu, pc_kcr3)); ASSYM(PC_UCR3, offsetof(struct pcpu, pc_ucr3)); ASSYM(PC_UCR3_LOAD_MASK, offsetof(struct pcpu, pc_ucr3_load_mask)); Index: sys/amd64/amd64/pmap.c =================================================================== --- sys/amd64/amd64/pmap.c +++ sys/amd64/amd64/pmap.c @@ -114,7 +114,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -535,22 +537,6 @@ static int pmap_pkru_deassign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva); static void pmap_pkru_deassign_all(pmap_t pmap); -static int -pmap_pcid_save_cnt_proc(SYSCTL_HANDLER_ARGS) -{ - int i; - uint64_t res; - - res = 0; - CPU_FOREACH(i) { - res += cpuid_to_pcpu[i]->pc_pm_save_cnt; - } - return (sysctl_handle_64(oidp, &res, 0, req)); -} -SYSCTL_PROC(_vm_pmap, OID_AUTO, pcid_save_cnt, CTLTYPE_U64 | CTLFLAG_RD | - CTLFLAG_MPSAFE, NULL, 0, pmap_pcid_save_cnt_proc, "QU", - "Count of saved TLB context on switch"); - static LIST_HEAD(, pmap_invl_gen) pmap_invl_gen_tracker = LIST_HEAD_INITIALIZER(&pmap_invl_gen_tracker); static struct mtx invl_gen_mtx; @@ -769,19 +755,30 @@ return (res); } +static COUNTER_U64_DEFINE_EARLY(pv_page_count); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_page_count, CTLFLAG_RD, + &pv_page_count, "Current number of allocated pv pages"); + +static COUNTER_U64_DEFINE_EARLY(pt_page_count); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pt_page_count, CTLFLAG_RD, + &pt_page_count, "Current number of allocated page table pages"); + #ifdef PV_STATS -static long invl_start_restart; -SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_start_restart, CTLFLAG_RD, - &invl_start_restart, 0, - ""); -static long invl_finish_restart; -SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_finish_restart, CTLFLAG_RD, - &invl_finish_restart, 0, - ""); + +static COUNTER_U64_DEFINE_EARLY(invl_start_restart); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_start_restart, + CTLFLAG_RD, &invl_start_restart, + "Number of delayed TLB invalidation request restarts"); + +static COUNTER_U64_DEFINE_EARLY(invl_finish_restart); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_finish_restart, CTLFLAG_RD, + &invl_finish_restart, + "Number of delayed TLB invalidation completion restarts"); + static int invl_max_qlen; SYSCTL_INT(_vm_pmap, OID_AUTO, invl_max_qlen, CTLFLAG_RD, &invl_max_qlen, 0, - ""); + "Maximum delayed TLB invalidation request queue length"); #endif #define di_delay locks_delay @@ -819,7 +816,7 @@ PV_STAT(i++); prevl = (uintptr_t)atomic_load_ptr(&p->next); if ((prevl & PMAP_INVL_GEN_NEXT_INVALID) != 0) { - PV_STAT(atomic_add_long(&invl_start_restart, 1)); + PV_STAT(counter_u64_add(invl_start_restart, 1)); lock_delay(&lda); goto again; } @@ -833,7 +830,7 @@ #endif if (!pmap_di_load_invl(p, &prev) || prev.next != NULL) { - PV_STAT(atomic_add_long(&invl_start_restart, 1)); + PV_STAT(counter_u64_add(invl_start_restart, 1)); lock_delay(&lda); goto again; } @@ -862,7 +859,7 @@ */ if (!pmap_di_store_invl(p, &prev, &new_prev)) { critical_exit(); - PV_STAT(atomic_add_long(&invl_start_restart, 1)); + PV_STAT(counter_u64_add(invl_start_restart, 1)); lock_delay(&lda); goto again; } @@ -926,7 +923,7 @@ for (p = &pmap_invl_gen_head; p != NULL; p = (void *)prevl) { prevl = (uintptr_t)atomic_load_ptr(&p->next); if ((prevl & PMAP_INVL_GEN_NEXT_INVALID) != 0) { - PV_STAT(atomic_add_long(&invl_finish_restart, 1)); + PV_STAT(counter_u64_add(invl_finish_restart, 1)); lock_delay(&lda); goto again; } @@ -939,7 +936,7 @@ * thread before us finished its DI and started it again. */ if (__predict_false(p == NULL)) { - PV_STAT(atomic_add_long(&invl_finish_restart, 1)); + PV_STAT(counter_u64_add(invl_finish_restart, 1)); lock_delay(&lda); goto again; } @@ -951,7 +948,7 @@ atomic_clear_ptr((uintptr_t *)&invl_gen->next, PMAP_INVL_GEN_NEXT_INVALID); critical_exit(); - PV_STAT(atomic_add_long(&invl_finish_restart, 1)); + PV_STAT(counter_u64_add(invl_finish_restart, 1)); lock_delay(&lda); goto again; } @@ -987,12 +984,15 @@ #endif #ifdef PV_STATS -static long invl_wait; -SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_wait, CTLFLAG_RD, &invl_wait, 0, +static COUNTER_U64_DEFINE_EARLY(invl_wait); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_wait, + CTLFLAG_RD, &invl_wait, "Number of times DI invalidation blocked pmap_remove_all/write"); -static long invl_wait_slow; -SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_wait_slow, CTLFLAG_RD, &invl_wait_slow, 0, - "Number of slow invalidation waits for lockless DI"); + +static COUNTER_U64_DEFINE_EARLY(invl_wait_slow); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_wait_slow, CTLFLAG_RD, + &invl_wait_slow, "Number of slow invalidation waits for lockless DI"); + #endif #ifdef NUMA @@ -1066,7 +1066,7 @@ while (*m_gen > pmap_invl_gen) { #ifdef PV_STATS if (!accounted) { - atomic_add_long(&invl_wait, 1); + counter_u64_add(invl_wait, 1); accounted = true; } #endif @@ -1086,7 +1086,7 @@ lock_delay_arg_init(&lda, &di_delay); while (*m_gen > atomic_load_long(&pmap_invl_gen_head.gen)) { if (fast || !pmap_invl_callout_inited) { - PV_STAT(atomic_add_long(&invl_wait, 1)); + PV_STAT(counter_u64_add(invl_wait, 1)); lock_delay(&lda); fast = false; } else { @@ -1119,7 +1119,7 @@ atomic_load_long(&pmap_invl_gen_head.gen)) { callout_reset(&pmap_invl_callout, 1, pmap_delayed_invl_callout_func, NULL); - PV_STAT(atomic_add_long(&invl_wait_slow, 1)); + PV_STAT(counter_u64_add(invl_wait_slow, 1)); pmap_delayed_invl_wait_block(m_gen, &pmap_invl_gen_head.gen); } @@ -1288,6 +1288,9 @@ struct spglist *free); static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); +static vm_page_t pmap_alloc_pt_page(vm_pindex_t, int); +static void pmap_free_pt_page(vm_page_t, bool); + /********************/ /* Inline functions */ /********************/ @@ -2459,28 +2462,28 @@ static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "2MB page mapping counters"); -static u_long pmap_pde_demotions; -SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD, - &pmap_pde_demotions, 0, "2MB page demotions"); +static COUNTER_U64_DEFINE_EARLY(pmap_pde_demotions); +SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, demotions, + CTLFLAG_RD, &pmap_pde_demotions, "2MB page demotions"); -static u_long pmap_pde_mappings; -SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD, - &pmap_pde_mappings, 0, "2MB page mappings"); +static COUNTER_U64_DEFINE_EARLY(pmap_pde_mappings); +SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD, + &pmap_pde_mappings, "2MB page mappings"); -static u_long pmap_pde_p_failures; -SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD, - &pmap_pde_p_failures, 0, "2MB page promotion failures"); +static COUNTER_U64_DEFINE_EARLY(pmap_pde_p_failures); +SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD, + &pmap_pde_p_failures, "2MB page promotion failures"); -static u_long pmap_pde_promotions; -SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD, - &pmap_pde_promotions, 0, "2MB page promotions"); +static COUNTER_U64_DEFINE_EARLY(pmap_pde_promotions); +SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD, + &pmap_pde_promotions, "2MB page promotions"); static SYSCTL_NODE(_vm_pmap, OID_AUTO, pdpe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "1GB page mapping counters"); -static u_long pmap_pdpe_demotions; -SYSCTL_ULONG(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD, - &pmap_pdpe_demotions, 0, "1GB page demotions"); +static COUNTER_U64_DEFINE_EARLY(pmap_pdpe_demotions); +SYSCTL_COUNTER_U64(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD, + &pmap_pdpe_demotions, "1GB page demotions"); /*************************************************** * Low level helper routines..... @@ -4016,6 +4019,8 @@ pmap_unwire_ptp(pmap, va, pml4pg, free); } + counter_u64_add(pt_page_count, -1); + /* * Put page on a list so that it is released after * *ALL* TLB shootdown is done @@ -4179,6 +4184,42 @@ pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, FALSE); } +/* Allocate a page table page and do related bookkeeping */ +static vm_page_t +pmap_alloc_pt_page(vm_pindex_t pindex, int flags) +{ + vm_page_t m; + + m = vm_page_alloc(NULL, pindex, flags | VM_ALLOC_NOOBJ); + if (__predict_false(m == NULL)) + return (NULL); + + counter_u64_add(pt_page_count, 1); + if ((flags & VM_ALLOC_ZERO) != 0 && (m->flags & PG_ZERO) == 0) + pmap_zero_page(m); + + return (m); +} + +static void +pmap_free_pt_page(vm_page_t m, bool zerofilled) +{ + /* + * This function assumes the page will need to be unwired, + * even though the counterpart allocation in pmap_alloc_pt_page() + * doesn't enforce VM_ALLOC_WIRED. However, all current uses + * of pmap_free_pt_page() require unwiring. The case in which + * a PT page doesn't require unwiring because its ref_count has + * naturally reached 0 is handled through _pmap_unwire_ptp(). + */ + vm_page_unwire_noq(m); + if (zerofilled) + vm_page_free_zero(m); + else + vm_page_free(m); + counter_u64_add(pt_page_count, -1); +} + /* * Initialize a preallocated and zeroed pmap structure, * such as one in a vmspace structure. @@ -4193,8 +4234,8 @@ /* * allocate the page directory page */ - pmltop_pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | - VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK); + pmltop_pg = pmap_alloc_pt_page(0, VM_ALLOC_NORMAL | VM_ALLOC_WIRED | + VM_ALLOC_ZERO | VM_ALLOC_WAITOK); pmltop_phys = VM_PAGE_TO_PHYS(pmltop_pg); pmap->pm_pmltop = (pml5_entry_t *)PHYS_TO_DMAP(pmltop_phys); @@ -4208,8 +4249,6 @@ pmap->pm_pmltopu = NULL; pmap->pm_type = pm_type; - if ((pmltop_pg->flags & PG_ZERO) == 0) - pagezero(pmap->pm_pmltop); /* * Do not install the host kernel mappings in the nested page @@ -4225,8 +4264,8 @@ else pmap_pinit_pml4(pmltop_pg); if ((curproc->p_md.md_flags & P_MD_KPTI) != 0) { - pmltop_pgu = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | - VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_WAITOK); + pmltop_pgu = pmap_alloc_pt_page(0, VM_ALLOC_WIRED | + VM_ALLOC_NORMAL | VM_ALLOC_WAITOK); pmap->pm_pmltopu = (pml4_entry_t *)PHYS_TO_DMAP( VM_PAGE_TO_PHYS(pmltop_pgu)); if (pmap_is_la57(pmap)) @@ -4411,11 +4450,9 @@ /* * Allocate a page table page. */ - if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | - VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) + m = pmap_alloc_pt_page(ptepindex, VM_ALLOC_WIRED | VM_ALLOC_ZERO); + if (m == NULL) return (NULL); - if ((m->flags & PG_ZERO) == 0) - pmap_zero_page(m); /* * Map the pagetable page into the process address space, if @@ -4443,8 +4480,7 @@ /* Wire up a new PDPE page */ pml4 = pmap_allocpte_getpml4(pmap, lockp, va, true); if (pml4 == NULL) { - vm_page_unwire_noq(m); - vm_page_free_zero(m); + pmap_free_pt_page(m, true); return (NULL); } KASSERT((*pml4 & PG_V) == 0, @@ -4471,8 +4507,7 @@ /* Wire up a new PDE page */ pdp = pmap_allocpte_getpdp(pmap, lockp, va, true); if (pdp == NULL) { - vm_page_unwire_noq(m); - vm_page_free_zero(m); + pmap_free_pt_page(m, true); return (NULL); } KASSERT((*pdp & PG_V) == 0, @@ -4482,8 +4517,7 @@ /* Wire up a new PTE page */ pdp = pmap_allocpte_getpdp(pmap, lockp, va, false); if (pdp == NULL) { - vm_page_unwire_noq(m); - vm_page_free_zero(m); + pmap_free_pt_page(m, true); return (NULL); } if ((*pdp & PG_V) == 0) { @@ -4492,8 +4526,7 @@ lockp, va) == NULL) { pmap_allocpte_free_unref(pmap, va, pmap_pml4e(pmap, va)); - vm_page_unwire_noq(m); - vm_page_free_zero(m); + pmap_free_pt_page(m, true); return (NULL); } } else { @@ -4511,6 +4544,7 @@ } pmap_resident_count_inc(pmap, 1); + return (m); } @@ -4671,14 +4705,12 @@ pmap->pm_pmltop[LMSPML4I + i] = 0; } - vm_page_unwire_noq(m); - vm_page_free_zero(m); + pmap_free_pt_page(m, true); if (pmap->pm_pmltopu != NULL) { m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap-> pm_pmltopu)); - vm_page_unwire_noq(m); - vm_page_free(m); + pmap_free_pt_page(m, false); } if (pmap->pm_type == PT_X86 && (cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0) @@ -4787,13 +4819,11 @@ pdpe = pmap_pdpe(kernel_pmap, kernel_vm_end); if ((*pdpe & X86_PG_V) == 0) { /* We need a new PDP entry */ - nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDPSHIFT, - VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | - VM_ALLOC_WIRED | VM_ALLOC_ZERO); + nkpg = pmap_alloc_pt_page(kernel_vm_end >> PDPSHIFT, + VM_ALLOC_WIRED | VM_ALLOC_INTERRUPT | + VM_ALLOC_ZERO); if (nkpg == NULL) panic("pmap_growkernel: no memory to grow kernel"); - if ((nkpg->flags & PG_ZERO) == 0) - pmap_zero_page(nkpg); paddr = VM_PAGE_TO_PHYS(nkpg); *pdpe = (pdp_entry_t)(paddr | X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M); @@ -4809,13 +4839,10 @@ continue; } - nkpg = vm_page_alloc(NULL, pmap_pde_pindex(kernel_vm_end), - VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | - VM_ALLOC_ZERO); + nkpg = pmap_alloc_pt_page(pmap_pde_pindex(kernel_vm_end), + VM_ALLOC_WIRED | VM_ALLOC_INTERRUPT | VM_ALLOC_ZERO); if (nkpg == NULL) panic("pmap_growkernel: no memory to grow kernel"); - if ((nkpg->flags & PG_ZERO) == 0) - pmap_zero_page(nkpg); paddr = VM_PAGE_TO_PHYS(nkpg); newpdir = paddr | X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M; pde_store(pde, newpdir); @@ -4852,28 +4879,39 @@ static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 }; #ifdef PV_STATS -static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; - -SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, - "Current number of pv entry chunks"); -SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, - "Current number of pv entry chunks allocated"); -SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, - "Current number of pv entry chunks frees"); -SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0, - "Number of times tried to get a chunk page but failed."); - -static long pv_entry_frees, pv_entry_allocs, pv_entry_count; -static int pv_entry_spare; - -SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0, - "Current number of pv entry frees"); -SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0, - "Current number of pv entry allocs"); -SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, - "Current number of pv entries"); -SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0, - "Current number of spare pv entries"); + +static COUNTER_U64_DEFINE_EARLY(pc_chunk_count); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, + &pc_chunk_count, "Current number of pv entry cnunks"); + +static COUNTER_U64_DEFINE_EARLY(pc_chunk_allocs); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, + &pc_chunk_allocs, "Total number of pv entry chunks allocated"); + +static COUNTER_U64_DEFINE_EARLY(pc_chunk_frees); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, + &pc_chunk_frees, "Total number of pv entry chunks freed"); + +static COUNTER_U64_DEFINE_EARLY(pc_chunk_tryfail); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, + &pc_chunk_tryfail, + "Number of failed attempts to get a pv entry chunk page"); + +static COUNTER_U64_DEFINE_EARLY(pv_entry_frees); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, + &pv_entry_frees, "Total number of pv entries freed"); + +static COUNTER_U64_DEFINE_EARLY(pv_entry_allocs); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, + &pv_entry_allocs, "Total number of pv entries allocated"); + +static COUNTER_U64_DEFINE_EARLY(pv_entry_count); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, + &pv_entry_count, "Current number of pv entries"); + +static COUNTER_U64_DEFINE_EARLY(pv_entry_spare); +SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, + &pv_entry_spare, "Current number of spare pv entries"); #endif static void @@ -5046,15 +5084,15 @@ } /* Every freed mapping is for a 4 KB page. */ pmap_resident_count_dec(pmap, freed); - PV_STAT(atomic_add_long(&pv_entry_frees, freed)); - PV_STAT(atomic_add_int(&pv_entry_spare, freed)); - PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); + PV_STAT(counter_u64_add(pv_entry_frees, freed)); + PV_STAT(counter_u64_add(pv_entry_spare, freed)); + PV_STAT(counter_u64_add(pv_entry_count, -freed)); TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1 && pc->pc_map[2] == PC_FREE2) { - PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); - PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); - PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); + PV_STAT(counter_u64_add(pv_entry_spare, -_NPCPV)); + PV_STAT(counter_u64_add(pc_chunk_count, -1)); + PV_STAT(counter_u64_add(pc_chunk_frees, 1)); /* Entire chunk is free; return it. */ m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); dump_drop_page(m_pc->phys_addr); @@ -5127,9 +5165,9 @@ int idx, field, bit; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - PV_STAT(atomic_add_long(&pv_entry_frees, 1)); - PV_STAT(atomic_add_int(&pv_entry_spare, 1)); - PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); + PV_STAT(counter_u64_add(pv_entry_frees, 1)); + PV_STAT(counter_u64_add(pv_entry_spare, 1)); + PV_STAT(counter_u64_add(pv_entry_count, -1)); pc = pv_to_chunk(pv); idx = pv - &pc->pc_pventry[0]; field = idx / 64; @@ -5153,9 +5191,10 @@ { vm_page_t m; - PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); - PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); - PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); + PV_STAT(counter_u64_add(pv_entry_spare, -_NPCPV)); + PV_STAT(counter_u64_add(pc_chunk_count, -1)); + PV_STAT(counter_u64_add(pc_chunk_frees, 1)); + counter_u64_add(pv_page_count, -1); /* entire chunk is free, return it */ m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); dump_drop_page(m->phys_addr); @@ -5218,7 +5257,7 @@ vm_page_t m; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); + PV_STAT(counter_u64_add(pv_entry_allocs, 1)); retry: pc = TAILQ_FIRST(&pmap->pm_pvchunk); if (pc != NULL) { @@ -5238,8 +5277,8 @@ TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); } - PV_STAT(atomic_add_long(&pv_entry_count, 1)); - PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); + PV_STAT(counter_u64_add(pv_entry_count, 1)); + PV_STAT(counter_u64_add(pv_entry_spare, -1)); return (pv); } } @@ -5248,15 +5287,16 @@ VM_ALLOC_WIRED); if (m == NULL) { if (lockp == NULL) { - PV_STAT(pc_chunk_tryfail++); + PV_STAT(counter_u64_add(pc_chunk_tryfail, 1)); return (NULL); } m = reclaim_pv_chunk(pmap, lockp); if (m == NULL) goto retry; - } - PV_STAT(atomic_add_int(&pc_chunk_count, 1)); - PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); + } else + counter_u64_add(pv_page_count, 1); + PV_STAT(counter_u64_add(pc_chunk_count, 1)); + PV_STAT(counter_u64_add(pc_chunk_allocs, 1)); dump_add_page(m->phys_addr); pc = (void *)PHYS_TO_DMAP(m->phys_addr); pc->pc_pmap = pmap; @@ -5269,8 +5309,8 @@ mtx_unlock(&pvc->pvc_lock); pv = &pc->pc_pventry[0]; TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); - PV_STAT(atomic_add_long(&pv_entry_count, 1)); - PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); + PV_STAT(counter_u64_add(pv_entry_count, 1)); + PV_STAT(counter_u64_add(pv_entry_spare, _NPCPV - 1)); return (pv); } @@ -5354,9 +5394,10 @@ if (m == NULL) goto retry; reclaimed = true; - } - PV_STAT(atomic_add_int(&pc_chunk_count, 1)); - PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); + } else + counter_u64_add(pv_page_count, 1); + PV_STAT(counter_u64_add(pc_chunk_count, 1)); + PV_STAT(counter_u64_add(pc_chunk_allocs, 1)); dump_add_page(m->phys_addr); pc = (void *)PHYS_TO_DMAP(m->phys_addr); pc->pc_pmap = pmap; @@ -5365,7 +5406,7 @@ pc->pc_map[2] = PC_FREE2; TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&new_tail[vm_page_domain(m)], pc, pc_lru); - PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV)); + PV_STAT(counter_u64_add(pv_entry_spare, _NPCPV)); /* * The reclaim might have freed a chunk from the current pmap. @@ -5440,7 +5481,7 @@ TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; /* Instantiate the remaining NPTEPG - 1 pv entries. */ - PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1)); + PV_STAT(counter_u64_add(pv_entry_allocs, NPTEPG - 1)); va_last = va + NBPDR - PAGE_SIZE; for (;;) { pc = TAILQ_FIRST(&pmap->pm_pvchunk); @@ -5470,8 +5511,8 @@ TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); } - PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1)); - PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1)); + PV_STAT(counter_u64_add(pv_entry_count, NPTEPG - 1)); + PV_STAT(counter_u64_add(pv_entry_spare, -(NPTEPG - 1))); } #if VM_NRESERVLEVEL > 0 @@ -5718,9 +5759,8 @@ * priority (VM_ALLOC_INTERRUPT). Otherwise, the * priority is normal. */ - mpte = vm_page_alloc(NULL, pmap_pde_pindex(va), - (in_kernel ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | - VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); + mpte = pmap_alloc_pt_page(pmap_pde_pindex(va), VM_ALLOC_WIRED | + (in_kernel ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL)); /* * If the allocation of the new page table page fails, @@ -5795,7 +5835,7 @@ if ((oldpde & PG_MANAGED) != 0) pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME, lockp); - atomic_add_long(&pmap_pde_demotions, 1); + counter_u64_add(pmap_pde_demotions, 1); CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx in pmap %p", va, pmap); return (TRUE); @@ -6517,7 +6557,7 @@ if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V) || !pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap, newpde))) { - atomic_add_long(&pmap_pde_p_failures, 1); + counter_u64_add(pmap_pde_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx" " in pmap %p", va, pmap); return; @@ -6542,7 +6582,7 @@ setpte: oldpte = *pte; if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) { - atomic_add_long(&pmap_pde_p_failures, 1); + counter_u64_add(pmap_pde_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx" " in pmap %p", va, pmap); return; @@ -6560,7 +6600,7 @@ (va & ~PDRMASK), pmap); } if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) { - atomic_add_long(&pmap_pde_p_failures, 1); + counter_u64_add(pmap_pde_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx" " in pmap %p", va, pmap); return; @@ -6580,7 +6620,7 @@ KASSERT(mpte->pindex == pmap_pde_pindex(va), ("pmap_promote_pde: page table page's pindex is wrong")); if (pmap_insert_pt_page(pmap, mpte, true)) { - atomic_add_long(&pmap_pde_p_failures, 1); + counter_u64_add(pmap_pde_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx in pmap %p", va, pmap); @@ -6606,7 +6646,7 @@ else pde_store(pde, PG_PROMOTED | PG_PS | newpde); - atomic_add_long(&pmap_pde_promotions, 1); + counter_u64_add(pmap_pde_promotions, 1); CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx" " in pmap %p", va, pmap); } @@ -7182,7 +7222,7 @@ */ pde_store(pde, newpde); - atomic_add_long(&pmap_pde_mappings, 1); + counter_u64_add(pmap_pde_mappings, 1); CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx in pmap %p", va, pmap); return (KERN_SUCCESS); @@ -7444,7 +7484,7 @@ pde_store(pde, pa | PG_PS | PG_M | PG_A | PG_U | PG_RW | PG_V); pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE); - atomic_add_long(&pmap_pde_mappings, 1); + counter_u64_add(pmap_pde_mappings, 1); } else { /* Continue on if the PDE is already valid. */ pdpg->ref_count--; @@ -7672,7 +7712,7 @@ *pde = srcptepaddr & ~PG_W; pmap_resident_count_inc(dst_pmap, NBPDR / PAGE_SIZE); - atomic_add_long(&pmap_pde_mappings, 1); + counter_u64_add(pmap_pde_mappings, 1); } else pmap_abort_ptp(dst_pmap, addr, dst_pdpg); continue; @@ -8157,9 +8197,9 @@ freed++; } } - PV_STAT(atomic_add_long(&pv_entry_frees, freed)); - PV_STAT(atomic_add_int(&pv_entry_spare, freed)); - PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); + PV_STAT(counter_u64_add(pv_entry_frees, freed)); + PV_STAT(counter_u64_add(pv_entry_spare, freed)); + PV_STAT(counter_u64_add(pv_entry_count, -freed)); if (allfree) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&free_chunks[pc_to_domain(pc)], pc, pc_list); @@ -9067,8 +9107,9 @@ oldpdpe = *pdpe; KASSERT((oldpdpe & (PG_PS | PG_V)) == (PG_PS | PG_V), ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); - if ((pdpg = vm_page_alloc(NULL, va >> PDPSHIFT, VM_ALLOC_INTERRUPT | - VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { + pdpg = pmap_alloc_pt_page(va >> PDPSHIFT, + VM_ALLOC_WIRED | VM_ALLOC_INTERRUPT); + if (pdpg == NULL) { CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx" " in pmap %p", va, pmap); return (FALSE); @@ -9100,7 +9141,7 @@ */ pmap_invalidate_page(pmap, (vm_offset_t)vtopde(va)); - pmap_pdpe_demotions++; + counter_u64_add(pmap_pdpe_demotions, 1); CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx" " in pmap %p", va, pmap); return (TRUE); @@ -9621,8 +9662,6 @@ PCPU_SET(kcr3, kcr3 | CR3_PCID_SAVE); PCPU_SET(ucr3, ucr3 | CR3_PCID_SAVE); - if (cached) - PCPU_INC(pm_save_cnt); pmap_activate_sw_pti_post(td, pmap); } @@ -9642,8 +9681,6 @@ load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid | cached); PCPU_SET(curpmap, pmap); - if (cached) - PCPU_INC(pm_save_cnt); } static void @@ -10084,13 +10121,7 @@ static vm_page_t pmap_large_map_getptp_unlocked(void) { - vm_page_t m; - - m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | - VM_ALLOC_ZERO); - if (m != NULL && (m->flags & PG_ZERO) == 0) - pmap_zero_page(m); - return (m); + return (pmap_alloc_pt_page(0, VM_ALLOC_NORMAL | VM_ALLOC_ZERO)); } static vm_page_t Index: sys/amd64/include/pcpu.h =================================================================== --- sys/amd64/include/pcpu.h +++ sys/amd64/include/pcpu.h @@ -76,7 +76,6 @@ struct system_segment_descriptor *pc_ldt; \ /* Pointer to the CPU TSS descriptor */ \ struct system_segment_descriptor *pc_tss; \ - uint64_t pc_pm_save_cnt; \ u_int pc_cmci_mask; /* MCx banks for CMCI */ \ uint64_t pc_dbreg[16]; /* ddb debugging regs */ \ uint64_t pc_pti_stack[PC_PTI_STACK_SZ]; \ @@ -89,7 +88,7 @@ uint32_t pc_ibpb_set; \ void *pc_mds_buf; \ void *pc_mds_buf64; \ - uint32_t pc_pad[2]; \ + uint32_t pc_pad[4]; \ uint8_t pc_mds_tmp[64]; \ u_int pc_ipi_bitmap; \ struct amd64tss pc_common_tss; \