Index: head/libexec/rpc.rstatd/rstat_proc.c =================================================================== --- head/libexec/rpc.rstatd/rstat_proc.c +++ head/libexec/rpc.rstatd/rstat_proc.c @@ -170,6 +170,7 @@ struct timeval tm, btm; int mib[6]; size_t len; + uint64_t val; int ifcount; #ifdef DEBUG @@ -229,11 +230,12 @@ #endif #define FETCH_CNT(stat, cnt) do { \ - len = sizeof((stat)); \ - if (sysctlbyname("vm.stats." #cnt , &(stat), &len, 0, 0) < 0) { \ - syslog(LOG_ERR, "sysctl(vm.stats." #cnt "): %m"); \ + len = sizeof(uint64_t); \ + if (sysctlbyname("vm.stats." #cnt , &val, &len, NULL, 0) < 0) { \ + syslog(LOG_ERR, "sysctl(vm.stats." #cnt "): %m"); \ exit(1); \ } \ + stat = val; \ } while (0) FETCH_CNT(stats_all.s1.v_pgpgin, vm.v_vnodepgsin); Index: head/sys/amd64/amd64/trap.c =================================================================== --- head/sys/amd64/amd64/trap.c +++ head/sys/amd64/amd64/trap.c @@ -176,7 +176,7 @@ register_t addr = 0; ksiginfo_t ksi; - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); type = frame->tf_trapno; #ifdef SMP Index: head/sys/amd64/include/atomic.h =================================================================== --- head/sys/amd64/include/atomic.h +++ head/sys/amd64/include/atomic.h @@ -348,7 +348,7 @@ * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in amd64/vm_machdep.c ensures that the value is correct. */ -#define OFFSETOF_MONITORBUF 0x180 +#define OFFSETOF_MONITORBUF 0x100 #if defined(SMP) static __inline void Index: head/sys/amd64/include/counter.h =================================================================== --- head/sys/amd64/include/counter.h +++ head/sys/amd64/include/counter.h @@ -31,7 +31,9 @@ #include -extern struct pcpu __pcpu[1]; +extern struct pcpu __pcpu[]; + +#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter #define counter_enter() do {} while (0) #define counter_exit() do {} while (0) Index: head/sys/amd64/include/pcpu.h =================================================================== --- head/sys/amd64/include/pcpu.h +++ head/sys/amd64/include/pcpu.h @@ -66,7 +66,7 @@ uint32_t pc_pcid_next; \ uint32_t pc_pcid_gen; \ uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ - char __pad[145] /* be divisor of PAGE_SIZE \ + char __pad[384] /* be divisor of PAGE_SIZE \ after cache alignment */ #define PC_DBREG_CMD_NONE 0 Index: head/sys/arm/arm/intr.c =================================================================== --- head/sys/arm/arm/intr.c +++ head/sys/arm/arm/intr.c @@ -185,7 +185,7 @@ struct intr_event *event; int i; - PCPU_INC(cnt.v_intr); + VM_CNT_INC(v_intr); i = -1; while ((i = arm_get_next_irq(i)) != -1) { intrcnt[i]++; Index: head/sys/arm/arm/trap-v4.c =================================================================== --- head/sys/arm/arm/trap-v4.c +++ head/sys/arm/arm/trap-v4.c @@ -204,7 +204,7 @@ td = curthread; p = td->td_proc; - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); /* Data abort came from user mode? */ user = TRAP_USERMODE(tf); @@ -615,7 +615,7 @@ td = curthread; p = td->td_proc; - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); if (TRAP_USERMODE(tf)) { td->td_frame = tf; Index: head/sys/arm/arm/trap-v6.c =================================================================== --- head/sys/arm/arm/trap-v6.c +++ head/sys/arm/arm/trap-v6.c @@ -291,7 +291,7 @@ void *onfault; #endif - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); td = curthread; fsr = (prefetch) ? cp15_ifsr_get(): cp15_dfsr_get(); Index: head/sys/arm/arm/undefined.c =================================================================== --- head/sys/arm/arm/undefined.c +++ head/sys/arm/arm/undefined.c @@ -201,7 +201,7 @@ if (__predict_true(frame->tf_spsr & PSR_F) == 0) enable_interrupts(PSR_F); - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); fault_pc = frame->tf_pc; Index: head/sys/arm/include/counter.h =================================================================== --- head/sys/arm/include/counter.h +++ head/sys/arm/include/counter.h @@ -32,6 +32,10 @@ #include #include +extern struct pcpu __pcpu[]; + +#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter + #define counter_enter() do {} while (0) #define counter_exit() do {} while (0) Index: head/sys/arm/include/pcpu.h =================================================================== --- head/sys/arm/include/pcpu.h +++ head/sys/arm/include/pcpu.h @@ -57,10 +57,10 @@ void *pc_qmap_pte2p; \ unsigned int pc_dbreg[32]; \ int pc_dbreg_cmd; \ - char __pad[27] + char __pad[155] #else #define PCPU_MD_FIELDS \ - char __pad[157] + char __pad[93] #endif #ifdef _KERNEL Index: head/sys/arm64/include/counter.h =================================================================== --- head/sys/arm64/include/counter.h +++ head/sys/arm64/include/counter.h @@ -32,6 +32,10 @@ #include #include +extern struct pcpu __pcpu[]; + +#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter + #define counter_enter() do {} while (0) #define counter_exit() do {} while (0) Index: head/sys/arm64/include/pcpu.h =================================================================== --- head/sys/arm64/include/pcpu.h +++ head/sys/arm64/include/pcpu.h @@ -39,7 +39,7 @@ u_int pc_acpi_id; /* ACPI CPU id */ \ u_int pc_midr; /* stored MIDR value */ \ uint64_t pc_clock; \ - char __pad[113] + char __pad[241] #ifdef _KERNEL Index: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c =================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c +++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c @@ -4575,8 +4575,8 @@ return (zfs_vm_pagerret_bad); } - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, count); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, count); lsize = PAGE_SIZE; if (IDX_TO_OFF(mlast->pindex) + lsize > object->un_pager.vnp.vnp_size) @@ -4758,8 +4758,8 @@ vm_page_undirty(ma[i]); } zfs_vmobject_wunlock(object); - PCPU_INC(cnt.v_vnodeout); - PCPU_ADD(cnt.v_vnodepgsout, ncount); + VM_CNT_INC(v_vnodeout); + VM_CNT_ADD(v_vnodepgsout, ncount); } dmu_tx_commit(tx); Index: head/sys/compat/linprocfs/linprocfs.c =================================================================== --- head/sys/compat/linprocfs/linprocfs.c +++ head/sys/compat/linprocfs/linprocfs.c @@ -425,17 +425,17 @@ } sbuf_printf(sb, "disk 0 0 0 0\n" - "page %u %u\n" - "swap %u %u\n" - "intr %u\n" - "ctxt %u\n" + "page %ju %ju\n" + "swap %ju %ju\n" + "intr %ju\n" + "ctxt %ju\n" "btime %lld\n", - vm_cnt.v_vnodepgsin, - vm_cnt.v_vnodepgsout, - vm_cnt.v_swappgsin, - vm_cnt.v_swappgsout, - vm_cnt.v_intr, - vm_cnt.v_swtch, + (uintmax_t)VM_CNT_FETCH(v_vnodepgsin), + (uintmax_t)VM_CNT_FETCH(v_vnodepgsout), + (uintmax_t)VM_CNT_FETCH(v_swappgsin), + (uintmax_t)VM_CNT_FETCH(v_swappgsout), + (uintmax_t)VM_CNT_FETCH(v_intr), + (uintmax_t)VM_CNT_FETCH(v_swtch), (long long)boottime.tv_sec); return (0); } Index: head/sys/fs/fuse/fuse_vnops.c =================================================================== --- head/sys/fs/fuse/fuse_vnops.c +++ head/sys/fs/fuse/fuse_vnops.c @@ -1793,8 +1793,8 @@ kva = (vm_offset_t)bp->b_data; pmap_qenter(kva, pages, npages); - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, npages); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, npages); count = npages << PAGE_SHIFT; iov.iov_base = (caddr_t)kva; @@ -1927,8 +1927,8 @@ kva = (vm_offset_t)bp->b_data; pmap_qenter(kva, pages, npages); - PCPU_INC(cnt.v_vnodeout); - PCPU_ADD(cnt.v_vnodepgsout, count); + VM_CNT_INC(v_vnodeout); + VM_CNT_ADD(v_vnodepgsout, count); iov.iov_base = (caddr_t)kva; iov.iov_len = count; Index: head/sys/fs/nfsclient/nfs_clbio.c =================================================================== --- head/sys/fs/nfsclient/nfs_clbio.c +++ head/sys/fs/nfsclient/nfs_clbio.c @@ -184,8 +184,8 @@ kva = (vm_offset_t) bp->b_data; pmap_qenter(kva, pages, npages); - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, npages); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, npages); count = npages << PAGE_SHIFT; iov.iov_base = (caddr_t) kva; @@ -320,8 +320,8 @@ } mtx_unlock(&np->n_mtx); - PCPU_INC(cnt.v_vnodeout); - PCPU_ADD(cnt.v_vnodepgsout, count); + VM_CNT_INC(v_vnodeout); + VM_CNT_ADD(v_vnodepgsout, count); iov.iov_base = unmapped_buf; iov.iov_len = count; Index: head/sys/fs/smbfs/smbfs_io.c =================================================================== --- head/sys/fs/smbfs/smbfs_io.c +++ head/sys/fs/smbfs/smbfs_io.c @@ -470,8 +470,8 @@ kva = (vm_offset_t) bp->b_data; pmap_qenter(kva, pages, npages); - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, npages); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, npages); count = npages << PAGE_SHIFT; iov.iov_base = (caddr_t) kva; @@ -595,8 +595,8 @@ kva = (vm_offset_t) bp->b_data; pmap_qenter(kva, pages, npages); - PCPU_INC(cnt.v_vnodeout); - PCPU_ADD(cnt.v_vnodepgsout, count); + VM_CNT_INC(v_vnodeout); + VM_CNT_ADD(v_vnodepgsout, count); iov.iov_base = (caddr_t) kva; iov.iov_len = count; Index: head/sys/i386/i386/trap.c =================================================================== --- head/sys/i386/i386/trap.c +++ head/sys/i386/i386/trap.c @@ -192,7 +192,7 @@ static int lastalert = 0; #endif - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); type = frame->tf_trapno; #ifdef SMP Index: head/sys/i386/include/atomic.h =================================================================== --- head/sys/i386/include/atomic.h +++ head/sys/i386/include/atomic.h @@ -45,7 +45,7 @@ * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in i386/vm_machdep.c ensures that the value is correct. */ -#define __OFFSETOF_MONITORBUF 0x180 +#define __OFFSETOF_MONITORBUF 0x80 static __inline void __mbk(void) Index: head/sys/i386/include/counter.h =================================================================== --- head/sys/i386/include/counter.h +++ head/sys/i386/include/counter.h @@ -36,6 +36,10 @@ #include #include +extern struct pcpu __pcpu[]; + +#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter + #define counter_enter() do { \ if ((cpu_feature & CPUID_CX8) == 0) \ critical_enter(); \ @@ -45,8 +49,6 @@ if ((cpu_feature & CPUID_CX8) == 0) \ critical_exit(); \ } while (0) - -extern struct pcpu __pcpu[MAXCPU]; static inline void counter_64_inc_8b(uint64_t *p, int64_t inc) Index: head/sys/i386/include/pcpu.h =================================================================== --- head/sys/i386/include/pcpu.h +++ head/sys/i386/include/pcpu.h @@ -68,7 +68,7 @@ caddr_t pc_cmap_addr2; \ vm_offset_t pc_qmap_addr; /* KVA for temporary mappings */\ uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ - char __pad[189] + char __pad[445] #ifdef _KERNEL Index: head/sys/kern/kern_fork.c =================================================================== --- head/sys/kern/kern_fork.c +++ head/sys/kern/kern_fork.c @@ -664,20 +664,20 @@ vm_forkproc(td, p2, td2, vm2, fr->fr_flags); if (fr->fr_flags == (RFFDG | RFPROC)) { - PCPU_INC(cnt.v_forks); - PCPU_ADD(cnt.v_forkpages, p2->p_vmspace->vm_dsize + + VM_CNT_INC(v_forks); + VM_CNT_ADD(v_forkpages, p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize); } else if (fr->fr_flags == (RFFDG | RFPROC | RFPPWAIT | RFMEM)) { - PCPU_INC(cnt.v_vforks); - PCPU_ADD(cnt.v_vforkpages, p2->p_vmspace->vm_dsize + + VM_CNT_INC(v_vforks); + VM_CNT_ADD(v_vforkpages, p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize); } else if (p1 == &proc0) { - PCPU_INC(cnt.v_kthreads); - PCPU_ADD(cnt.v_kthreadpages, p2->p_vmspace->vm_dsize + + VM_CNT_INC(v_kthreads); + VM_CNT_ADD(v_kthreadpages, p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize); } else { - PCPU_INC(cnt.v_rforks); - PCPU_ADD(cnt.v_rforkpages, p2->p_vmspace->vm_dsize + + VM_CNT_INC(v_rforks); + VM_CNT_ADD(v_rforkpages, p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize); } Index: head/sys/kern/kern_intr.c =================================================================== --- head/sys/kern/kern_intr.c +++ head/sys/kern/kern_intr.c @@ -1156,7 +1156,7 @@ ih->ih_need = 1; if (!(flags & SWI_DELAY)) { - PCPU_INC(cnt.v_soft); + VM_CNT_INC(v_soft); #ifdef INTR_FILTER error = intr_event_schedule_thread(ie, ie->ie_thread); #else Index: head/sys/kern/kern_synch.c =================================================================== --- head/sys/kern/kern_synch.c +++ head/sys/kern/kern_synch.c @@ -426,7 +426,7 @@ td->td_incruntime += runtime; PCPU_SET(switchtime, new_switchtime); td->td_generation++; /* bump preempt-detect counter */ - PCPU_INC(cnt.v_swtch); + VM_CNT_INC(v_swtch); PCPU_SET(switchticks, ticks); CTR4(KTR_PROC, "mi_switch: old thread %ld (td_sched %p, pid %ld, %s)", td->td_tid, td_get_sched(td), td->td_proc->p_pid, td->td_name); Index: head/sys/kern/kern_thread.c =================================================================== --- head/sys/kern/kern_thread.c +++ head/sys/kern/kern_thread.c @@ -546,7 +546,7 @@ td->td_incruntime += runtime; PCPU_SET(switchtime, new_switchtime); PCPU_SET(switchticks, ticks); - PCPU_INC(cnt.v_swtch); + VM_CNT_INC(v_swtch); /* Save our resource usage in our process. */ td->td_ru.ru_nvcsw++; Index: head/sys/kern/subr_intr.c =================================================================== --- head/sys/kern/subr_intr.c +++ head/sys/kern/subr_intr.c @@ -292,7 +292,7 @@ KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__)); - PCPU_INC(cnt.v_intr); + VM_CNT_INC(v_intr); critical_enter(); td = curthread; oldframe = td->td_intr_frame; Index: head/sys/kern/subr_syscall.c =================================================================== --- head/sys/kern/subr_syscall.c +++ head/sys/kern/subr_syscall.c @@ -58,7 +58,7 @@ struct proc *p; int error, traced; - PCPU_INC(cnt.v_syscall); + VM_CNT_INC(v_syscall); p = td->td_proc; td->td_pticks = 0; Index: head/sys/kern/subr_trap.c =================================================================== --- head/sys/kern/subr_trap.c +++ head/sys/kern/subr_trap.c @@ -234,7 +234,7 @@ td->td_flags &= ~(TDF_ASTPENDING | TDF_NEEDSIGCHK | TDF_NEEDSUSPCHK | TDF_NEEDRESCHED | TDF_ALRMPEND | TDF_PROFPEND | TDF_MACPEND); thread_unlock(td); - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); if (td->td_cowgen != p->p_cowgen) thread_cow_update(td); Index: head/sys/kern/vfs_bio.c =================================================================== --- head/sys/kern/vfs_bio.c +++ head/sys/kern/vfs_bio.c @@ -4769,8 +4769,8 @@ pgsin += pgsin_a; if (rahead != NULL) *rahead = pgsin_a; - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, pgsin); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, pgsin); br_flags = (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0) ? GB_UNMAPPED : 0; Index: head/sys/mips/include/counter.h =================================================================== --- head/sys/mips/include/counter.h +++ head/sys/mips/include/counter.h @@ -34,6 +34,8 @@ #include #endif +#define EARLY_COUNTER &((struct pcpu *)pcpu_space)->pc_early_dummy_counter + #define counter_enter() critical_enter() #define counter_exit() critical_exit() Index: head/sys/mips/include/intr_machdep.h =================================================================== --- head/sys/mips/include/intr_machdep.h +++ head/sys/mips/include/intr_machdep.h @@ -71,6 +71,6 @@ { if (counter) atomic_add_long(counter, 1); - PCPU_INC(cnt.v_intr); + VM_CNT_INC(v_intr); } #endif /* !_MACHINE_INTR_MACHDEP_H_ */ Index: head/sys/mips/include/pcpu.h =================================================================== --- head/sys/mips/include/pcpu.h +++ head/sys/mips/include/pcpu.h @@ -45,11 +45,11 @@ #ifdef __mips_n64 #define PCPU_MD_MIPS64_FIELDS \ PCPU_MD_COMMON_FIELDS \ - char __pad[53] + char __pad[245] #else #define PCPU_MD_MIPS32_FIELDS \ PCPU_MD_COMMON_FIELDS \ - char __pad[189] + char __pad[125] #endif #ifdef __mips_n64 Index: head/sys/powerpc/include/counter.h =================================================================== --- head/sys/powerpc/include/counter.h +++ head/sys/powerpc/include/counter.h @@ -34,6 +34,10 @@ #include #endif +extern struct pcpu __pcpu[]; + +#define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter + #ifdef __powerpc64__ #define counter_enter() do {} while (0) @@ -78,8 +82,6 @@ #endif #define counter_u64_add_protected(c, i) counter_u64_add(c, i) - -extern struct pcpu __pcpu[MAXCPU]; static inline void counter_u64_add(counter_u64_t c, int64_t inc) Index: head/sys/powerpc/include/pcpu.h =================================================================== --- head/sys/powerpc/include/pcpu.h +++ head/sys/powerpc/include/pcpu.h @@ -57,7 +57,7 @@ vm_offset_t pc_qmap_addr; \ struct pvo_entry *pc_qmap_pvo; \ struct mtx pc_qmap_lock; \ - /* char __pad[0] */ + char __pad[128] #define PCPU_MD_AIM64_FIELDS \ struct slb pc_slb[64]; \ @@ -67,7 +67,7 @@ vm_offset_t pc_qmap_addr; \ struct pvo_entry *pc_qmap_pvo; \ struct mtx pc_qmap_lock; \ - char __pad[1121 - sizeof(struct mtx)] + char __pad[1345] #ifdef __powerpc64__ #define PCPU_MD_AIM_FIELDS PCPU_MD_AIM64_FIELDS @@ -81,9 +81,9 @@ #define BOOKE_TLBSAVE_LEN (BOOKE_TLB_SAVELEN * BOOKE_TLB_MAXNEST) #ifdef __powerpc64__ -#define BOOKE_PCPU_PAD 773 +#define BOOKE_PCPU_PAD 901 #else -#define BOOKE_PCPU_PAD 173 +#define BOOKE_PCPU_PAD 429 #endif #define PCPU_MD_BOOKE_FIELDS \ register_t pc_booke_critsave[BOOKE_CRITSAVE_LEN]; \ Index: head/sys/powerpc/powerpc/trap.c =================================================================== --- head/sys/powerpc/powerpc/trap.c +++ head/sys/powerpc/powerpc/trap.c @@ -171,7 +171,7 @@ u_int ucode; ksiginfo_t ksi; - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); td = curthread; p = td->td_proc; Index: head/sys/sparc64/include/counter.h =================================================================== --- head/sys/sparc64/include/counter.h +++ head/sys/sparc64/include/counter.h @@ -34,6 +34,9 @@ #include #endif +extern struct pcpu dummy_pcpu[]; +#define EARLY_COUNTER &dummy_pcpu[0].pc_early_dummy_counter + #define counter_enter() critical_enter() #define counter_exit() critical_exit() Index: head/sys/sparc64/include/pcpu.h =================================================================== --- head/sys/sparc64/include/pcpu.h +++ head/sys/sparc64/include/pcpu.h @@ -62,7 +62,7 @@ u_int pc_tlb_ctx; \ u_int pc_tlb_ctx_max; \ u_int pc_tlb_ctx_min; \ - char __pad[397] + char __pad[653] #ifdef _KERNEL Index: head/sys/sparc64/sparc64/exception.S =================================================================== --- head/sys/sparc64/sparc64/exception.S +++ head/sys/sparc64/sparc64/exception.S @@ -2476,9 +2476,8 @@ inc %l1 stx %l1, [%l0] - lduw [PCPU(CNT) + V_INTR], %l0 - inc %l0 - stw %l0, [PCPU(CNT) + V_INTR] + call counter_intr_inc + nop ba,a %xcc, tl0_ret nop @@ -2989,11 +2988,8 @@ add %l5, %l4, %l4 ldx [%l4], %l5 inc %l5 - stx %l5, [%l4] - - lduw [PCPU(CNT) + V_INTR], %l4 - inc %l4 - stw %l4, [PCPU(CNT) + V_INTR] + call counter_intr_inc + stx %l5, [%l4] ldx [%sp + SPOFF + CCFSZ + TF_Y], %l4 Index: head/sys/sparc64/sparc64/genassym.c =================================================================== --- head/sys/sparc64/sparc64/genassym.c +++ head/sys/sparc64/sparc64/genassym.c @@ -134,7 +134,6 @@ ASSYM(PC_IRHEAD, offsetof(struct pcpu, pc_irhead)); ASSYM(PC_IRTAIL, offsetof(struct pcpu, pc_irtail)); ASSYM(PC_IRFREE, offsetof(struct pcpu, pc_irfree)); -ASSYM(PC_CNT, offsetof(struct pcpu, pc_cnt)); ASSYM(PC_SIZEOF, sizeof(struct pcpu)); ASSYM(PC_CACHE, offsetof(struct pcpu, pc_cache)); Index: head/sys/sparc64/sparc64/intr_machdep.c =================================================================== --- head/sys/sparc64/sparc64/intr_machdep.c +++ head/sys/sparc64/sparc64/intr_machdep.c @@ -123,6 +123,7 @@ static void intr_stray_vector(void *); static int intrcnt_setname(const char *, int); static void intrcnt_updatename(int, const char *, int); +void counter_intr_inc(void); static void intrcnt_updatename(int vec, const char *name, int ispil) @@ -449,6 +450,19 @@ intrcnt_updatename(vec, iv->iv_event->ie_fullname, 0); sx_xunlock(&intr_table_lock); return (error); +} + +/* + * Do VM_CNT_INC(intr), being in the interrupt context already. This is + * called from assembly. + * To avoid counter_enter() and appropriate assertion, unwrap VM_CNT_INC() + * and hardcode the actual increment. + */ +void +counter_intr_inc(void) +{ + + *(uint64_t *)zpcpu_get(vm_cnt.v_intr) += 1; } #ifdef SMP Index: head/sys/sparc64/sparc64/machdep.c =================================================================== --- head/sys/sparc64/sparc64/machdep.c +++ head/sys/sparc64/sparc64/machdep.c @@ -126,6 +126,7 @@ void *dpcpu0; char pcpu0[PCPU_PAGES * PAGE_SIZE]; +struct pcpu dummy_pcpu[MAXCPU]; struct trapframe frame0; vm_offset_t kstack0; Index: head/sys/sparc64/sparc64/trap.c =================================================================== --- head/sys/sparc64/sparc64/trap.c +++ head/sys/sparc64/sparc64/trap.c @@ -267,7 +267,7 @@ trap_msg[tf->tf_type & ~T_KERNEL], (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil)); - PCPU_INC(cnt.v_trap); + VM_CNT_INC(v_trap); if ((tf->tf_tstate & TSTATE_PRIV) == 0) { KASSERT(td != NULL, ("trap: curthread NULL")); Index: head/sys/sys/pcpu.h =================================================================== --- head/sys/sys/pcpu.h +++ head/sys/sys/pcpu.h @@ -43,7 +43,6 @@ #include #include #include -#include #include #include @@ -158,7 +157,6 @@ u_int pc_cpuid; /* This cpu number */ STAILQ_ENTRY(pcpu) pc_allcpu; struct lock_list_entry *pc_spinlocks; - struct vmmeter pc_cnt; /* VM stats counters */ long pc_cp_time[CPUSTATES]; /* statclock ticks */ struct device *pc_device; void *pc_netisr; /* netisr SWI cookie */ @@ -166,6 +164,7 @@ int pc_domain; /* Memory domain. */ struct rm_queue pc_rm_queue; /* rmlock list of trackers */ uintptr_t pc_dynamic; /* Dynamic per-cpu data area */ + uint64_t pc_early_dummy_counter; /* Startup time counter(9) */ /* * Keep MD fields last, so that CPU-specific variations on a Index: head/sys/sys/vmmeter.h =================================================================== --- head/sys/sys/vmmeter.h +++ head/sys/sys/vmmeter.h @@ -39,51 +39,85 @@ */ #define MAXSLP 20 +/* Systemwide totals computed every five seconds. */ +struct vmtotal { + int16_t t_rq; /* length of the run queue */ + int16_t t_dw; /* jobs in ``disk wait'' (neg priority) */ + int16_t t_pw; /* jobs in page wait */ + int16_t t_sl; /* jobs sleeping in core */ + int16_t t_sw; /* swapped out runnable/short block jobs */ + int32_t t_vm; /* total virtual memory */ + int32_t t_avm; /* active virtual memory */ + int32_t t_rm; /* total real memory in use */ + int32_t t_arm; /* active real memory */ + int32_t t_vmshr; /* shared virtual memory */ + int32_t t_avmshr; /* active shared virtual memory */ + int32_t t_rmshr; /* shared real memory */ + int32_t t_armshr; /* active shared real memory */ + int32_t t_free; /* free memory pages */ +}; + +#if defined(_KERNEL) || defined(_WANT_VMMETER) +#include + /* * System wide statistics counters. * Locking: * a - locked by atomic operations * c - constant after initialization * f - locked by vm_page_queue_free_mtx - * p - locked by being in the PCPU and atomicity respect to interrupts + * p - uses counter(9) * q - changes are synchronized by the corresponding vm_pagequeue lock */ struct vmmeter { /* * General system activity. */ - u_int v_swtch; /* (p) context switches */ - u_int v_trap; /* (p) calls to trap */ - u_int v_syscall; /* (p) calls to syscall() */ - u_int v_intr; /* (p) device interrupts */ - u_int v_soft; /* (p) software interrupts */ + counter_u64_t v_swtch; /* (p) context switches */ + counter_u64_t v_trap; /* (p) calls to trap */ + counter_u64_t v_syscall; /* (p) calls to syscall() */ + counter_u64_t v_intr; /* (p) device interrupts */ + counter_u64_t v_soft; /* (p) software interrupts */ /* * Virtual memory activity. */ - u_int v_vm_faults; /* (p) address memory faults */ - u_int v_io_faults; /* (p) page faults requiring I/O */ - u_int v_cow_faults; /* (p) copy-on-writes faults */ - u_int v_cow_optim; /* (p) optimized copy-on-writes faults */ - u_int v_zfod; /* (p) pages zero filled on demand */ - u_int v_ozfod; /* (p) optimized zero fill pages */ - u_int v_swapin; /* (p) swap pager pageins */ - u_int v_swapout; /* (p) swap pager pageouts */ - u_int v_swappgsin; /* (p) swap pager pages paged in */ - u_int v_swappgsout; /* (p) swap pager pages paged out */ - u_int v_vnodein; /* (p) vnode pager pageins */ - u_int v_vnodeout; /* (p) vnode pager pageouts */ - u_int v_vnodepgsin; /* (p) vnode_pager pages paged in */ - u_int v_vnodepgsout; /* (p) vnode pager pages paged out */ - u_int v_intrans; /* (p) intransit blocking page faults */ - u_int v_reactivated; /* (p) pages reactivated by the pagedaemon */ - u_int v_pdwakeups; /* (p) times daemon has awaken from sleep */ - u_int v_pdpages; /* (p) pages analyzed by daemon */ - u_int v_pdshortfalls; /* (p) page reclamation shortfalls */ + counter_u64_t v_vm_faults; /* (p) address memory faults */ + counter_u64_t v_io_faults; /* (p) page faults requiring I/O */ + counter_u64_t v_cow_faults; /* (p) copy-on-writes faults */ + counter_u64_t v_cow_optim; /* (p) optimized COW faults */ + counter_u64_t v_zfod; /* (p) pages zero filled on demand */ + counter_u64_t v_ozfod; /* (p) optimized zero fill pages */ + counter_u64_t v_swapin; /* (p) swap pager pageins */ + counter_u64_t v_swapout; /* (p) swap pager pageouts */ + counter_u64_t v_swappgsin; /* (p) swap pager pages paged in */ + counter_u64_t v_swappgsout; /* (p) swap pager pages paged out */ + counter_u64_t v_vnodein; /* (p) vnode pager pageins */ + counter_u64_t v_vnodeout; /* (p) vnode pager pageouts */ + counter_u64_t v_vnodepgsin; /* (p) vnode_pager pages paged in */ + counter_u64_t v_vnodepgsout; /* (p) vnode pager pages paged out */ + counter_u64_t v_intrans; /* (p) intransit blocking page faults */ + counter_u64_t v_reactivated; /* (p) reactivated by the pagedaemon */ + counter_u64_t v_pdwakeups; /* (p) times daemon has awaken */ + counter_u64_t v_pdpages; /* (p) pages analyzed by daemon */ + counter_u64_t v_pdshortfalls; /* (p) page reclamation shortfalls */ - u_int v_dfree; /* (p) pages freed by daemon */ - u_int v_pfree; /* (p) pages freed by exiting processes */ - u_int v_tfree; /* (p) total pages freed */ + counter_u64_t v_dfree; /* (p) pages freed by daemon */ + counter_u64_t v_pfree; /* (p) pages freed by processes */ + counter_u64_t v_tfree; /* (p) total pages freed */ /* + * Fork/vfork/rfork activity. + */ + counter_u64_t v_forks; /* (p) fork() calls */ + counter_u64_t v_vforks; /* (p) vfork() calls */ + counter_u64_t v_rforks; /* (p) rfork() calls */ + counter_u64_t v_kthreads; /* (p) fork() calls by kernel */ + counter_u64_t v_forkpages; /* (p) pages affected by fork() */ + counter_u64_t v_vforkpages; /* (p) pages affected by vfork() */ + counter_u64_t v_rforkpages; /* (p) pages affected by rfork() */ + counter_u64_t v_kthreadpages; /* (p) ... and by kernel fork() */ +#define VM_METER_NCOUNTERS \ + (offsetof(struct vmmeter, v_page_size) / sizeof(counter_u64_t)) + /* * Distribution of page usages. */ u_int v_page_size; /* (c) page size in bytes */ @@ -100,24 +134,18 @@ u_int v_pageout_free_min; /* (c) min pages reserved for kernel */ u_int v_interrupt_free_min; /* (c) reserved pages for int code */ u_int v_free_severe; /* (c) severe page depletion point */ - /* - * Fork/vfork/rfork activity. - */ - u_int v_forks; /* (p) fork() calls */ - u_int v_vforks; /* (p) vfork() calls */ - u_int v_rforks; /* (p) rfork() calls */ - u_int v_kthreads; /* (p) fork() calls by kernel */ - u_int v_forkpages; /* (p) VM pages affected by fork() */ - u_int v_vforkpages; /* (p) VM pages affected by vfork() */ - u_int v_rforkpages; /* (p) VM pages affected by rfork() */ - u_int v_kthreadpages; /* (p) VM pages affected by fork() by kernel */ }; +#endif /* _KERNEL || _WANT_VMMETER */ + #ifdef _KERNEL extern struct vmmeter vm_cnt; - extern u_int vm_pageout_wakeup_thresh; +#define VM_CNT_ADD(var, x) counter_u64_add(vm_cnt.var, x) +#define VM_CNT_INC(var) VM_CNT_ADD(var, 1) +#define VM_CNT_FETCH(var) counter_u64_fetch(vm_cnt.var) + /* * Return TRUE if we are under our severe low-free-pages threshold * @@ -189,33 +217,5 @@ return (vm_paging_target()); } - -/* - * Obtain the value of a per-CPU counter. - */ -#define VM_METER_PCPU_CNT(member) \ - vm_meter_cnt(__offsetof(struct vmmeter, member)) - -u_int vm_meter_cnt(size_t); - -#endif - -/* systemwide totals computed every five seconds */ -struct vmtotal { - int16_t t_rq; /* length of the run queue */ - int16_t t_dw; /* jobs in ``disk wait'' (neg priority) */ - int16_t t_pw; /* jobs in page wait */ - int16_t t_sl; /* jobs sleeping in core */ - int16_t t_sw; /* swapped out runnable/short block jobs */ - int32_t t_vm; /* total virtual memory */ - int32_t t_avm; /* active virtual memory */ - int32_t t_rm; /* total real memory in use */ - int32_t t_arm; /* active real memory */ - int32_t t_vmshr; /* shared virtual memory */ - int32_t t_avmshr; /* active shared virtual memory */ - int32_t t_rmshr; /* shared real memory */ - int32_t t_armshr; /* active shared real memory */ - int32_t t_free; /* free memory pages */ -}; - -#endif +#endif /* _KERNEL */ +#endif /* _SYS_VMMETER_H_ */ Index: head/sys/vm/swap_pager.c =================================================================== --- head/sys/vm/swap_pager.c +++ head/sys/vm/swap_pager.c @@ -1180,8 +1180,8 @@ bp->b_pgbefore = rbehind != NULL ? *rbehind : 0; bp->b_pgafter = rahead != NULL ? *rahead : 0; - PCPU_INC(cnt.v_swapin); - PCPU_ADD(cnt.v_swappgsin, count); + VM_CNT_INC(v_swapin); + VM_CNT_ADD(v_swappgsin, count); /* * perform the I/O. NOTE!!! bp cannot be considered valid after @@ -1205,7 +1205,7 @@ VM_OBJECT_WLOCK(object); while ((m[0]->oflags & VPO_SWAPINPROG) != 0) { m[0]->oflags |= VPO_SWAPSLEEP; - PCPU_INC(cnt.v_intrans); + VM_CNT_INC(v_intrans); if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP, "swread", hz * 20)) { printf( @@ -1393,8 +1393,8 @@ bp->b_dirtyoff = 0; bp->b_dirtyend = bp->b_bcount; - PCPU_INC(cnt.v_swapout); - PCPU_ADD(cnt.v_swappgsout, bp->b_npages); + VM_CNT_INC(v_swapout); + VM_CNT_ADD(v_swappgsout, bp->b_npages); /* * We unconditionally set rtvals[] to VM_PAGER_PEND so that we Index: head/sys/vm/vm_fault.c =================================================================== --- head/sys/vm/vm_fault.c +++ head/sys/vm/vm_fault.c @@ -497,7 +497,7 @@ boolean_t wired; /* Passed by reference. */ bool dead, growstack, hardfault, is_first_object_locked; - PCPU_INC(cnt.v_vm_faults); + VM_CNT_INC(v_vm_faults); fs.vp = NULL; faultcount = 0; nera = -1; @@ -673,7 +673,7 @@ } vm_object_pip_wakeup(fs.object); VM_OBJECT_WUNLOCK(fs.object); - PCPU_INC(cnt.v_intrans); + VM_CNT_INC(v_intrans); vm_object_deallocate(fs.first_object); goto RetryFault; } @@ -999,9 +999,9 @@ if ((fs.m->flags & PG_ZERO) == 0) { pmap_zero_page(fs.m); } else { - PCPU_INC(cnt.v_ozfod); + VM_CNT_INC(v_ozfod); } - PCPU_INC(cnt.v_zfod); + VM_CNT_INC(v_zfod); fs.m->valid = VM_PAGE_BITS_ALL; /* Don't try to prefault neighboring pages. */ faultcount = 1; @@ -1095,7 +1095,7 @@ vm_page_xbusy(fs.m); fs.first_m = fs.m; fs.m = NULL; - PCPU_INC(cnt.v_cow_optim); + VM_CNT_INC(v_cow_optim); } else { /* * Oh, well, lets copy it. @@ -1131,7 +1131,7 @@ fs.m = fs.first_m; if (!is_first_object_locked) VM_OBJECT_WLOCK(fs.object); - PCPU_INC(cnt.v_cow_faults); + VM_CNT_INC(v_cow_faults); curthread->td_cow++; } else { prot &= ~VM_PROT_WRITE; @@ -1246,7 +1246,7 @@ */ unlock_and_deallocate(&fs); if (hardfault) { - PCPU_INC(cnt.v_io_faults); + VM_CNT_INC(v_io_faults); curthread->td_ru.ru_majflt++; #ifdef RACCT if (racct_enable && fs.object->type == OBJT_VNODE) { Index: head/sys/vm/vm_meter.c =================================================================== --- head/sys/vm/vm_meter.c +++ head/sys/vm/vm_meter.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -55,8 +56,53 @@ #include #include -struct vmmeter vm_cnt; +struct vmmeter vm_cnt = { + .v_swtch = EARLY_COUNTER, + .v_trap = EARLY_COUNTER, + .v_syscall = EARLY_COUNTER, + .v_intr = EARLY_COUNTER, + .v_soft = EARLY_COUNTER, + .v_vm_faults = EARLY_COUNTER, + .v_io_faults = EARLY_COUNTER, + .v_cow_faults = EARLY_COUNTER, + .v_cow_optim = EARLY_COUNTER, + .v_zfod = EARLY_COUNTER, + .v_ozfod = EARLY_COUNTER, + .v_swapin = EARLY_COUNTER, + .v_swapout = EARLY_COUNTER, + .v_swappgsin = EARLY_COUNTER, + .v_swappgsout = EARLY_COUNTER, + .v_vnodein = EARLY_COUNTER, + .v_vnodeout = EARLY_COUNTER, + .v_vnodepgsin = EARLY_COUNTER, + .v_vnodepgsout = EARLY_COUNTER, + .v_intrans = EARLY_COUNTER, + .v_reactivated = EARLY_COUNTER, + .v_pdwakeups = EARLY_COUNTER, + .v_pdpages = EARLY_COUNTER, + .v_pdshortfalls = EARLY_COUNTER, + .v_dfree = EARLY_COUNTER, + .v_pfree = EARLY_COUNTER, + .v_tfree = EARLY_COUNTER, + .v_forks = EARLY_COUNTER, + .v_vforks = EARLY_COUNTER, + .v_rforks = EARLY_COUNTER, + .v_kthreads = EARLY_COUNTER, + .v_forkpages = EARLY_COUNTER, + .v_vforkpages = EARLY_COUNTER, + .v_rforkpages = EARLY_COUNTER, + .v_kthreadpages = EARLY_COUNTER, +}; +static void +vmcounter_startup(void) +{ + counter_u64_t *cnt = (counter_u64_t *)&vm_cnt; + + COUNTER_ARRAY_ALLOC(cnt, VM_METER_NCOUNTERS, M_WAITOK); +} +SYSINIT(counter, SI_SUB_CPU, SI_ORDER_FOURTH + 1, vmcounter_startup, NULL); + SYSCTL_UINT(_vm, VM_V_FREE_MIN, v_free_min, CTLFLAG_RW, &vm_cnt.v_free_min, 0, "Minimum low-free-pages threshold"); SYSCTL_UINT(_vm, VM_V_FREE_TARGET, v_free_target, @@ -210,40 +256,6 @@ return (sysctl_handle_opaque(oidp, &total, sizeof(total), req)); } -/* - * vm_meter_cnt() - accumulate statistics from all cpus and the global cnt - * structure. - * - * The vmmeter structure is now per-cpu as well as global. Those - * statistics which can be kept on a per-cpu basis (to avoid cache - * stalls between cpus) can be moved to the per-cpu vmmeter. Remaining - * statistics, such as v_free_reserved, are left in the global - * structure. - */ -u_int -vm_meter_cnt(size_t offset) -{ - struct pcpu *pcpu; - u_int count; - int i; - - count = *(u_int *)((char *)&vm_cnt + offset); - CPU_FOREACH(i) { - pcpu = pcpu_find(i); - count += *(u_int *)((char *)&pcpu->pc_cnt + offset); - } - return (count); -} - -static int -cnt_sysctl(SYSCTL_HANDLER_ARGS) -{ - u_int count; - - count = vm_meter_cnt((char *)arg1 - (char *)&vm_cnt); - return (SYSCTL_OUT(req, &count, sizeof(count))); -} - SYSCTL_PROC(_vm, VM_TOTAL, vmtotal, CTLTYPE_OPAQUE|CTLFLAG_RD|CTLFLAG_MPSAFE, 0, sizeof(struct vmtotal), vmtotal, "S,vmtotal", "System virtual memory statistics"); @@ -255,9 +267,7 @@ SYSCTL_NODE(_vm_stats, OID_AUTO, misc, CTLFLAG_RW, 0, "VM meter misc stats"); #define VM_STATS(parent, var, descr) \ - SYSCTL_PROC(parent, OID_AUTO, var, \ - CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, &vm_cnt.var, 0, \ - cnt_sysctl, "IU", descr) + SYSCTL_COUNTER_U64(parent, OID_AUTO, var, CTLFLAG_RD, &vm_cnt.var, descr) #define VM_STATS_VM(var, descr) VM_STATS(_vm_stats_vm, var, descr) #define VM_STATS_SYS(var, descr) VM_STATS(_vm_stats_sys, var, descr) @@ -288,19 +298,6 @@ VM_STATS_VM(v_dfree, "Pages freed by pagedaemon"); VM_STATS_VM(v_pfree, "Pages freed by exiting processes"); VM_STATS_VM(v_tfree, "Total pages freed"); -VM_STATS_VM(v_page_size, "Page size in bytes"); -VM_STATS_VM(v_page_count, "Total number of pages in system"); -VM_STATS_VM(v_free_reserved, "Pages reserved for deadlock"); -VM_STATS_VM(v_free_target, "Pages desired free"); -VM_STATS_VM(v_free_min, "Minimum low-free-pages threshold"); -VM_STATS_VM(v_free_count, "Free pages"); -VM_STATS_VM(v_wire_count, "Wired pages"); -VM_STATS_VM(v_active_count, "Active pages"); -VM_STATS_VM(v_inactive_target, "Desired inactive pages"); -VM_STATS_VM(v_inactive_count, "Inactive pages"); -VM_STATS_VM(v_laundry_count, "Pages eligible for laundering"); -VM_STATS_VM(v_pageout_free_min, "Min pages reserved for kernel"); -VM_STATS_VM(v_interrupt_free_min, "Reserved pages for interrupt code"); VM_STATS_VM(v_forks, "Number of fork() calls"); VM_STATS_VM(v_vforks, "Number of vfork() calls"); VM_STATS_VM(v_rforks, "Number of rfork() calls"); @@ -309,6 +306,23 @@ VM_STATS_VM(v_vforkpages, "VM pages affected by vfork()"); VM_STATS_VM(v_rforkpages, "VM pages affected by rfork()"); VM_STATS_VM(v_kthreadpages, "VM pages affected by fork() by kernel"); + +#define VM_STATS_UINT(var, descr) \ + SYSCTL_UINT(_vm_stats_vm, OID_AUTO, var, CTLFLAG_RD, &vm_cnt.var, 0, descr) +VM_STATS_UINT(v_page_size, "Page size in bytes"); +VM_STATS_UINT(v_page_count, "Total number of pages in system"); +VM_STATS_UINT(v_free_reserved, "Pages reserved for deadlock"); +VM_STATS_UINT(v_free_target, "Pages desired free"); +VM_STATS_UINT(v_free_min, "Minimum low-free-pages threshold"); +VM_STATS_UINT(v_free_count, "Free pages"); +VM_STATS_UINT(v_wire_count, "Wired pages"); +VM_STATS_UINT(v_active_count, "Active pages"); +VM_STATS_UINT(v_inactive_target, "Desired inactive pages"); +VM_STATS_UINT(v_inactive_count, "Inactive pages"); +VM_STATS_UINT(v_laundry_count, "Pages eligible for laundering"); +VM_STATS_UINT(v_pageout_free_min, "Min pages reserved for kernel"); +VM_STATS_UINT(v_interrupt_free_min, "Reserved pages for interrupt code"); +VM_STATS_UINT(v_free_severe, "Severe page depletion point"); #ifdef COMPAT_FREEBSD11 /* Index: head/sys/vm/vm_object.c =================================================================== --- head/sys/vm/vm_object.c +++ head/sys/vm/vm_object.c @@ -765,7 +765,7 @@ p->object = NULL; if (p->wire_count == 0) { vm_page_free(p); - PCPU_INC(cnt.v_pfree); + VM_CNT_INC(v_pfree); } vm_page_unlock(p); } Index: head/sys/vm/vm_page.c =================================================================== --- head/sys/vm/vm_page.c +++ head/sys/vm/vm_page.c @@ -2748,7 +2748,7 @@ } else KASSERT(m->queue == PQ_NONE, ("vm_page_free_toq: unmanaged page %p is queued", m)); - PCPU_INC(cnt.v_tfree); + VM_CNT_INC(v_tfree); if (vm_page_sbusied(m)) panic("vm_page_free: freeing busy page %p", m); Index: head/sys/vm/vm_pageout.c =================================================================== --- head/sys/vm/vm_pageout.c +++ head/sys/vm/vm_pageout.c @@ -667,7 +667,7 @@ goto unlock_return; if (vm_page_busied(p)) continue; - PCPU_INC(cnt.v_pdpages); + VM_CNT_INC(v_pdpages); vm_page_lock(p); if (p->wire_count != 0 || p->hold_count != 0 || !pmap_page_exists_quick(pmap, p)) { @@ -1003,7 +1003,7 @@ } if (act_delta != 0) { if (object->ref_count != 0) { - PCPU_INC(cnt.v_reactivated); + VM_CNT_INC(v_reactivated); vm_page_activate(m); /* @@ -1052,7 +1052,7 @@ if (m->dirty == 0) { free_page: vm_page_free(m); - PCPU_INC(cnt.v_dfree); + VM_CNT_INC(v_dfree); } else if ((object->flags & OBJ_DEAD) == 0) { if (object->type != OBJT_SWAP && object->type != OBJT_DEFAULT) @@ -1187,7 +1187,7 @@ KASSERT(shortfall_cycle >= 0, ("negative cycle %d", shortfall_cycle)); launder = 0; - wakeups = VM_METER_PCPU_CNT(v_pdwakeups); + wakeups = VM_CNT_FETCH(v_pdwakeups); /* * First determine whether we need to launder pages to meet a @@ -1378,7 +1378,7 @@ KASSERT(queue_locked, ("unlocked inactive queue")); KASSERT(vm_page_inactive(m), ("Inactive queue %p", m)); - PCPU_INC(cnt.v_pdpages); + VM_CNT_INC(v_pdpages); next = TAILQ_NEXT(m, plinks.q); /* @@ -1476,7 +1476,7 @@ } if (act_delta != 0) { if (object->ref_count != 0) { - PCPU_INC(cnt.v_reactivated); + VM_CNT_INC(v_reactivated); vm_page_activate(m); /* @@ -1521,7 +1521,7 @@ if (m->dirty == 0) { free_page: vm_page_free(m); - PCPU_INC(cnt.v_dfree); + VM_CNT_INC(v_dfree); --page_shortage; } else if ((object->flags & OBJ_DEAD) == 0) vm_page_launder(m); @@ -1551,7 +1551,7 @@ if (pq->pq_cnt > 0 || atomic_load_acq_int(&swapdev_enabled)) { if (page_shortage > 0) { vm_laundry_request = VM_LAUNDRY_SHORTFALL; - PCPU_INC(cnt.v_pdshortfalls); + VM_CNT_INC(v_pdshortfalls); } else if (vm_laundry_request != VM_LAUNDRY_SHORTFALL) vm_laundry_request = VM_LAUNDRY_BACKGROUND; wakeup(&vm_laundry_request); @@ -1635,7 +1635,7 @@ * The count for page daemon pages is updated after checking * the page for eligibility. */ - PCPU_INC(cnt.v_pdpages); + VM_CNT_INC(v_pdpages); /* * Check to see "how much" the page has been used. @@ -2036,7 +2036,7 @@ if (mtx_sleep(&vm_pageout_wanted, &vm_page_queue_free_mtx, PDROP | PVM, "psleep", hz) == 0) { - PCPU_INC(cnt.v_pdwakeups); + VM_CNT_INC(v_pdwakeups); pass = 1; } else pass = 0; Index: head/sys/vm/vnode_pager.c =================================================================== --- head/sys/vm/vnode_pager.c +++ head/sys/vm/vnode_pager.c @@ -799,8 +799,8 @@ relpbuf(bp, freecnt); VM_OBJECT_WLOCK(object); for (i = 0; i < count; i++) { - PCPU_INC(cnt.v_vnodein); - PCPU_INC(cnt.v_vnodepgsin); + VM_CNT_INC(v_vnodein); + VM_CNT_INC(v_vnodepgsin); error = vnode_pager_input_old(object, m[i]); if (error) break; @@ -819,8 +819,8 @@ if (pagesperblock == 0) { relpbuf(bp, freecnt); for (i = 0; i < count; i++) { - PCPU_INC(cnt.v_vnodein); - PCPU_INC(cnt.v_vnodepgsin); + VM_CNT_INC(v_vnodein); + VM_CNT_INC(v_vnodepgsin); error = vnode_pager_input_smlfs(object, m[i]); if (error) break; @@ -1030,8 +1030,8 @@ (uintmax_t)blkno0, (uintmax_t)bp->b_blkno)); atomic_add_long(&runningbufspace, bp->b_runningbufspace); - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, bp->b_npages); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, bp->b_npages); if (iodone != NULL) { /* async */ bp->b_pgiodone = iodone; @@ -1276,8 +1276,8 @@ auio.uio_td = (struct thread *) 0; error = VOP_WRITE(vp, &auio, vnode_pager_putpages_ioflags(flags), curthread->td_ucred); - PCPU_INC(cnt.v_vnodeout); - PCPU_ADD(cnt.v_vnodepgsout, ncount); + VM_CNT_INC(v_vnodeout); + VM_CNT_ADD(v_vnodepgsout, ncount); ppscheck = 0; if (error) { Index: head/sys/x86/acpica/srat.c =================================================================== --- head/sys/x86/acpica/srat.c +++ head/sys/x86/acpica/srat.c @@ -31,6 +31,7 @@ #include "opt_vm.h" #include +#include #include #include #include Index: head/sys/x86/x86/intr_machdep.c =================================================================== --- head/sys/x86/x86/intr_machdep.c +++ head/sys/x86/x86/intr_machdep.c @@ -246,7 +246,7 @@ * processed too. */ (*isrc->is_count)++; - PCPU_INC(cnt.v_intr); + VM_CNT_INC(v_intr); ie = isrc->is_event; Index: head/usr.bin/top/machine.c =================================================================== --- head/usr.bin/top/machine.c +++ head/usr.bin/top/machine.c @@ -516,7 +516,7 @@ static int swapavail = 0; static int swapfree = 0; static long bufspace = 0; - static int nspgsin, nspgsout; + static uint64_t nspgsin, nspgsout; GETSYSCTL("vfs.bufspace", bufspace); GETSYSCTL("vm.stats.vm.v_active_count", memory_stats[0]); Index: head/usr.bin/vmstat/vmstat.c =================================================================== --- head/usr.bin/vmstat/vmstat.c +++ head/usr.bin/vmstat/vmstat.c @@ -54,6 +54,7 @@ #include #include #include +#define _WANT_VMMETER #include #include @@ -127,7 +128,57 @@ static char **specified_devices; static devstat_select_mode select_mode; -static struct vmmeter sum, osum; +static struct __vmmeter { + uint64_t v_swtch; + uint64_t v_trap; + uint64_t v_syscall; + uint64_t v_intr; + uint64_t v_soft; + uint64_t v_vm_faults; + uint64_t v_io_faults; + uint64_t v_cow_faults; + uint64_t v_cow_optim; + uint64_t v_zfod; + uint64_t v_ozfod; + uint64_t v_swapin; + uint64_t v_swapout; + uint64_t v_swappgsin; + uint64_t v_swappgsout; + uint64_t v_vnodein; + uint64_t v_vnodeout; + uint64_t v_vnodepgsin; + uint64_t v_vnodepgsout; + uint64_t v_intrans; + uint64_t v_reactivated; + uint64_t v_pdwakeups; + uint64_t v_pdpages; + uint64_t v_pdshortfalls; + uint64_t v_dfree; + uint64_t v_pfree; + uint64_t v_tfree; + uint64_t v_forks; + uint64_t v_vforks; + uint64_t v_rforks; + uint64_t v_kthreads; + uint64_t v_forkpages; + uint64_t v_vforkpages; + uint64_t v_rforkpages; + uint64_t v_kthreadpages; + u_int v_page_size; + u_int v_page_count; + u_int v_free_reserved; + u_int v_free_target; + u_int v_free_min; + u_int v_free_count; + u_int v_wire_count; + u_int v_active_count; + u_int v_inactive_target; + u_int v_inactive_count; + u_int v_laundry_count; + u_int v_pageout_free_min; + u_int v_interrupt_free_min; + u_int v_free_severe; +} sum, osum; #define VMSTAT_DEFAULT_LINES 20 /* Default number of `winlines'. */ volatile sig_atomic_t wresized; /* Tty resized, when non-zero. */ @@ -448,93 +499,50 @@ } static void -fill_pcpu(struct pcpu ***pcpup, int* maxcpup) +fill_vmmeter(struct __vmmeter *vmmp) { struct pcpu **pcpu; - int maxcpu, i; - *pcpup = NULL; - - if (kd == NULL) - return; - - maxcpu = kvm_getmaxcpu(kd); - if (maxcpu < 0) - xo_errx(1, "kvm_getmaxcpu: %s", kvm_geterr(kd)); - - pcpu = calloc(maxcpu, sizeof(struct pcpu *)); - if (pcpu == NULL) - xo_err(1, "calloc"); - - for (i = 0; i < maxcpu; i++) { - pcpu[i] = kvm_getpcpu(kd, i); - if (pcpu[i] == (struct pcpu *)-1) - xo_errx(1, "kvm_getpcpu: %s", kvm_geterr(kd)); - } - - *maxcpup = maxcpu; - *pcpup = pcpu; -} - -static void -free_pcpu(struct pcpu **pcpu, int maxcpu) -{ - int i; - - for (i = 0; i < maxcpu; i++) - free(pcpu[i]); - free(pcpu); -} - -static void -fill_vmmeter(struct vmmeter *vmmp) -{ - struct pcpu **pcpu; - int maxcpu, i; - if (kd != NULL) { - kread(X_SUM, vmmp, sizeof(*vmmp)); - fill_pcpu(&pcpu, &maxcpu); - for (i = 0; i < maxcpu; i++) { - if (pcpu[i] == NULL) - continue; -#define ADD_FROM_PCPU(i, name) \ - vmmp->name += pcpu[i]->pc_cnt.name - ADD_FROM_PCPU(i, v_swtch); - ADD_FROM_PCPU(i, v_trap); - ADD_FROM_PCPU(i, v_syscall); - ADD_FROM_PCPU(i, v_intr); - ADD_FROM_PCPU(i, v_soft); - ADD_FROM_PCPU(i, v_vm_faults); - ADD_FROM_PCPU(i, v_io_faults); - ADD_FROM_PCPU(i, v_cow_faults); - ADD_FROM_PCPU(i, v_cow_optim); - ADD_FROM_PCPU(i, v_zfod); - ADD_FROM_PCPU(i, v_ozfod); - ADD_FROM_PCPU(i, v_swapin); - ADD_FROM_PCPU(i, v_swapout); - ADD_FROM_PCPU(i, v_swappgsin); - ADD_FROM_PCPU(i, v_swappgsout); - ADD_FROM_PCPU(i, v_vnodein); - ADD_FROM_PCPU(i, v_vnodeout); - ADD_FROM_PCPU(i, v_vnodepgsin); - ADD_FROM_PCPU(i, v_vnodepgsout); - ADD_FROM_PCPU(i, v_intrans); - ADD_FROM_PCPU(i, v_tfree); - ADD_FROM_PCPU(i, v_forks); - ADD_FROM_PCPU(i, v_vforks); - ADD_FROM_PCPU(i, v_rforks); - ADD_FROM_PCPU(i, v_kthreads); - ADD_FROM_PCPU(i, v_forkpages); - ADD_FROM_PCPU(i, v_vforkpages); - ADD_FROM_PCPU(i, v_rforkpages); - ADD_FROM_PCPU(i, v_kthreadpages); -#undef ADD_FROM_PCPU - } - free_pcpu(pcpu, maxcpu); + struct vmmeter vm_cnt; + + kread(X_SUM, &vm_cnt, sizeof(vm_cnt)); +#define GET_COUNTER(name) \ + vmmp->name = kvm_counter_u64_fetch(kd, (u_long)vm_cnt.name) + GET_COUNTER(v_swtch); + GET_COUNTER(v_trap); + GET_COUNTER(v_syscall); + GET_COUNTER(v_intr); + GET_COUNTER(v_soft); + GET_COUNTER(v_vm_faults); + GET_COUNTER(v_io_faults); + GET_COUNTER(v_cow_faults); + GET_COUNTER(v_cow_optim); + GET_COUNTER(v_zfod); + GET_COUNTER(v_ozfod); + GET_COUNTER(v_swapin); + GET_COUNTER(v_swapout); + GET_COUNTER(v_swappgsin); + GET_COUNTER(v_swappgsout); + GET_COUNTER(v_vnodein); + GET_COUNTER(v_vnodeout); + GET_COUNTER(v_vnodepgsin); + GET_COUNTER(v_vnodepgsout); + GET_COUNTER(v_intrans); + GET_COUNTER(v_tfree); + GET_COUNTER(v_forks); + GET_COUNTER(v_vforks); + GET_COUNTER(v_rforks); + GET_COUNTER(v_kthreads); + GET_COUNTER(v_forkpages); + GET_COUNTER(v_vforkpages); + GET_COUNTER(v_rforkpages); + GET_COUNTER(v_kthreadpages); +#undef GET_COUNTER } else { - size_t size = sizeof(unsigned int); + size_t size = sizeof(uint64_t); + #define GET_VM_STATS(cat, name) \ mysysctl("vm.stats." #cat "." #name, &vmmp->name, &size, NULL, 0) /* sys */