- Remove 'struct vmmeter' from 'struct pcpu', leaving only global vmmeter

  in place.  To do per-cpu stats, convert all fields that previously were
  maintained in the vmmeters that sit in pcpus to counter(9).
- Since some vmmeter stats may be touched at very early stages of boot,
  before we have set up UMA and we can do counter_u64_alloc(), provide an
  early counter mechanism:
  o Leave one spare uint64_t in struct pcpu, named pc_early_dummy_counter.
  o Point counter(9) fields of vmmeter to pcpu[0].pc_early_dummy_counter,
    so that at early stages of boot, before counters are allocated we already
    point to a counter that can be safely written to.
  o For sparc64 that required a whole dummy pcpu[MAXCPU] array.

Further related changes:
- Don't include vmmeter.h into pcpu.h.
- vm.stats.vm.v_swappgsout and vm.stats.vm.v_swappgsin changed to 64-bit,
  to match kernel representation.
- struct vmmeter hidden under _KERNEL, and only vmstat(1) is an exclusion.

This is based on benno@'s 4-year old patch:
https://lists.freebsd.org/pipermail/freebsd-arch/2013-July/014471.html

Reviewed by:	kib, gallatin, marius, lidl
Differential Revision:	https://reviews.freebsd.org/D10156

7 files changed, 97 insertions, 83 deletions

diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c
index 33b3ab1359a9..76b69e5053e9 100644
--- a/sys/vm/swap_pager.c
+++ b/sys/vm/swap_pager.c
@@ -1180,8 +1180,8 @@ swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,
 	bp->b_pgbefore = rbehind != NULL ? *rbehind : 0;
 	bp->b_pgafter = rahead != NULL ? *rahead : 0;
 
-	PCPU_INC(cnt.v_swapin);
-	PCPU_ADD(cnt.v_swappgsin, count);
+	VM_CNT_INC(v_swapin);
+	VM_CNT_ADD(v_swappgsin, count);
 
 	/*
 	 * perform the I/O.  NOTE!!!  bp cannot be considered valid after
@@ -1205,7 +1205,7 @@ swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,
 	VM_OBJECT_WLOCK(object);
 	while ((m[0]->oflags & VPO_SWAPINPROG) != 0) {
 		m[0]->oflags |= VPO_SWAPSLEEP;
-		PCPU_INC(cnt.v_intrans);
+		VM_CNT_INC(v_intrans);
 		if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP,
 		    "swread", hz * 20)) {
 			printf(
@@ -1393,8 +1393,8 @@ swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
 		bp->b_dirtyoff = 0;
 		bp->b_dirtyend = bp->b_bcount;
 
-		PCPU_INC(cnt.v_swapout);
-		PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
+		VM_CNT_INC(v_swapout);
+		VM_CNT_ADD(v_swappgsout, bp->b_npages);
 
 		/*
 		 * We unconditionally set rtvals[] to VM_PAGER_PEND so that we
diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index 2349e4938c01..0e584e5f0cb7 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -497,7 +497,7 @@ vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
 	boolean_t wired;	/* Passed by reference. */
 	bool dead, growstack, hardfault, is_first_object_locked;
 
-	PCPU_INC(cnt.v_vm_faults);
+	VM_CNT_INC(v_vm_faults);
 	fs.vp = NULL;
 	faultcount = 0;
 	nera = -1;
@@ -673,7 +673,7 @@ RetryFault:;
 				}
 				vm_object_pip_wakeup(fs.object);
 				VM_OBJECT_WUNLOCK(fs.object);
-				PCPU_INC(cnt.v_intrans);
+				VM_CNT_INC(v_intrans);
 				vm_object_deallocate(fs.first_object);
 				goto RetryFault;
 			}
@@ -999,9 +999,9 @@ readrest:
 			if ((fs.m->flags & PG_ZERO) == 0) {
 				pmap_zero_page(fs.m);
 			} else {
-				PCPU_INC(cnt.v_ozfod);
+				VM_CNT_INC(v_ozfod);
 			}
-			PCPU_INC(cnt.v_zfod);
+			VM_CNT_INC(v_zfod);
 			fs.m->valid = VM_PAGE_BITS_ALL;
 			/* Don't try to prefault neighboring pages. */
 			faultcount = 1;
@@ -1095,7 +1095,7 @@ readrest:
 				vm_page_xbusy(fs.m);
 				fs.first_m = fs.m;
 				fs.m = NULL;
-				PCPU_INC(cnt.v_cow_optim);
+				VM_CNT_INC(v_cow_optim);
 			} else {
 				/*
 				 * Oh, well, lets copy it.
@@ -1131,7 +1131,7 @@ readrest:
 			fs.m = fs.first_m;
 			if (!is_first_object_locked)
 				VM_OBJECT_WLOCK(fs.object);
-			PCPU_INC(cnt.v_cow_faults);
+			VM_CNT_INC(v_cow_faults);
 			curthread->td_cow++;
 		} else {
 			prot &= ~VM_PROT_WRITE;
@@ -1246,7 +1246,7 @@ readrest:
 	 */
 	unlock_and_deallocate(&fs);
 	if (hardfault) {
-		PCPU_INC(cnt.v_io_faults);
+		VM_CNT_INC(v_io_faults);
 		curthread->td_ru.ru_majflt++;
 #ifdef RACCT
 		if (racct_enable && fs.object->type == OBJT_VNODE) {
diff --git a/sys/vm/vm_meter.c b/sys/vm/vm_meter.c
index 5f1789de50e5..5f4cd46ab1e4 100644
--- a/sys/vm/vm_meter.c
+++ b/sys/vm/vm_meter.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
+#include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
 #include <sys/resource.h>
@@ -55,7 +56,52 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_object.h>
 #include <sys/sysctl.h>
 
-struct vmmeter vm_cnt;
+struct vmmeter vm_cnt = {
+	.v_swtch = EARLY_COUNTER,
+	.v_trap = EARLY_COUNTER,
+	.v_syscall = EARLY_COUNTER,
+	.v_intr = EARLY_COUNTER,
+	.v_soft = EARLY_COUNTER,
+	.v_vm_faults = EARLY_COUNTER,
+	.v_io_faults = EARLY_COUNTER,
+	.v_cow_faults = EARLY_COUNTER,
+	.v_cow_optim = EARLY_COUNTER,
+	.v_zfod = EARLY_COUNTER,
+	.v_ozfod = EARLY_COUNTER,
+	.v_swapin = EARLY_COUNTER,
+	.v_swapout = EARLY_COUNTER,
+	.v_swappgsin = EARLY_COUNTER,
+	.v_swappgsout = EARLY_COUNTER,
+	.v_vnodein = EARLY_COUNTER,
+	.v_vnodeout = EARLY_COUNTER,
+	.v_vnodepgsin = EARLY_COUNTER,
+	.v_vnodepgsout = EARLY_COUNTER,
+	.v_intrans = EARLY_COUNTER,
+	.v_reactivated = EARLY_COUNTER,
+	.v_pdwakeups = EARLY_COUNTER,
+	.v_pdpages = EARLY_COUNTER,
+	.v_pdshortfalls = EARLY_COUNTER,
+	.v_dfree = EARLY_COUNTER,
+	.v_pfree = EARLY_COUNTER,
+	.v_tfree = EARLY_COUNTER,
+	.v_forks = EARLY_COUNTER,
+	.v_vforks = EARLY_COUNTER,
+	.v_rforks = EARLY_COUNTER,
+	.v_kthreads = EARLY_COUNTER,
+	.v_forkpages = EARLY_COUNTER,
+	.v_vforkpages = EARLY_COUNTER,
+	.v_rforkpages = EARLY_COUNTER,
+	.v_kthreadpages = EARLY_COUNTER,
+};
+
+static void
+vmcounter_startup(void)
+{
+	counter_u64_t *cnt = (counter_u64_t *)&vm_cnt;
+
+	COUNTER_ARRAY_ALLOC(cnt, VM_METER_NCOUNTERS, M_WAITOK);
+}
+SYSINIT(counter, SI_SUB_CPU, SI_ORDER_FOURTH + 1, vmcounter_startup, NULL);
 
 SYSCTL_UINT(_vm, VM_V_FREE_MIN, v_free_min,
 	CTLFLAG_RW, &vm_cnt.v_free_min, 0, "Minimum low-free-pages threshold");
@@ -210,40 +256,6 @@ vmtotal(SYSCTL_HANDLER_ARGS)
 	return (sysctl_handle_opaque(oidp, &total, sizeof(total), req));
 }
 
-/*
- * vm_meter_cnt() -	accumulate statistics from all cpus and the global cnt
- *			structure.
- *
- *	The vmmeter structure is now per-cpu as well as global.  Those
- *	statistics which can be kept on a per-cpu basis (to avoid cache
- *	stalls between cpus) can be moved to the per-cpu vmmeter.  Remaining
- *	statistics, such as v_free_reserved, are left in the global
- *	structure.
- */
-u_int
-vm_meter_cnt(size_t offset)
-{
-	struct pcpu *pcpu;
-	u_int count;
-	int i;
-
-	count = *(u_int *)((char *)&vm_cnt + offset);
-	CPU_FOREACH(i) {
-		pcpu = pcpu_find(i);
-		count += *(u_int *)((char *)&pcpu->pc_cnt + offset);
-	}
-	return (count);
-}
-
-static int
-cnt_sysctl(SYSCTL_HANDLER_ARGS)
-{
-	u_int count;
-
-	count = vm_meter_cnt((char *)arg1 - (char *)&vm_cnt);
-	return (SYSCTL_OUT(req, &count, sizeof(count)));
-}
-
 SYSCTL_PROC(_vm, VM_TOTAL, vmtotal, CTLTYPE_OPAQUE|CTLFLAG_RD|CTLFLAG_MPSAFE,
     0, sizeof(struct vmtotal), vmtotal, "S,vmtotal", 
     "System virtual memory statistics");
@@ -255,9 +267,7 @@ static SYSCTL_NODE(_vm_stats, OID_AUTO, vm, CTLFLAG_RW, 0,
 SYSCTL_NODE(_vm_stats, OID_AUTO, misc, CTLFLAG_RW, 0, "VM meter misc stats");
 
 #define	VM_STATS(parent, var, descr) \
-	SYSCTL_PROC(parent, OID_AUTO, var, \
-	    CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, &vm_cnt.var, 0,	\
-	    cnt_sysctl, "IU", descr)
+    SYSCTL_COUNTER_U64(parent, OID_AUTO, var, CTLFLAG_RD, &vm_cnt.var, descr)
 #define	VM_STATS_VM(var, descr)		VM_STATS(_vm_stats_vm, var, descr)
 #define	VM_STATS_SYS(var, descr)	VM_STATS(_vm_stats_sys, var, descr)
 
@@ -288,19 +298,6 @@ VM_STATS_VM(v_pdshortfalls, "Page reclamation shortfalls");
 VM_STATS_VM(v_dfree, "Pages freed by pagedaemon");
 VM_STATS_VM(v_pfree, "Pages freed by exiting processes");
 VM_STATS_VM(v_tfree, "Total pages freed");
-VM_STATS_VM(v_page_size, "Page size in bytes");
-VM_STATS_VM(v_page_count, "Total number of pages in system");
-VM_STATS_VM(v_free_reserved, "Pages reserved for deadlock");
-VM_STATS_VM(v_free_target, "Pages desired free");
-VM_STATS_VM(v_free_min, "Minimum low-free-pages threshold");
-VM_STATS_VM(v_free_count, "Free pages");
-VM_STATS_VM(v_wire_count, "Wired pages");
-VM_STATS_VM(v_active_count, "Active pages");
-VM_STATS_VM(v_inactive_target, "Desired inactive pages");
-VM_STATS_VM(v_inactive_count, "Inactive pages");
-VM_STATS_VM(v_laundry_count, "Pages eligible for laundering");
-VM_STATS_VM(v_pageout_free_min, "Min pages reserved for kernel");
-VM_STATS_VM(v_interrupt_free_min, "Reserved pages for interrupt code");
 VM_STATS_VM(v_forks, "Number of fork() calls");
 VM_STATS_VM(v_vforks, "Number of vfork() calls");
 VM_STATS_VM(v_rforks, "Number of rfork() calls");
@@ -310,6 +307,23 @@ VM_STATS_VM(v_vforkpages, "VM pages affected by vfork()");
 VM_STATS_VM(v_rforkpages, "VM pages affected by rfork()");
 VM_STATS_VM(v_kthreadpages, "VM pages affected by fork() by kernel");
 
+#define	VM_STATS_UINT(var, descr)	\
+    SYSCTL_UINT(_vm_stats_vm, OID_AUTO, var, CTLFLAG_RD, &vm_cnt.var, 0, descr)
+VM_STATS_UINT(v_page_size, "Page size in bytes");
+VM_STATS_UINT(v_page_count, "Total number of pages in system");
+VM_STATS_UINT(v_free_reserved, "Pages reserved for deadlock");
+VM_STATS_UINT(v_free_target, "Pages desired free");
+VM_STATS_UINT(v_free_min, "Minimum low-free-pages threshold");
+VM_STATS_UINT(v_free_count, "Free pages");
+VM_STATS_UINT(v_wire_count, "Wired pages");
+VM_STATS_UINT(v_active_count, "Active pages");
+VM_STATS_UINT(v_inactive_target, "Desired inactive pages");
+VM_STATS_UINT(v_inactive_count, "Inactive pages");
+VM_STATS_UINT(v_laundry_count, "Pages eligible for laundering");
+VM_STATS_UINT(v_pageout_free_min, "Min pages reserved for kernel");
+VM_STATS_UINT(v_interrupt_free_min, "Reserved pages for interrupt code");
+VM_STATS_UINT(v_free_severe, "Severe page depletion point");
+
 #ifdef COMPAT_FREEBSD11
 /*
  * Provide compatibility sysctls for the benefit of old utilities which exit
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index d57984b38826..022fbd57c882 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -765,7 +765,7 @@ vm_object_terminate(vm_object_t object)
 		p->object = NULL;
 		if (p->wire_count == 0) {
 			vm_page_free(p);
-			PCPU_INC(cnt.v_pfree);
+			VM_CNT_INC(v_pfree);
 		}
 		vm_page_unlock(p);
 	}
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 5aeefbf1c09a..e6befe15b915 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -2748,7 +2748,7 @@ vm_page_free_toq(vm_page_t m)
 	} else
 		KASSERT(m->queue == PQ_NONE,
 		    ("vm_page_free_toq: unmanaged page %p is queued", m));
-	PCPU_INC(cnt.v_tfree);
+	VM_CNT_INC(v_tfree);
 
 	if (vm_page_sbusied(m))
 		panic("vm_page_free: freeing busy page %p", m);
diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index 6c94f370971c..2e08fdf9d403 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -667,7 +667,7 @@ vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object,
 				goto unlock_return;
 			if (vm_page_busied(p))
 				continue;
-			PCPU_INC(cnt.v_pdpages);
+			VM_CNT_INC(v_pdpages);
 			vm_page_lock(p);
 			if (p->wire_count != 0 || p->hold_count != 0 ||
 			    !pmap_page_exists_quick(pmap, p)) {
@@ -1003,7 +1003,7 @@ scan:
 		}
 		if (act_delta != 0) {
 			if (object->ref_count != 0) {
-				PCPU_INC(cnt.v_reactivated);
+				VM_CNT_INC(v_reactivated);
 				vm_page_activate(m);
 
 				/*
@@ -1052,7 +1052,7 @@ scan:
 		if (m->dirty == 0) {
 free_page:
 			vm_page_free(m);
-			PCPU_INC(cnt.v_dfree);
+			VM_CNT_INC(v_dfree);
 		} else if ((object->flags & OBJ_DEAD) == 0) {
 			if (object->type != OBJT_SWAP &&
 			    object->type != OBJT_DEFAULT)
@@ -1187,7 +1187,7 @@ vm_pageout_laundry_worker(void *arg)
 		KASSERT(shortfall_cycle >= 0,
 		    ("negative cycle %d", shortfall_cycle));
 		launder = 0;
-		wakeups = VM_METER_PCPU_CNT(v_pdwakeups);
+		wakeups = VM_CNT_FETCH(v_pdwakeups);
 
 		/*
 		 * First determine whether we need to launder pages to meet a
@@ -1378,7 +1378,7 @@ vm_pageout_scan(struct vm_domain *vmd, int pass)
 		KASSERT(queue_locked, ("unlocked inactive queue"));
 		KASSERT(vm_page_inactive(m), ("Inactive queue %p", m));
 
-		PCPU_INC(cnt.v_pdpages);
+		VM_CNT_INC(v_pdpages);
 		next = TAILQ_NEXT(m, plinks.q);
 
 		/*
@@ -1476,7 +1476,7 @@ unlock_page:
 		}
 		if (act_delta != 0) {
 			if (object->ref_count != 0) {
-				PCPU_INC(cnt.v_reactivated);
+				VM_CNT_INC(v_reactivated);
 				vm_page_activate(m);
 
 				/*
@@ -1521,7 +1521,7 @@ unlock_page:
 		if (m->dirty == 0) {
 free_page:
 			vm_page_free(m);
-			PCPU_INC(cnt.v_dfree);
+			VM_CNT_INC(v_dfree);
 			--page_shortage;
 		} else if ((object->flags & OBJ_DEAD) == 0)
 			vm_page_launder(m);
@@ -1551,7 +1551,7 @@ drop_page:
 		if (pq->pq_cnt > 0 || atomic_load_acq_int(&swapdev_enabled)) {
 			if (page_shortage > 0) {
 				vm_laundry_request = VM_LAUNDRY_SHORTFALL;
-				PCPU_INC(cnt.v_pdshortfalls);
+				VM_CNT_INC(v_pdshortfalls);
 			} else if (vm_laundry_request != VM_LAUNDRY_SHORTFALL)
 				vm_laundry_request = VM_LAUNDRY_BACKGROUND;
 			wakeup(&vm_laundry_request);
@@ -1635,7 +1635,7 @@ drop_page:
 		 * The count for page daemon pages is updated after checking
 		 * the page for eligibility.
 		 */
-		PCPU_INC(cnt.v_pdpages);
+		VM_CNT_INC(v_pdpages);
 
 		/*
 		 * Check to see "how much" the page has been used.
@@ -2036,7 +2036,7 @@ vm_pageout_worker(void *arg)
 			if (mtx_sleep(&vm_pageout_wanted,
 			    &vm_page_queue_free_mtx, PDROP | PVM, "psleep",
 			    hz) == 0) {
-				PCPU_INC(cnt.v_pdwakeups);
+				VM_CNT_INC(v_pdwakeups);
 				pass = 1;
 			} else
 				pass = 0;
diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c
index f52016d68f00..aba50c6a5b58 100644
--- a/sys/vm/vnode_pager.c
+++ b/sys/vm/vnode_pager.c
@@ -799,8 +799,8 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count,
 		relpbuf(bp, freecnt);
 		VM_OBJECT_WLOCK(object);
 		for (i = 0; i < count; i++) {
-			PCPU_INC(cnt.v_vnodein);
-			PCPU_INC(cnt.v_vnodepgsin);
+			VM_CNT_INC(v_vnodein);
+			VM_CNT_INC(v_vnodepgsin);
 			error = vnode_pager_input_old(object, m[i]);
 			if (error)
 				break;
@@ -819,8 +819,8 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count,
 	if (pagesperblock == 0) {
 		relpbuf(bp, freecnt);
 		for (i = 0; i < count; i++) {
-			PCPU_INC(cnt.v_vnodein);
-			PCPU_INC(cnt.v_vnodepgsin);
+			VM_CNT_INC(v_vnodein);
+			VM_CNT_INC(v_vnodepgsin);
 			error = vnode_pager_input_smlfs(object, m[i]);
 			if (error)
 				break;
@@ -1030,8 +1030,8 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count,
 	    (uintmax_t)blkno0, (uintmax_t)bp->b_blkno));
 
 	atomic_add_long(&runningbufspace, bp->b_runningbufspace);
-	PCPU_INC(cnt.v_vnodein);
-	PCPU_ADD(cnt.v_vnodepgsin, bp->b_npages);
+	VM_CNT_INC(v_vnodein);
+	VM_CNT_ADD(v_vnodepgsin, bp->b_npages);
 
 	if (iodone != NULL) { /* async */
 		bp->b_pgiodone = iodone;
@@ -1276,8 +1276,8 @@ vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *ma, int bytecount,
 	auio.uio_td = (struct thread *) 0;
 	error = VOP_WRITE(vp, &auio, vnode_pager_putpages_ioflags(flags),
 	    curthread->td_ucred);
-	PCPU_INC(cnt.v_vnodeout);
-	PCPU_ADD(cnt.v_vnodepgsout, ncount);
+	VM_CNT_INC(v_vnodeout);
+	VM_CNT_ADD(v_vnodepgsout, ncount);
 
 	ppscheck = 0;
 	if (error) {