diff options
author | Gleb Smirnoff <glebius@FreeBSD.org> | 2017-04-17 17:34:47 +0000 |
---|---|---|
committer | Gleb Smirnoff <glebius@FreeBSD.org> | 2017-04-17 17:34:47 +0000 |
commit | 83c9dea1bac40c7c7cbde4ccb3d747134311ab5a (patch) | |
tree | 7679e220e254a60031cd36e1421cb9c844a72521 /sys/vm | |
parent | 21d5d37ba4c0131d6c141695366e266e32cc3bc1 (diff) | |
download | src-83c9dea1bac40c7c7cbde4ccb3d747134311ab5a.tar.gz src-83c9dea1bac40c7c7cbde4ccb3d747134311ab5a.zip |
- Remove 'struct vmmeter' from 'struct pcpu', leaving only global vmmeter
in place. To do per-cpu stats, convert all fields that previously were
maintained in the vmmeters that sit in pcpus to counter(9).
- Since some vmmeter stats may be touched at very early stages of boot,
before we have set up UMA and we can do counter_u64_alloc(), provide an
early counter mechanism:
o Leave one spare uint64_t in struct pcpu, named pc_early_dummy_counter.
o Point counter(9) fields of vmmeter to pcpu[0].pc_early_dummy_counter,
so that at early stages of boot, before counters are allocated we already
point to a counter that can be safely written to.
o For sparc64 that required a whole dummy pcpu[MAXCPU] array.
Further related changes:
- Don't include vmmeter.h into pcpu.h.
- vm.stats.vm.v_swappgsout and vm.stats.vm.v_swappgsin changed to 64-bit,
to match kernel representation.
- struct vmmeter hidden under _KERNEL, and only vmstat(1) is an exclusion.
This is based on benno@'s 4-year old patch:
https://lists.freebsd.org/pipermail/freebsd-arch/2013-July/014471.html
Reviewed by: kib, gallatin, marius, lidl
Differential Revision: https://reviews.freebsd.org/D10156
Notes
Notes:
svn path=/head/; revision=317061
Diffstat (limited to 'sys/vm')
-rw-r--r-- | sys/vm/swap_pager.c | 10 | ||||
-rw-r--r-- | sys/vm/vm_fault.c | 14 | ||||
-rw-r--r-- | sys/vm/vm_meter.c | 116 | ||||
-rw-r--r-- | sys/vm/vm_object.c | 2 | ||||
-rw-r--r-- | sys/vm/vm_page.c | 2 | ||||
-rw-r--r-- | sys/vm/vm_pageout.c | 20 | ||||
-rw-r--r-- | sys/vm/vnode_pager.c | 16 |
7 files changed, 97 insertions, 83 deletions
diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c index 33b3ab1359a9..76b69e5053e9 100644 --- a/sys/vm/swap_pager.c +++ b/sys/vm/swap_pager.c @@ -1180,8 +1180,8 @@ swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, bp->b_pgbefore = rbehind != NULL ? *rbehind : 0; bp->b_pgafter = rahead != NULL ? *rahead : 0; - PCPU_INC(cnt.v_swapin); - PCPU_ADD(cnt.v_swappgsin, count); + VM_CNT_INC(v_swapin); + VM_CNT_ADD(v_swappgsin, count); /* * perform the I/O. NOTE!!! bp cannot be considered valid after @@ -1205,7 +1205,7 @@ swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, VM_OBJECT_WLOCK(object); while ((m[0]->oflags & VPO_SWAPINPROG) != 0) { m[0]->oflags |= VPO_SWAPSLEEP; - PCPU_INC(cnt.v_intrans); + VM_CNT_INC(v_intrans); if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP, "swread", hz * 20)) { printf( @@ -1393,8 +1393,8 @@ swap_pager_putpages(vm_object_t object, vm_page_t *m, int count, bp->b_dirtyoff = 0; bp->b_dirtyend = bp->b_bcount; - PCPU_INC(cnt.v_swapout); - PCPU_ADD(cnt.v_swappgsout, bp->b_npages); + VM_CNT_INC(v_swapout); + VM_CNT_ADD(v_swappgsout, bp->b_npages); /* * We unconditionally set rtvals[] to VM_PAGER_PEND so that we diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index 2349e4938c01..0e584e5f0cb7 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -497,7 +497,7 @@ vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, boolean_t wired; /* Passed by reference. */ bool dead, growstack, hardfault, is_first_object_locked; - PCPU_INC(cnt.v_vm_faults); + VM_CNT_INC(v_vm_faults); fs.vp = NULL; faultcount = 0; nera = -1; @@ -673,7 +673,7 @@ RetryFault:; } vm_object_pip_wakeup(fs.object); VM_OBJECT_WUNLOCK(fs.object); - PCPU_INC(cnt.v_intrans); + VM_CNT_INC(v_intrans); vm_object_deallocate(fs.first_object); goto RetryFault; } @@ -999,9 +999,9 @@ readrest: if ((fs.m->flags & PG_ZERO) == 0) { pmap_zero_page(fs.m); } else { - PCPU_INC(cnt.v_ozfod); + VM_CNT_INC(v_ozfod); } - PCPU_INC(cnt.v_zfod); + VM_CNT_INC(v_zfod); fs.m->valid = VM_PAGE_BITS_ALL; /* Don't try to prefault neighboring pages. */ faultcount = 1; @@ -1095,7 +1095,7 @@ readrest: vm_page_xbusy(fs.m); fs.first_m = fs.m; fs.m = NULL; - PCPU_INC(cnt.v_cow_optim); + VM_CNT_INC(v_cow_optim); } else { /* * Oh, well, lets copy it. @@ -1131,7 +1131,7 @@ readrest: fs.m = fs.first_m; if (!is_first_object_locked) VM_OBJECT_WLOCK(fs.object); - PCPU_INC(cnt.v_cow_faults); + VM_CNT_INC(v_cow_faults); curthread->td_cow++; } else { prot &= ~VM_PROT_WRITE; @@ -1246,7 +1246,7 @@ readrest: */ unlock_and_deallocate(&fs); if (hardfault) { - PCPU_INC(cnt.v_io_faults); + VM_CNT_INC(v_io_faults); curthread->td_ru.ru_majflt++; #ifdef RACCT if (racct_enable && fs.object->type == OBJT_VNODE) { diff --git a/sys/vm/vm_meter.c b/sys/vm/vm_meter.c index 5f1789de50e5..5f4cd46ab1e4 100644 --- a/sys/vm/vm_meter.c +++ b/sys/vm/vm_meter.c @@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/kernel.h> #include <sys/lock.h> +#include <sys/malloc.h> #include <sys/mutex.h> #include <sys/proc.h> #include <sys/resource.h> @@ -55,7 +56,52 @@ __FBSDID("$FreeBSD$"); #include <vm/vm_object.h> #include <sys/sysctl.h> -struct vmmeter vm_cnt; +struct vmmeter vm_cnt = { + .v_swtch = EARLY_COUNTER, + .v_trap = EARLY_COUNTER, + .v_syscall = EARLY_COUNTER, + .v_intr = EARLY_COUNTER, + .v_soft = EARLY_COUNTER, + .v_vm_faults = EARLY_COUNTER, + .v_io_faults = EARLY_COUNTER, + .v_cow_faults = EARLY_COUNTER, + .v_cow_optim = EARLY_COUNTER, + .v_zfod = EARLY_COUNTER, + .v_ozfod = EARLY_COUNTER, + .v_swapin = EARLY_COUNTER, + .v_swapout = EARLY_COUNTER, + .v_swappgsin = EARLY_COUNTER, + .v_swappgsout = EARLY_COUNTER, + .v_vnodein = EARLY_COUNTER, + .v_vnodeout = EARLY_COUNTER, + .v_vnodepgsin = EARLY_COUNTER, + .v_vnodepgsout = EARLY_COUNTER, + .v_intrans = EARLY_COUNTER, + .v_reactivated = EARLY_COUNTER, + .v_pdwakeups = EARLY_COUNTER, + .v_pdpages = EARLY_COUNTER, + .v_pdshortfalls = EARLY_COUNTER, + .v_dfree = EARLY_COUNTER, + .v_pfree = EARLY_COUNTER, + .v_tfree = EARLY_COUNTER, + .v_forks = EARLY_COUNTER, + .v_vforks = EARLY_COUNTER, + .v_rforks = EARLY_COUNTER, + .v_kthreads = EARLY_COUNTER, + .v_forkpages = EARLY_COUNTER, + .v_vforkpages = EARLY_COUNTER, + .v_rforkpages = EARLY_COUNTER, + .v_kthreadpages = EARLY_COUNTER, +}; + +static void +vmcounter_startup(void) +{ + counter_u64_t *cnt = (counter_u64_t *)&vm_cnt; + + COUNTER_ARRAY_ALLOC(cnt, VM_METER_NCOUNTERS, M_WAITOK); +} +SYSINIT(counter, SI_SUB_CPU, SI_ORDER_FOURTH + 1, vmcounter_startup, NULL); SYSCTL_UINT(_vm, VM_V_FREE_MIN, v_free_min, CTLFLAG_RW, &vm_cnt.v_free_min, 0, "Minimum low-free-pages threshold"); @@ -210,40 +256,6 @@ vmtotal(SYSCTL_HANDLER_ARGS) return (sysctl_handle_opaque(oidp, &total, sizeof(total), req)); } -/* - * vm_meter_cnt() - accumulate statistics from all cpus and the global cnt - * structure. - * - * The vmmeter structure is now per-cpu as well as global. Those - * statistics which can be kept on a per-cpu basis (to avoid cache - * stalls between cpus) can be moved to the per-cpu vmmeter. Remaining - * statistics, such as v_free_reserved, are left in the global - * structure. - */ -u_int -vm_meter_cnt(size_t offset) -{ - struct pcpu *pcpu; - u_int count; - int i; - - count = *(u_int *)((char *)&vm_cnt + offset); - CPU_FOREACH(i) { - pcpu = pcpu_find(i); - count += *(u_int *)((char *)&pcpu->pc_cnt + offset); - } - return (count); -} - -static int -cnt_sysctl(SYSCTL_HANDLER_ARGS) -{ - u_int count; - - count = vm_meter_cnt((char *)arg1 - (char *)&vm_cnt); - return (SYSCTL_OUT(req, &count, sizeof(count))); -} - SYSCTL_PROC(_vm, VM_TOTAL, vmtotal, CTLTYPE_OPAQUE|CTLFLAG_RD|CTLFLAG_MPSAFE, 0, sizeof(struct vmtotal), vmtotal, "S,vmtotal", "System virtual memory statistics"); @@ -255,9 +267,7 @@ static SYSCTL_NODE(_vm_stats, OID_AUTO, vm, CTLFLAG_RW, 0, SYSCTL_NODE(_vm_stats, OID_AUTO, misc, CTLFLAG_RW, 0, "VM meter misc stats"); #define VM_STATS(parent, var, descr) \ - SYSCTL_PROC(parent, OID_AUTO, var, \ - CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, &vm_cnt.var, 0, \ - cnt_sysctl, "IU", descr) + SYSCTL_COUNTER_U64(parent, OID_AUTO, var, CTLFLAG_RD, &vm_cnt.var, descr) #define VM_STATS_VM(var, descr) VM_STATS(_vm_stats_vm, var, descr) #define VM_STATS_SYS(var, descr) VM_STATS(_vm_stats_sys, var, descr) @@ -288,19 +298,6 @@ VM_STATS_VM(v_pdshortfalls, "Page reclamation shortfalls"); VM_STATS_VM(v_dfree, "Pages freed by pagedaemon"); VM_STATS_VM(v_pfree, "Pages freed by exiting processes"); VM_STATS_VM(v_tfree, "Total pages freed"); -VM_STATS_VM(v_page_size, "Page size in bytes"); -VM_STATS_VM(v_page_count, "Total number of pages in system"); -VM_STATS_VM(v_free_reserved, "Pages reserved for deadlock"); -VM_STATS_VM(v_free_target, "Pages desired free"); -VM_STATS_VM(v_free_min, "Minimum low-free-pages threshold"); -VM_STATS_VM(v_free_count, "Free pages"); -VM_STATS_VM(v_wire_count, "Wired pages"); -VM_STATS_VM(v_active_count, "Active pages"); -VM_STATS_VM(v_inactive_target, "Desired inactive pages"); -VM_STATS_VM(v_inactive_count, "Inactive pages"); -VM_STATS_VM(v_laundry_count, "Pages eligible for laundering"); -VM_STATS_VM(v_pageout_free_min, "Min pages reserved for kernel"); -VM_STATS_VM(v_interrupt_free_min, "Reserved pages for interrupt code"); VM_STATS_VM(v_forks, "Number of fork() calls"); VM_STATS_VM(v_vforks, "Number of vfork() calls"); VM_STATS_VM(v_rforks, "Number of rfork() calls"); @@ -310,6 +307,23 @@ VM_STATS_VM(v_vforkpages, "VM pages affected by vfork()"); VM_STATS_VM(v_rforkpages, "VM pages affected by rfork()"); VM_STATS_VM(v_kthreadpages, "VM pages affected by fork() by kernel"); +#define VM_STATS_UINT(var, descr) \ + SYSCTL_UINT(_vm_stats_vm, OID_AUTO, var, CTLFLAG_RD, &vm_cnt.var, 0, descr) +VM_STATS_UINT(v_page_size, "Page size in bytes"); +VM_STATS_UINT(v_page_count, "Total number of pages in system"); +VM_STATS_UINT(v_free_reserved, "Pages reserved for deadlock"); +VM_STATS_UINT(v_free_target, "Pages desired free"); +VM_STATS_UINT(v_free_min, "Minimum low-free-pages threshold"); +VM_STATS_UINT(v_free_count, "Free pages"); +VM_STATS_UINT(v_wire_count, "Wired pages"); +VM_STATS_UINT(v_active_count, "Active pages"); +VM_STATS_UINT(v_inactive_target, "Desired inactive pages"); +VM_STATS_UINT(v_inactive_count, "Inactive pages"); +VM_STATS_UINT(v_laundry_count, "Pages eligible for laundering"); +VM_STATS_UINT(v_pageout_free_min, "Min pages reserved for kernel"); +VM_STATS_UINT(v_interrupt_free_min, "Reserved pages for interrupt code"); +VM_STATS_UINT(v_free_severe, "Severe page depletion point"); + #ifdef COMPAT_FREEBSD11 /* * Provide compatibility sysctls for the benefit of old utilities which exit diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index d57984b38826..022fbd57c882 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -765,7 +765,7 @@ vm_object_terminate(vm_object_t object) p->object = NULL; if (p->wire_count == 0) { vm_page_free(p); - PCPU_INC(cnt.v_pfree); + VM_CNT_INC(v_pfree); } vm_page_unlock(p); } diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 5aeefbf1c09a..e6befe15b915 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -2748,7 +2748,7 @@ vm_page_free_toq(vm_page_t m) } else KASSERT(m->queue == PQ_NONE, ("vm_page_free_toq: unmanaged page %p is queued", m)); - PCPU_INC(cnt.v_tfree); + VM_CNT_INC(v_tfree); if (vm_page_sbusied(m)) panic("vm_page_free: freeing busy page %p", m); diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 6c94f370971c..2e08fdf9d403 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -667,7 +667,7 @@ vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object, goto unlock_return; if (vm_page_busied(p)) continue; - PCPU_INC(cnt.v_pdpages); + VM_CNT_INC(v_pdpages); vm_page_lock(p); if (p->wire_count != 0 || p->hold_count != 0 || !pmap_page_exists_quick(pmap, p)) { @@ -1003,7 +1003,7 @@ scan: } if (act_delta != 0) { if (object->ref_count != 0) { - PCPU_INC(cnt.v_reactivated); + VM_CNT_INC(v_reactivated); vm_page_activate(m); /* @@ -1052,7 +1052,7 @@ scan: if (m->dirty == 0) { free_page: vm_page_free(m); - PCPU_INC(cnt.v_dfree); + VM_CNT_INC(v_dfree); } else if ((object->flags & OBJ_DEAD) == 0) { if (object->type != OBJT_SWAP && object->type != OBJT_DEFAULT) @@ -1187,7 +1187,7 @@ vm_pageout_laundry_worker(void *arg) KASSERT(shortfall_cycle >= 0, ("negative cycle %d", shortfall_cycle)); launder = 0; - wakeups = VM_METER_PCPU_CNT(v_pdwakeups); + wakeups = VM_CNT_FETCH(v_pdwakeups); /* * First determine whether we need to launder pages to meet a @@ -1378,7 +1378,7 @@ vm_pageout_scan(struct vm_domain *vmd, int pass) KASSERT(queue_locked, ("unlocked inactive queue")); KASSERT(vm_page_inactive(m), ("Inactive queue %p", m)); - PCPU_INC(cnt.v_pdpages); + VM_CNT_INC(v_pdpages); next = TAILQ_NEXT(m, plinks.q); /* @@ -1476,7 +1476,7 @@ unlock_page: } if (act_delta != 0) { if (object->ref_count != 0) { - PCPU_INC(cnt.v_reactivated); + VM_CNT_INC(v_reactivated); vm_page_activate(m); /* @@ -1521,7 +1521,7 @@ unlock_page: if (m->dirty == 0) { free_page: vm_page_free(m); - PCPU_INC(cnt.v_dfree); + VM_CNT_INC(v_dfree); --page_shortage; } else if ((object->flags & OBJ_DEAD) == 0) vm_page_launder(m); @@ -1551,7 +1551,7 @@ drop_page: if (pq->pq_cnt > 0 || atomic_load_acq_int(&swapdev_enabled)) { if (page_shortage > 0) { vm_laundry_request = VM_LAUNDRY_SHORTFALL; - PCPU_INC(cnt.v_pdshortfalls); + VM_CNT_INC(v_pdshortfalls); } else if (vm_laundry_request != VM_LAUNDRY_SHORTFALL) vm_laundry_request = VM_LAUNDRY_BACKGROUND; wakeup(&vm_laundry_request); @@ -1635,7 +1635,7 @@ drop_page: * The count for page daemon pages is updated after checking * the page for eligibility. */ - PCPU_INC(cnt.v_pdpages); + VM_CNT_INC(v_pdpages); /* * Check to see "how much" the page has been used. @@ -2036,7 +2036,7 @@ vm_pageout_worker(void *arg) if (mtx_sleep(&vm_pageout_wanted, &vm_page_queue_free_mtx, PDROP | PVM, "psleep", hz) == 0) { - PCPU_INC(cnt.v_pdwakeups); + VM_CNT_INC(v_pdwakeups); pass = 1; } else pass = 0; diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index f52016d68f00..aba50c6a5b58 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -799,8 +799,8 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count, relpbuf(bp, freecnt); VM_OBJECT_WLOCK(object); for (i = 0; i < count; i++) { - PCPU_INC(cnt.v_vnodein); - PCPU_INC(cnt.v_vnodepgsin); + VM_CNT_INC(v_vnodein); + VM_CNT_INC(v_vnodepgsin); error = vnode_pager_input_old(object, m[i]); if (error) break; @@ -819,8 +819,8 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count, if (pagesperblock == 0) { relpbuf(bp, freecnt); for (i = 0; i < count; i++) { - PCPU_INC(cnt.v_vnodein); - PCPU_INC(cnt.v_vnodepgsin); + VM_CNT_INC(v_vnodein); + VM_CNT_INC(v_vnodepgsin); error = vnode_pager_input_smlfs(object, m[i]); if (error) break; @@ -1030,8 +1030,8 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count, (uintmax_t)blkno0, (uintmax_t)bp->b_blkno)); atomic_add_long(&runningbufspace, bp->b_runningbufspace); - PCPU_INC(cnt.v_vnodein); - PCPU_ADD(cnt.v_vnodepgsin, bp->b_npages); + VM_CNT_INC(v_vnodein); + VM_CNT_ADD(v_vnodepgsin, bp->b_npages); if (iodone != NULL) { /* async */ bp->b_pgiodone = iodone; @@ -1276,8 +1276,8 @@ vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *ma, int bytecount, auio.uio_td = (struct thread *) 0; error = VOP_WRITE(vp, &auio, vnode_pager_putpages_ioflags(flags), curthread->td_ucred); - PCPU_INC(cnt.v_vnodeout); - PCPU_ADD(cnt.v_vnodepgsout, ncount); + VM_CNT_INC(v_vnodeout); + VM_CNT_ADD(v_vnodepgsout, ncount); ppscheck = 0; if (error) { |