Add two new kernel options to control memory locality on NUMA hardware.

 - UMA_XDOMAIN enables an additional per-cpu bucket for freed memory that
   was freed on a different domain from where it was allocated.  This is
   only used for UMA_ZONE_NUMA (first-touch) zones.
 - UMA_FIRSTTOUCH sets the default UMA policy to be first-touch for all
   zones.  This tries to maintain locality for kernel memory.

Reviewed by:	gallatin, alc, kib
Tested by:	pho, gallatin
Sponsored by:	Netflix
Differential Revision:	https://reviews.freebsd.org/D20929

4 files changed, 11 insertions, 1 deletions

diff --git a/lib/libmemstat/memstat.c b/lib/libmemstat/memstat.c
index 4e97ba6779df..b3788661ae95 100644
--- a/lib/libmemstat/memstat.c
+++ b/lib/libmemstat/memstat.c
@@ -326,6 +326,13 @@ memstat_get_sleeps(const struct memory_type *mtp)
 	return (mtp->mt_sleeps);
 }
 
+uint64_t
+memstat_get_xdomain(const struct memory_type *mtp)
+{
+
+	return (mtp->mt_xdomain);
+}
+
 void *
 memstat_get_caller_pointer(const struct memory_type *mtp, int index)
 {
diff --git a/lib/libmemstat/memstat.h b/lib/libmemstat/memstat.h
index 98fc99f078f6..27f1405fa8f9 100644
--- a/lib/libmemstat/memstat.h
+++ b/lib/libmemstat/memstat.h
@@ -136,6 +136,7 @@ uint64_t	 memstat_get_count(const struct memory_type *mtp);
 uint64_t	 memstat_get_free(const struct memory_type *mtp);
 uint64_t	 memstat_get_failures(const struct memory_type *mtp);
 uint64_t	 memstat_get_sleeps(const struct memory_type *mtp);
+uint64_t	 memstat_get_xdomain(const struct memory_type *mtp);
 void		*memstat_get_caller_pointer(const struct memory_type *mtp,
 		    int index);
 void		 memstat_set_caller_pointer(struct memory_type *mtp,
diff --git a/lib/libmemstat/memstat_internal.h b/lib/libmemstat/memstat_internal.h
index 12345d9f884c..26a9e64f8f05 100644
--- a/lib/libmemstat/memstat_internal.h
+++ b/lib/libmemstat/memstat_internal.h
@@ -69,6 +69,7 @@ struct memory_type {
 	uint64_t	 mt_free;	/* Number of cached free items. */
 	uint64_t	 mt_failures;	/* Number of allocation failures. */
 	uint64_t	 mt_sleeps;	/* Number of allocation sleeps. */
+	uint64_t	 mt_xdomain;	/* Number of cross domain sleeps. */
 
 	/*
 	 * Caller-owned memory.
diff --git a/lib/libmemstat/memstat_uma.c b/lib/libmemstat/memstat_uma.c
index 2575fb704521..4aff9f7ae1fa 100644
--- a/lib/libmemstat/memstat_uma.c
+++ b/lib/libmemstat/memstat_uma.c
@@ -201,6 +201,7 @@ retry:
 		mtp->mt_numfrees = uthp->uth_frees;
 		mtp->mt_failures = uthp->uth_fails;
 		mtp->mt_sleeps = uthp->uth_sleeps;
+		mtp->mt_xdomain = uthp->uth_xdomain;
 
 		for (j = 0; j < maxcpus; j++) {
 			upsp = (struct uma_percpu_stat *)p;
@@ -423,11 +424,11 @@ memstat_kvm_uma(struct memory_type_list *list, void *kvm_handle)
 			mtp->mt_failures = kvm_counter_u64_fetch(kvm,
 			    (unsigned long )uz.uz_fails);
 			mtp->mt_sleeps = uz.uz_sleeps;
-
 			/* See comment above in memstat_sysctl_uma(). */
 			if (mtp->mt_numallocs < mtp->mt_numfrees)
 				mtp->mt_numallocs = mtp->mt_numfrees;
 
+			mtp->mt_xdomain = uz.uz_xdomain;
 			if (kz.uk_flags & UMA_ZFLAG_INTERNAL)
 				goto skip_percpu;
 			for (i = 0; i < mp_maxid + 1; i++) {