mm, sysctl: make NUMA stats configurable

This is the second step which introduces a tunable interface that allow
numa stats configurable for optimizing zone_statistics(), as suggested
by Dave Hansen and Ying Huang.

=========================================================================

When page allocation performance becomes a bottleneck and you can
tolerate some possible tool breakage and decreased numa counter
precision, you can do:

	echo 0 > /proc/sys/vm/numa_stat

In this case, numa counter update is ignored.  We can see about
*4.8%*(185->176) drop of cpu cycles per single page allocation and
reclaim on Jesper's page_bench01 (single thread) and *8.1%*(343->315)
drop of cpu cycles per single page allocation and reclaim on Jesper's
page_bench03 (88 threads) running on a 2-Socket Broadwell-based server
(88 threads, 126G memory).

Benchmark link provided by Jesper D Brouer (increase loop times to
10000000):

  https://github.com/netoptimizer/prototype-kernel/tree/master/kernel/mm/bench

=========================================================================

When page allocation performance is not a bottleneck and you want all
tooling to work, you can do:

	echo 1 > /proc/sys/vm/numa_stat

This is system default setting.

Many thanks to Michal Hocko, Dave Hansen, Ying Huang and Vlastimil Babka
for comments to help improve the original patch.

[keescook@chromium.org: make sure mutex is a global static]
Link: http://lkml.kernel.org/r/20171107213809.GA4314@beast
Link: http://lkml.kernel.org/r/1508290927-8518-1-git-send-email-kemi.wang@intel.com
Signed-off-by: Kemi Wang <kemi.wang@intel.com>
Signed-off-by: Kees Cook <keescook@chromium.org>
Reported-by: Jesper Dangaard Brouer <brouer@redhat.com>
Suggested-by: Dave Hansen <dave.hansen@intel.com>
Suggested-by: Ying Huang <ying.huang@intel.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Michal Hocko <mhocko@suse.com>
Cc: "Luis R . Rodriguez" <mcgrof@kernel.org>
Cc: Kees Cook <keescook@chromium.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Christopher Lameter <cl@linux.com>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Tim Chen <tim.c.chen@intel.com>
Cc: Andi Kleen <andi.kleen@intel.com>
Cc: Aaron Lu <aaron.lu@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/Documentation/sysctl/vm.txt b/Documentation/sysctl/vm.txt
index 3e579740b49fd6762ed577253bd01802d4112fc5..055c8b3e101805e3cf66288a914d757efed35a48 100644 (file)
--- a/Documentation/sysctl/vm.txt
+++ b/Documentation/sysctl/vm.txt
@@ -58,6 +58,7 @@ Currently, these files are in /proc/sys/vm:
 - percpu_pagelist_fraction
 - stat_interval
 - stat_refresh
+- numa_stat
 - swappiness
 - user_reserve_kbytes
 - vfs_cache_pressure
@@ -799,6 +800,21 @@ with no ill effects: errors and warnings on these stats are suppressed.)
 
 ==============================================================
 
+numa_stat
+
+This interface allows runtime configuration of numa statistics.
+
+When page allocation performance becomes a bottleneck and you can tolerate
+some possible tool breakage and decreased numa counter precision, you can
+do:
+       echo 0 > /proc/sys/vm/numa_stat
+
+When page allocation performance is not a bottleneck and you want all
+tooling to work, you can do:
+       echo 1 > /proc/sys/vm/numa_stat
+
+==============================================================
+
 swappiness
 
 This control is used to define how aggressive the kernel will swap

diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index 1e0cb72e0598bb668f1207c6e031f2b731321187..1779c9817b39aca623b88ad6b0d2d363899379e5 100644 (file)
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -7,9 +7,19 @@
 #include <linux/mmzone.h>
 #include <linux/vm_event_item.h>
 #include <linux/atomic.h>
+#include <linux/static_key.h>
 
 extern int sysctl_stat_interval;
 
+#ifdef CONFIG_NUMA
+#define ENABLE_NUMA_STAT   1
+#define DISABLE_NUMA_STAT   0
+extern int sysctl_vm_numa_stat;
+DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
+extern int sysctl_vm_numa_stat_handler(struct ctl_table *table,
+               int write, void __user *buffer, size_t *length, loff_t *ppos);
+#endif
+
 #ifdef CONFIG_VM_EVENT_COUNTERS
 /*
  * Light weight per cpu counter implementation.

diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 7638e2f7fff8f295827c0747000e3bf395a1c61c..4a13a389e99b243a5e261d3b3a12b67e2e681c11 100644 (file)
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -1356,6 +1356,15 @@ static struct ctl_table vm_table[] = {
                .mode           = 0644,
                .proc_handler   = &hugetlb_mempolicy_sysctl_handler,
        },
+       {
+               .procname               = "numa_stat",
+               .data                   = &sysctl_vm_numa_stat,
+               .maxlen                 = sizeof(int),
+               .mode                   = 0644,
+               .proc_handler   = sysctl_vm_numa_stat_handler,
+               .extra1                 = &zero,
+               .extra2                 = &one,
+       },
 #endif
         {
                .procname       = "hugetlb_shm_group",

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index dad166b736ba452c431e1dd981db9e0f04fa3ee1..4ce44d3ff03da6a6f09a9b92c9bb0750aaf3b7a4 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1915,6 +1915,9 @@ static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
        struct page *page;
 
        page = __alloc_pages(gfp, order, nid);
+       /* skip NUMA_INTERLEAVE_HIT counter update if numa stats is disabled */
+       if (!static_branch_likely(&vm_numa_stat_key))
+               return page;
        if (page && page_to_nid(page) == nid) {
                preempt_disable();
                __inc_numa_state(page_zone(page), NUMA_INTERLEAVE_HIT);

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 7ca668e946e5c9077e71a5705aa9ee1e7474b6c1..67f523c4711a2e8d0de4e0599c77cfabdda82efd 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -82,6 +82,8 @@ DEFINE_PER_CPU(int, numa_node);
 EXPORT_PER_CPU_SYMBOL(numa_node);
 #endif
 
+DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
+
 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
 /*
  * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
@@ -2777,6 +2779,10 @@ static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)
 #ifdef CONFIG_NUMA
        enum numa_stat_item local_stat = NUMA_LOCAL;
 
+       /* skip numa counters update if numa stats is disabled */
+       if (!static_branch_likely(&vm_numa_stat_key))
+               return;
+
        if (z->node != numa_node_id())
                local_stat = NUMA_OTHER;
 

diff --git a/mm/vmstat.c b/mm/vmstat.c
index 7d11554861e4d05898719c60f2b39bfdf82c5fc8..40b2db6db6b16df89ff5c313cab83c95b8c0f38e 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -32,6 +32,77 @@
 
 #define NUMA_STATS_THRESHOLD (U16_MAX - 2)
 
+#ifdef CONFIG_NUMA
+int sysctl_vm_numa_stat = ENABLE_NUMA_STAT;
+
+/* zero numa counters within a zone */
+static void zero_zone_numa_counters(struct zone *zone)
+{
+       int item, cpu;
+
+       for (item = 0; item < NR_VM_NUMA_STAT_ITEMS; item++) {
+               atomic_long_set(&zone->vm_numa_stat[item], 0);
+               for_each_online_cpu(cpu)
+                       per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item]
+                                               = 0;
+       }
+}
+
+/* zero numa counters of all the populated zones */
+static void zero_zones_numa_counters(void)
+{
+       struct zone *zone;
+
+       for_each_populated_zone(zone)
+               zero_zone_numa_counters(zone);
+}
+
+/* zero global numa counters */
+static void zero_global_numa_counters(void)
+{
+       int item;
+
+       for (item = 0; item < NR_VM_NUMA_STAT_ITEMS; item++)
+               atomic_long_set(&vm_numa_stat[item], 0);
+}
+
+static void invalid_numa_statistics(void)
+{
+       zero_zones_numa_counters();
+       zero_global_numa_counters();
+}
+
+static DEFINE_MUTEX(vm_numa_stat_lock);
+
+int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
+               void __user *buffer, size_t *length, loff_t *ppos)
+{
+       int ret, oldval;
+
+       mutex_lock(&vm_numa_stat_lock);
+       if (write)
+               oldval = sysctl_vm_numa_stat;
+       ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
+       if (ret || !write)
+               goto out;
+
+       if (oldval == sysctl_vm_numa_stat)
+               goto out;
+       else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) {
+               static_branch_enable(&vm_numa_stat_key);
+               pr_info("enable numa statistics\n");
+       } else {
+               static_branch_disable(&vm_numa_stat_key);
+               invalid_numa_statistics();
+               pr_info("disable numa statistics, and clear numa counters\n");
+       }
+
+out:
+       mutex_unlock(&vm_numa_stat_lock);
+       return ret;
+}
+#endif
+
 #ifdef CONFIG_VM_EVENT_COUNTERS
 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
 EXPORT_PER_CPU_SYMBOL(vm_event_states);