17 Dec
2021
17 Dec
'21
10:13 p.m.
From: Huang Ying <ying.huang(a)intel.com>
It isn't easy for the administrator to determine the hot threshold.
So in this patch, a method to adjust the hot threshold automatically
is implemented. The basic idea is to control the number of the
candidate promotion pages to match the promotion rate limit. If the
hint page fault latency of a page is less than the hot threshold, we
will try to promote the page, and the page is called the candidate
promotion page.
If the number of the candidate promotion pages in the statistics
interval is much more than the promotion rate limit, the hot threshold
will be decreased to reduce the number of the candidate promotion
pages. Otherwise, the hot threshold will be increased to increase the
number of the candidate promotion pages.
To make the above method works, in each statistics interval, the total
number of the pages to check (on which the hint page faults occur) and
the hot/cold distribution need to be stable. Because the page tables
are scanned linearly in NUMA balancing, but the hot/cold distribution
isn't uniform along the address, the statistics interval should be
larger than the NUMA balancing scan period. So in the patch, the max
scan period is used as statistics interval and it works well in our
tests.
The sysctl knob kernel.numa_balancing_hot_threshold_ms becomes the
initial value and max value of the hot threshold.
Signed-off-by: "Huang, Ying" <ying.huang(a)intel.com>
Cc: Andrew Morton <akpm(a)linux-foundation.org>
Cc: Michal Hocko <mhocko(a)suse.com>
Cc: Rik van Riel <riel(a)surriel.com>
Cc: Mel Gorman <mgorman(a)techsingularity.net>
Cc: Peter Zijlstra <peterz(a)infradead.org>
Cc: Dave Hansen <dave.hansen(a)linux.intel.com>
Cc: Yang Shi <shy828301(a)gmail.com>
Cc: Zi Yan <ziy(a)nvidia.com>
Cc: Wei Xu <weixugc(a)google.com>
Cc: osalvador <osalvador(a)suse.de>
Cc: Shakeel Butt <shakeelb(a)google.com>
Cc: Hasan Al Maruf <hasanalmaruf(a)fb.com>
Cc: linux-kernel(a)vger.kernel.org
Cc: linux-mm(a)kvack.org
Signed-off-by: zhongjiang-ali <zhongjiang-ali(a)linux.alibaba.com>
---
include/linux/mmzone.h | 3 +++
include/linux/sched/sysctl.h | 3 +++
kernel/sched/core.c | 15 +++++++++++
kernel/sched/fair.c | 64 ++++++++++++++++++++++++++++++++++++++++----
kernel/sysctl.c | 3 ++-
5 files changed, 82 insertions(+), 6 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 4463451..fcc2c18 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -760,6 +760,9 @@ struct hugepage_reclaim {
#ifdef CONFIG_NUMA_BALANCING
unsigned long numa_ts;
unsigned long numa_nr_candidate;
+ unsigned long numa_threshold_ts;
+ unsigned long numa_threshold_nr_candidate;
+ unsigned long numa_threshold;
#endif
/* Fields commonly accessed by the page reclaim scanner */
diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h
index bda72b8..ac11582 100644
--- a/include/linux/sched/sysctl.h
+++ b/include/linux/sched/sysctl.h
@@ -103,6 +103,9 @@ extern int sysctl_numa_balancing(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp,
loff_t *ppos);
+int sysctl_numa_balancing_threshold(struct ctl_table *table, int write, void *buffer,
+ size_t *lenp, loff_t *ppos);
+
extern int sysctl_schedstats(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp,
loff_t *ppos);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5b78a1a..2f77c69 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2189,6 +2189,18 @@ void set_numabalancing_state(bool enabled)
}
#ifdef CONFIG_PROC_SYSCTL
+static void reset_memory_tiering(void)
+{
+ struct pglist_data *pgdat;
+
+ for_each_online_pgdat(pgdat) {
+ pgdat->numa_threshold = 0;
+ pgdat->numa_threshold_nr_candidate =
+ node_page_state(pgdat, PGPROMOTE_CANDIDATE);
+ pgdat->numa_threshold_ts = jiffies;
+ }
+}
+
int sysctl_numa_balancing(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
@@ -2205,6 +2217,9 @@ int sysctl_numa_balancing(struct ctl_table *table, int write,
if (err < 0)
return err;
if (write) {
+ if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
+ (state & NUMA_BALANCING_MEMORY_TIERING))
+ reset_memory_tiering();
sysctl_numa_balancing_mode = state;
__set_numabalancing_state(state);
}
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index e102fab..143d689 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -2904,6 +2904,54 @@ static bool numa_migration_check_rate_limit(struct pglist_data
*pgdat,
return true;
}
+int sysctl_numa_balancing_threshold(struct ctl_table *table, int write, void *buffer,
+ size_t *lenp, loff_t *ppos)
+{
+ int err;
+ struct pglist_data *pgdat;
+
+ if (write && !capable(CAP_SYS_ADMIN))
+ return -EPERM;
+
+ err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+ if (err < 0 || !write)
+ return err;
+
+ for_each_online_pgdat(pgdat)
+ pgdat->numa_threshold = 0;
+
+ return err;
+}
+
+#define NUMA_MIGRATION_ADJUST_STEPS 16
+
+static void numa_migration_adjust_threshold(struct pglist_data *pgdat,
+ unsigned long rate_limit,
+ unsigned long ref_th)
+{
+ unsigned long now = jiffies, last_th_ts, th_period;
+ unsigned long unit_th, th;
+ unsigned long nr_cand, ref_cand, diff_cand;
+
+ th_period = msecs_to_jiffies(sysctl_numa_balancing_scan_period_max);
+ last_th_ts = pgdat->numa_threshold_ts;
+ if (now > last_th_ts + th_period &&
+ cmpxchg(&pgdat->numa_threshold_ts, last_th_ts, now) == last_th_ts) {
+ ref_cand = rate_limit *
+ sysctl_numa_balancing_scan_period_max / 1000;
+ nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
+ diff_cand = nr_cand - pgdat->numa_threshold_nr_candidate;
+ unit_th = ref_th / NUMA_MIGRATION_ADJUST_STEPS;
+ th = pgdat->numa_threshold ? : ref_th;
+ if (diff_cand > ref_cand * 11 / 10)
+ th = max(th - unit_th, unit_th);
+ else if (diff_cand < ref_cand * 9 / 10)
+ th = min(th + unit_th, ref_th);
+ pgdat->numa_threshold_nr_candidate = nr_cand;
+ pgdat->numa_threshold = th;
+ }
+}
+
bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
int src_nid, int dst_cpu)
{
@@ -2918,19 +2966,25 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page
* page,
if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
!node_is_toptier(src_nid)) {
struct pglist_data *pgdat;
- unsigned long rate_limit, latency, th;
+ unsigned long rate_limit, latency, th, def_th;
pgdat = NODE_DATA(dst_nid);
- if (pgdat_free_space_enough(pgdat))
+ if (pgdat_free_space_enough(pgdat)) {
+ /* workload changed, reset hot threshold */
+ pgdat->numa_threshold = 0;
return true;
+ }
+
+ def_th = sysctl_numa_balancing_hot_threshold;
+ rate_limit =
+ sysctl_numa_balancing_rate_limit << (20 - PAGE_SHIFT);
+ numa_migration_adjust_threshold(pgdat, rate_limit, def_th);
- th = sysctl_numa_balancing_hot_threshold;
+ th = pgdat->numa_threshold ? : def_th;
latency = numa_hint_fault_latency(page);
if (latency > th)
return false;
- rate_limit =
- sysctl_numa_balancing_rate_limit << (20 - PAGE_SHIFT);
return numa_migration_check_rate_limit(pgdat, rate_limit,
1UL << compound_order(page));
}
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 32e8218..0fe1ea1 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -461,7 +461,8 @@ static int sysrq_sysctl_handler(struct ctl_table *table, int write,
.data = &sysctl_numa_balancing_hot_threshold,
.maxlen = sizeof(unsigned int),
.mode = 0644,
- .proc_handler = proc_dointvec,
+ .proc_handler = sysctl_numa_balancing_threshold,
+ .extra1 = &zero,
},
{
.procname = "numa_balancing_rate_limit_mbps",
--
1.8.3.1