From: Huang Ying &lt;ying.huang(a)intel.com&gt; ANBZ: #80 cherry-picked from https://git.kernel.org/pub/scm/linux/kernel/git/vishal/tiering.git/commit/?… Signed-off-by: "Huang, Ying" &lt;ying.huang(a)intel.com&gt; Signed-off-by: Baolin Wang &lt;baolin.wang(a)linux.alibaba.com&gt; --- include/linux/mmzone.h | 1 + kernel/sched/fair.c | 6 ++++-- mm/vmstat.c | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index fcc2c18..30b42ef 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -209,6 +209,7 @@ enum node_stat_item { #ifdef CONFIG_NUMA_BALANCING PGPROMOTE_SUCCESS, /* promote successfully */ PGPROMOTE_CANDIDATE, /* candidate pages to promote */ + PROMOTE_THRESHOLD, #endif NR_VM_NODE_STAT_ITEMS }; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index fc954e1..f938351 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2930,7 +2930,7 @@ static void numa_migration_adjust_threshold(struct pglist_data *pgdat, unsigned long ref_th) { unsigned long now = jiffies, last_th_ts, th_period; - unsigned long unit_th, th; + unsigned long unit_th, th, oth; unsigned long nr_cand, ref_cand, diff_cand; th_period = msecs_to_jiffies(sysctl_numa_balancing_scan_period_max); @@ -2942,7 +2942,8 @@ static void numa_migration_adjust_threshold(struct pglist_data *pgdat, nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); diff_cand = nr_cand - pgdat->numa_threshold_nr_candidate; unit_th = ref_th / NUMA_MIGRATION_ADJUST_STEPS; - th = pgdat->numa_threshold ? : ref_th; + oth = pgdat->numa_threshold; + th = oth ? : ref_th; if (diff_cand > ref_cand * 11 / 10) th = max(th - unit_th, unit_th); else if (diff_cand < ref_cand * 9 / 10) @@ -2950,6 +2951,7 @@ static void numa_migration_adjust_threshold(struct pglist_data *pgdat, pgdat->numa_threshold_nr_candidate = nr_cand; pgdat->numa_threshold = th; trace_autonuma_threshold(pgdat->node_id, diff_cand, th); + mod_node_page_state(pgdat, PROMOTE_THRESHOLD, th - oth); } } diff --git a/mm/vmstat.c b/mm/vmstat.c index d921c42d..acd049b 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1189,6 +1189,7 @@ int fragmentation_index(struct zone *zone, unsigned int order) #ifdef CONFIG_NUMA_BALANCING "pgpromote_success", "pgpromote_candidate", + "promote_threshold", #endif /* enum writeback_stat_item counters */ -- 1.8.3.1

From: Feng Tang &lt;feng.tang(a)intel.com&gt; ANBZ: #80 cherry-picked from https://git.kernel.org/pub/scm/linux/kernel/git/vishal/tiering.git/commit/?… Now, migrate_misplaced_page() can be used for the mapped pages only. We want to use it for the unmapped file cache pages in the following patches. So the "VMA" parameter (not available for the unmapped pages) checking in the function is loosed. Signed-off-by: Feng Tang &lt;feng.tang(a)intel.com&gt; Signed-off-by: Baolin Wang &lt;baolin.wang(a)linux.alibaba.com&gt; --- mm/migrate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/migrate.c b/mm/migrate.c index f263a4a..15838b3 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2029,7 +2029,7 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, * Don't migrate file pages that are mapped in multiple processes * with execute permissions as they are probably shared libraries. */ - if (page_mapcount(page) != 1 && page_is_file_cache(page) && + if (vma && page_mapcount(page) != 1 && page_is_file_cache(page) && (vma->vm_flags & VM_EXEC)) goto out; -- 1.8.3.1

From: Huang Ying &lt;ying.huang(a)intel.com&gt; ANBZ: #80 cherry-picked from https://git.kernel.org/pub/scm/linux/kernel/git/vishal/tiering.git/commit/?… Commonly used file systems such as ext4 and xfs can migrate dirty file cache pages with MIGRATE_ASYNC. So enable the dirty file cache pages migration in AutoNUMA to optimize the page placement for these pages. The patch improves the read/write bandwidth of fio benchmark with 80:20 read/write ratio and normal random distribution by 11.3% on a 2 socket Intel server with Optane DC Persistent Memory Model. Signed-off-by: "Huang, Ying" &lt;ying.huang(a)intel.com&gt; Signed-off-by: Baolin Wang &lt;baolin.wang(a)linux.alibaba.com&gt; --- mm/migrate.c | 7 ------- mm/mprotect.c | 8 -------- 2 files changed, 15 deletions(-) diff --git a/mm/migrate.c b/mm/migrate.c index c7632c9..4a57837 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2060,13 +2060,6 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, (vma->vm_flags & VM_EXEC)) goto out; - /* - * Also do not migrate dirty pages as not all filesystems can move - * dirty pages in MIGRATE_ASYNC mode which is a waste of cycles. - */ - if (page_is_file_cache(page) && PageDirty(page)) - goto out; - isolated = numamigrate_isolate_page(pgdat, page); if (!isolated) goto out; diff --git a/mm/mprotect.c b/mm/mprotect.c index 3072213..6525e96 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -93,14 +93,6 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, page_mapcount(page) != 1) continue; - /* - * While migration can move some dirty pages, - * it cannot move them all from MIGRATE_ASYNC - * context. - */ - if (page_is_file_cache(page) && PageDirty(page)) - continue; - /* Avoid TLB flush if possible */ if (pte_protnone(oldpte)) continue; -- 1.8.3.1

From: Huang Ying &lt;ying.huang(a)intel.com&gt; ANBZ: #80 cherry-picked from https://git.kernel.org/pub/scm/linux/kernel/git/vishal/tiering.git/commit/?… To distinguish the file pages demotion from the anonymous pages demotion. This can be used to check the balance between the file/anonymous pages demotion. Signed-off-by: "Huang, Ying" &lt;ying.huang(a)intel.com&gt; Signed-off-by: Baolin Wang &lt;baolin.wang(a)linux.alibaba.com&gt; --- include/linux/vm_event_item.h | 1 + mm/vmscan.c | 4 ++++ mm/vmstat.c | 1 + 3 files changed, 6 insertions(+) diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index 8984994..d5419b3 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -34,6 +34,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, PGSTEAL_DIRECT, PGDEMOTE_KSWAPD, PGDEMOTE_DIRECT, + PGDEMOTE_FILE, PGSCAN_KSWAPD, PGSCAN_DIRECT, PGSCAN_DIRECT_THROTTLE, diff --git a/mm/vmscan.c b/mm/vmscan.c index ab79a20..fa9e268 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1135,6 +1135,7 @@ static unsigned int demote_page_list(struct list_head *demote_pages, int target_nid = next_demotion_node(pgdat->node_id); unsigned int nr_succeeded; int err; + bool file_lru; if (list_empty(demote_pages)) return 0; @@ -1142,6 +1143,7 @@ static unsigned int demote_page_list(struct list_head *demote_pages, if (target_nid == NUMA_NO_NODE) return 0; + file_lru = page_is_file_cache(lru_to_page(demote_pages)); /* Demotion ignores all cpuset and mempolicy settings */ err = migrate_pages(demote_pages, alloc_demote_page, NULL, target_nid, MIGRATE_ASYNC, MR_DEMOTION, @@ -1151,6 +1153,8 @@ static unsigned int demote_page_list(struct list_head *demote_pages, __count_vm_events(PGDEMOTE_KSWAPD, nr_succeeded); else __count_vm_events(PGDEMOTE_DIRECT, nr_succeeded); + if (file_lru) + __count_vm_events(PGDEMOTE_FILE, nr_succeeded); return nr_succeeded; } diff --git a/mm/vmstat.c b/mm/vmstat.c index a92a7ad..3ca0d88 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1222,6 +1222,7 @@ int fragmentation_index(struct zone *zone, unsigned int order) "pgsteal_direct", "pgdemote_kswapd", "pgdemote_direct", + "pgdemote_file", "pgscan_kswapd", "pgscan_direct", "pgscan_direct_throttle", -- 1.8.3.1

From: Huang Ying &lt;ying.huang(a)intel.com&gt; ANBZ: #80 cherry-picked from https://git.kernel.org/pub/scm/linux/kernel/git/vishal/tiering.git/commit/?… When THP is promoted from the slow memory node to the fast memory node, migrate_balanced_pgdat() is called to check whether there's enough free space on the target node. But originally, only the total number of the free pages is considered, but the order of the free pages isn't considered. So it's possible that there's no free page with expected order, although the total number of the free pages is enough. So that the page demotion for the fast memory node isn't initiated to make the high order pages available. To solve the above issue, the page order is considered in migrate_balanced_pgdat(). TODO: test results? Signed-off-by: "Huang, Ying" &lt;ying.huang(a)intel.com&gt; Signed-off-by: Baolin Wang &lt;baolin.wang(a)linux.alibaba.com&gt; --- mm/migrate.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/mm/migrate.c b/mm/migrate.c index 0ac0e86..e9c022e 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1949,8 +1949,7 @@ static int kernel_move_pages(pid_t pid, unsigned long nr_pages, * Returns true if this is a safe migration target node for misplaced NUMA * pages. Currently it only checks the watermarks which crude */ -static bool migrate_balanced_pgdat(struct pglist_data *pgdat, - unsigned long nr_migrate_pages) +static bool migrate_balanced_pgdat(struct pglist_data *pgdat, int order) { int z; @@ -1960,10 +1959,9 @@ static bool migrate_balanced_pgdat(struct pglist_data *pgdat, if (!populated_zone(zone)) continue; - /* Avoid waking kswapd by allocating pages_to_migrate pages. */ - if (!zone_watermark_ok(zone, 0, - high_wmark_pages(zone) + - nr_migrate_pages, + /* Avoid waking kswapd by allocating pages to migrate. */ + if (!zone_watermark_ok(zone, order, + high_wmark_pages(zone), 0, 0)) continue; return true; @@ -1994,7 +1992,7 @@ static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) VM_BUG_ON_PAGE(order && !PageTransHuge(page), page); /* Avoid migrating to a node that is nearly full */ - if (!migrate_balanced_pgdat(pgdat, 1UL << compound_order(page))) { + if (!migrate_balanced_pgdat(pgdat, order)) { int z; if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) || -- 1.8.3.1

From: Feng Tang &lt;feng.tang(a)intel.com&gt; ANBZ: #80 cherry-picked from https://git.kernel.org/pub/scm/linux/kernel/git/vishal/tiering.git/commit/?… page migration is a key function in mm core, used by memory compaction, cma, large order page allocation and PMEM promotion/demotion. And it could fail due to many reasons, like no enough memory in target node, page refcount being abnormal etc. Adding counters into /proc/vmstat will give the detailed info about why and where it fails, which will help debugging and analyzing. Reviewed-by: Dave Hansen &lt;dave.hansen(a)intel.com&gt; Signed-off-by: Feng Tang &lt;feng.tang(a)intel.com&gt; Signed-off-by: Baolin Wang &lt;baolin.wang(a)linux.alibaba.com&gt; --- include/linux/vm_event_item.h | 4 ++++ mm/migrate.c | 22 +++++++++++++++++++--- mm/vmstat.c | 4 ++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index d5419b3..ab9c658 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -59,6 +59,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, #endif #ifdef CONFIG_MIGRATION PGMIGRATE_SUCCESS, PGMIGRATE_FAIL, + PGMIGRATE_DST_NODE_FULL_FAIL, + PGMIGRATE_NUMA_ISOLATE_FAIL, + PGMIGRATE_NOMEM_FAIL, + PGMIGRATE_REFCOUNT_FAIL, #endif #ifdef CONFIG_COMPACTION COMPACTMIGRATE_SCANNED, COMPACTFREE_SCANNED, diff --git a/mm/migrate.c b/mm/migrate.c index e9c022e..7e6dcef 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -457,6 +457,9 @@ int migrate_page_move_mapping(struct address_space *mapping, expected_count += is_device_private_page(page); expected_count += is_device_public_page(page); + if (page_count(page) != expected_count) + count_vm_events(PGMIGRATE_REFCOUNT_FAIL, hpage_nr_pages(page)); + if (!mapping) { /* Anonymous page without mapping */ if (page_count(page) != expected_count) @@ -1508,6 +1511,9 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, } } nr_failed++; + + count_vm_events(PGMIGRATE_NOMEM_FAIL, + hpage_nr_pages(page)); goto out; case -EAGAIN: retry++; @@ -1995,6 +2001,9 @@ static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) if (!migrate_balanced_pgdat(pgdat, order)) { int z; + count_vm_events(PGMIGRATE_DST_NODE_FULL_FAIL, + hpage_nr_pages(page)); + if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) || !numa_demotion_enabled) return 0; @@ -2065,8 +2074,11 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, goto out; isolated = numamigrate_isolate_page(pgdat, page); - if (!isolated) + if (!isolated) { + count_vm_events(PGMIGRATE_NUMA_ISOLATE_FAIL, + hpage_nr_pages(page)); goto out; + } list_add(&page->lru, &migratepages); nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page, @@ -2117,14 +2129,18 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, unsigned long mmun_end = mmun_start + HPAGE_PMD_SIZE; isolated = numamigrate_isolate_page(pgdat, page); - if (!isolated) + if (!isolated) { + count_vm_events(PGMIGRATE_NUMA_ISOLATE_FAIL, HPAGE_PMD_NR); goto out_fail; + } new_page = alloc_pages_node(node, (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE), HPAGE_PMD_ORDER); - if (!new_page) + if (!new_page) { + count_vm_events(PGMIGRATE_NOMEM_FAIL, HPAGE_PMD_NR); goto out_fail; + } prep_transhuge_page(new_page); /* Prepare a page as a migration target */ diff --git a/mm/vmstat.c b/mm/vmstat.c index 3ca0d88..5651e09 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1257,6 +1257,10 @@ int fragmentation_index(struct zone *zone, unsigned int order) #ifdef CONFIG_MIGRATION "pgmigrate_success", "pgmigrate_fail", + "pgmigrate_fail_dst_node_fail", + "pgmigrate_fail_numa_isolate_fail", + "pgmigrate_fail_nomem_fail", + "pgmigrate_fail_refcount_fail", #endif #ifdef CONFIG_COMPACTION "compact_migrate_scanned", -- 1.8.3.1

From: Huang Ying &lt;ying.huang(a)intel.com&gt; ANBZ: #80 cherry-picked from https://git.kernel.org/pub/scm/linux/kernel/git/vishal/tiering.git/commit/?… In current implementation, to adjust the memory tiering promotion threshold more quickly, we use the fixed steps (16) to adjust. But now we can reduce the threshold (more strictly) much more quickly, so we can use some other way to adjust the threshold to enlarge the range of the threshold and to make the threshold more accurate. In this patch, if the number of the pages that pass the threshold is more than 110% of the target number, we will decrease the threshold to 90% of its original value until 1ms. And we will increase the threshold to 110% of its original value vice versa. In this way, the minimal threshold is 1 ms (vs. 62ms by default originally). And the possible values of threshold becomes much more. Both make it possible to adjust the threshold more accurately. One downside is that if the PMEM pages become much colder suddenly, we will need more time to adjust the threshold to adapt to that. So that the promotion throughput will be lower than expected. But if the PMEM pages become colder, it’s less urgent to promote them too. So it should be OK to promote them slower. Eventually, the threshold can be adjusted to the proper value. In the test with the pmbench memory accessing benchmark on a 2-socket server machine with Optane DCPMM, the promotion threshold can be adjusted to a smaller value (33 ms vs. 62 ms originally). The pmbench score decreases 2.8%, mainly because the promotion throughput decreases 4.9%. Because the number of the pages that can be promoted isn't distributed evenly vs. time, but the ratelimit is enforced equally vs. time. Will resolve the issue with the following patches in the patchset. Signed-off-by: "Huang, Ying" &lt;ying.huang(a)intel.com&gt; Signed-off-by: Baolin Wang &lt;baolin.wang(a)linux.alibaba.com&gt; --- kernel/sched/fair.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5806316..74dec2e 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2927,14 +2927,12 @@ int sysctl_numa_balancing_threshold(struct ctl_table *table, int write, void *bu return err; } -#define NUMA_MIGRATION_ADJUST_STEPS 16 - static void numa_migration_adjust_threshold(struct pglist_data *pgdat, unsigned long rate_limit, unsigned long ref_th) { unsigned long now = jiffies, last_th_ts, th_period; - unsigned long unit_th, th, oth; + unsigned long th, oth; unsigned long last_nr_cand, nr_cand, ref_cand, diff_cand; th_period = msecs_to_jiffies(sysctl_numa_balancing_scan_period_max); @@ -2952,13 +2950,15 @@ static void numa_migration_adjust_threshold(struct pglist_data *pgdat, } pgdat->numa_threshold_ts = now; pgdat->numa_threshold_nr_candidate = nr_cand; - unit_th = ref_th / NUMA_MIGRATION_ADJUST_STEPS; oth = pgdat->numa_threshold; th = oth ? : ref_th; - if (diff_cand > ref_cand * 11 / 10) - th = max(th - unit_th, unit_th); - else if (diff_cand < ref_cand * 9 / 10) - th = min(th + unit_th, ref_th); + if (diff_cand > ref_cand * 11 / 10) { + th = min(th * 9 / 10, th - 1); + th = max(th, 1UL); + } else if (diff_cand < ref_cand * 9 / 10) { + th = max(th * 11 / 10, th + 1); + th = min(th, ref_th * 2); + } pgdat->numa_threshold = th; spin_unlock(&pgdat->numa_lock); trace_autonuma_threshold(pgdat->node_id, diff_cand, th); -- 1.8.3.1

From: Huang Ying &lt;ying.huang(a)intel.com&gt; ANBZ: #80 cherry-picked from https://git.kernel.org/pub/scm/linux/kernel/git/vishal/tiering.git/commit/?… The write performance of PMEM is much worse than its read performance. So even if a write-mostly pages is colder than a read-mostly pages, it is usually better to put the write-mostly pages in DRAM and read-mostly pages in PMEM. To give write-mostly pages more opportunity to be promoted to DRAM, in this patch, the hot threshold for write hint page fault is doubled (easier to be promoted). Signed-off-by: "Huang, Ying" &lt;ying.huang(a)intel.com&gt; Cc: Andrew Morton &lt;akpm(a)linux-foundation.org&gt; Cc: Michal Hocko &lt;mhocko(a)suse.com&gt; Cc: Rik van Riel &lt;riel(a)redhat.com&gt; Cc: Mel Gorman &lt;mgorman(a)suse.de&gt; Cc: Peter Zijlstra &lt;peterz(a)infradead.org&gt; Cc: Ingo Molnar &lt;mingo(a)kernel.org&gt; Cc: Dave Hansen &lt;dave.hansen(a)linux.intel.com&gt; Cc: Dan Williams &lt;dan.j.williams(a)intel.com&gt; Cc: linux-kernel(a)vger.kernel.org Cc: linux-mm(a)kvack.org Signed-off-by: Baolin Wang &lt;baolin.wang(a)linux.alibaba.com&gt; --- include/linux/mempolicy.h | 4 ++-- include/linux/migrate.h | 1 + include/linux/sched/numa_balancing.h | 5 +++-- kernel/sched/fair.c | 4 +++- mm/filemap.c | 3 ++- mm/huge_memory.c | 2 +- mm/memory.c | 5 ++++- mm/mempolicy.c | 6 ++++-- mm/migrate.c | 3 ++- 9 files changed, 22 insertions(+), 11 deletions(-) diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 9f08c25..5453883 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -202,7 +202,7 @@ static inline bool vma_migratable(struct vm_area_struct *vma) return true; } -extern int mpol_misplaced(struct page *, struct vm_area_struct *, unsigned long); +extern int mpol_misplaced(struct page *, struct vm_area_struct *, unsigned long, int); extern void mpol_put_task_policy(struct task_struct *); #else @@ -300,7 +300,7 @@ static inline int mpol_parse_str(char *str, struct mempolicy **mpol) #endif static inline int mpol_misplaced(struct page *page, struct vm_area_struct *vma, - unsigned long address) + unsigned long address, int flags) { return -1; /* no node preference */ } diff --git a/include/linux/migrate.h b/include/linux/migrate.h index c9bf8b2..8c94650 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -31,6 +31,7 @@ enum migrate_reason { /* promote_file_page() flags */ #define PFP_LOCKED 0x1 +#define PFP_WRITE 0x2 /* In mm/debug.c; also keep sync with include/trace/events/migrate.h */ extern char *migrate_reason_names[MR_TYPES]; diff --git a/include/linux/sched/numa_balancing.h b/include/linux/sched/numa_balancing.h index 3988762..f5608ea 100644 --- a/include/linux/sched/numa_balancing.h +++ b/include/linux/sched/numa_balancing.h @@ -14,6 +14,7 @@ #define TNF_SHARED 0x04 #define TNF_FAULT_LOCAL 0x08 #define TNF_MIGRATE_FAIL 0x10 +#define TNF_WRITE 0x40 #ifdef CONFIG_NUMA_BALANCING extern void task_numa_fault(int last_node, int node, int pages, int flags); @@ -21,7 +22,7 @@ extern void set_numabalancing_state(bool enabled); extern void task_numa_free(struct task_struct *p, bool final); extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page, - int src_nid, int dst_cpu); + int src_nid, int dst_cpu, int flags); #else static inline void task_numa_fault(int last_node, int node, int pages, int flags) @@ -38,7 +39,7 @@ static inline void task_numa_free(struct task_struct *p, bool final) { } static inline bool should_numa_migrate_memory(struct task_struct *p, - struct page *page, int src_nid, int dst_cpu) + struct page *page, int src_nid, int dst_cpu, int flags) { return true; } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 62c9953..0184145 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2974,7 +2974,7 @@ static void numa_migration_adjust_threshold(struct pglist_data *pgdat, } bool should_numa_migrate_memory(struct task_struct *p, struct page * page, - int src_nid, int dst_cpu) + int src_nid, int dst_cpu, int flags) { struct numa_group *ng = deref_curr_numa_group(p); int dst_nid = cpu_to_node(dst_cpu); @@ -3002,6 +3002,8 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page * page, numa_migration_adjust_threshold(pgdat, rate_limit, def_th); th = pgdat->numa_threshold ? : def_th; + if (flags & TNF_WRITE) + th *= 2; latency = numa_hint_fault_latency(page); if (latency > th) return false; diff --git a/mm/filemap.c b/mm/filemap.c index ad863b5..ebd32fd 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3264,7 +3264,8 @@ struct page *grab_cache_page_write_begin(struct address_space *mapping, if (page) { wait_for_stable_page(page); - if (!promotion_tried && promote_file_page(page, PFP_LOCKED)) { + if (!promotion_tried && + promote_file_page(page, PFP_LOCKED | PFP_WRITE)) { promotion_tried = true; goto find_page; } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 0a9126c..98ab428 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1853,7 +1853,7 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd) * page_table_lock if at all possible */ page_locked = trylock_page(page); - target_nid = mpol_misplaced(page, vma, haddr); + target_nid = mpol_misplaced(page, vma, haddr, flags); if (target_nid == -1) { /* If the page was locked, there are no parallel migrations */ if (page_locked) diff --git a/mm/memory.c b/mm/memory.c index 2100f65..c0ece4f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3860,7 +3860,7 @@ static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma, *flags |= TNF_FAULT_LOCAL; } - return mpol_misplaced(page, vma, addr); + return mpol_misplaced(page, vma, addr, *flags); } static vm_fault_t do_numa_page(struct vm_fault *vmf) @@ -3899,6 +3899,9 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) ptep_modify_prot_commit(vma->vm_mm, vmf->address, vmf->pte, pte); update_mmu_cache(vma, vmf->address, vmf->pte); + if (vmf->flags & FAULT_FLAG_WRITE) + flags |= TNF_WRITE; + page = vm_normal_page(vma, vmf->address, pte); if (!page) { pte_unmap_unlock(vmf->pte, vmf->ptl); diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 1a1593c..077a0eb 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -2388,7 +2388,8 @@ static void sp_free(struct sp_node *n) * Policy determination "mimics" alloc_page_vma(). * Called from fault path where we know the vma and faulting address. */ -int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr) +int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr, + int flags) { struct mempolicy *pol; struct zoneref *z; @@ -2442,7 +2443,8 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long if (pol->flags & MPOL_F_MORON) { polnid = thisnid; - if (!should_numa_migrate_memory(current, page, curnid, thiscpu)) + if (!should_numa_migrate_memory(current, page, curnid, thiscpu, + flags)) goto out; } diff --git a/mm/migrate.c b/mm/migrate.c index 7e6dcef..6d25ea0 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -50,6 +50,7 @@ #include <linux/memory.h> #include <linux/random.h> #include <linux/sched/sysctl.h> +#include <linux/sched/numa_balancing.h> #include <asm/tlbflush.h> @@ -1194,7 +1195,7 @@ bool promote_file_page(struct page *page, int flags) node_is_toptier(nid)) return false; - nid = mpol_misplaced(page, NULL, 0); + nid = mpol_misplaced(page, NULL, 0, (flags & PFP_WRITE) ? TNF_WRITE : 0); if (nid == NUMA_NO_NODE) return false; -- 1.8.3.1