2273977dc9c143a722b34cd3335f519f42d061f5
[openwrt/staging/981213.git] /
1 From fc0e3b06e0f19917b7ecad7967a72f61d4743644 Mon Sep 17 00:00:00 2001
2 From: "T.J. Alumbaugh" <talumbau@google.com>
3 Date: Wed, 18 Jan 2023 00:18:26 +0000
4 Subject: [PATCH 17/19] UPSTREAM: mm: multi-gen LRU: improve walk_pmd_range()
5
6 Improve readability of walk_pmd_range() and walk_pmd_range_locked().
7
8 Link: https://lkml.kernel.org/r/20230118001827.1040870-7-talumbau@google.com
9 Change-Id: Ia084fbf53fe989673b7804ca8ca520af12d7d52a
10 Signed-off-by: T.J. Alumbaugh <talumbau@google.com>
11 Cc: Yu Zhao <yuzhao@google.com>
12 Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
13 (cherry picked from commit b5ff4133617d0eced35b685da0bd0929dd9fabb7)
14 Bug: 274865848
15 Signed-off-by: T.J. Mercier <tjmercier@google.com>
16 ---
17 mm/vmscan.c | 40 ++++++++++++++++++++--------------------
18 1 file changed, 20 insertions(+), 20 deletions(-)
19
20 --- a/mm/vmscan.c
21 +++ b/mm/vmscan.c
22 @@ -3980,8 +3980,8 @@ restart:
23 }
24
25 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
26 -static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,
27 - struct mm_walk *args, unsigned long *bitmap, unsigned long *start)
28 +static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
29 + struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
30 {
31 int i;
32 pmd_t *pmd;
33 @@ -3994,18 +3994,19 @@ static void walk_pmd_range_locked(pud_t
34 VM_WARN_ON_ONCE(pud_leaf(*pud));
35
36 /* try to batch at most 1+MIN_LRU_BATCH+1 entries */
37 - if (*start == -1) {
38 - *start = next;
39 + if (*first == -1) {
40 + *first = addr;
41 + bitmap_zero(bitmap, MIN_LRU_BATCH);
42 return;
43 }
44
45 - i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start);
46 + i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first);
47 if (i && i <= MIN_LRU_BATCH) {
48 __set_bit(i - 1, bitmap);
49 return;
50 }
51
52 - pmd = pmd_offset(pud, *start);
53 + pmd = pmd_offset(pud, *first);
54
55 ptl = pmd_lockptr(args->mm, pmd);
56 if (!spin_trylock(ptl))
57 @@ -4016,15 +4017,16 @@ static void walk_pmd_range_locked(pud_t
58 do {
59 unsigned long pfn;
60 struct folio *folio;
61 - unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start;
62 +
63 + /* don't round down the first address */
64 + addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first;
65
66 pfn = get_pmd_pfn(pmd[i], vma, addr);
67 if (pfn == -1)
68 goto next;
69
70 if (!pmd_trans_huge(pmd[i])) {
71 - if (arch_has_hw_nonleaf_pmd_young() &&
72 - get_cap(LRU_GEN_NONLEAF_YOUNG))
73 + if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG))
74 pmdp_test_and_clear_young(vma, addr, pmd + i);
75 goto next;
76 }
77 @@ -4053,12 +4055,11 @@ next:
78 arch_leave_lazy_mmu_mode();
79 spin_unlock(ptl);
80 done:
81 - *start = -1;
82 - bitmap_zero(bitmap, MIN_LRU_BATCH);
83 + *first = -1;
84 }
85 #else
86 -static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,
87 - struct mm_walk *args, unsigned long *bitmap, unsigned long *start)
88 +static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
89 + struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
90 {
91 }
92 #endif
93 @@ -4071,9 +4072,9 @@ static void walk_pmd_range(pud_t *pud, u
94 unsigned long next;
95 unsigned long addr;
96 struct vm_area_struct *vma;
97 - unsigned long pos = -1;
98 + unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)];
99 + unsigned long first = -1;
100 struct lru_gen_mm_walk *walk = args->private;
101 - unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {};
102
103 VM_WARN_ON_ONCE(pud_leaf(*pud));
104
105 @@ -4115,18 +4116,17 @@ restart:
106 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
107 continue;
108
109 - walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);
110 + walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
111 continue;
112 }
113 #endif
114 walk->mm_stats[MM_NONLEAF_TOTAL]++;
115
116 - if (arch_has_hw_nonleaf_pmd_young() &&
117 - get_cap(LRU_GEN_NONLEAF_YOUNG)) {
118 + if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG)) {
119 if (!pmd_young(val))
120 continue;
121
122 - walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);
123 + walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
124 }
125
126 if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i))
127 @@ -4143,7 +4143,7 @@ restart:
128 update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i);
129 }
130
131 - walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos);
132 + walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
133
134 if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
135 goto restart;