projects / openwrt / staging / linusw.git / blob
? search:


02dd920e54cb419fefbde541149810b7e98ff083
[openwrt/staging/linusw.git] /
1 From fc0e3b06e0f19917b7ecad7967a72f61d4743644 Mon Sep 17 00:00:00 2001
2 From: "T.J. Alumbaugh" <talumbau@google.com>
3 Date: Wed, 18 Jan 2023 00:18:26 +0000
4 Subject: [PATCH 17/19] UPSTREAM: mm: multi-gen LRU: improve walk_pmd_range()
5
6 Improve readability of walk_pmd_range() and walk_pmd_range_locked().
7
8 Link: https://lkml.kernel.org/r/20230118001827.1040870-7-talumbau@google.com
9 Change-Id: Ia084fbf53fe989673b7804ca8ca520af12d7d52a
10 Signed-off-by: T.J. Alumbaugh <talumbau@google.com>
11 Cc: Yu Zhao <yuzhao@google.com>
12 Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
13 (cherry picked from commit b5ff4133617d0eced35b685da0bd0929dd9fabb7)
14 Bug: 274865848
15 Signed-off-by: T.J. Mercier <tjmercier@google.com>
16 ---
17 mm/vmscan.c | 40 ++++++++++++++++++++--------------------
18 1 file changed, 20 insertions(+), 20 deletions(-)
19
20 diff --git a/mm/vmscan.c b/mm/vmscan.c
21 index d5d6f8d94f58a..8f496c2e670a9 100644
22 --- a/mm/vmscan.c
23 +++ b/mm/vmscan.c
24 @@ -3980,8 +3980,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
25 }
26
27 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
28 -static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,
29 - struct mm_walk *args, unsigned long *bitmap, unsigned long *start)
30 +static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
31 + struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
32 {
33 int i;
34 pmd_t *pmd;
35 @@ -3994,18 +3994,19 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
36 VM_WARN_ON_ONCE(pud_leaf(*pud));
37
38 /* try to batch at most 1+MIN_LRU_BATCH+1 entries */
39 - if (*start == -1) {
40 - *start = next;
41 + if (*first == -1) {
42 + *first = addr;
43 + bitmap_zero(bitmap, MIN_LRU_BATCH);
44 return;
45 }
46
47 - i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start);
48 + i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first);
49 if (i && i <= MIN_LRU_BATCH) {
50 __set_bit(i - 1, bitmap);
51 return;
52 }
53
54 - pmd = pmd_offset(pud, *start);
55 + pmd = pmd_offset(pud, *first);
56
57 ptl = pmd_lockptr(args->mm, pmd);
58 if (!spin_trylock(ptl))
59 @@ -4016,15 +4017,16 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
60 do {
61 unsigned long pfn;
62 struct folio *folio;
63 - unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start;
64 +
65 + /* don't round down the first address */
66 + addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first;
67
68 pfn = get_pmd_pfn(pmd[i], vma, addr);
69 if (pfn == -1)
70 goto next;
71
72 if (!pmd_trans_huge(pmd[i])) {
73 - if (arch_has_hw_nonleaf_pmd_young() &&
74 - get_cap(LRU_GEN_NONLEAF_YOUNG))
75 + if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG))
76 pmdp_test_and_clear_young(vma, addr, pmd + i);
77 goto next;
78 }
79 @@ -4053,12 +4055,11 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
80 arch_leave_lazy_mmu_mode();
81 spin_unlock(ptl);
82 done:
83 - *start = -1;
84 - bitmap_zero(bitmap, MIN_LRU_BATCH);
85 + *first = -1;
86 }
87 #else
88 -static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,
89 - struct mm_walk *args, unsigned long *bitmap, unsigned long *start)
90 +static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
91 + struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
92 {
93 }
94 #endif
95 @@ -4071,9 +4072,9 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
96 unsigned long next;
97 unsigned long addr;
98 struct vm_area_struct *vma;
99 - unsigned long pos = -1;
100 + unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)];
101 + unsigned long first = -1;
102 struct lru_gen_mm_walk *walk = args->private;
103 - unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {};
104
105 VM_WARN_ON_ONCE(pud_leaf(*pud));
106
107 @@ -4115,18 +4116,17 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
108 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
109 continue;
110
111 - walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);
112 + walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
113 continue;
114 }
115 #endif
116 walk->mm_stats[MM_NONLEAF_TOTAL]++;
117
118 - if (arch_has_hw_nonleaf_pmd_young() &&
119 - get_cap(LRU_GEN_NONLEAF_YOUNG)) {
120 + if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG)) {
121 if (!pmd_young(val))
122 continue;
123
124 - walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);
125 + walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
126 }
127
128 if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i))
129 @@ -4143,7 +4143,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
130 update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i);
131 }
132
133 - walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos);
134 + walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
135
136 if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
137 goto restart;
138 --
139 2.40.1
140
Staging tree of Linus Walleij