mmu_notifier: call mmu_notifier_invalidate_range() from VMM

Add calls to the new mmu_notifier_invalidate_range() function to all
places in the VMM that need it.

Signed-off-by: Joerg Roedel <jroedel@suse.de>
Reviewed-by: Andrea Arcangeli <aarcange@redhat.com>
Reviewed-by: Jérôme Glisse <jglisse@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Rik van Riel <riel@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <jweiner@redhat.com>
Cc: Jay Cornwall <Jay.Cornwall@amd.com>
Cc: Oded Gabbay <Oded.Gabbay@amd.com>
Cc: Suravee Suthikulpanit <Suravee.Suthikulpanit@amd.com>
Cc: Jesse Barnes <jbarnes@virtuousgeek.org>
Cc: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Oded Gabbay <oded.gabbay@amd.com>

diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index 17907908d1df992c142f1aadad1340f3a4b6f120..966da2b4b8033335e07b68e7d8599330d263e170 100644 (file)
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -284,6 +284,44 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
        __young;                                                        \
 })
 
+#define        ptep_clear_flush_notify(__vma, __address, __ptep)               \
+({                                                                     \
+       unsigned long ___addr = __address & PAGE_MASK;                  \
+       struct mm_struct *___mm = (__vma)->vm_mm;                       \
+       pte_t ___pte;                                                   \
+                                                                       \
+       ___pte = ptep_clear_flush(__vma, __address, __ptep);            \
+       mmu_notifier_invalidate_range(___mm, ___addr,                   \
+                                       ___addr + PAGE_SIZE);           \
+                                                                       \
+       ___pte;                                                         \
+})
+
+#define pmdp_clear_flush_notify(__vma, __haddr, __pmd)                 \
+({                                                                     \
+       unsigned long ___haddr = __haddr & HPAGE_PMD_MASK;              \
+       struct mm_struct *___mm = (__vma)->vm_mm;                       \
+       pmd_t ___pmd;                                                   \
+                                                                       \
+       ___pmd = pmdp_clear_flush(__vma, __haddr, __pmd);               \
+       mmu_notifier_invalidate_range(___mm, ___haddr,                  \
+                                     ___haddr + HPAGE_PMD_SIZE);       \
+                                                                       \
+       ___pmd;                                                         \
+})
+
+#define pmdp_get_and_clear_notify(__mm, __haddr, __pmd)                        \
+({                                                                     \
+       unsigned long ___haddr = __haddr & HPAGE_PMD_MASK;              \
+       pmd_t ___pmd;                                                   \
+                                                                       \
+       ___pmd = pmdp_get_and_clear(__mm, __haddr, __pmd);              \
+       mmu_notifier_invalidate_range(__mm, ___haddr,                   \
+                                     ___haddr + HPAGE_PMD_SIZE);       \
+                                                                       \
+       ___pmd;                                                         \
+})
+
 /*
  * set_pte_at_notify() sets the pte _after_ running the notifier.
  * This is safe to start by updating the secondary MMUs, because the primary MMU
@@ -362,6 +400,9 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
 
 #define ptep_clear_flush_young_notify ptep_clear_flush_young
 #define pmdp_clear_flush_young_notify pmdp_clear_flush_young
+#define        ptep_clear_flush_notify ptep_clear_flush
+#define pmdp_clear_flush_notify pmdp_clear_flush
+#define pmdp_get_and_clear_notify pmdp_get_and_clear
 #define set_pte_at_notify set_pte_at
 
 #endif /* CONFIG_MMU_NOTIFIER */

diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 1d0af8a2c6469bda46438dbd8383cbf535d65077..bc143cf56cab79affc8549054b04112956de08e7 100644 (file)
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -193,7 +193,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
        }
 
        flush_cache_page(vma, addr, pte_pfn(*ptep));
-       ptep_clear_flush(vma, addr, ptep);
+       ptep_clear_flush_notify(vma, addr, ptep);
        set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
 
        page_remove_rmap(page);

diff --git a/mm/fremap.c b/mm/fremap.c
index 72b8fa36143328f728a44de1b5cd53b629653a0a..9129013732d72b323251ab680b0db39f2961aba9 100644 (file)
--- a/mm/fremap.c
+++ b/mm/fremap.c
@@ -37,7 +37,7 @@ static void zap_pte(struct mm_struct *mm, struct vm_area_struct *vma,
 
        if (pte_present(pte)) {
                flush_cache_page(vma, addr, pte_pfn(pte));
-               pte = ptep_clear_flush(vma, addr, ptep);
+               pte = ptep_clear_flush_notify(vma, addr, ptep);
                page = vm_normal_page(vma, addr, pte);
                if (page) {
                        if (pte_dirty(pte))

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index de984159cf0b8a0be6a73b9f421e9a0b4f75c599..1d89526ed53196576cfbf2f00f220eebe3f1511a 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1036,7 +1036,7 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
                goto out_free_pages;
        VM_BUG_ON_PAGE(!PageHead(page), page);
 
-       pmdp_clear_flush(vma, haddr, pmd);
+       pmdp_clear_flush_notify(vma, haddr, pmd);
        /* leave pmd empty until pte is filled */
 
        pgtable = pgtable_trans_huge_withdraw(mm, pmd);
@@ -1179,7 +1179,7 @@ alloc:
                pmd_t entry;
                entry = mk_huge_pmd(new_page, vma->vm_page_prot);
                entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
-               pmdp_clear_flush(vma, haddr, pmd);
+               pmdp_clear_flush_notify(vma, haddr, pmd);
                page_add_new_anon_rmap(new_page, vma, haddr);
                mem_cgroup_commit_charge(new_page, memcg, false);
                lru_cache_add_active_or_unevictable(new_page, vma);
@@ -1512,7 +1512,7 @@ int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
                pmd_t entry;
                ret = 1;
                if (!prot_numa) {
-                       entry = pmdp_get_and_clear(mm, addr, pmd);
+                       entry = pmdp_get_and_clear_notify(mm, addr, pmd);
                        if (pmd_numa(entry))
                                entry = pmd_mknonnuma(entry);
                        entry = pmd_modify(entry, newprot);
@@ -1644,6 +1644,7 @@ static int __split_huge_page_splitting(struct page *page,
                 * serialize against split_huge_page*.
                 */
                pmdp_splitting_flush(vma, address, pmd);
+
                ret = 1;
                spin_unlock(ptl);
        }
@@ -2834,7 +2835,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
        pmd_t _pmd;
        int i;
 
-       pmdp_clear_flush(vma, haddr, pmd);
+       pmdp_clear_flush_notify(vma, haddr, pmd);
        /* leave pmd empty until pte is filled */
 
        pgtable = pgtable_trans_huge_withdraw(mm, pmd);

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 9fd722769927f9e5bb5e03fb6516db7a5f7c8f42..2e6add04fa1b37bd451f8bbd89e6cec6ab26f614 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -2598,8 +2598,11 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
                        }
                        set_huge_pte_at(dst, addr, dst_pte, entry);
                } else {
-                       if (cow)
+                       if (cow) {
                                huge_ptep_set_wrprotect(src, addr, src_pte);
+                               mmu_notifier_invalidate_range(src, mmun_start,
+                                                                  mmun_end);
+                       }
                        entry = huge_ptep_get(src_pte);
                        ptepage = pte_page(entry);
                        get_page(ptepage);
@@ -2899,6 +2902,7 @@ retry_avoidcopy:
 
                /* Break COW */
                huge_ptep_clear_flush(vma, address, ptep);
+               mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
                set_huge_pte_at(mm, address, ptep,
                                make_huge_pte(vma, new_page, 1));
                page_remove_rmap(old_page);
@@ -3374,6 +3378,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
         * and that page table be reused and filled with junk.
         */
        flush_tlb_range(vma, start, end);
+       mmu_notifier_invalidate_range(mm, start, end);
        mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
        mmu_notifier_invalidate_range_end(mm, start, end);
 

diff --git a/mm/ksm.c b/mm/ksm.c
index 6b2e337bc03c7d6c5fce5e33023bdb5d2f111789..d247efab5073abfaeb9c11e33fb87a2fbb8ebdb0 100644 (file)
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -892,7 +892,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
                 * this assure us that no O_DIRECT can happen after the check
                 * or in the middle of the check.
                 */
-               entry = ptep_clear_flush(vma, addr, ptep);
+               entry = ptep_clear_flush_notify(vma, addr, ptep);
                /*
                 * Check that no O_DIRECT or similar I/O is in progress on the
                 * page
@@ -960,7 +960,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
        page_add_anon_rmap(kpage, vma, addr);
 
        flush_cache_page(vma, addr, pte_pfn(*ptep));
-       ptep_clear_flush(vma, addr, ptep);
+       ptep_clear_flush_notify(vma, addr, ptep);
        set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
 
        page_remove_rmap(page);

diff --git a/mm/memory.c b/mm/memory.c
index 3e503831e042a6aa7b96d2608ecb570dfffa0aa7..655fd3d34bb0908e6e18d7b084342f6132f85521 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -238,6 +238,7 @@ static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
 {
        tlb->need_flush = 0;
        tlb_flush(tlb);
+       mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end);
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
        tlb_table_flush(tlb);
 #endif
@@ -2234,7 +2235,7 @@ gotten:
                 * seen in the presence of one thread doing SMC and another
                 * thread doing COW.
                 */
-               ptep_clear_flush(vma, address, page_table);
+               ptep_clear_flush_notify(vma, address, page_table);
                page_add_new_anon_rmap(new_page, vma, address);
                mem_cgroup_commit_charge(new_page, memcg, false);
                lru_cache_add_active_or_unevictable(new_page, vma);

diff --git a/mm/migrate.c b/mm/migrate.c
index 01439953abf548690ac17f1b994511b587e9326a..41945cb0ca38d318d88a4361ae01ddf2dea4a7b1 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1854,7 +1854,7 @@ fail_putback:
         */
        flush_cache_range(vma, mmun_start, mmun_end);
        page_add_anon_rmap(new_page, vma, mmun_start);
-       pmdp_clear_flush(vma, mmun_start, pmd);
+       pmdp_clear_flush_notify(vma, mmun_start, pmd);
        set_pmd_at(mm, mmun_start, pmd, entry);
        flush_tlb_range(vma, mmun_start, mmun_end);
        update_mmu_cache_pmd(vma, address, &entry);
@@ -1862,6 +1862,7 @@ fail_putback:
        if (page_count(page) != 2) {
                set_pmd_at(mm, mmun_start, pmd, orig_entry);
                flush_tlb_range(vma, mmun_start, mmun_end);
+               mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
                update_mmu_cache_pmd(vma, address, &entry);
                page_remove_rmap(new_page);
                goto fail_putback;

diff --git a/mm/rmap.c b/mm/rmap.c
index 19886fb2f13aac6a659ba1ae8b9e4db2f8efa7a4..d3eb1e02d1c681fd21e1f1752b0327349b24accd 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1378,7 +1378,7 @@ static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount,
 
                /* Nuke the page table entry. */
                flush_cache_page(vma, address, pte_pfn(*pte));
-               pteval = ptep_clear_flush(vma, address, pte);
+               pteval = ptep_clear_flush_notify(vma, address, pte);
 
                /* If nonlinear, store the file page offset in the pte. */
                if (page->index != linear_page_index(vma, address)) {