mm, drm/i915: mark pinned shmemfs pages as unevictable

The i915 driver uses shmemfs to allocate backing storage for gem
objects. These shmemfs pages can be pinned (increased ref count) by
shmem_read_mapping_page_gfp(). When a lot of pages are pinned, vmscan
wastes a lot of time scanning these pinned pages. In some extreme case,
all pages in the inactive anon lru are pinned, and only the inactive
anon lru is scanned due to inactive_ratio, the system cannot swap and
invokes the oom-killer. Mark these pinned pages as unevictable to speed
up vmscan.

Export pagevec API check_move_unevictable_pages().

This patch was inspired by Chris Wilson's change [1].

[1]: https://patchwork.kernel.org/patch/9768741/

Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Dave Hansen <dave.hansen@intel.com>
Signed-off-by: Kuo-Hsin Yang <vovoy@chromium.org>
Acked-by: Michal Hocko <mhocko@suse.com> # mm part
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Acked-by: Dave Hansen <dave.hansen@intel.com>
Acked-by: Andrew Morton <akpm@linux-foundation.org>
Link: https://patchwork.freedesktop.org/patch/msgid/20181106132324.17390-1-chris@chris-wilson.co.uk
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>

diff --git a/Documentation/vm/unevictable-lru.rst b/Documentation/vm/unevictable-lru.rst
index fdd84cb8d511f4aef71f18f281ba629ad025d17f..b8e29f977f2d1f3a2898444bc9720c39ab9eaa4f 100644 (file)
--- a/Documentation/vm/unevictable-lru.rst
+++ b/Documentation/vm/unevictable-lru.rst
@@ -143,7 +143,7 @@ using a number of wrapper functions:
        Query the address space, and return true if it is completely
        unevictable.
 
-These are currently used in two places in the kernel:
+These are currently used in three places in the kernel:
 
  (1) By ramfs to mark the address spaces of its inodes when they are created,
      and this mark remains for the life of the inode.
@@ -154,6 +154,10 @@ These are currently used in two places in the kernel:
      swapped out; the application must touch the pages manually if it wants to
      ensure they're in memory.
 
+ (3) By the i915 driver to mark pinned address space until it's unpinned. The
+     amount of unevictable memory marked by i915 driver is roughly the bounded
+     object size in debugfs/dri/0/i915_gem_objects.
+
 
 Detecting Unevictable Pages
 ---------------------------

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 347b3836c80950ba39090b8caded4b1c5d22d1fb..5b80b0c14aed7a9db3fc9dc7231bee2af54aabbd 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -2382,11 +2382,23 @@ void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
        invalidate_mapping_pages(mapping, 0, (loff_t)-1);
 }
 
+/*
+ * Move pages to appropriate lru and release the pagevec, decrementing the
+ * ref count of those pages.
+ */
+static void check_release_pagevec(struct pagevec *pvec)
+{
+       check_move_unevictable_pages(pvec);
+       __pagevec_release(pvec);
+       cond_resched();
+}
+
 static void
 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
                              struct sg_table *pages)
 {
        struct sgt_iter sgt_iter;
+       struct pagevec pvec;
        struct page *page;
 
        __i915_gem_object_release_shmem(obj, pages, true);
@@ -2396,6 +2408,9 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
        if (i915_gem_object_needs_bit17_swizzle(obj))
                i915_gem_object_save_bit_17_swizzle(obj, pages);
 
+       mapping_clear_unevictable(file_inode(obj->base.filp)->i_mapping);
+
+       pagevec_init(&pvec);
        for_each_sgt_page(page, sgt_iter, pages) {
                if (obj->mm.dirty)
                        set_page_dirty(page);
@@ -2403,9 +2418,11 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
                if (obj->mm.madv == I915_MADV_WILLNEED)
                        mark_page_accessed(page);
 
-               put_page(page);
-               cond_resched();
+               if (!pagevec_add(&pvec, page))
+                       check_release_pagevec(&pvec);
        }
+       if (pagevec_count(&pvec))
+               check_release_pagevec(&pvec);
        obj->mm.dirty = false;
 
        sg_free_table(pages);
@@ -2526,6 +2543,7 @@ static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
        unsigned long last_pfn = 0;     /* suppress gcc warning */
        unsigned int max_segment = i915_sg_segment_size();
        unsigned int sg_page_sizes;
+       struct pagevec pvec;
        gfp_t noreclaim;
        int ret;
 
@@ -2561,6 +2579,7 @@ rebuild_st:
         * Fail silently without starting the shrinker
         */
        mapping = obj->base.filp->f_mapping;
+       mapping_set_unevictable(mapping);
        noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
        noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
 
@@ -2675,8 +2694,14 @@ rebuild_st:
 err_sg:
        sg_mark_end(sg);
 err_pages:
-       for_each_sgt_page(page, sgt_iter, st)
-               put_page(page);
+       mapping_clear_unevictable(mapping);
+       pagevec_init(&pvec);
+       for_each_sgt_page(page, sgt_iter, st) {
+               if (!pagevec_add(&pvec, page))
+                       check_release_pagevec(&pvec);
+       }
+       if (pagevec_count(&pvec))
+               check_release_pagevec(&pvec);
        sg_free_table(st);
        kfree(st);
 

diff --git a/include/linux/swap.h b/include/linux/swap.h
index 8e2c11e692baf9bba6380d63888cfbf5a206cf0b..6c95df96c9aa288902fe57545d58ecd4ccb79cb1 100644 (file)
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -18,6 +18,8 @@ struct notifier_block;
 
 struct bio;
 
+struct pagevec;
+
 #define SWAP_FLAG_PREFER       0x8000  /* set if swap priority specified */
 #define SWAP_FLAG_PRIO_MASK    0x7fff
 #define SWAP_FLAG_PRIO_SHIFT   0
@@ -373,7 +375,7 @@ static inline int node_reclaim(struct pglist_data *pgdat, gfp_t mask,
 #endif
 
 extern int page_evictable(struct page *page);
-extern void check_move_unevictable_pages(struct page **, int nr_pages);
+extern void check_move_unevictable_pages(struct pagevec *pvec);
 
 extern int kswapd_run(int nid);
 extern void kswapd_stop(int nid);

diff --git a/mm/shmem.c b/mm/shmem.c
index 446942677cd4b88fad61e9a869a901bc66fadc41..0c3b005a59eb4b28dce083bd88eb503d13b9216a 100644 (file)
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -781,7 +781,7 @@ void shmem_unlock_mapping(struct address_space *mapping)
                        break;
                index = indices[pvec.nr - 1] + 1;
                pagevec_remove_exceptionals(&pvec);
-               check_move_unevictable_pages(pvec.pages, pvec.nr);
+               check_move_unevictable_pages(&pvec);
                pagevec_release(&pvec);
                cond_resched();
        }

diff --git a/mm/vmscan.c b/mm/vmscan.c
index c7ce2c1612259c45896bae9739a288966a28c970..0dbc493026a27d4f59b7ca6d83a4c864340f99a1 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -46,6 +46,7 @@
 #include <linux/delayacct.h>
 #include <linux/sysctl.h>
 #include <linux/oom.h>
+#include <linux/pagevec.h>
 #include <linux/prefetch.h>
 #include <linux/printk.h>
 #include <linux/dax.h>
@@ -4162,17 +4163,16 @@ int page_evictable(struct page *page)
        return ret;
 }
 
-#ifdef CONFIG_SHMEM
 /**
- * check_move_unevictable_pages - check pages for evictability and move to appropriate zone lru list
- * @pages:     array of pages to check
- * @nr_pages:  number of pages to check
+ * check_move_unevictable_pages - check pages for evictability and move to
+ * appropriate zone lru list
+ * @pvec: pagevec with lru pages to check
  *
- * Checks pages for evictability and moves them to the appropriate lru list.
- *
- * This function is only used for SysV IPC SHM_UNLOCK.
+ * Checks pages for evictability, if an evictable page is in the unevictable
+ * lru list, moves it to the appropriate evictable lru list. This function
+ * should be only used for lru pages.
  */
-void check_move_unevictable_pages(struct page **pages, int nr_pages)
+void check_move_unevictable_pages(struct pagevec *pvec)
 {
        struct lruvec *lruvec;
        struct pglist_data *pgdat = NULL;
@@ -4180,8 +4180,8 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)
        int pgrescued = 0;
        int i;
 
-       for (i = 0; i < nr_pages; i++) {
-               struct page *page = pages[i];
+       for (i = 0; i < pvec->nr; i++) {
+               struct page *page = pvec->pages[i];
                struct pglist_data *pagepgdat = page_pgdat(page);
 
                pgscanned++;
@@ -4213,4 +4213,4 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)
                spin_unlock_irq(&pgdat->lru_lock);
        }
 }
-#endif /* CONFIG_SHMEM */
+EXPORT_SYMBOL_GPL(check_move_unevictable_pages);