From a8cff4c8283af35546339c9ada5a90a70fe4a075 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 10 Jun 2019 15:54:30 +0100 Subject: [PATCH] drm/i915: Promote i915->mm.obj_lock to be irqsafe The intent is to be able to update the mm.lists from inside an irqsoff section (e.g. from a softirq rcu workqueue), ergo we need to make the i915->mm.obj_lock irqsafe. v2: can_discard_pages() ensures we are shrinkable v3: Beware shadowing of 'flags' Fixes: 3b4fa9640ccd ("drm/i915: Track the purgeable objects on a separate eviction list") Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=110869 Signed-off-by: Chris Wilson Cc: Joonas Lahtinen Cc: Matthew Auld Reviewed-by: Matthew Auld Link: https://patchwork.freedesktop.org/patch/msgid/20190610145430.17717-1-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/gem/i915_gem_domain.c | 23 +++++++----- drivers/gpu/drm/i915/gem/i915_gem_object.c | 12 ++++--- drivers/gpu/drm/i915/gem/i915_gem_pages.c | 16 ++++++--- drivers/gpu/drm/i915/gem/i915_gem_shrinker.c | 38 ++++++++++---------- drivers/gpu/drm/i915/gem/i915_gem_stolen.c | 5 +-- drivers/gpu/drm/i915/i915_debugfs.c | 10 +++--- drivers/gpu/drm/i915/i915_gem.c | 8 +++-- drivers/gpu/drm/i915/i915_vma.c | 17 +++++---- 8 files changed, 80 insertions(+), 49 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c index e5deae62681f..31929220b90f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c @@ -475,15 +475,20 @@ static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) } mutex_unlock(&i915->ggtt.vm.mutex); - if (i915_gem_object_is_shrinkable(obj) && - obj->mm.madv == I915_MADV_WILLNEED) { - struct list_head *list; - - spin_lock(&i915->mm.obj_lock); - list = obj->bind_count ? - &i915->mm.bound_list : &i915->mm.unbound_list; - list_move_tail(&obj->mm.link, list); - spin_unlock(&i915->mm.obj_lock); + if (i915_gem_object_is_shrinkable(obj)) { + unsigned long flags; + + spin_lock_irqsave(&i915->mm.obj_lock, flags); + + if (obj->mm.madv == I915_MADV_WILLNEED) { + struct list_head *list; + + list = obj->bind_count ? + &i915->mm.bound_list : &i915->mm.unbound_list; + list_move_tail(&obj->mm.link, list); + } + + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); } } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index a0bc8f7ab780..d02a1aff2058 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -207,9 +207,11 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, */ if (i915_gem_object_has_pages(obj) && i915_gem_object_is_shrinkable(obj)) { - spin_lock(&i915->mm.obj_lock); + unsigned long flags; + + spin_lock_irqsave(&i915->mm.obj_lock, flags); list_del_init(&obj->mm.link); - spin_unlock(&i915->mm.obj_lock); + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); } mutex_unlock(&i915->drm.struct_mutex); @@ -330,9 +332,11 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj) obj->mm.madv = I915_MADV_DONTNEED; if (i915_gem_object_has_pages(obj)) { - spin_lock(&i915->mm.obj_lock); + unsigned long flags; + + spin_lock_irqsave(&i915->mm.obj_lock, flags); list_move_tail(&obj->mm.link, &i915->mm.purge_list); - spin_unlock(&i915->mm.obj_lock); + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); } } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index 7e64fd6bc19b..7ff907d6d0c6 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -57,11 +57,15 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg)); if (i915_gem_object_is_shrinkable(obj)) { - spin_lock(&i915->mm.obj_lock); + unsigned long flags; + + spin_lock_irqsave(&i915->mm.obj_lock, flags); + i915->mm.shrink_count++; i915->mm.shrink_memory += obj->base.size; list_add(&obj->mm.link, &i915->mm.unbound_list); - spin_unlock(&i915->mm.obj_lock); + + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); } } @@ -151,11 +155,15 @@ __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) return pages; if (i915_gem_object_is_shrinkable(obj)) { - spin_lock(&i915->mm.obj_lock); + unsigned long flags; + + spin_lock_irqsave(&i915->mm.obj_lock, flags); + list_del(&obj->mm.link); i915->mm.shrink_count--; i915->mm.shrink_memory -= obj->base.size; - spin_unlock(&i915->mm.obj_lock); + + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); } if (obj->mm.mapping) { diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c index d71e630c6fb8..70a4c9d3c098 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c @@ -138,7 +138,7 @@ unsigned long i915_gem_shrink(struct drm_i915_private *i915, unsigned long target, unsigned long *nr_scanned, - unsigned flags) + unsigned int shrink) { const struct { struct list_head *list; @@ -154,7 +154,7 @@ i915_gem_shrink(struct drm_i915_private *i915, unsigned long scanned = 0; bool unlock; - if (!shrinker_lock(i915, flags, &unlock)) + if (!shrinker_lock(i915, shrink, &unlock)) return 0; /* @@ -166,12 +166,12 @@ i915_gem_shrink(struct drm_i915_private *i915, * We don't care about errors here; if we cannot wait upon the GPU, * we will free as much as we can and hope to get a second chance. */ - if (flags & I915_SHRINK_ACTIVE) + if (shrink & I915_SHRINK_ACTIVE) i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED, MAX_SCHEDULE_TIMEOUT); - trace_i915_gem_shrink(i915, target, flags); + trace_i915_gem_shrink(i915, target, shrink); i915_retire_requests(i915); /* @@ -179,10 +179,10 @@ i915_gem_shrink(struct drm_i915_private *i915, * device just to recover a little memory. If absolutely necessary, * we will force the wake during oom-notifier. */ - if (flags & I915_SHRINK_BOUND) { + if (shrink & I915_SHRINK_BOUND) { wakeref = intel_runtime_pm_get_if_in_use(i915); if (!wakeref) - flags &= ~I915_SHRINK_BOUND; + shrink &= ~I915_SHRINK_BOUND; } /* @@ -207,8 +207,9 @@ i915_gem_shrink(struct drm_i915_private *i915, for (phase = phases; phase->list; phase++) { struct list_head still_in_list; struct drm_i915_gem_object *obj; + unsigned long flags; - if ((flags & phase->bit) == 0) + if ((shrink & phase->bit) == 0) continue; INIT_LIST_HEAD(&still_in_list); @@ -220,50 +221,50 @@ i915_gem_shrink(struct drm_i915_private *i915, * to be able to shrink their pages, so they remain on * the unbound/bound list until actually freed. */ - spin_lock(&i915->mm.obj_lock); + spin_lock_irqsave(&i915->mm.obj_lock, flags); while (count < target && (obj = list_first_entry_or_null(phase->list, typeof(*obj), mm.link))) { list_move_tail(&obj->mm.link, &still_in_list); - if (flags & I915_SHRINK_VMAPS && + if (shrink & I915_SHRINK_VMAPS && !is_vmalloc_addr(obj->mm.mapping)) continue; - if (!(flags & I915_SHRINK_ACTIVE) && + if (!(shrink & I915_SHRINK_ACTIVE) && (i915_gem_object_is_active(obj) || i915_gem_object_is_framebuffer(obj))) continue; - if (!(flags & I915_SHRINK_BOUND) && + if (!(shrink & I915_SHRINK_BOUND) && READ_ONCE(obj->bind_count)) continue; if (!can_release_pages(obj)) continue; - spin_unlock(&i915->mm.obj_lock); + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); if (unsafe_drop_pages(obj)) { /* May arrive from get_pages on another bo */ mutex_lock_nested(&obj->mm.lock, I915_MM_SHRINKER); if (!i915_gem_object_has_pages(obj)) { - try_to_writeback(obj, flags); + try_to_writeback(obj, shrink); count += obj->base.size >> PAGE_SHIFT; } mutex_unlock(&obj->mm.lock); } scanned += obj->base.size >> PAGE_SHIFT; - spin_lock(&i915->mm.obj_lock); + spin_lock_irqsave(&i915->mm.obj_lock, flags); } list_splice_tail(&still_in_list, phase->list); - spin_unlock(&i915->mm.obj_lock); + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); } - if (flags & I915_SHRINK_BOUND) + if (shrink & I915_SHRINK_BOUND) intel_runtime_pm_put(i915, wakeref); i915_retire_requests(i915); @@ -379,6 +380,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) struct drm_i915_gem_object *obj; unsigned long unevictable, bound, unbound, freed_pages; intel_wakeref_t wakeref; + unsigned long flags; freed_pages = 0; with_intel_runtime_pm(i915, wakeref) @@ -392,7 +394,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) * being pointed to by hardware. */ unbound = bound = unevictable = 0; - spin_lock(&i915->mm.obj_lock); + spin_lock_irqsave(&i915->mm.obj_lock, flags); list_for_each_entry(obj, &i915->mm.unbound_list, mm.link) { if (!can_release_pages(obj)) unevictable += obj->base.size >> PAGE_SHIFT; @@ -405,7 +407,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) else bound += obj->base.size >> PAGE_SHIFT; } - spin_unlock(&i915->mm.obj_lock); + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); if (freed_pages || unbound || bound) pr_info("Purging GPU memory, %lu pages freed, " diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index 432037e5d0a5..8f619ad2fab8 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -613,6 +613,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv struct drm_i915_gem_object *obj; struct drm_mm_node *stolen; struct i915_vma *vma; + unsigned long flags; int ret; if (!drm_mm_initialized(&dev_priv->mm.stolen)) @@ -689,10 +690,10 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv list_move_tail(&vma->vm_link, &ggtt->vm.bound_list); mutex_unlock(&ggtt->vm.mutex); - spin_lock(&dev_priv->mm.obj_lock); + spin_lock_irqsave(&dev_priv->mm.obj_lock, flags); GEM_BUG_ON(i915_gem_object_is_shrinkable(obj)); obj->bind_count++; - spin_unlock(&dev_priv->mm.obj_lock); + spin_unlock_irqrestore(&dev_priv->mm.obj_lock, flags); return obj; diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index f212241a2758..2e79f0e4c5af 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -269,6 +269,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) struct drm_i915_gem_object *obj; u64 total_obj_size, total_gtt_size; unsigned long total, count, n; + unsigned long flags; int ret; total = READ_ONCE(dev_priv->mm.shrink_count); @@ -282,7 +283,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) total_obj_size = total_gtt_size = count = 0; - spin_lock(&dev_priv->mm.obj_lock); + spin_lock_irqsave(&dev_priv->mm.obj_lock, flags); list_for_each_entry(obj, &dev_priv->mm.bound_list, mm.link) { if (count == total) break; @@ -305,7 +306,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) objects[count++] = obj; total_obj_size += obj->base.size; } - spin_unlock(&dev_priv->mm.obj_lock); + spin_unlock_irqrestore(&dev_priv->mm.obj_lock, flags); sort(objects, count, sizeof(*objects), obj_rank_by_stolen, NULL); @@ -457,6 +458,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data) u64 size, mapped_size, purgeable_size, dpy_size, huge_size; struct drm_i915_gem_object *obj; unsigned int page_sizes = 0; + unsigned long flags; char buf[80]; int ret; @@ -469,7 +471,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data) purgeable_size = purgeable_count = 0; huge_size = huge_count = 0; - spin_lock(&dev_priv->mm.obj_lock); + spin_lock_irqsave(&dev_priv->mm.obj_lock, flags); list_for_each_entry(obj, &dev_priv->mm.unbound_list, mm.link) { size += obj->base.size; ++count; @@ -518,7 +520,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data) page_sizes |= obj->mm.page_sizes.sg; } } - spin_unlock(&dev_priv->mm.obj_lock); + spin_unlock_irqrestore(&dev_priv->mm.obj_lock, flags); seq_printf(m, "%u bound objects, %llu bytes\n", count, size); diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 9f2e213c6046..e980c1ee3dcf 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -1138,7 +1138,10 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, struct list_head *list; if (i915_gem_object_is_shrinkable(obj)) { - spin_lock(&i915->mm.obj_lock); + unsigned long flags; + + spin_lock_irqsave(&i915->mm.obj_lock, flags); + if (obj->mm.madv != I915_MADV_WILLNEED) list = &i915->mm.purge_list; else if (obj->bind_count) @@ -1146,7 +1149,8 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, else list = &i915->mm.unbound_list; list_move_tail(&obj->mm.link, list); - spin_unlock(&i915->mm.obj_lock); + + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); } } diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index f6ac8394da77..80050f6a0893 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -80,11 +80,14 @@ static void vma_print_allocator(struct i915_vma *vma, const char *reason) static void obj_bump_mru(struct drm_i915_gem_object *obj) { struct drm_i915_private *i915 = to_i915(obj->base.dev); + unsigned long flags; + + spin_lock_irqsave(&i915->mm.obj_lock, flags); - spin_lock(&i915->mm.obj_lock); if (obj->bind_count) list_move_tail(&obj->mm.link, &i915->mm.bound_list); - spin_unlock(&i915->mm.obj_lock); + + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); obj->mm.dirty = true; /* be paranoid */ } @@ -678,8 +681,9 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) if (vma->obj) { struct drm_i915_gem_object *obj = vma->obj; + unsigned long flags; - spin_lock(&dev_priv->mm.obj_lock); + spin_lock_irqsave(&dev_priv->mm.obj_lock, flags); if (i915_gem_object_is_shrinkable(obj)) list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list); @@ -687,7 +691,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) obj->bind_count++; assert_bind_count(obj); - spin_unlock(&dev_priv->mm.obj_lock); + spin_unlock_irqrestore(&dev_priv->mm.obj_lock, flags); } return 0; @@ -721,8 +725,9 @@ i915_vma_remove(struct i915_vma *vma) */ if (vma->obj) { struct drm_i915_gem_object *obj = vma->obj; + unsigned long flags; - spin_lock(&i915->mm.obj_lock); + spin_lock_irqsave(&i915->mm.obj_lock, flags); GEM_BUG_ON(obj->bind_count == 0); if (--obj->bind_count == 0 && @@ -730,7 +735,7 @@ i915_vma_remove(struct i915_vma *vma) obj->mm.madv == I915_MADV_WILLNEED) list_move_tail(&obj->mm.link, &i915->mm.unbound_list); - spin_unlock(&i915->mm.obj_lock); + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); /* * And finally now the object is completely decoupled from this -- 2.30.2