drm/ttm: implement LRU add callbacks v2

This allows fine grained control for the driver where to add a BO into the LRU.

v2: fix typo in comment

Reviewed-by: Sinclair Yeh <syeh@vmware.com>
Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 7f5fc29354f702b980936b0936649602422ece0c..d580ccccd2b868e098230d39950bf21e5626bfe1 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -922,6 +922,8 @@ static struct ttm_bo_driver amdgpu_bo_driver = {
        .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
        .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
        .io_mem_free = &amdgpu_ttm_io_mem_free,
+       .lru_tail = &ttm_bo_default_lru_tail,
+       .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int amdgpu_ttm_init(struct amdgpu_device *adev)

diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c
index 08f82eae69398509a7daf55d55a4a94a4e57b6f4..59f2f93b6f84859ea04c9d092df3f276d2f6a7a6 100644 (file)
--- a/drivers/gpu/drm/ast/ast_ttm.c
+++ b/drivers/gpu/drm/ast/ast_ttm.c
@@ -245,6 +245,8 @@ struct ttm_bo_driver ast_bo_driver = {
        .verify_access = ast_bo_verify_access,
        .io_mem_reserve = &ast_ttm_io_mem_reserve,
        .io_mem_free = &ast_ttm_io_mem_free,
+       .lru_tail = &ttm_bo_default_lru_tail,
+       .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int ast_mm_init(struct ast_private *ast)

diff --git a/drivers/gpu/drm/bochs/bochs_mm.c b/drivers/gpu/drm/bochs/bochs_mm.c
index d812ad014da5c407476eb01246bdb9bd2474b2c1..24a30f64ed2d8df912982da78fd69051afe6b6e2 100644 (file)
--- a/drivers/gpu/drm/bochs/bochs_mm.c
+++ b/drivers/gpu/drm/bochs/bochs_mm.c
@@ -212,6 +212,8 @@ struct ttm_bo_driver bochs_bo_driver = {
        .verify_access = bochs_bo_verify_access,
        .io_mem_reserve = &bochs_ttm_io_mem_reserve,
        .io_mem_free = &bochs_ttm_io_mem_free,
+       .lru_tail = &ttm_bo_default_lru_tail,
+       .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int bochs_mm_init(struct bochs_device *bochs)

diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c
index dfffd528517a53b1783e8f5c22d1f622d1e7d537..6768b7b1af32d98bc26c259870ac471983e4a62d 100644 (file)
--- a/drivers/gpu/drm/cirrus/cirrus_ttm.c
+++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c
@@ -245,6 +245,8 @@ struct ttm_bo_driver cirrus_bo_driver = {
        .verify_access = cirrus_bo_verify_access,
        .io_mem_reserve = &cirrus_ttm_io_mem_reserve,
        .io_mem_free = &cirrus_ttm_io_mem_free,
+       .lru_tail = &ttm_bo_default_lru_tail,
+       .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int cirrus_mm_init(struct cirrus_device *cirrus)

diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c
index 05108b505fbfa1382ad071ab80e4357ad4db4ae2..9d5083d0f1eec7a0c84c2f4e9bb7746e4f21e785 100644 (file)
--- a/drivers/gpu/drm/mgag200/mgag200_ttm.c
+++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c
@@ -245,6 +245,8 @@ struct ttm_bo_driver mgag200_bo_driver = {
        .verify_access = mgag200_bo_verify_access,
        .io_mem_reserve = &mgag200_ttm_io_mem_reserve,
        .io_mem_free = &mgag200_ttm_io_mem_free,
+       .lru_tail = &ttm_bo_default_lru_tail,
+       .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int mgag200_mm_init(struct mga_device *mdev)

diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 5fe500033a958e1a024df0183148c1bbdae8d43d..74a8a2ca89ee103e1e22ba245d56f3e0c499d14d 100644 (file)
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -1611,6 +1611,8 @@ struct ttm_bo_driver nouveau_bo_driver = {
        .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
        .io_mem_reserve = &nouveau_ttm_io_mem_reserve,
        .io_mem_free = &nouveau_ttm_io_mem_free,
+       .lru_tail = &ttm_bo_default_lru_tail,
+       .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 struct nvkm_vma *

diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index 953412766416f379e04a521f2c9f91a1078c8e12..0738d74c8d049faf18f0153378de60a1978a49e7 100644 (file)
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -384,6 +384,8 @@ static struct ttm_bo_driver qxl_bo_driver = {
        .io_mem_reserve = &qxl_ttm_io_mem_reserve,
        .io_mem_free = &qxl_ttm_io_mem_free,
        .move_notify = &qxl_bo_move_notify,
+       .lru_tail = &ttm_bo_default_lru_tail,
+       .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int qxl_ttm_init(struct qxl_device *qdev)

diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 7dddfdce85e6be56f5d6bab8787fad82aa120809..81032ca571479971fe98dc8452c2658351517e4b 100644 (file)
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -863,6 +863,8 @@ static struct ttm_bo_driver radeon_bo_driver = {
        .fault_reserve_notify = &radeon_bo_fault_reserve_notify,
        .io_mem_reserve = &radeon_ttm_io_mem_reserve,
        .io_mem_free = &radeon_ttm_io_mem_free,
+       .lru_tail = &ttm_bo_default_lru_tail,
+       .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int radeon_ttm_init(struct radeon_device *rdev)

diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 309a72ee7fb4a1a00c11ca68d40625918a83b817..301e2371c34f5c260e95c9a1fc7b2122329497cf 100644 (file)
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -164,7 +164,6 @@ static void ttm_bo_release_list(struct kref *list_kref)
 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 {
        struct ttm_bo_device *bdev = bo->bdev;
-       struct ttm_mem_type_manager *man;
 
        lockdep_assert_held(&bo->resv->lock.base);
 
@@ -172,12 +171,11 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 
                BUG_ON(!list_empty(&bo->lru));
 
-               man = &bdev->man[bo->mem.mem_type];
-               list_add_tail(&bo->lru, &man->lru);
+               list_add(&bo->lru, bdev->driver->lru_tail(bo));
                kref_get(&bo->list_kref);
 
                if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) {
-                       list_add_tail(&bo->swap, &bo->glob->swap_lru);
+                       list_add(&bo->swap, bdev->driver->swap_lru_tail(bo));
                        kref_get(&bo->list_kref);
                }
        }
@@ -230,7 +228,6 @@ EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
 {
        struct ttm_bo_device *bdev = bo->bdev;
-       struct ttm_mem_type_manager *man;
 
        lockdep_assert_held(&bo->resv->lock.base);
 
@@ -242,15 +239,29 @@ void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
                list_del_init(&bo->lru);
 
        } else {
-               if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG))
-                       list_move_tail(&bo->swap, &bo->glob->swap_lru);
+               if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) {
+                       list_del(&bo->swap);
+                       list_add(&bo->swap, bdev->driver->swap_lru_tail(bo));
+               }
 
-               man = &bdev->man[bo->mem.mem_type];
-               list_move_tail(&bo->lru, &man->lru);
+               list_del(&bo->lru);
+               list_add(&bo->lru, bdev->driver->lru_tail(bo));
        }
 }
 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
 
+struct list_head *ttm_bo_default_lru_tail(struct ttm_buffer_object *bo)
+{
+       return bo->bdev->man[bo->mem.mem_type].lru.prev;
+}
+EXPORT_SYMBOL(ttm_bo_default_lru_tail);
+
+struct list_head *ttm_bo_default_swap_lru_tail(struct ttm_buffer_object *bo)
+{
+       return bo->glob->swap_lru.prev;
+}
+EXPORT_SYMBOL(ttm_bo_default_swap_lru_tail);
+
 /*
  * Call bo->mutex locked.
  */

diff --git a/drivers/gpu/drm/virtio/virtgpu_ttm.c b/drivers/gpu/drm/virtio/virtgpu_ttm.c
index 9fd924cd2b7fa2cd37df124a32818a20bffd8bab..a0580815629f52a4c6dc61a22ca330d977991570 100644 (file)
--- a/drivers/gpu/drm/virtio/virtgpu_ttm.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ttm.c
@@ -426,6 +426,8 @@ static struct ttm_bo_driver virtio_gpu_bo_driver = {
        .io_mem_free = &virtio_gpu_ttm_io_mem_free,
        .move_notify = &virtio_gpu_bo_move_notify,
        .swap_notify = &virtio_gpu_bo_swap_notify,
+       .lru_tail = &ttm_bo_default_lru_tail,
+       .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };
 
 int virtio_gpu_ttm_init(struct virtio_gpu_device *vgdev)

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
index e09423d75b6b8d0f08543383a88bf9d6e4392d85..78b75ee3c93141d0d7733636aac665ca9caa9f06 100644 (file)
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
@@ -857,4 +857,6 @@ struct ttm_bo_driver vmw_bo_driver = {
        .fault_reserve_notify = &vmw_ttm_fault_reserve_notify,
        .io_mem_reserve = &vmw_ttm_io_mem_reserve,
        .io_mem_free = &vmw_ttm_io_mem_free,
+       .lru_tail = &ttm_bo_default_lru_tail,
+       .swap_lru_tail = &ttm_bo_default_swap_lru_tail,
 };

diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index 542f9212727cbd0c5e85f488d139243c85878fc8..513f7f96b80a90e530a4c74179f943a0dff8c40f 100644 (file)
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -440,6 +440,12 @@ struct ttm_bo_driver {
         * Called with LRU lock held immediately before the removal.
         */
        void (*lru_removal)(struct ttm_buffer_object *bo);
+
+       /**
+        * Return the list_head after which a BO should be inserted in the LRU.
+        */
+       struct list_head *(*lru_tail)(struct ttm_buffer_object *bo);
+       struct list_head *(*swap_lru_tail)(struct ttm_buffer_object *bo);
 };
 
 /**
@@ -757,6 +763,9 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man);
 extern void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo);
 extern void ttm_bo_add_to_lru(struct ttm_buffer_object *bo);
 
+struct list_head *ttm_bo_default_lru_tail(struct ttm_buffer_object *bo);
+struct list_head *ttm_bo_default_swap_lru_tail(struct ttm_buffer_object *bo);
+
 /**
  * __ttm_bo_reserve:
  *