drm/amdkfd: change system memory overcommit limit

It is to improve system limit by:
1. replacing userptrlimit with a total memory limit that
conunts TTM memory usage and userptr usage.
2. counting acc size for all BOs.

Signed-off-by: Eric Huang <JinHuiEric.Huang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 84e4c1e4d1095823eec074801769001480110153..f3129b912714b0cac63cd2d60469e35d1778795e 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -46,9 +46,9 @@
 /* Impose limit on how much memory KFD can use */
 static struct {
        uint64_t max_system_mem_limit;
-       uint64_t max_userptr_mem_limit;
+       uint64_t max_ttm_mem_limit;
        int64_t system_mem_used;
-       int64_t userptr_mem_used;
+       int64_t ttm_mem_used;
        spinlock_t mem_limit_lock;
 } kfd_mem_limit;
 
@@ -90,8 +90,8 @@ static bool check_if_add_bo_to_vm(struct amdgpu_vm *avm,
 }
 
 /* Set memory usage limits. Current, limits are
- *  System (kernel) memory - 3/8th System RAM
- *  Userptr memory - 3/4th System RAM
+ *  System (TTM + userptr) memory - 3/4th System RAM
+ *  TTM memory - 3/8th System RAM
  */
 void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
 {
@@ -103,48 +103,54 @@ void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
        mem *= si.mem_unit;
 
        spin_lock_init(&kfd_mem_limit.mem_limit_lock);
-       kfd_mem_limit.max_system_mem_limit = (mem >> 1) - (mem >> 3);
-       kfd_mem_limit.max_userptr_mem_limit = mem - (mem >> 2);
-       pr_debug("Kernel memory limit %lluM, userptr limit %lluM\n",
+       kfd_mem_limit.max_system_mem_limit = (mem >> 1) + (mem >> 2);
+       kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3);
+       pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n",
                (kfd_mem_limit.max_system_mem_limit >> 20),
-               (kfd_mem_limit.max_userptr_mem_limit >> 20));
+               (kfd_mem_limit.max_ttm_mem_limit >> 20));
 }
 
 static int amdgpu_amdkfd_reserve_system_mem_limit(struct amdgpu_device *adev,
-                                             uint64_t size, u32 domain)
+               uint64_t size, u32 domain, bool sg)
 {
-       size_t acc_size;
+       size_t acc_size, system_mem_needed, ttm_mem_needed;
        int ret = 0;
 
        acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
                                       sizeof(struct amdgpu_bo));
 
        spin_lock(&kfd_mem_limit.mem_limit_lock);
+
        if (domain == AMDGPU_GEM_DOMAIN_GTT) {
-               if (kfd_mem_limit.system_mem_used + (acc_size + size) >
-                       kfd_mem_limit.max_system_mem_limit) {
-                       ret = -ENOMEM;
-                       goto err_no_mem;
-               }
-               kfd_mem_limit.system_mem_used += (acc_size + size);
-       } else if (domain == AMDGPU_GEM_DOMAIN_CPU) {
-               if ((kfd_mem_limit.system_mem_used + acc_size >
-                       kfd_mem_limit.max_system_mem_limit) ||
-                       (kfd_mem_limit.userptr_mem_used + (size + acc_size) >
-                       kfd_mem_limit.max_userptr_mem_limit)) {
-                       ret = -ENOMEM;
-                       goto err_no_mem;
-               }
-               kfd_mem_limit.system_mem_used += acc_size;
-               kfd_mem_limit.userptr_mem_used += size;
+               /* TTM GTT memory */
+               system_mem_needed = acc_size + size;
+               ttm_mem_needed = acc_size + size;
+       } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) {
+               /* Userptr */
+               system_mem_needed = acc_size + size;
+               ttm_mem_needed = acc_size;
+       } else {
+               /* VRAM and SG */
+               system_mem_needed = acc_size;
+               ttm_mem_needed = acc_size;
+       }
+
+       if ((kfd_mem_limit.system_mem_used + system_mem_needed >
+               kfd_mem_limit.max_system_mem_limit) ||
+               (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
+               kfd_mem_limit.max_ttm_mem_limit))
+               ret = -ENOMEM;
+       else {
+               kfd_mem_limit.system_mem_used += system_mem_needed;
+               kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
        }
-err_no_mem:
+
        spin_unlock(&kfd_mem_limit.mem_limit_lock);
        return ret;
 }
 
 static void unreserve_system_mem_limit(struct amdgpu_device *adev,
-                                      uint64_t size, u32 domain)
+               uint64_t size, u32 domain, bool sg)
 {
        size_t acc_size;
 
@@ -154,14 +160,18 @@ static void unreserve_system_mem_limit(struct amdgpu_device *adev,
        spin_lock(&kfd_mem_limit.mem_limit_lock);
        if (domain == AMDGPU_GEM_DOMAIN_GTT) {
                kfd_mem_limit.system_mem_used -= (acc_size + size);
-       } else if (domain == AMDGPU_GEM_DOMAIN_CPU) {
+               kfd_mem_limit.ttm_mem_used -= (acc_size + size);
+       } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) {
+               kfd_mem_limit.system_mem_used -= (acc_size + size);
+               kfd_mem_limit.ttm_mem_used -= acc_size;
+       } else {
                kfd_mem_limit.system_mem_used -= acc_size;
-               kfd_mem_limit.userptr_mem_used -= size;
+               kfd_mem_limit.ttm_mem_used -= acc_size;
        }
        WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
                  "kfd system memory accounting unbalanced");
-       WARN_ONCE(kfd_mem_limit.userptr_mem_used < 0,
-                 "kfd userptr memory accounting unbalanced");
+       WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
+                 "kfd TTM memory accounting unbalanced");
 
        spin_unlock(&kfd_mem_limit.mem_limit_lock);
 }
@@ -171,16 +181,22 @@ void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo)
        spin_lock(&kfd_mem_limit.mem_limit_lock);
 
        if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) {
-               kfd_mem_limit.system_mem_used -= bo->tbo.acc_size;
-               kfd_mem_limit.userptr_mem_used -= amdgpu_bo_size(bo);
+               kfd_mem_limit.system_mem_used -=
+                       (bo->tbo.acc_size + amdgpu_bo_size(bo));
+               kfd_mem_limit.ttm_mem_used -= bo->tbo.acc_size;
        } else if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT) {
                kfd_mem_limit.system_mem_used -=
                        (bo->tbo.acc_size + amdgpu_bo_size(bo));
+               kfd_mem_limit.ttm_mem_used -=
+                       (bo->tbo.acc_size + amdgpu_bo_size(bo));
+       } else {
+               kfd_mem_limit.system_mem_used -= bo->tbo.acc_size;
+               kfd_mem_limit.ttm_mem_used -= bo->tbo.acc_size;
        }
        WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
                  "kfd system memory accounting unbalanced");
-       WARN_ONCE(kfd_mem_limit.userptr_mem_used < 0,
-                 "kfd userptr memory accounting unbalanced");
+       WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
+                 "kfd TTM memory accounting unbalanced");
 
        spin_unlock(&kfd_mem_limit.mem_limit_lock);
 }
@@ -1219,10 +1235,11 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 
        amdgpu_sync_create(&(*mem)->sync);
 
-       ret = amdgpu_amdkfd_reserve_system_mem_limit(adev, size, alloc_domain);
+       ret = amdgpu_amdkfd_reserve_system_mem_limit(adev, size,
+                                                    alloc_domain, false);
        if (ret) {
                pr_debug("Insufficient system memory\n");
-               goto err_reserve_system_mem;
+               goto err_reserve_limit;
        }
 
        pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s\n",
@@ -1270,10 +1287,10 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 allocate_init_user_pages_failed:
        amdgpu_bo_unref(&bo);
        /* Don't unreserve system mem limit twice */
-       goto err_reserve_system_mem;
+       goto err_reserve_limit;
 err_bo_create:
-       unreserve_system_mem_limit(adev, size, alloc_domain);
-err_reserve_system_mem:
+       unreserve_system_mem_limit(adev, size, alloc_domain, false);
+err_reserve_limit:
        mutex_destroy(&(*mem)->lock);
        kfree(*mem);
        return ret;