From 1438f15fd06f0ee8978a13e7fa24c1795cbfd419 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Wed, 23 Sep 2015 15:08:33 -0400 Subject: [PATCH] Fix VM range for dGPU local memory Currently, Kernel imposes a limit on VM. Thunk should be aware of it. This fix is required till Kernel VM limit is sorted. For now both "Host Access" memory and "Local Memory" share the same VM range. Change-Id: I5a9220face20df9ede2b78bd6201a01dd2ea70e0 Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 61b540a1d8..652b640482 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -482,6 +482,8 @@ void fmm_print(uint32_t gpu_id) manageble_aperture_print(&gpu_mem[i].gpuvm_aperture); printf("Scratch aperture:\n"); manageble_aperture_print(&gpu_mem[i].scratch_aperture); + printf("dGPU aperture:\n"); + manageble_aperture_print(&gpu_mem[i].dgpu_aperture); } } #else @@ -583,13 +585,18 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) if (gpu_mem_id < 0) return NULL; - aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; - - flags = KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE; if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) { flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE; /* Alignment is needed to match a workaround for a VI HW bug in the kernel */ MemorySizeInBytes = (MemorySizeInBytes + 0x7fffULL) & ~0x7fffULL; + /* + * TODO: Once VA limit is raised from 0x200000000 (8GB) use gpuvm_aperture. + * In that way the host access range won't be used for local memory + */ + aperture = &gpu_mem[gpu_mem_id].dgpu_aperture; + } else { + flags = KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE; + aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; } return __fmm_allocate_device(gpu_id, MemorySizeInBytes, @@ -1076,13 +1083,36 @@ static void *reserve_address(void *addr, long long unsigned int len) } #define ADDRESS_RANGE_LIMIT_MASK 0xFFFFFFFFFF +#define AMDGPU_SYSFS_VM_SIZE "/sys/module/amdgpu/parameters/vm_size" + +/* + * TODO: Provide a cleaner interface via topology + */ +static HSAKMT_STATUS get_dgpu_vm_limit(uint32_t *vm_size_in_gb) +{ + FILE *fd; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + fd = fopen(AMDGPU_SYSFS_VM_SIZE, "r"); + if (!fd) + return HSAKMT_STATUS_ERROR; + if (fscanf(fd, "%ul", vm_size_in_gb) != 1) { + ret = HSAKMT_STATUS_ERROR; + goto err; + } + +err: + fclose(fd); + return ret; +} static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) { bool found; HSAKMT_STATUS ret; void *addr, *ret_addr; - HSAuint64 max_len; + HSAuint64 max_len, max_vm_limit; + uint32_t max_vm_limit_in_gb; long long unsigned int temp; uint32_t gpu_id; HsaNodeProperties props; @@ -1122,9 +1152,19 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) if (base) *base = ret_addr; dgpu_shared_aperture_base = ret_addr; + + ret = get_dgpu_vm_limit(&max_vm_limit_in_gb); + if (ret != HSAKMT_STATUS_SUCCESS) { + printf("Error! Unable to find vm_size for gGPU\n"); + return ret; + } + max_vm_limit = (HSAuint64)max_vm_limit_in_gb << 30; + if (((long long unsigned int)ret_addr + max_len) < max_vm_limit) + max_vm_limit = ((long long unsigned int)ret_addr + max_len); + if (limit) - *limit = (void *)((long long unsigned int)ret_addr + max_len); - dgpu_shared_aperture_limit = (void *)((long long unsigned int)ret_addr + max_len); + *limit = (void *)max_vm_limit; + dgpu_shared_aperture_limit = (void *)max_vm_limit; is_dgpu_mem_init = true; return HSAKMT_STATUS_SUCCESS; }