From 05b5ef35b356379245fa6d25e443268200ba5526 Mon Sep 17 00:00:00 2001 From: "Bill(Shuzhou) Liu" Date: Thu, 17 Aug 2023 14:36:03 -0500 Subject: [PATCH] Fallback to kfd node when VRAM sysfs not available The driver may not expose VRAM sysfs in certain system. Add a fallback to it. Change-Id: Ib3be71b4f4d2c79318d5026b0a97f3657d8a97b6 [ROCm/amdsmi commit: a10f00bf5742f3d710242726310e09f5235adf1b] --- projects/amdsmi/include/rocm_smi/kfd_ioctl.h | 10 ++ .../amdsmi/include/rocm_smi/rocm_smi_kfd.h | 4 + projects/amdsmi/src/rocm_smi.cc | 19 ++++ projects/amdsmi/src/rocm_smi_kfd.cc | 92 +++++++++++++++++++ 4 files changed, 125 insertions(+) diff --git a/projects/amdsmi/include/rocm_smi/kfd_ioctl.h b/projects/amdsmi/include/rocm_smi/kfd_ioctl.h index 5817833eae..3b781ce129 100755 --- a/projects/amdsmi/include/rocm_smi/kfd_ioctl.h +++ b/projects/amdsmi/include/rocm_smi/kfd_ioctl.h @@ -36,6 +36,12 @@ struct kfd_ioctl_get_version_args { __u32 minor_version; /* from KFD */ }; +struct kfd_ioctl_get_available_memory_args { + __u64 available; /* from KFD */ + __u32 gpu_id; /* to KFD */ + __u32 pad; +}; + /* For kfd_ioctl_create_queue_args.queue_type. */ #define KFD_IOC_QUEUE_TYPE_COMPUTE 0x0 #define KFD_IOC_QUEUE_TYPE_SDMA 0x1 @@ -726,6 +732,10 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_CROSS_MEMORY_COPY \ AMDKFD_IOWR(0x22, struct kfd_ioctl_cross_memory_copy_args) + +#define AMDKFD_IOC_AVAILABLE_MEMORY \ + AMDKFD_IOWR(0x23, struct kfd_ioctl_get_available_memory_args) + #define AMDKFD_COMMAND_START 0x01 #undef AMDKFD_COMMAND_END #define AMDKFD_COMMAND_END 0x22 diff --git a/projects/amdsmi/include/rocm_smi/rocm_smi_kfd.h b/projects/amdsmi/include/rocm_smi/rocm_smi_kfd.h index a0c8f5fe2d..9cf8fd8e40 100755 --- a/projects/amdsmi/include/rocm_smi/rocm_smi_kfd.h +++ b/projects/amdsmi/include/rocm_smi/rocm_smi_kfd.h @@ -80,6 +80,10 @@ class KFDNode { uint32_t amdgpu_dev_index(void) const {return amdgpu_dev_index_;} void set_amdgpu_dev_index(uint32_t val) {amdgpu_dev_index_ = val;} + // Get memory from kfd + int get_total_memory(uint64_t* total); + int get_used_memory(uint64_t* used); + private: uint32_t node_indx_; uint32_t amdgpu_dev_index_; diff --git a/projects/amdsmi/src/rocm_smi.cc b/projects/amdsmi/src/rocm_smi.cc index 5ae4895c9d..38c0023466 100755 --- a/projects/amdsmi/src/rocm_smi.cc +++ b/projects/amdsmi/src/rocm_smi.cc @@ -3077,6 +3077,14 @@ rsmi_dev_memory_total_get(uint32_t dv_ind, rsmi_memory_type_t mem_type, DEVICE_MUTEX ret = get_dev_value_int(mem_type_file, dv_ind, total); + // Fallback to KFD reported memory if VRAM total is 0 + if (mem_type == RSMI_MEM_TYPE_VRAM && *total == 0) { + GET_DEV_AND_KFDNODE_FROM_INDX + if (kfd_node->get_total_memory(total) == 0 && *total > 0) { + return RSMI_STATUS_SUCCESS; + } + } + return ret; CATCH } @@ -3113,6 +3121,17 @@ rsmi_dev_memory_usage_get(uint32_t dv_ind, rsmi_memory_type_t mem_type, DEVICE_MUTEX ret = get_dev_value_int(mem_type_file, dv_ind, used); + // Fallback to KFD reported memory if no VRAM + if (mem_type == RSMI_MEM_TYPE_VRAM && *used == 0) { + GET_DEV_AND_KFDNODE_FROM_INDX + uint64_t total = 0; + ret = get_dev_value_int(amd::smi::kDevMemTotVRAM, dv_ind, &total); + if (total != 0) return ret; // do not need to fallback + if ( kfd_node->get_used_memory(used) == 0 ) { + return RSMI_STATUS_SUCCESS; + } + } + return ret; CATCH } diff --git a/projects/amdsmi/src/rocm_smi_kfd.cc b/projects/amdsmi/src/rocm_smi_kfd.cc index 13aed64588..092bcb3414 100755 --- a/projects/amdsmi/src/rocm_smi_kfd.cc +++ b/projects/amdsmi/src/rocm_smi_kfd.cc @@ -43,6 +43,9 @@ #include #include +#include +#include +#include #include #include @@ -770,6 +773,95 @@ KFDNode::get_io_link_bandwidth(uint32_t node_to, uint64_t *max_bandwidth, return 0; } +// /sys/class/kfd/kfd/topology/nodes/*/mem_banks/*/properties +// size_in_bytes 68702699520 +int KFDNode::get_total_memory(uint64_t* total) { + if (total == nullptr) return EINVAL; + *total = 0; + + std::string f_path = kKFDNodesPathRoot; + f_path += "/"; + f_path += std::to_string(node_indx_); + f_path += "/mem_banks"; + + auto kfd_node_dir = opendir(f_path.c_str()); + if (kfd_node_dir == nullptr) { + return errno; + } + auto dentry = readdir(kfd_node_dir); + while (dentry != nullptr) { + if (dentry->d_name[0] == '.') { + dentry = readdir(kfd_node_dir); + continue; + } + + if (!is_number(dentry->d_name)) { + dentry = readdir(kfd_node_dir); + continue; + } + + // read "size_in_bytes 68702699520" line + const std::string size_in_bytes_property = "size_in_bytes "; + std::string memory_bank_file = f_path + "/" + + dentry->d_name + "/properties"; + std::ifstream fs(memory_bank_file); + if (!fs) { + dentry = readdir(kfd_node_dir); + continue; + } + std::string line; + while (std::getline(fs, line)) { + if (line.substr(0, size_in_bytes_property.length()) + == size_in_bytes_property) { + auto bytes = line.substr(size_in_bytes_property.length()); + try { + *total += std::stol(bytes); + break; + } catch(...) { + dentry = readdir(kfd_node_dir); + continue; + } + } + } // end loop for lines in property file + } // end loop for mem_bank directory + + if (closedir(kfd_node_dir)) { + std::string err_str = "Failed to close KFD node directory "; + err_str += f_path; + err_str += "."; + perror(err_str.c_str()); + return 1; + } + return 0; +} + +// ioctl on kfd node device +int KFDNode::get_used_memory(uint64_t* used) { + if (used == nullptr) return EINVAL; + static const char *kPathKFDIoctl = "/dev/kfd"; + + int kfd_fd = open(kPathKFDIoctl, O_RDWR | O_CLOEXEC); + if (kfd_fd <= 0) { + return 1; + } + struct kfd_ioctl_get_available_memory_args mem = {0, 0, 0}; + mem.gpu_id = gpu_id_; + if (ioctl(kfd_fd, AMDKFD_IOC_AVAILABLE_MEMORY , &mem) != 0) { + close(kfd_fd); + return 1; + } + close(kfd_fd); + + // used = total - available + uint64_t total = 0; + int ret = get_total_memory(&total); + if (ret == 0 && total > 0 && mem.available < total) { + *used = total - mem.available; + return 0; + } + + return 1; +} } // namespace smi } // namespace amd