diff --git a/projects/rocm-smi-lib/.editorconfig b/projects/rocm-smi-lib/.editorconfig index 3e3944a39c..86ca34a181 100644 --- a/projects/rocm-smi-lib/.editorconfig +++ b/projects/rocm-smi-lib/.editorconfig @@ -4,11 +4,6 @@ # top-most EditorConfig file root = true -# Unix-style newlines with a newline ending every file and no stray whitespaces -[*] -end_of_line = lf -trim_trailing_whitespace = true - # Matches multiple files with brace expansion notation # Set default charset [*.{c,cc,cpp,h,hh,hpp}] diff --git a/projects/rocm-smi-lib/include/rocm_smi/kfd_ioctl.h b/projects/rocm-smi-lib/include/rocm_smi/kfd_ioctl.h index 5817833eae..3b781ce129 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/kfd_ioctl.h +++ b/projects/rocm-smi-lib/include/rocm_smi/kfd_ioctl.h @@ -36,6 +36,12 @@ struct kfd_ioctl_get_version_args { __u32 minor_version; /* from KFD */ }; +struct kfd_ioctl_get_available_memory_args { + __u64 available; /* from KFD */ + __u32 gpu_id; /* to KFD */ + __u32 pad; +}; + /* For kfd_ioctl_create_queue_args.queue_type. */ #define KFD_IOC_QUEUE_TYPE_COMPUTE 0x0 #define KFD_IOC_QUEUE_TYPE_SDMA 0x1 @@ -726,6 +732,10 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_CROSS_MEMORY_COPY \ AMDKFD_IOWR(0x22, struct kfd_ioctl_cross_memory_copy_args) + +#define AMDKFD_IOC_AVAILABLE_MEMORY \ + AMDKFD_IOWR(0x23, struct kfd_ioctl_get_available_memory_args) + #define AMDKFD_COMMAND_START 0x01 #undef AMDKFD_COMMAND_END #define AMDKFD_COMMAND_END 0x22 diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h index 6c0e1b9d60..053447b501 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h @@ -1468,6 +1468,8 @@ rsmi_status_t rsmi_dev_unique_id_get(uint32_t dv_ind, uint64_t *id); * written * * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call. + * @retval ::RSMI_STATUS_UNEXPECTED_DATA Data read or provided was not as + * expected * */ rsmi_status_t @@ -2370,6 +2372,8 @@ rsmi_status_t rsmi_dev_mem_overdrive_level_get(uint32_t dv_ind, uint32_t *od); * @retval ::RSMI_STATUS_NOT_SUPPORTED installed software or hardware does not * support this function with the given arguments * @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid + * @retval ::RSMI_STATUS_UNEXPECTED_DATA Data read or provided was not as + * expected * */ rsmi_status_t rsmi_dev_gpu_clk_freq_get(uint32_t dv_ind, diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h index a0c8f5fe2d..9cf8fd8e40 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h @@ -80,6 +80,10 @@ class KFDNode { uint32_t amdgpu_dev_index(void) const {return amdgpu_dev_index_;} void set_amdgpu_dev_index(uint32_t val) {amdgpu_dev_index_ = val;} + // Get memory from kfd + int get_total_memory(uint64_t* total); + int get_used_memory(uint64_t* used); + private: uint32_t node_indx_; uint32_t amdgpu_dev_index_; diff --git a/projects/rocm-smi-lib/rocm_smi/example/rocm_smi_example.cc b/projects/rocm-smi-lib/rocm_smi/example/rocm_smi_example.cc index 9e9019e2b8..0e78debb91 100755 --- a/projects/rocm-smi-lib/rocm_smi/example/rocm_smi_example.cc +++ b/projects/rocm-smi-lib/rocm_smi/example/rocm_smi_example.cc @@ -577,7 +577,7 @@ static rsmi_status_t test_set_freq(uint32_t dv_ind) { static void print_frequencies(rsmi_frequencies_t *f) { assert(f != nullptr); for (uint32_t j = 0; j < f->num_supported; ++j) { - std::cout << "\t** " << j << ": " << f->frequency[j]; + std::cout << "\t** " << j << ": " << std::to_string(f->frequency[j]); if (j == f->current) { std::cout << " *"; } @@ -777,6 +777,14 @@ int main() { std::cout << f.num_supported << std::endl; print_frequencies(&f); + ret = rsmi_dev_gpu_clk_freq_get(i, RSMI_CLK_TYPE_SOC, &f); + CHK_RSMI_NOT_SUPPORTED_OR_UNEXPECTED_DATA_RET(ret) + std::cout << "\t**Supported GPU clock frequencies (SOC clk): "; + std::cout << f.num_supported << std::endl; + std::cout << "\t**Current value (SOC clk): "; + std::cout << f.current << std::endl; + print_frequencies(&f); + std::cout << "\t**Monitor name: "; char name[128]; ret = rsmi_dev_name_get(i, name, 128); diff --git a/projects/rocm-smi-lib/src/rocm_smi.cc b/projects/rocm-smi-lib/src/rocm_smi.cc index 5ae4895c9d..b3347a155d 100755 --- a/projects/rocm-smi-lib/src/rocm_smi.cc +++ b/projects/rocm-smi-lib/src/rocm_smi.cc @@ -1065,6 +1065,8 @@ static rsmi_status_t get_frequencies(amd::smi::DevInfoTypes type, rsmi_clk_type_ if (f == nullptr) { return RSMI_STATUS_INVALID_ARGS; } + memset(f, 0, sizeof(rsmi_frequencies_t)); + f->current=0; ret = GetDevValueVec(type, dv_ind, &val_vec); if (ret != RSMI_STATUS_SUCCESS) { @@ -1114,6 +1116,7 @@ static rsmi_status_t get_frequencies(amd::smi::DevInfoTypes type, rsmi_clk_type_ // assert(f->current < f->num_supported); if (f->current >= f->num_supported) { f->current = -1; + return RSMI_STATUS_UNEXPECTED_DATA; } return RSMI_STATUS_SUCCESS; @@ -1748,7 +1751,7 @@ rsmi_dev_gpu_clk_freq_set(uint32_t dv_ind, TRY std::ostringstream ss; - ss << __PRETTY_FUNCTION__ << "| ======= start ======="; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; LOG_TRACE(ss); REQUIRE_ROOT_ACCESS DEVICE_MUTEX @@ -3077,6 +3080,14 @@ rsmi_dev_memory_total_get(uint32_t dv_ind, rsmi_memory_type_t mem_type, DEVICE_MUTEX ret = get_dev_value_int(mem_type_file, dv_ind, total); + // Fallback to KFD reported memory if VRAM total is 0 + if (mem_type == RSMI_MEM_TYPE_VRAM && *total == 0) { + GET_DEV_AND_KFDNODE_FROM_INDX + if (kfd_node->get_total_memory(total) == 0 && *total > 0) { + return RSMI_STATUS_SUCCESS; + } + } + return ret; CATCH } @@ -3113,6 +3124,17 @@ rsmi_dev_memory_usage_get(uint32_t dv_ind, rsmi_memory_type_t mem_type, DEVICE_MUTEX ret = get_dev_value_int(mem_type_file, dv_ind, used); + // Fallback to KFD reported memory if no VRAM + if (mem_type == RSMI_MEM_TYPE_VRAM && *used == 0) { + GET_DEV_AND_KFDNODE_FROM_INDX + uint64_t total = 0; + ret = get_dev_value_int(amd::smi::kDevMemTotVRAM, dv_ind, &total); + if (total != 0) return ret; // do not need to fallback + if ( kfd_node->get_used_memory(used) == 0 ) { + return RSMI_STATUS_SUCCESS; + } + } + return ret; CATCH } @@ -3231,8 +3253,9 @@ rsmi_status_string(rsmi_status_t status, const char **status_string) { break; case RSMI_STATUS_UNEXPECTED_DATA: - *status_string = "RSMI_STATUS_UNEXPECTED_DATA: Data (usually from reading" - " a file) was not of the type that was expected"; + *status_string = "RSMI_STATUS_UNEXPECTED_DATA: Data read (usually from " + "a file) or provided to function is " + "not what was expected"; break; case RSMI_STATUS_BUSY: diff --git a/projects/rocm-smi-lib/src/rocm_smi_kfd.cc b/projects/rocm-smi-lib/src/rocm_smi_kfd.cc index 13aed64588..092bcb3414 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_kfd.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_kfd.cc @@ -43,6 +43,9 @@ #include #include +#include +#include +#include #include #include @@ -770,6 +773,95 @@ KFDNode::get_io_link_bandwidth(uint32_t node_to, uint64_t *max_bandwidth, return 0; } +// /sys/class/kfd/kfd/topology/nodes/*/mem_banks/*/properties +// size_in_bytes 68702699520 +int KFDNode::get_total_memory(uint64_t* total) { + if (total == nullptr) return EINVAL; + *total = 0; + + std::string f_path = kKFDNodesPathRoot; + f_path += "/"; + f_path += std::to_string(node_indx_); + f_path += "/mem_banks"; + + auto kfd_node_dir = opendir(f_path.c_str()); + if (kfd_node_dir == nullptr) { + return errno; + } + auto dentry = readdir(kfd_node_dir); + while (dentry != nullptr) { + if (dentry->d_name[0] == '.') { + dentry = readdir(kfd_node_dir); + continue; + } + + if (!is_number(dentry->d_name)) { + dentry = readdir(kfd_node_dir); + continue; + } + + // read "size_in_bytes 68702699520" line + const std::string size_in_bytes_property = "size_in_bytes "; + std::string memory_bank_file = f_path + "/" + + dentry->d_name + "/properties"; + std::ifstream fs(memory_bank_file); + if (!fs) { + dentry = readdir(kfd_node_dir); + continue; + } + std::string line; + while (std::getline(fs, line)) { + if (line.substr(0, size_in_bytes_property.length()) + == size_in_bytes_property) { + auto bytes = line.substr(size_in_bytes_property.length()); + try { + *total += std::stol(bytes); + break; + } catch(...) { + dentry = readdir(kfd_node_dir); + continue; + } + } + } // end loop for lines in property file + } // end loop for mem_bank directory + + if (closedir(kfd_node_dir)) { + std::string err_str = "Failed to close KFD node directory "; + err_str += f_path; + err_str += "."; + perror(err_str.c_str()); + return 1; + } + return 0; +} + +// ioctl on kfd node device +int KFDNode::get_used_memory(uint64_t* used) { + if (used == nullptr) return EINVAL; + static const char *kPathKFDIoctl = "/dev/kfd"; + + int kfd_fd = open(kPathKFDIoctl, O_RDWR | O_CLOEXEC); + if (kfd_fd <= 0) { + return 1; + } + struct kfd_ioctl_get_available_memory_args mem = {0, 0, 0}; + mem.gpu_id = gpu_id_; + if (ioctl(kfd_fd, AMDKFD_IOC_AVAILABLE_MEMORY , &mem) != 0) { + close(kfd_fd); + return 1; + } + close(kfd_fd); + + // used = total - available + uint64_t total = 0; + int ret = get_total_memory(&total); + if (ret == 0 && total > 0 && mem.available < total) { + *used = total - mem.available; + return 0; + } + + return 1; +} } // namespace smi } // namespace amd diff --git a/projects/rocm-smi-lib/tests/rocm_smi_test/functional/volt_freq_curv_read.cc b/projects/rocm-smi-lib/tests/rocm_smi_test/functional/volt_freq_curv_read.cc index c7a27a04f7..84f77d6afc 100755 --- a/projects/rocm-smi-lib/tests/rocm_smi_test/functional/volt_freq_curv_read.cc +++ b/projects/rocm-smi-lib/tests/rocm_smi_test/functional/volt_freq_curv_read.cc @@ -171,7 +171,7 @@ void TestVoltCurvRead::Run(void) { CHK_ERR_ASRT(err) // Verify api support checking functionality is working err = rsmi_dev_od_volt_info_get(i, nullptr); - ASSERT_EQ(err, RSMI_STATUS_NOT_SUPPORTED); + ASSERT_EQ(err, RSMI_STATUS_INVALID_ARGS); } if (err == RSMI_STATUS_SUCCESS) { diff --git a/projects/rocm-smi-lib/tests/rocm_smi_test/rsmitst.exclude b/projects/rocm-smi-lib/tests/rocm_smi_test/rsmitst.exclude index f0ea842f7f..7d229b68f4 100644 --- a/projects/rocm-smi-lib/tests/rocm_smi_test/rsmitst.exclude +++ b/projects/rocm-smi-lib/tests/rocm_smi_test/rsmitst.exclude @@ -57,7 +57,13 @@ $BLACKLIST_ALL_ASICS\ "rsmitstReadWrite.TestPerfLevelReadWrite" # SWDEV-391407 -FILTER[aqua_vanjaram]=\ +FILTER[90400]=\ +$BLACKLIST_ALL_ASICS\ +"rsmitstReadOnly.TestVoltCurvRead:"\ +"rsmitstReadOnly.TestFrequenciesRead:"\ +"rsmitstReadWrite.TestFrequenciesReadWrite:"\ +"rsmitstReadWrite.TestPowerReadWrite" +FILTER[90401]=\ $BLACKLIST_ALL_ASICS\ "rsmitstReadOnly.TestVoltCurvRead:"\ "rsmitstReadOnly.TestFrequenciesRead:"\