From 68155baed536b3394f5cdec4cf63805b6b0c2e1d Mon Sep 17 00:00:00 2001 From: Chris Freehill Date: Fri, 10 Jul 2020 00:18:22 -0500 Subject: [PATCH] Handle un-readable kfd properties files Some systems have kfd sysfs properties entries that are unreadable--for example, when a multi-gpu system is dividing the gpus among containers, each container may only be able to access certain gpus. Previously, all kfd topology node properties entries were assumed to be valid. Now, we check for readability before declaring them "valid". Fixes SWDEV-240169 Also: * remove an assertion that would happen when no pcie device identifier files are found on the system. * fix cpplint issues Change-Id: I74321b685159dd2628c890b33c39ad82988cb9dd --- include/rocm_smi/rocm_smi.h | 5 +++-- src/rocm_smi.cc | 3 --- src/rocm_smi_kfd.cc | 24 ++++++++++++++++++++- src/rocm_smi_main.cc | 12 +++++++++++ tests/rocm_smi_test/functional/volt_read.cc | 5 +++-- 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/include/rocm_smi/rocm_smi.h b/include/rocm_smi/rocm_smi.h index 5373e8f830..8908bac7b4 100755 --- a/include/rocm_smi/rocm_smi.h +++ b/include/rocm_smi/rocm_smi.h @@ -1799,8 +1799,9 @@ rsmi_status_t rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type, * @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid * */ -rsmi_status_t rsmi_dev_volt_metric_get(uint32_t dv_ind, rsmi_voltage_type_t sensor_type, - rsmi_voltage_metric_t metric, int64_t *voltage); +rsmi_status_t rsmi_dev_volt_metric_get(uint32_t dv_ind, + rsmi_voltage_type_t sensor_type, + rsmi_voltage_metric_t metric, int64_t *voltage); /** @} */ // end of PhysQuer /*****************************************************************************/ diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index 07b04ab861..ec0221ca8d 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -1545,9 +1545,6 @@ static rsmi_status_t get_dev_name_from_id(uint32_t dv_ind, char *name, } if (val_str.size() == 0) { - // We should have already returned if we were looking for - // device or subdivce - assert(typ == NAME_STR_VENDOR); return get_backup_name(vendor_id, name, len); } size_t ct = val_str.copy(name, len); diff --git a/src/rocm_smi_kfd.cc b/src/rocm_smi_kfd.cc index cf77bfcedb..7d6011aec9 100755 --- a/src/rocm_smi_kfd.cc +++ b/src/rocm_smi_kfd.cc @@ -154,6 +154,22 @@ static int OpenKFDNodeFile(uint32_t dev_id, std::string node_file, return 0; } +bool KFDNodeSupported(uint32_t node_indx) { + std::ifstream fs; + bool ret = true; + int err; + err = OpenKFDNodeFile(node_indx, "properties", &fs); + + if (err == ENOENT) { + return false; + } + if (fs.peek() == std::ifstream::traits_type::eof()) { + ret = false; + } + fs.close(); + return ret; +} + int ReadKFDDeviceProperties(uint32_t kfd_node_id, std::vector *retVec) { std::string line; @@ -175,7 +191,7 @@ int ReadKFDDeviceProperties(uint32_t kfd_node_id, if (retVec->size() == 0) { fs.close(); - return 0; + return ENOENT; } // Remove any *trailing* empty (whitespace) lines while (retVec->back().find_first_not_of(" \t\n\v\f\r") == std::string::npos) { @@ -471,6 +487,12 @@ int DiscoverKFDNodes(std::map> *nodes) { } node_indx = std::stoi(dentry->d_name); + + if (!KFDNodeSupported(node_indx)) { + dentry = readdir(kfd_node_dir); + continue; + } + node = std::shared_ptr(new KFDNode(node_indx)); node->Initialize(); diff --git a/src/rocm_smi_main.cc b/src/rocm_smi_main.cc index 3ade5c0afb..cd4fc9110b 100755 --- a/src/rocm_smi_main.cc +++ b/src/rocm_smi_main.cc @@ -293,6 +293,18 @@ RocmSMI::Initialize(uint64_t flags) { std::shared_ptr dev; + // Remove any drm nodes that don't have a corresponding readable kfd node. + // kfd nodes will not be added if their properties file is not readable. + auto dev_iter = monitor_devices_.begin(); + while (dev_iter != monitor_devices_.end()) { + uint64_t bdfid = (*dev_iter)->bdfid(); + if (tmp_map.find(bdfid) == tmp_map.end()) { + dev_iter = monitor_devices_.erase(dev_iter); + continue; + } + dev_iter++; + } + // 1. construct kfd_node_map_ with gpu_id as key and *Device as value // 2. for each kfd node, write the corresponding dv_ind // 3. for each amdgpu device, write the corresponding gpu_id diff --git a/tests/rocm_smi_test/functional/volt_read.cc b/tests/rocm_smi_test/functional/volt_read.cc index 7398a5a2db..a8be24e03f 100644 --- a/tests/rocm_smi_test/functional/volt_read.cc +++ b/tests/rocm_smi_test/functional/volt_read.cc @@ -132,8 +132,9 @@ void TestVoltRead::Run(void) { }; for (uint32_t i = RSMI_VOLT_TYPE_FIRST; i <= RSMI_VOLT_TYPE_LAST; ++i) { IF_VERB(STANDARD) { - std::cout << "\t** **********" << GetVoltSensorNameStr(static_cast(i)) << - " Voltage **********" << std::endl; + std::cout << "\t** **********" << + GetVoltSensorNameStr(static_cast(i)) << + " Voltage **********" << std::endl; } print_volt_metric(RSMI_VOLT_CURRENT, "Current Voltage"); print_volt_metric(RSMI_VOLT_MAX, "Voltage max value");