Add rsmi_compute_process_gpus_get()

Given a process ID, give the device indices that process is currently using. Also: * made corrections to how RSMI, amdgpu (ie, "card#") and KFD indicies translate from one another * add a few missing error codes to rsmi_status_string() * fix some formatting Change-Id: Icd2cae66bb4fec768da96af7cf9cf8b8b66ec7f9 [ROCm/amdsmi commit: 2d6e15190c]
2020-02-15 20:39:07 -06:00
parent 40dbb98b41
commit abd01b37eb
12 changed files with 604 additions and 109 deletions
@@ -53,6 +53,7 @@
 #include <bitset>
 #include <cstdint>
 #include <unordered_map>
+#include <unordered_set>
 #include <map>
 #include <fstream>
 #include <iostream>
@@ -108,6 +109,16 @@ static rsmi_status_t handleException() {
  std::shared_ptr<amd::smi::Device> dev = smi.monitor_devices()[dv_ind]; \
  assert(dev != nullptr);

+
+#define GET_DEV_AND_KFDNODE_FROM_INDX \
+  GET_DEV_FROM_INDX \
+  std::shared_ptr<amd::smi::KFDNode> kfd_node; \
+  if (smi.kfd_node_map().find(dev->kfd_gpu_id()) == \
+                                                 smi.kfd_node_map().end()) { \
+    return RSMI_INITIALIZATION_ERROR; \
+  } \
+  kfd_node = smi.kfd_node_map()[dev->kfd_gpu_id()];
+
 #define REQUIRE_ROOT_ACCESS \
    if (amd::smi::RocmSMI::getInstance().euid()) { \
      return RSMI_STATUS_PERMISSION; \
@@ -168,6 +179,7 @@ static rsmi_status_t errno_to_rsmi_status(uint32_t err) {
    case EISDIR: return RSMI_STATUS_FILE_ERROR;
    case EINTR:  return RSMI_STATUS_INTERRUPT;
    case EIO:    return RSMI_STATUS_UNEXPECTED_SIZE;
+    case ENXIO:  return RSMI_STATUS_UNEXPECTED_DATA;
    default:     return RSMI_STATUS_UNKNOWN_ERROR;
  }
 }
@@ -208,7 +220,6 @@ static uint64_t get_multiplier_from_str(char units_char) {
 */
 static uint64_t freq_string_to_int(const std::vector<std::string> &freq_lines,
                                bool *is_curr, uint32_t lanes[], uint32_t i) {
-
  assert(i < freq_lines.size());
  if (i >= freq_lines.size()) {
    throw amd::smi::rsmi_exception(RSMI_STATUS_INPUT_OUT_OF_BOUNDS,
@@ -696,26 +707,15 @@ rsmi_status_t
 rsmi_dev_pci_id_get(uint32_t dv_ind, uint64_t *bdfid) {
  TRY

-  CHK_SUPPORT_NAME_ONLY(bdfid)
+  GET_DEV_AND_KFDNODE_FROM_INDX
+  CHK_API_SUPPORT_ONLY(bdfid, RSMI_DEFAULT_VARIANT, RSMI_DEFAULT_VARIANT)
  DEVICE_MUTEX

  *bdfid = dev->bdfid();

-  int32_t ret = dev->populateKFDNodeProperties();
-
-  if (ret) {
-    return errno_to_rsmi_status(errno);
-  }
-
  uint64_t domain = 0;

-  ret = dev->getKFDNodeProperty(amd::smi::kDevKFDNodePropDomain, &domain);
-
-  if (ret == EINVAL) {
-    // "domain" is not found in properties file; just go with the 16 bit
-    // domain already found
-    return RSMI_STATUS_SUCCESS;
-  }
+  kfd_node->get_property_value("domain", &domain);

  // Replace the 16 bit domain originally set like this:
  // BDFID = ((<DOMAIN> & 0xffff) << 32) | ((<BUS> & 0xff) << 8) |
@@ -2329,15 +2329,30 @@ rsmi_status_string(rsmi_status_t status, const char **status_string) {
                             " the call";
      break;

+    case RSMI_STATUS_INTERRUPT:
+      *status_string = "An interrupt occurred while executing the function";
+      break;
+
+    case RSMI_STATUS_UNEXPECTED_SIZE:
+      *status_string = "Data (usually from reading a file) was out of"
+                                              " range from what was expected";
+      break;
+
+    case RSMI_STATUS_NO_DATA:
+      *status_string = "No data was found (usually from reading a file) "
+                                                    "where data was expected";
+      break;
+
+    case RSMI_STATUS_UNEXPECTED_DATA:
+      *status_string = "Data (usually from reading a file) was not of the "
+                                                     "type that was expected";
+      break;
+
    case RSMI_STATUS_UNKNOWN_ERROR:
      *status_string = "An unknown error prevented the call from completing"
                          " successfully";
      break;

-    case RSMI_STATUS_INTERRUPT:
-      *status_string = "An interrupt occurred while executing the function";
-      break;
-
    default:
      *status_string = "An unknown error occurred";
      return RSMI_STATUS_UNKNOWN_ERROR;
@@ -2696,6 +2711,49 @@ rsmi_compute_process_info_get(rsmi_process_info_t *procs,
  CATCH
 }

+rsmi_status_t
+rsmi_compute_process_gpus_get(uint32_t pid, uint32_t *dv_indices,
+                                                      uint32_t *num_devices) {
+  TRY
+
+  if (num_devices == nullptr) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+
+  std::unordered_set<uint64_t> gpu_set;
+  int err = amd::smi::GetProcessGPUs(pid, &gpu_set);
+
+  if (err) {
+    return errno_to_rsmi_status(err);
+  }
+
+  uint32_t i = 0;
+  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();
+
+  if (dv_indices != nullptr) {
+    for (auto it = gpu_set.begin(); i < *num_devices && it != gpu_set.end();
+                                                                  ++it, ++i) {
+      uint64_t gpu_id_val = (*it);
+      dv_indices[i] = smi.kfd_node_map()[gpu_id_val]->amdgpu_dev_index();
+    }
+  }
+
+  if (dv_indices && *num_devices < gpu_set.size()) {
+    // In this case, *num_devices should already hold the number of items
+    // written to dv_devices. We just have to let the caller know there's more.
+    return RSMI_STATUS_INSUFFICIENT_SIZE;
+  }
+
+  *num_devices = static_cast<uint32_t>(gpu_set.size());
+  if (gpu_set.size() > smi.monitor_devices().size()) {
+    return RSMI_STATUS_UNEXPECTED_SIZE;
+  }
+
+  return RSMI_STATUS_SUCCESS;
+
+  CATCH
+}
+
 rsmi_status_t
 rsmi_dev_memory_reserved_pages_get(uint32_t dv_ind, uint32_t *num_pages,
                                          rsmi_retired_page_record_t *records) {
@@ -368,7 +368,7 @@ amd::smi::evt::Event::stopCounter(void) {
  return 0;
 }

-static long
+static ssize_t
 readn(int fd, void *buf, size_t n) {
  ssize_t left = n;
  ssize_t bytes;
@@ -777,53 +777,6 @@ int Device::readDevInfo(DevInfoTypes type, std::string *val) {
  return 0;
 }

-int Device::populateKFDNodeProperties(bool force_update) {
-  int ret;
-
-  std::vector<std::string> propVec;
-
-  if (kfdNodePropMap_.size() > 0 && !force_update) {
-    return 0;
-  }
-
-  ret = ReadKFDDeviceProperties(index_, &propVec);
-
-  if (ret) {
-    return ret;
-  }
-
-  std::string key_str;
-  // std::string val_str;
-  uint64_t val_int;  // Assume all properties are unsigned integers for now
-  std::istringstream fs;
-
-  for (uint32_t i = 0; i < propVec.size(); ++i) {
-    fs.str(propVec[i]);
-    fs >> key_str;
-    fs >> val_int;
-
-    kfdNodePropMap_[key_str] = val_int;
-
-    fs.str("");
-    fs.clear();
-  }
-
-  return 0;
-}
-
-int Device::getKFDNodeProperty(DevKFDNodePropTypes prop, uint64_t *val) {
-  assert(val != nullptr);
-  assert(kDevKFDPropNameMap.find(prop) != kDevKFDPropNameMap.end());
-
-  const char *prop_name = kDevKFDPropNameMap.at(prop);
-  if (kfdNodePropMap_.find(prop_name) == kfdNodePropMap_.end()) {
-    return EINVAL;
-  }
-
-  *val = kfdNodePropMap_.at(prop_name);
-  return 0;
-}
-
 void Device::DumpSupportedFunctions(void) {
  SupportedFuncMapIt func_iter = supported_funcs_.begin();

@@ -47,7 +47,7 @@

 #include <algorithm>
 #include <string>
-#include <map>
+#include <unordered_set>
 #include <fstream>
 #include <cstdint>
 #include <iostream>
@@ -57,6 +57,7 @@
 #include "rocm_smi/rocm_smi.h"
 #include "rocm_smi/rocm_smi_exception.h"
 #include "rocm_smi/rocm_smi_utils.h"
+#include "rocm_smi/rocm_smi_device.h"

 namespace amd {
 namespace smi {
@@ -66,25 +67,75 @@ static const char *kKFDNodesPathRoot = "/sys/class/kfd/kfd/topology/nodes";
 // Sysfs file names
 static const char *kKFDPasidFName = "pasid";

+
+
+// KFD Node Property strings
+// static const char *kKFDNodePropCPU_CORES_COUNTStr =    "cpu_cores_count";
+// static const char *kKFDNodePropSIMD_COUNTStr =         "simd_count";
+// static const char *kKFDNodePropMEM_BANKS_COUNTStr =    "mem_banks_count";
+// static const char *kKFDNodePropCACHES_COUNTStr =       "caches_count";
+// static const char *kKFDNodePropIO_LINKS_COUNTStr =     "io_links_count";
+// static const char *kKFDNodePropCPU_CORE_ID_BASEStr =   "cpu_core_id_base";
+// static const char *kKFDNodePropSIMD_ID_BASEStr =       "simd_id_base";
+// static const char *kKFDNodePropMAX_WAVES_PER_SIMDStr = "max_waves_per_simd";
+// static const char *kKFDNodePropLDS_SIZE_IN_KBStr =     "lds_size_in_kb";
+// static const char *kKFDNodePropGDS_SIZE_IN_KBStr =     "gds_size_in_kb";
+// static const char *kKFDNodePropNUM_GWSStr =            "num_gws";
+// static const char *kKFDNodePropWAVE_FRONT_SIZEStr =    "wave_front_size";
+// static const char *kKFDNodePropARRAY_COUNTStr =        "array_count";
+// static const char *kKFDNodePropSIMD_ARRAYS_PER_ENGINEStr =
+//                                                   "simd_arrays_per_engine";
+// static const char *kKFDNodePropCU_PER_SIMD_ARRAYStr =  "cu_per_simd_array";
+// static const char *kKFDNodePropSIMD_PER_CUStr =        "simd_per_cu";
+// static const char *kKFDNodePropMAX_SLOTS_SCRATCH_CUStr =
+//                                                     "max_slots_scratch_cu";
+// static const char *kKFDNodePropVENDOR_IDStr =          "vendor_id";
+// static const char *kKFDNodePropDEVICE_IDStr =          "device_id";
+static const char *kKFDNodePropLOCATION_IDStr =          "location_id";
+static const char *kKFDNodePropDOMAINStr =               "domain";
+// static const char *kKFDNodePropDRM_RENDER_MINORStr =   "drm_render_minor";
+// static const char *kKFDNodePropHIVE_IDStr =            "hive_id";
+// static const char *kKFDNodePropNUM_SDMA_ENGINESStr =   "num_sdma_engines";
+// static const char *kKFDNodePropNUM_SDMA_XGMI_ENGINESStr =
+//                                                   "num_sdma_xgmi_engines";
+// static const char *kKFDNodePropNUM_SDMA_QUEUES_PER_ENGINEStr =
+//                                              "num_sdma_queues_per_engine";
+// static const char *kKFDNodePropNUM_CP_QUEUESStr =      "num_cp_queues";
+// static const char *kKFDNodePropMAX_ENGINE_CLK_FCOMPUTEStr =
+//                                                 "max_engine_clk_fcompute";
+// static const char *kKFDNodePropLOCAL_MEM_SIZEStr =     "local_mem_size";
+// static const char *kKFDNodePropFW_VERSIONStr =         "fw_version";
+// static const char *kKFDNodePropCAPABILITYStr =         "capability";
+// static const char *kKFDNodePropDEBUG_PROPStr =         "debug_prop";
+// static const char *kKFDNodePropSDMA_FW_VERSIOStr =     "sdma_fw_versio";
+// static const char *kKFDNodePropMAX_ENGINE_CLK_CCOMPUTEStr =
+//                                                "max_engine_clk_ccompute";
+
 static bool is_number(const std::string &s) {
  return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
 }

-int ReadKFDDeviceProperties(uint32_t dev_id,
-                                           std::vector<std::string> *retVec) {
+static std::string KFDDevicePath(uint32_t dev_id) {
+  std::string node_path = kKFDNodesPathRoot;
+  node_path += '/';
+  node_path += std::to_string(dev_id);
+  return node_path;
+}
+
+static int OpenKFDNodeFile(uint32_t dev_id, std::string node_file,
+                                                          std::ifstream *fs) {
  std::string line;
  int ret;
-  std::ifstream fs;
-  std::string properties_path = kKFDNodesPathRoot;
+  std::string f_path;
  bool reg_file;

-  assert(retVec != nullptr);
+  assert(fs != nullptr);

-  properties_path += '/';
-  properties_path += std::to_string(dev_id);
-  properties_path += "/properties";
+  f_path = KFDDevicePath(dev_id);
+  f_path += "/";
+  f_path += node_file;

-  ret = isRegularFile(properties_path, &reg_file);
+  ret = isRegularFile(f_path, &reg_file);

  if (ret != 0) {
    return ret;
@@ -93,26 +144,104 @@ int ReadKFDDeviceProperties(uint32_t dev_id,
    return ENOENT;
  }

-  fs.open(properties_path);
+  fs->open(f_path);

-  if (!fs.is_open()) {
+  if (!fs->is_open()) {
      return errno;
  }

+  return 0;
+}
+
+int ReadKFDDeviceProperties(uint32_t kfd_node_id,
+                                           std::vector<std::string> *retVec) {
+  std::string line;
+  int ret;
+  std::ifstream fs;
+  std::string properties_path;
+
+  assert(retVec != nullptr);
+
+  ret = OpenKFDNodeFile(kfd_node_id, "properties", &fs);
+
+  if (ret) {
+    return ret;
+  }
+
  while (std::getline(fs, line)) {
    retVec->push_back(line);
  }

  if (retVec->size() == 0) {
+    fs.close();
    return 0;
  }
  // Remove any *trailing* empty (whitespace) lines
  while (retVec->back().find_first_not_of(" \t\n\v\f\r") == std::string::npos) {
    retVec->pop_back();
  }
+
+  fs.close();
  return 0;
 }

+static int ReadKFDGpuId(uint32_t kfd_node_id, uint64_t *gpu_id) {
+  std::string line;
+  int ret;
+  std::ifstream fs;
+  std::string gpu_id_str;
+
+  assert(gpu_id != nullptr);
+
+  ret = OpenKFDNodeFile(kfd_node_id, "gpu_id", &fs);
+
+  if (ret) {
+    fs.close();
+    return ret;
+  }
+
+  std::stringstream ss;
+  ss << fs.rdbuf();
+  fs.close();
+
+  gpu_id_str = ss.str();
+
+  gpu_id_str.erase(std::remove(gpu_id_str.begin(), gpu_id_str.end(), '\n'),
+                                                            gpu_id_str.end());
+
+  if (!is_number(gpu_id_str)) {
+    return ENXIO;
+  }
+
+  *gpu_id = std::stoi(gpu_id_str);
+  return 0;
+}
+
+static int ReadKFDGpuName(uint32_t kfd_node_id, std::string *gpu_name) {
+  std::string line;
+  int ret;
+  std::ifstream fs;
+
+  assert(gpu_name != nullptr);
+
+  ret = OpenKFDNodeFile(kfd_node_id, "name", &fs);
+
+  if (ret) {
+    fs.close();
+    return ret;
+  }
+
+  std::stringstream ss;
+  ss << fs.rdbuf();
+  fs.close();
+
+  *gpu_name = ss.str();
+
+  gpu_name->erase(std::remove(gpu_name->begin(), gpu_name->end(), '\n'),
+                                                             gpu_name->end());
+
+  return 0;
+}

 int GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated,
                                                  uint32_t *num_procs_found) {
@@ -128,7 +257,7 @@ int GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated,
  }
  auto dentry = readdir(proc_dir);

-  std::string prod_id_str;
+  std::string proc_id_str;
  std::string tmp;

  while (dentry != nullptr) {
@@ -137,29 +266,32 @@ int GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated,
      continue;
    }

-    prod_id_str = dentry->d_name;
-    assert(is_number(prod_id_str) && "Unexpected file name in kfd/proc dir");
-    if (!is_number(prod_id_str)) {
+    proc_id_str = dentry->d_name;
+    assert(is_number(proc_id_str) && "Unexpected file name in kfd/proc dir");
+    if (!is_number(proc_id_str)) {
+      dentry = readdir(proc_dir);
      continue;
    }
    if (procs && *num_procs_found < num_allocated) {
      int err;
      std::string tmp;

-      procs[*num_procs_found].process_id = std::stoi(prod_id_str);
+      procs[*num_procs_found].process_id = std::stoi(proc_id_str);

      std::string pasid_str_path = kKFDProcPathRoot;
      pasid_str_path += "/";
-      pasid_str_path += prod_id_str;
+      pasid_str_path += proc_id_str;
      pasid_str_path += "/";
      pasid_str_path += kKFDPasidFName;

      err = ReadSysfsStr(pasid_str_path, &tmp);
      if (err) {
-        return err;
+        dentry = readdir(proc_dir);
+        continue;
      }
      assert(is_number(tmp) && "Unexpected value in pasid file");
      if (!is_number(tmp)) {
+        closedir(proc_dir);
        return EINVAL;
      }
      procs[*num_procs_found].pasid = std::stoi(tmp);
@@ -176,6 +308,81 @@ int GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated,
  return 0;
 }

+// Read the gpuid files found in all the <queue id> dirs and put them in
+// gpus_found.
+// Directory structure:
+//     /sys/class/kfd/kfd/proc/<pid>/queues/<queue id>/gpuid
+
+int GetProcessGPUs(uint32_t pid, std::unordered_set<uint64_t> *gpu_set) {
+  int err;
+
+  assert(gpu_set != nullptr);
+  if (gpu_set == nullptr) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+  errno = 0;
+
+  std::string queues_dir = kKFDProcPathRoot;
+  queues_dir += "/";
+  queues_dir += std::to_string(pid);
+  queues_dir += "/queues";
+
+  auto queues_dir_hd = opendir(queues_dir.c_str());
+
+  if (queues_dir_hd == nullptr) {
+    std::string err_str = "Unable to open queues directory for process ";
+    err_str += std::to_string(pid);
+    perror(err_str.c_str());
+    return ESRCH;
+  }
+
+  auto q_dentry = readdir(queues_dir_hd);
+
+  std::string q_gpu_id_str;
+  std::string q_dir;
+
+  std::string tmp;
+
+  while (q_dentry != nullptr) {
+    if (q_dentry->d_name[0] == '.') {
+      q_dentry = readdir(queues_dir_hd);
+      continue;
+    }
+
+    if (!is_number(q_dentry->d_name)) {
+      q_dentry = readdir(queues_dir_hd);
+      continue;
+    }
+
+    q_gpu_id_str = queues_dir + '/' + q_dentry->d_name + "/gpuid";
+
+    err = ReadSysfsStr(q_gpu_id_str, &tmp);
+    if (err) {
+      q_dentry = readdir(queues_dir_hd);
+      continue;
+    }
+
+    uint64_t val;
+    try {
+      val = std::stoi(tmp);
+    } catch (...) {
+      std::cerr << "Error; read invalid data: " << tmp << " from " <<
+                                                    q_gpu_id_str << std::endl;
+      closedir(queues_dir_hd);
+      return ENXIO;  // Return "no such device" if we read an invalid gpu id
+    }
+    gpu_set->insert(val);
+
+    q_dentry = readdir(queues_dir_hd);
+  }
+
+  errno = 0;
+  if (closedir(queues_dir_hd)) {
+    return errno;
+  }
+  return 0;
+}
+
 int GetProcessInfoForPID(uint32_t pid, rsmi_process_info_t *proc) {
  assert(proc != nullptr);
  int err;
@@ -208,5 +415,138 @@ int GetProcessInfoForPID(uint32_t pid, rsmi_process_info_t *proc) {
  return 0;
 }

+int DiscoverKFDNodes(std::map<uint64_t, std::shared_ptr<KFDNode>> *nodes) {
+  assert(nodes != nullptr);
+
+  if (nodes == nullptr) {
+    return EINVAL;
+  }
+  assert(nodes->size() == 0);
+
+  nodes->clear();
+
+  std::shared_ptr<KFDNode> node;
+  uint32_t node_indx;
+
+  auto kfd_node_dir = opendir(kKFDNodesPathRoot);
+  assert(kfd_node_dir != nullptr);
+
+  auto dentry = readdir(kfd_node_dir);
+  while (dentry != nullptr) {
+    if (dentry->d_name[0] == '.') {
+      dentry = readdir(kfd_node_dir);
+      continue;
+    }
+
+    if (!is_number(dentry->d_name)) {
+      dentry = readdir(kfd_node_dir);
+      continue;
+    }
+
+    node_indx = std::stoi(dentry->d_name);
+    node = std::shared_ptr<KFDNode>(new KFDNode(node_indx));
+
+    node->Initialize();
+
+    if (node->gpu_id() == 0) {
+      // Don't add; this is a cpu node.
+      dentry = readdir(kfd_node_dir);
+      continue;
+    }
+
+    uint64_t kfd_gpu_node_bus_fn;
+    uint64_t kfd_gpu_node_domain;
+    int ret;
+    ret =
+      node->get_property_value(kKFDNodePropLOCATION_IDStr,
+                                                        &kfd_gpu_node_bus_fn);
+    if (ret != 0) {
+      closedir(kfd_node_dir);
+      return ret;
+    }
+    ret =
+        node->get_property_value(kKFDNodePropDOMAINStr, &kfd_gpu_node_domain);
+    if (ret != 0) {
+      closedir(kfd_node_dir);
+      return ret;
+    }
+
+    uint64_t kfd_bdfid =
+                       (kfd_gpu_node_domain << 32) | (kfd_gpu_node_bus_fn);
+    (*nodes)[kfd_bdfid] = node;
+
+    dentry = readdir(kfd_node_dir);
+  }
+
+  if (closedir(kfd_node_dir)) {
+    return 1;
+  }
+  return 0;
+}
+
+KFDNode::~KFDNode() {
+}
+
+int KFDNode::ReadProperties(void) {
+  int ret;
+
+  std::vector<std::string> propVec;
+
+  assert(properties_.size() == 0);
+  if (properties_.size() > 0) {
+    return 0;
+  }
+
+  ret = ReadKFDDeviceProperties(node_indx_, &propVec);
+
+  if (ret) {
+    return ret;
+  }
+
+  std::string key_str;
+  // std::string val_str;
+  uint64_t val_int;  // Assume all properties are unsigned integers for now
+  std::istringstream fs;
+
+  for (uint32_t i = 0; i < propVec.size(); ++i) {
+    fs.str(propVec[i]);
+    fs >> key_str;
+    fs >> val_int;
+
+    properties_[key_str] = val_int;
+
+    fs.str("");
+    fs.clear();
+  }
+
+  return 0;
+}
+
+int
+KFDNode::Initialize(void) {
+  int ret = 0;
+  ret = ReadProperties();
+  if (ret) {return ret;}
+
+  ret = ReadKFDGpuId(node_indx_, &gpu_id_);
+  if (ret) {return ret;}
+
+  ret = ReadKFDGpuName(node_indx_, &name_);
+
+  return ret;
+}
+int
+KFDNode::get_property_value(std::string property, uint64_t *value) {
+  assert(value != nullptr);
+  if (value == nullptr) {
+    return EINVAL;
+  }
+  if (properties_.find(property) == properties_.end()) {
+    return EINVAL;
+  }
+  *value = properties_[property];
+  return 0;
+}
+
 }  // namespace smi
 }  // namespace amd
@@ -55,12 +55,14 @@
 #include <utility>
 #include <functional>
 #include <cerrno>
+#include <unordered_map>

 #include "rocm_smi/rocm_smi.h"
 #include "rocm_smi/rocm_smi_device.h"
 #include "rocm_smi/rocm_smi_main.h"
 #include "rocm_smi/rocm_smi_exception.h"
 #include "rocm_smi/rocm_smi_utils.h"
+#include "rocm_smi/rocm_smi_kfd.h"

 static const char *kPathDRMRoot = "/sys/class/drm";
 static const char *kPathHWMonRoot = "/sys/class/hwmon";
@@ -253,9 +255,9 @@ RocmSMI::Initialize(uint64_t flags) {
      ++i;
  }

-  // DiscoverDevices() will seach for devices and monitors and update internal
-  // data structures.
-  DiscoverDevices();
+  // DiscoverAmdgpuDevices() will seach for devices and monitors and update
+  // internal data structures.
+  DiscoverAmdgpuDevices();

  // IterateSMIDevices will iterate through all the known devices and apply
  // the provided call-back to each device found.
@@ -264,7 +266,34 @@ RocmSMI::Initialize(uint64_t flags) {

  if (ret != 0) {
    throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR,
-                                      "Failed to initialize rocm_smi library.");
+               "Failed to initialize rocm_smi library (amdgpu node discovery.");
+  }
+
+  std::map<uint64_t, std::shared_ptr<KFDNode>> tmp_map;
+  ret = DiscoverKFDNodes(&tmp_map);
+  if (ret != 0) {
+    throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR,
+                 "Failed to initialize rocm_smi library (KFD node discovery).");
+  }
+
+  std::shared_ptr<amd::smi::Device> dev;
+
+  // 1. construct kfd_node_map_ with gpu_id as key and *Device as value
+  // 2. for each kfd node, write the corresponding dv_ind
+  // 3. for each amdgpu device, write the corresponding gpu_id
+  for (uint32_t dv_ind = 0; dv_ind < s_monitor_devices.size(); ++dv_ind) {
+    dev = s_monitor_devices[dv_ind];
+    uint64_t bdfid = dev->bdfid();
+    assert(tmp_map.find(bdfid) != tmp_map.end());
+    if (tmp_map.find(bdfid) == tmp_map.end()) {
+      throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR,
+                   "amdgpu device bdfid has no KFD matching node");
+    }
+
+    tmp_map[bdfid]->set_amdgpu_dev_index(dv_ind);
+    uint64_t gpu_id = tmp_map[bdfid]->gpu_id();
+    dev->set_kfd_gpu_id(gpu_id);
+    kfd_node_map_[gpu_id] = tmp_map[bdfid];
  }
 }

@@ -345,10 +374,11 @@ RocmSMI::AddToDeviceList(std::string dev_name) {
  }

  std::string d_name = dev_name;
-  uint32_t d_index = GetDeviceIndex(d_name);
+  uint32_t card_indx = GetDeviceIndex(d_name);
  dev->set_drm_render_minor(GetDrmRenderMinor(dev_path));
-  dev->set_index(d_index);
-  GetSupportedEventGroups(d_index, dev->supported_event_groups());
+  dev->set_card_index(card_indx);
+  GetSupportedEventGroups(card_indx, dev->supported_event_groups());
+
  devices_.push_back(dev);

  return;
@@ -381,7 +411,7 @@ static bool isAMDGPU(std::string dev_path) {
  return false;
 }

-uint32_t RocmSMI::DiscoverDevices(void) {
+uint32_t RocmSMI::DiscoverAmdgpuDevices(void) {
  auto ret = 0;

  // If this gets called more than once, clear previous findings.
@@ -303,6 +303,7 @@ static int get_supported_sensors(std::string dir_path, std::string fn_reg_ex,
  std::string::size_type pos = fn_reg_ex.find('#');

  if (pos == std::string::npos) {
+    closedir(hwmon_dir);
    return -1;
  }
  fn_reg_ex.erase(pos, 1);
@@ -326,12 +327,16 @@ static int get_supported_sensors(std::string dir_path, std::string fn_reg_ex,
      assert(errno == 0);
      assert(*endptr == '\0');
      if (errno) {
+        closedir(hwmon_dir);
        return -2;
      }
      sensors->push_back(mon_val);
    }
    dentry = readdir(hwmon_dir);
  }
+  if (closedir(hwmon_dir)) {
+    return errno;
+  }
  return 0;
 }