Make bdfid use 32 bit domain if possible

If the 32-bit domain is found in the kfd node properties for
a device, then it will be used when constructing the bdfid.
If it's not present, it will continue to use the 16 bit version.

Also, whether or not 32b or 16b are used for the domain, the
domain will now be placed in the upper 32b of the 64b bdfid.

* Fixed some unrelated doxygen issues

Change-Id: Icb5116daa1ab45ee305bdbe6cd5df5736dd3ffa3


[ROCm/rocm_smi_lib commit: 469af303d6]

projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h

@@ -328,7 +328,7 @@ typedef enum {
 /**
  * @brief Pre-set Profile Selections. These bitmasks can be AND'd with the
  * ::rsmi_power_profile_status_t.available_profiles returned from
- * ::rsmi_dev_power_profile_presets_get() to determine which power profiles
+ * ::rsmi_dev_power_profile_presets_get to determine which power profiles
  * are supported by the system.
  */
 typedef enum {
@@ -996,6 +996,19 @@ rsmi_dev_pci_bandwidth_get(uint32_t dv_ind, rsmi_pcie_bandwidth_t *bandwidth);
  *  (BDFID) associated with device @p dv_ind to the value pointed to by
  *  @p bdfid.
  *
+ *  The format of @p bdfid will be as follows:
+ *
+ *      BDFID = ((DOMAIN & 0xffffffff) << 32) | ((BUS & 0xff) << 8) |
+ *                                   ((DEVICE & 0x1f) <<3 ) | (FUNCTION & 0x7)
+ *
+ *  | Name     | Field   |
+ *  ---------- | ------- |
+ *  | Domain   | [64:32] |
+ *  | Reserved | [31:16] |
+ *  | Bus      | [15: 8] |
+ *  | Device   | [ 7: 3] |
+ *  | Function | [ 2: 0] |
+ *
  *  @param[in] dv_ind a device index
  *
  *  @param[inout] bdfid a pointer to uint64_t to which the device bdfid value
@@ -1851,9 +1864,9 @@ rsmi_dev_firmware_version_get(uint32_t dv_ind, rsmi_fw_block_t block,
  *
  * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call.
  *
- * ::RSMI_NOT_SUPPORTED will be returned if either ECC is not enabled for the
- * specified block @p block, or if there is no kernel support for ECC for
- * that block.
+ * ::RSMI_STATUS_NOT_SUPPORTED will be returned if either ECC is not enabled
+ * for the specified block @p block, or if there is no kernel support for ECC
+ * for that block.
  *
  */
 rsmi_status_t rsmi_dev_ecc_count_get(uint32_t dv_ind,

projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h

@@ -51,6 +51,7 @@
 #include <cstdint>
 #include <vector>
 #include <unordered_set>
+#include <map>
 
 #include "rocm_smi/rocm_smi_monitor.h"
 #include "rocm_smi/rocm_smi_power_mon.h"
@@ -64,6 +65,38 @@ extern "C" {
 namespace amd {
 namespace smi {
 
+enum DevKFDNodePropTypes {
+  kDevKFDNodePropCachesCnt,
+  kDevKFDNodePropIoLinksCnt,
+  kDevKFDNodePropCPUCoreIdBase,
+  kDevKFDNodePropSimdIdBase,
+  kDevKFDNodePropMaxWavePerSimd,
+  kDevKFDNodePropLdsSz,
+  kDevKFDNodePropGdsSz,
+  kDevKFDNodePropNumGWS,
+  kDevKFDNodePropWaveFrontSize,
+  kDevKFDNodePropArrCnt,
+  kDevKFDNodePropSimdArrPerEng,
+  kDevKFDNodePropCuPerSimdArr,
+  kDevKFDNodePropSimdPerCU,
+  kDevKFDNodePropMaxSlotsScratchCu,
+  kDevKFDNodePropVendorId,
+  kDevKFDNodePropDeviceId,
+  kDevKFDNodePropLocationId,
+  kDevKFDNodePropDrmRenderMinor,
+  kDevKFDNodePropHiveId,
+  kDevKFDNodePropNumSdmaEngines,
+  kDevKFDNodePropNumSdmaXgmiEngs,
+  kDevKFDNodePropMaxEngClkFComp,
+  kDevKFDNodePropLocMemSz,
+  kDevKFDNodePropFwVer,
+  kDevKFDNodePropCapability,
+  kDevKFDNodePropDbgProp,
+  kDevKFDNodePropSdmaFwVer,
+  kDevKFDNodePropMaxEngClkCComp,
+  kDevKFDNodePropDomain,
+};
+
 enum DevInfoTypes {
   kDevPerfLevel,
   kDevOverDriveLevel,
@@ -139,6 +172,9 @@ class Device {
     int readDevInfo(DevInfoTypes type, std::vector<std::string> *retVec);
     int writeDevInfo(DevInfoTypes type, uint64_t val);
     int writeDevInfo(DevInfoTypes type, std::string val);
+    int populateKFDNodeProperties(bool force_update = false);
+    int getKFDNodeProperty(DevKFDNodePropTypes prop, uint64_t *val);
+
     uint32_t index(void) const {return index_;}
     void set_index(uint32_t index) {index_ = index;}
     uint32_t drm_render_minor(void) const {return drm_render_minor_;}
@@ -146,7 +182,6 @@ class Device {
     static rsmi_dev_perf_level perfLvlStrToEnum(std::string s);
     uint64_t bdfid(void) const {return bdfid_;}
     void set_bdfid(uint64_t val) {bdfid_ = val;}
-    uint64_t get_bdfid(void) const {return bdfid_;}
     pthread_mutex_t *mutex(void) {return mutex_.ptr;}
     evt::dev_evt_grp_set_t* supported_event_groups(void) {
                                              return &supported_event_groups_;}
@@ -169,6 +204,7 @@ class Device {
     uint64_t bdfid_;
     std::unordered_set<rsmi_event_group_t,
                        evt::RSMIEventGrpHashFunction> supported_event_groups_;
+    std::map<std::string, uint64_t> kfdNodePropMap_;
 };
 
 }  // namespace smi

projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h

@@ -43,6 +43,9 @@
 #ifndef INCLUDE_ROCM_SMI_ROCM_SMI_KFD_H_
 #define INCLUDE_ROCM_SMI_ROCM_SMI_KFD_H_
 
+#include <string>
+#include <vector>
+
 #include "rocm_smi/rocm_smi.h"
 
 namespace amd {
@@ -54,6 +57,9 @@ GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated,
 int
 GetProcessInfoForPID(uint32_t pid, rsmi_process_info_t *proc);
 
+int
+ReadKFDDeviceProperties(uint32_t dev_id, std::vector<std::string> *retVec);
+
 }  // namespace smi
 }  // namespace amd
 

projects/rocm-smi-lib/src/rocm_smi.cc

@@ -621,7 +621,35 @@ rsmi_dev_pci_id_get(uint32_t dv_ind, uint64_t *bdfid) {
 
   DEVICE_MUTEX
 
-  *bdfid = dev->get_bdfid();
+  *bdfid = dev->bdfid();
+
+  int32_t ret = dev->populateKFDNodeProperties();
+
+  if (ret) {
+    return errno_to_rsmi_status(errno);
+  }
+
+  uint64_t domain = 0;
+
+  ret = dev->getKFDNodeProperty(amd::smi::kDevKFDNodePropDomain, &domain);
+
+  if (ret == EINVAL) {
+    // "domain" is not found in properties file; just go with the 16 bit
+    // domain already found
+    return RSMI_STATUS_SUCCESS;
+  }
+
+  // Replace the 16 bit domain originally set like this:
+  // BDFID = ((<DOMAIN> & 0xffff) << 32) | ((<BUS> & 0xff) << 8) |
+  //                        ((device& 0x1f) <<3 ) | (function & 0x7)
+  // with this:
+  // BDFID = ((<DOMAIN> & 0xffffffff) << 32) | ((<BUS> & 0xff) << 8) |
+  //                        ((device& 0x1f) <<3 ) | (function & 0x7)
+
+  assert((domain & 0xFFFFFFFF00000000) == 0);
+  (*bdfid) &= 0xFFFF;  // Clear out the old 16 bit domain
+  *bdfid |= (domain & 0xFFFFFFFF) << 32;
+
   return RSMI_STATUS_SUCCESS;
   CATCH
 }

projects/rocm-smi-lib/src/rocm_smi_device.cc

@@ -61,6 +61,7 @@
 #include "rocm_smi/rocm_smi.h"
 #include "rocm_smi/rocm_smi_exception.h"
 #include "rocm_smi/rocm_smi_utils.h"
+#include "rocm_smi/rocm_smi_kfd.h"
 
 extern "C" {
 #include "shared_mutex.h"  // NOLINT
@@ -139,7 +140,71 @@ static const char *kDevFwVersionUvdFName = "fw_version/uvd_fw_version";
 static const char *kDevFwVersionVceFName = "fw_version/vce_fw_version";
 static const char *kDevFwVersionVcnFName = "fw_version/vcn_fw_version";
 
+static const char *kDevKFDNodePropCachesCntSName = "caches_count";
+static const char *kDevKFDNodePropIoLinksCntSName = "io_links_count";
+static const char *kDevKFDNodePropCPUCoreIdBaseSName = "cpu_core_id_base";
+static const char *kDevKFDNodePropSimdIdBaseSName = "simd_id_base";
+static const char *kDevKFDNodePropMaxWavePerSimdSName = "max_waves_per_simd";
+static const char *kDevKFDNodePropLdsSzSName = "lds_size_in_kb";
+static const char *kDevKFDNodePropGdsSzSName = "gds_size_in_kb";
+static const char *kDevKFDNodePropNumGWSSName = "num_gws";
+static const char *kDevKFDNodePropWaveFrontSizeSName = "wave_front_size";
+static const char *kDevKFDNodePropArrCntSName = "array_count";
+static const char *kDevKFDNodePropSimdArrPerEngSName = "simd_arrays_per_engine";
+static const char *kDevKFDNodePropCuPerSimdArrSName = "cu_per_simd_array";
+static const char *kDevKFDNodePropSimdPerCUSName = "simd_per_cu";
+static const char *kDevKFDNodePropMaxSlotsScratchCuSName =
+                                                       "max_slots_scratch_cu";
+static const char *kDevKFDNodePropVendorIdSName = "vendor_id";
+static const char *kDevKFDNodePropDeviceIdSName = "device_id";
+static const char *kDevKFDNodePropLocationIdSName = "location_id";
+static const char *kDevKFDNodePropDrmRenderMinorSName = "drm_render_minor";
+static const char *kDevKFDNodePropHiveIdSName = "hive_id";
+static const char *kDevKFDNodePropNumSdmaEnginesSName = "num_sdma_engines";
+static const char *kDevKFDNodePropNumSdmaXgmiEngsSName =
+                                                      "num_sdma_xgmi_engines";
+static const char *kDevKFDNodePropMaxEngClkFCompSName =
+                                                    "max_engine_clk_fcompute";
+static const char *kDevKFDNodePropLocMemSzSName = "local_mem_size";
+static const char *kDevKFDNodePropFwVerSName = "fw_version";
+static const char *kDevKFDNodePropCapabilitySName = "capability";
+static const char *kDevKFDNodePropDbgPropSName = "debug_prop";
+static const char *kDevKFDNodePropSdmaFwVerSName = "sdma_fw_version";
+static const char *kDevKFDNodePropMaxEngClkCCompSName =
+                                                    "max_engine_clk_ccompute";
+static const char *kDevKFDNodePropDomainSName = "domain";
 
+static const std::map<DevKFDNodePropTypes, const char *> kDevKFDPropNameMap = {
+    {kDevKFDNodePropCachesCnt, kDevKFDNodePropCachesCntSName},
+    {kDevKFDNodePropIoLinksCnt, kDevKFDNodePropIoLinksCntSName},
+    {kDevKFDNodePropCPUCoreIdBase, kDevKFDNodePropCPUCoreIdBaseSName},
+    {kDevKFDNodePropSimdIdBase, kDevKFDNodePropSimdIdBaseSName},
+    {kDevKFDNodePropMaxWavePerSimd, kDevKFDNodePropMaxWavePerSimdSName},
+    {kDevKFDNodePropLdsSz, kDevKFDNodePropLdsSzSName},
+    {kDevKFDNodePropGdsSz, kDevKFDNodePropGdsSzSName},
+    {kDevKFDNodePropNumGWS, kDevKFDNodePropNumGWSSName},
+    {kDevKFDNodePropWaveFrontSize, kDevKFDNodePropWaveFrontSizeSName},
+    {kDevKFDNodePropArrCnt, kDevKFDNodePropArrCntSName},
+    {kDevKFDNodePropSimdArrPerEng, kDevKFDNodePropSimdArrPerEngSName},
+    {kDevKFDNodePropCuPerSimdArr, kDevKFDNodePropCuPerSimdArrSName},
+    {kDevKFDNodePropSimdPerCU, kDevKFDNodePropSimdPerCUSName},
+    {kDevKFDNodePropMaxSlotsScratchCu, kDevKFDNodePropMaxSlotsScratchCuSName},
+    {kDevKFDNodePropVendorId, kDevKFDNodePropVendorIdSName},
+    {kDevKFDNodePropDeviceId, kDevKFDNodePropDeviceIdSName},
+    {kDevKFDNodePropLocationId, kDevKFDNodePropLocationIdSName},
+    {kDevKFDNodePropDrmRenderMinor, kDevKFDNodePropDrmRenderMinorSName},
+    {kDevKFDNodePropHiveId, kDevKFDNodePropHiveIdSName},
+    {kDevKFDNodePropNumSdmaEngines, kDevKFDNodePropNumSdmaEnginesSName},
+    {kDevKFDNodePropNumSdmaXgmiEngs, kDevKFDNodePropNumSdmaXgmiEngsSName},
+    {kDevKFDNodePropMaxEngClkFComp, kDevKFDNodePropMaxEngClkFCompSName},
+    {kDevKFDNodePropLocMemSz, kDevKFDNodePropLocMemSzSName},
+    {kDevKFDNodePropFwVer, kDevKFDNodePropFwVerSName},
+    {kDevKFDNodePropCapability, kDevKFDNodePropCapabilitySName},
+    {kDevKFDNodePropDbgProp, kDevKFDNodePropDbgPropSName},
+    {kDevKFDNodePropSdmaFwVer, kDevKFDNodePropSdmaFwVerSName},
+    {kDevKFDNodePropMaxEngClkCComp, kDevKFDNodePropMaxEngClkCCompSName},
+    {kDevKFDNodePropDomain, kDevKFDNodePropDomainSName},
+};
 static const std::map<DevInfoTypes, const char *> kDevAttribNameMap = {
     {kDevPerfLevel, kDevPerfLevelFName},
     {kDevOverDriveLevel, kDevOverDriveLevelFName},
@@ -520,6 +585,53 @@ int Device::readDevInfo(DevInfoTypes type, std::string *val) {
   return 0;
 }
 
+int Device::populateKFDNodeProperties(bool force_update) {
+  int ret;
+
+  std::vector<std::string> propVec;
+
+  if (kfdNodePropMap_.size() > 0 && !force_update) {
+    return 0;
+  }
+
+  ret = ReadKFDDeviceProperties(index_, &propVec);
+
+  if (ret) {
+    return ret;
+  }
+
+  std::string key_str;
+  // std::string val_str;
+  uint64_t val_int;  // Assume all properties are unsigned integers for now
+  std::istringstream fs;
+
+  for (uint32_t i = 0; i < propVec.size(); ++i) {
+    fs.str(propVec[i]);
+    fs >> key_str;
+    fs >> val_int;
+
+    kfdNodePropMap_[key_str] = val_int;
+
+    fs.str("");
+    fs.clear();
+  }
+
+  return 0;
+}
+
+int Device::getKFDNodeProperty(DevKFDNodePropTypes prop, uint64_t *val) {
+  assert(val != nullptr);
+  assert(kDevKFDPropNameMap.find(prop) != kDevKFDPropNameMap.end());
+
+  const char *prop_name = kDevKFDPropNameMap.at(prop);
+  if (kfdNodePropMap_.find(prop_name) == kfdNodePropMap_.end()) {
+    return EINVAL;
+  }
+
+  *val = kfdNodePropMap_.at(prop_name);
+  return 0;
+}
+
 #undef RET_IF_NONZERO
 }  // namespace smi
 }  // namespace amd

projects/rocm-smi-lib/src/rocm_smi_kfd.cc

@@ -62,7 +62,7 @@ namespace amd {
 namespace smi {
 
 static const char *kKFDProcPathRoot = "/sys/class/kfd/kfd/proc";
-
+static const char *kKFDNodesPathRoot = "/sys/class/kfd/kfd/topology/nodes";
 // Sysfs file names
 static const char *kKFDPasidFName = "pasid";
 
@@ -70,6 +70,50 @@ static bool is_number(const std::string &s) {
   return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
 }
 
+int ReadKFDDeviceProperties(uint32_t dev_id,
+                                           std::vector<std::string> *retVec) {
+  std::string line;
+  int ret;
+  std::ifstream fs;
+  std::string properties_path = kKFDNodesPathRoot;
+  bool reg_file;
+
+  assert(retVec != nullptr);
+
+  properties_path += '/';
+  properties_path += std::to_string(dev_id);
+  properties_path += "/properties";
+
+  ret = isRegularFile(properties_path, &reg_file);
+
+  if (ret != 0) {
+    return ret;
+  }
+  if (!reg_file) {
+    return ENOENT;
+  }
+
+  fs.open(properties_path);
+
+  if (!fs.is_open()) {
+      return errno;
+  }
+
+  while (std::getline(fs, line)) {
+    retVec->push_back(line);
+  }
+
+  if (retVec->size() == 0) {
+    return 0;
+  }
+  // Remove any *trailing* empty (whitespace) lines
+  while (retVec->back().find_first_not_of(" \t\n\v\f\r") == std::string::npos) {
+    retVec->pop_back();
+  }
+  return 0;
+}
+
+
 int GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated,
                                                   uint32_t *num_procs_found) {
   assert(num_procs_found != nullptr);

projects/rocm-smi-lib/src/rocm_smi_main.cc

@@ -73,7 +73,6 @@ static const char *kAMDMonitorTypes[] = {"radeon", "amdgpu", ""};
 namespace amd {
 namespace smi {
 
-
 static uint32_t GetDeviceIndex(const std::string s) {
   std::string t = s;
   size_t tmp = t.find_last_not_of("0123456789");
@@ -123,12 +122,13 @@ static int SameDevice(const std::string fileA, const std::string fileB) {
 //  XXXX:XX:XX.X,
 //  domain:bus:device.function
 //
-//  where X is a hex integer (lower case is expected)
-static bool is_bdfid_path_str(const std::string in_name, uint64_t *bdfid) {
+//  where X is a hex integer (lower case is expected). If so, write the value
+//  to bdfid
+static bool bdfid_from_path(const std::string in_name, uint64_t *bdfid) {
   char *p = nullptr;
   char *name_start;
   char name[13] = {'\0'};
-  uint32_t tmp;
+  uint64_t tmp;
 
   assert(bdfid != nullptr);
 
@@ -139,7 +139,7 @@ static bool is_bdfid_path_str(const std::string in_name, uint64_t *bdfid) {
   tmp = in_name.copy(name, 12);
   assert(tmp == 12);
 
-  // BDFID = ((<DOMAIN> & 0xffff) << 13) | ((<BUS> & 0x1f) << 8) |
+  // BDFID = ((<DOMAIN> & 0xffff) << 32) | ((<BUS> & 0xff) << 8) |
             //                        ((device& 0x1f) <<3 ) | (function & 0x7)
   *bdfid = 0;
   name_start = name;
@@ -150,7 +150,7 @@ static bool is_bdfid_path_str(const std::string in_name, uint64_t *bdfid) {
   if (*p != ':' || p - name_start != 4) {
     return false;
   }
-  *bdfid |= tmp << 13;
+  *bdfid |= tmp << 32;
 
   // Match this: xxxx:XX:xx.x
   p++;  // Skip past ':'
@@ -205,7 +205,7 @@ static uint32_t ConstructBDFID(std::string path, uint64_t *bdfid) {
     slash_i = tpath_str.find_last_of('/', end_i);
     tmp = tpath_str.substr(slash_i + 1, end_i - slash_i);
 
-    if (is_bdfid_path_str(tmp, bdfid)) {
+    if (bdfid_from_path(tmp, bdfid)) {
       return 0;
     }
     end_i = slash_i - 1;
@@ -335,7 +335,6 @@ RocmSMI::AddToDeviceList(std::string dev_name) {
   uint32_t d_index = GetDeviceIndex(d_name);
   dev->set_drm_render_minor(GetDrmRenderMinor(dev_path));
   dev->set_index(d_index);
-
   GetSupportedEventGroups(d_index, dev->supported_event_groups());
   devices_.push_back(dev);