Partition EBUSY with RSMI_STATUS_BUSY & invalid GPU Metrics check

* Updates: - [API/CLI] rsmi_dev_*_partition_set & rsmi_dev_*_partition_reset - exposed RSMI_STATUS_BUSY for EBUSY writes + cleaned up accidental map insertions (maplookup[] can insert values that are not in the map, map.at(key) fixes this potential issue) - [API] rsmi_dev_gpu_metrics_info_get() - returns RSMI_STATUS_NOT_SUPPORTED for unsupported metric tables outside of 1v1/1v2/1v3 - [API] writeDevInfoStr() - exposes RSMI_STATUS_BUSY for EBUSY write errors; kept backward compatibility for other writes which do not care about these states - [API] rsmi_dev_od_volt_info_get() & rsmi_dev_od_volt_curve_regions_get() have better logging + Expose more details on why they are erroring - [Utils/logs/example] Expose AMD GPU gfx target version to aid in system troubleshooting - [Utils] Added test methods that look at od volt freq & regions into here - for easier access across several tests - [Utils] Updated getRSMIStatusString(new argument - fullstatus; default to true for backwards compatibility) -> true shows shortened RSMI STATUS response - [Utils] Added splitString to cut out noisy return responses (used in getRSMIStatusString(), when fullstatus = true) - [Utils] Added getFileCreationDate() to expose build date of the library - helpful for local builds or experimental builds - [Utils] Macro cleanup - [Example] Added a few gpu_metric checks - helpful for upcoming updates - [Device] SYSFS/DebugFS - now have better r/w displayed in logs - [LOGS] Expose library build date - see above for details - [Tests] Add more warnings/errors to test builds - [Tests] Moved up Partition tests for ordered test runs - helped identify issues with GPU BUSY writes - [Tests] compute_partition_read_write - handles RSMI_STATUS_BUSY with waits for busy status found & cleaned up how we checked for partition changes - with RSMI responses exposed more clearly - [Tests] perf_determinism - multi gpu now properly runs through with full resets as needed - [Tests] volt_freq_curv_read - better error handling with more verbose output Change-Id: Ie94c6abb6a9aab95c345996d3ad3843cf6734977 Signed-off-by: Charis Poag <Charis.Poag@amd.com> [ROCm/rocm_smi_lib commit: 57b6135e54]
2023-10-23 21:37:31 -05:00
parent e142177077
commit e89751e202
18 changed files with 701 additions and 215 deletions
@@ -929,6 +929,9 @@ rsmi_status_t
 rsmi_perf_determinism_mode_set(uint32_t dv_ind, uint64_t clkvalue) {
  TRY
  DEVICE_MUTEX
+  std::ostringstream ss;
+  ss << __PRETTY_FUNCTION__ << " | ======= start =======";
+  LOG_TRACE(ss);

  // Set perf. level to performance determinism so that we can then set the power profile
  rsmi_status_t ret = rsmi_dev_perf_level_set_v1(dv_ind,
@@ -1510,6 +1513,9 @@ rsmi_status_t rsmi_dev_od_volt_info_set(uint32_t dv_ind, uint32_t vpoint,

 static void get_vc_region(uint32_t start_ind,
                std::vector<std::string> *val_vec, rsmi_freq_volt_region_t *p) {
+  std::ostringstream ss;
+  ss << __PRETTY_FUNCTION__ << " | ======= start =======";
+  LOG_TRACE(ss);
  assert(p != nullptr);
  assert(val_vec != nullptr);
  THROW_IF_NULLPTR_DEREF(p)
@@ -1520,6 +1526,9 @@ static void get_vc_region(uint32_t start_ind,
  assert((*val_vec)[kOD_OD_RANGE_label_array_index] == "OD_RANGE:");
  if ((val_vec->size() < kOD_OD_RANGE_label_array_index + 2)  ||
      ((*val_vec)[kOD_OD_RANGE_label_array_index] != "OD_RANGE:") ) {
+    ss << __PRETTY_FUNCTION__ << " | ======= end ======= | returning "
+       << getRSMIStatusString(RSMI_STATUS_UNEXPECTED_DATA);
+    LOG_TRACE(ss);
    throw amd::smi::rsmi_exception(RSMI_STATUS_UNEXPECTED_DATA, __FUNCTION__);
  }
  od_value_pair_str_to_range((*val_vec)[start_ind], &p->freq_range);
@@ -1539,6 +1548,7 @@ static rsmi_status_t get_od_clk_volt_curve_regions(uint32_t dv_ind,
  TRY
  std::vector<std::string> val_vec;
  rsmi_status_t ret;
+  std::ostringstream ss;

  assert(num_regions != nullptr);
  assert(p != nullptr);
@@ -1547,12 +1557,20 @@ static rsmi_status_t get_od_clk_volt_curve_regions(uint32_t dv_ind,

  ret = GetDevValueVec(amd::smi::kDevPowerODVoltage, dv_ind, &val_vec);
  if (ret != RSMI_STATUS_SUCCESS) {
+    ss << __PRETTY_FUNCTION__
+       << " | Issue: could not retreive kDevPowerODVoltage" << "; returning "
+       << getRSMIStatusString(ret);
+    LOG_ERROR(ss);
    return ret;
  }

  // This is a work-around to handle systems where kDevPowerODVoltage is not
  // fully supported yet.
  if (val_vec.size() < 2) {
+    ss << __PRETTY_FUNCTION__
+       << " | Issue: val_vec.size() < 2" << "; returning "
+       << getRSMIStatusString(RSMI_STATUS_NOT_YET_IMPLEMENTED);
+    LOG_ERROR(ss);
    return RSMI_STATUS_NOT_YET_IMPLEMENTED;
  }

@@ -1560,8 +1578,17 @@ static rsmi_status_t get_od_clk_volt_curve_regions(uint32_t dv_ind,
  assert((val_vec_size - kOD_VDDC_CURVE_start_index) > 0);
  assert((val_vec_size - kOD_VDDC_CURVE_start_index)%2 == 0);

+  ss << __PRETTY_FUNCTION__
+     << " | val_vec_size = " << std::dec
+     << val_vec_size
+     << " | kOD_VDDC_CURVE_start_index = " << kOD_VDDC_CURVE_start_index;
+  LOG_DEBUG(ss);
  if (((val_vec_size - kOD_VDDC_CURVE_start_index) <= 0)  ||
      (((val_vec_size - kOD_VDDC_CURVE_start_index)%2 != 0))) {
+    ss << __PRETTY_FUNCTION__ << " | Issue: od vdd curve returned unexpected "
+       << "data" << "; returning "
+       << getRSMIStatusString(RSMI_STATUS_UNEXPECTED_SIZE);
+    LOG_ERROR(ss);
    throw amd::smi::rsmi_exception(RSMI_STATUS_UNEXPECTED_SIZE, __FUNCTION__);
  }

@@ -2749,6 +2776,9 @@ rsmi_dev_od_volt_info_get(uint32_t dv_ind, rsmi_od_volt_freq_data_t *odv) {
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  DEVICE_MUTEX
+  if (odv == nullptr) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
  CHK_SUPPORT_NAME_ONLY(odv)
  rsmi_status_t ret = get_od_clk_volt_info(dv_ind, odv);

@@ -2779,7 +2809,7 @@ rsmi_status_t rsmi_dev_od_volt_curve_regions_get(uint32_t dv_ind,
                     uint32_t *num_regions, rsmi_freq_volt_region_t *buffer) {
  TRY
  std::ostringstream ss;
-  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
+  ss << __PRETTY_FUNCTION__ << " | ======= start =======";
  LOG_TRACE(ss);

  CHK_SUPPORT_NAME_ONLY((num_regions == nullptr || buffer == nullptr) ?
@@ -2791,6 +2821,12 @@ rsmi_status_t rsmi_dev_od_volt_curve_regions_get(uint32_t dv_ind,
  DEVICE_MUTEX
  rsmi_status_t ret = get_od_clk_volt_curve_regions(dv_ind, num_regions,
                                                                      buffer);
+  if (*num_regions == 0) {
+    ret = RSMI_STATUS_NOT_SUPPORTED;
+  }
+  ss << __PRETTY_FUNCTION__ << " | ======= end ======= | returning "
+     << getRSMIStatusString(ret);
+  LOG_TRACE(ss);
  return ret;
  CATCH
 }
@@ -4468,7 +4504,7 @@ get_compute_partition(uint32_t dv_ind, std::string &compute_partition) {
    return ret;
  }

-  switch (mapStringToRSMIComputePartitionTypes[compute_partition_str]) {
+  switch (mapStringToRSMIComputePartitionTypes.at(compute_partition_str)) {
    case RSMI_COMPUTE_PARTITION_CPX:
    case RSMI_COMPUTE_PARTITION_SPX:
    case RSMI_COMPUTE_PARTITION_DPX:
@@ -4585,9 +4621,12 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS
+  if (!amd::smi::is_sudo_user()) {
+    return RSMI_STATUS_PERMISSION;
+  }
  DEVICE_MUTEX
  std::string newComputePartitionStr
-              = mapRSMIToStringComputePartitionTypes[compute_partition];
+              = mapRSMIToStringComputePartitionTypes.at(compute_partition);
  std::string currentComputePartition;

  switch (compute_partition) {
@@ -4605,6 +4644,7 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
         << " | Device #: " << dv_ind
         << " | Type: "
         << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
+         << " | Data: " << newComputePartitionStr
         << " | Cause: requested setting was invalid"
         << " | Returning = "
         << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS) << " |";
@@ -4623,6 +4663,7 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
       << " | Device #: " << dv_ind
       << " | Type: "
       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
+       << " | Data: " << newComputePartitionStr
       << " | Cause: not an available compute partition setting"
       << " | Returning = "
       << getRSMIStatusString(available_ret) << " |";
@@ -4650,7 +4691,7 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
    return ret_get;
  }
  rsmi_compute_partition_type_t currRSMIComputePartition
-    = mapStringToRSMIComputePartitionTypes[currentComputePartition];
+    = mapStringToRSMIComputePartitionTypes.at(currentComputePartition);
  if (currRSMIComputePartition == compute_partition) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
@@ -4665,6 +4706,15 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
    return RSMI_STATUS_SUCCESS;
  }

+  ss <<  __PRETTY_FUNCTION__ << " | about to try writing |"
+     << newComputePartitionStr
+     << "| size of string = " << newComputePartitionStr.size()
+     << "| size of c-string = "<< std::dec
+     << sizeof(newComputePartitionStr.c_str())/sizeof(newComputePartitionStr[0])
+     << "| sizeof string = " << std::dec
+     << sizeof(newComputePartitionStr);
+  LOG_DEBUG(ss);
+
  GET_DEV_FROM_INDX
  int ret = dev->writeDevInfo(amd::smi::kDevComputePartition,
                              newComputePartitionStr);
@@ -4699,7 +4749,7 @@ static rsmi_status_t get_memory_partition(uint32_t dv_ind,
    return ret;
  }

-  switch (mapStringToMemoryPartitionTypes[val_str]) {
+  switch (mapStringToMemoryPartitionTypes.at(val_str)) {
    case RSMI_MEMORY_PARTITION_NPS1:
    case RSMI_MEMORY_PARTITION_NPS2:
    case RSMI_MEMORY_PARTITION_NPS4:
@@ -4755,7 +4805,7 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
  }

  std::string newMemoryPartition
-              = mapRSMIToStringMemoryPartitionTypes[memory_partition];
+              = mapRSMIToStringMemoryPartitionTypes.at(memory_partition);
  std::string currentMemoryPartition;

  switch (memory_partition) {
@@ -4798,7 +4848,7 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
    return ret_get;
  }
  rsmi_memory_partition_type_t currRSMIMemoryPartition
-    = mapStringToMemoryPartitionTypes[currentMemoryPartition];
+    = mapStringToMemoryPartitionTypes.at(currentMemoryPartition);
  if (currRSMIMemoryPartition == memory_partition) {
    ss << __PRETTY_FUNCTION__
     << " | ======= end ======= "
@@ -4942,7 +4992,7 @@ rsmi_status_t rsmi_dev_compute_partition_reset(uint32_t dv_ind) {
  // Likely due to device not supporting it
  if (bootState != "UNKNOWN") {
    rsmi_compute_partition_type_t compute_partition =
-      mapStringToRSMIComputePartitionTypes[bootState];
+      mapStringToRSMIComputePartitionTypes.at(bootState);
    ret = rsmi_dev_compute_partition_set(dv_ind, compute_partition);
  }
  ss << __PRETTY_FUNCTION__
@@ -4981,7 +5031,7 @@ rsmi_status_t rsmi_dev_memory_partition_reset(uint32_t dv_ind) {
  // Likely due to device not supporting it
  if (bootState != "UNKNOWN") {
    rsmi_memory_partition_type_t memory_partition =
-      mapStringToMemoryPartitionTypes[bootState];
+      mapStringToMemoryPartitionTypes.at(bootState);
    ret = rsmi_dev_memory_partition_set(dv_ind, memory_partition);
  }
  ss << __PRETTY_FUNCTION__
@@ -598,14 +598,17 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) {

  int ret = isRegularFile(sysfs_path, &reg_file);
  if (ret != 0) {
-    ss << "File did not exist - SYSFS file (" << sysfs_path
+    ss << __PRETTY_FUNCTION__ << " | Issue: File did not exist - SYSFS file ("
+       << sysfs_path
       << ") for DevInfoInfoType (" << RocmSMI::devInfoTypesStrings.at(type)
       << "), returning " << std::to_string(ret);
    LOG_ERROR(ss);
    return ret;
  }
  if (!reg_file) {
-    ss << "File is not a regular file - SYSFS file (" << sysfs_path << ") for "
+    ss << __PRETTY_FUNCTION__
+       << " | Issue: File is not a regular file - SYSFS file ("
+       << sysfs_path << ") for "
       << "DevInfoInfoType (" << RocmSMI::devInfoTypesStrings.at(type) << "),"
       << " returning ENOENT (" << std::strerror(ENOENT) << ")";
    LOG_ERROR(ss);
@@ -615,7 +618,8 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) {
  fs->open(sysfs_path);

  if (!fs->is_open()) {
-    ss << "Could not open - SYSFS file (" << sysfs_path << ") for "
+    ss << __PRETTY_FUNCTION__
+       << " | Issue: Could not open - SYSFS file (" << sysfs_path << ") for "
       << "DevInfoInfoType (" << RocmSMI::devInfoTypesStrings.at(type) << "), "
       << ", returning " << std::to_string(errno) << " ("
       << std::strerror(errno) << ")";
@@ -623,7 +627,8 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) {
    return errno;
  }

-  ss << "Successfully opened SYSFS file (" << sysfs_path
+  ss << __PRETTY_FUNCTION__ << " | Successfully opened SYSFS file ("
+     << sysfs_path
     << ") for DevInfoInfoType (" << RocmSMI::devInfoTypesStrings.at(type)
     << ")";
  LOG_INFO(ss);
@@ -671,32 +676,51 @@ int Device::readDevInfoStr(DevInfoTypes type, std::string *retStr) {
  ret = openSysfsFileStream(type, &fs);
  if (ret != 0) {
    ss << "Could not read device info string for DevInfoType ("
-     << RocmSMI::devInfoTypesStrings.at(type)<< "), returning "
+     << RocmSMI::devInfoTypesStrings.at(type) << "), returning "
     << std::to_string(ret);
    LOG_ERROR(ss);
    return ret;
  }

  fs >> *retStr;
-  std::string info = "Successfully read device info string for DevInfoType (" +
-                      RocmSMI::devInfoTypesStrings.at(type) + "): " +
-                      *retStr;
-  LOG_INFO(info);
  fs.close();
-
+  ss << __PRETTY_FUNCTION__
+     << "Successfully read device info string for DevInfoType (" +
+            RocmSMI::devInfoTypesStrings.at(type) + "): " + *retStr
+     << " | "
+     << (fs.is_open() ? " File stream is opened" : " File stream is closed")
+     << " | " << (fs.bad() ? "[ERROR] Bad read operation" :
+     "[GOOD] No bad bit read, successful read operation")
+     << " | " << (fs.fail() ? "[ERROR] Failed read - format error" :
+     "[GOOD] No fail - Successful read operation")
+     << " | " << (fs.eof() ? "[ERROR] Failed read - EOF error" :
+     "[GOOD] No eof error - Successful read operation")
+     << " | " << (fs.good() ? "[GOOD] read good - Successful read operation" :
+     "[ERROR] Failed read - good error");
+  LOG_INFO(ss);
  return 0;
 }

-int Device::writeDevInfoStr(DevInfoTypes type, std::string valStr) {
-  auto tempPath = path_;
+int Device::writeDevInfoStr(DevInfoTypes type, std::string valStr,
+                           bool returnWriteErr) {
+  // returnWriteErr = false, backwards compatability (old calls)
+  // returnWriteErr = true, improvement - allows us to detect errors
+  // when writing to file
+  // (such as EBUSY)
+  auto sysfs_path = path_;
+  sysfs_path += "/device/";
+  sysfs_path += kDevAttribNameMap.at(type);
  std::ofstream fs;
  int ret;
  std::ostringstream ss;

-  fs.rdbuf()->pubsetbuf(nullptr,0);
+  fs.flush();
+  fs.rdbuf()->pubsetbuf(0, 0);
  ret = openSysfsFileStream(type, &fs, valStr.c_str());
  if (ret != 0) {
-    ss << "Could not write device info string (" << valStr
+    fs.close();
+    ss << __PRETTY_FUNCTION__ << " | Issue: Could not open fileStream; "
+       << "Could not write device info string (" << valStr
       << ") for DevInfoType (" << RocmSMI::devInfoTypesStrings.at(type)
       << "), returning " << std::to_string(ret);
    LOG_ERROR(ss);
@@ -705,19 +729,39 @@ int Device::writeDevInfoStr(DevInfoTypes type, std::string valStr) {

  // We'll catch any exceptions in rocm_smi.cc code.
  if (fs << valStr) {
+    fs.flush();
+    fs.close();
    ss << "Successfully wrote device info string (" << valStr
       << ") for DevInfoType (" << RocmSMI::devInfoTypesStrings.at(type)
       << "), returning RSMI_STATUS_SUCCESS";
    LOG_INFO(ss);
    ret = RSMI_STATUS_SUCCESS;
  } else {
-    ss << "Could not write device info string (" << valStr
+    if (returnWriteErr) {
+      ret = errno;
+    } else {
+      ret = RSMI_STATUS_NOT_SUPPORTED;
+    }
+    fs.flush();
+    fs.close();
+    ss << __PRETTY_FUNCTION__ << " | Issue: Could not write to file; "
+       << "Could not write device info string (" << valStr
       << ") for DevInfoType (" << RocmSMI::devInfoTypesStrings.at(type)
-       << "), returning RSMI_STATUS_NOT_SUPPORTED";
+       << "), returning " << getRSMIStatusString(ErrnoToRsmiStatus(ret));
+    ss << " | "
+       << (fs.is_open() ? "[ERROR] File stream open" :
+          "[GOOD] File stream closed")
+       << " | " << (fs.bad() ? "[ERROR] Bad write operation" :
+                    "[GOOD] No bad bit write, successful write operation")
+       << " | " << (fs.fail() ? "[ERROR] Failed write - format error" :
+                    "[GOOD] No fail - Successful write operation")
+       << " | " << (fs.eof() ? "[ERROR] Failed write - EOF error" :
+                    "[GOOD] No eof error - Successful write operation")
+       << " | " << (fs.good() ?
+                   "[GOOD] Write good - Successful write operation" :
+                   "[ERROR] Failed write - good error");
    LOG_ERROR(ss);
-    ret = RSMI_STATUS_NOT_SUPPORTED;
  }
-  fs.close();

  return ret;
 }
@@ -756,6 +800,9 @@ int Device::writeDevInfo(DevInfoTypes type, uint64_t val) {
 }

 int Device::writeDevInfo(DevInfoTypes type, std::string val) {
+  auto sysfs_path = path_;
+  sysfs_path += "/device/";
+  sysfs_path += kDevAttribNameMap.at(type);
  switch (type) {
    case kDevGPUMClk:
    case kDevDCEFClk:
@@ -764,9 +811,10 @@ int Device::writeDevInfo(DevInfoTypes type, std::string val) {
    case kDevPCIEClk:
    case kDevPowerODVoltage:
    case kDevSOCClk:
+      return writeDevInfoStr(type, val);
    case kDevComputePartition:
    case kDevMemoryPartition:
-      return writeDevInfoStr(type, val);
+      return writeDevInfoStr(type, val, true);

    default:
      return EINVAL;
@@ -899,6 +947,7 @@ int Device::readDevInfo(DevInfoTypes type, uint64_t *val) {
  std::string tempStr;
  int ret;
  int tmp_val;
+  std::ostringstream ss;

  switch (type) {
    case kDevDevID:
@@ -496,6 +496,12 @@ rsmi_dev_gpu_metrics_info_get(uint32_t dv_ind, rsmi_gpu_metrics_t *smu) {
  // a specific version.
  *smu = {};

+  uint8_t dev_content_revision = dev->gpu_metrics_ver().content_revision;
+  if (dev_content_revision != RSMI_GPU_METRICS_API_CONTENT_VER_1 ||
+      dev_content_revision != RSMI_GPU_METRICS_API_CONTENT_VER_2 ||
+      dev_content_revision != RSMI_GPU_METRICS_API_CONTENT_VER_3) {
+    return RSMI_STATUS_NOT_SUPPORTED;
+  }
  if (dev->gpu_metrics_ver().content_revision ==
                                        RSMI_GPU_METRICS_API_CONTENT_VER_1) {
        ret = GetDevBinaryBlob(amd::smi::kDevGpuMetrics, dv_ind,
@@ -971,5 +971,26 @@ int get_gpu_id(uint32_t node, uint64_t *gpu_id) {
  return retVal;
 }

+// /sys/class/kfd/kfd/topology/nodes/*/properties | grep gfx_target_version
+int KFDNode::get_gfx_target_version(uint64_t *gfx_target_version) {
+  std::ostringstream ss;
+  std::string properties_path = "/sys/class/kfd/kfd/topology/nodes/"
+    + std::to_string(this->node_indx_) + "/properties";
+  uint64_t gfx_version = 0;
+  int ret = read_node_properties(this->node_indx_, "gfx_target_version",
+                                 &gfx_version);
+  *gfx_target_version = gfx_version;
+  ss << __PRETTY_FUNCTION__
+     << " | File: " << properties_path
+     << " | Successfully read node #" << std::to_string(this->node_indx_)
+     << " for gfx_target_version"
+     << " | Data (gfx_target_version) *gfx_target_version = "
+     << std::to_string(*gfx_target_version)
+     << " | return = " << std::to_string(ret)
+     << " | ";
+  LOG_DEBUG(ss);
+  return ret;
+}
+
 }  // namespace smi
 }  // namespace amd
@@ -445,6 +445,12 @@ RocmSMI::Initialize(uint64_t flags) {
    // store each device boot partition state, if file doesn't exist
    dev->storeDevicePartitions(dv_ind);
  }
+
+  // Assists displaying GPU information after device enumeration
+  // Otherwise GPU related info will not be discoverable
+  if (ROCmLogging::Logger::getInstance()->isLoggerEnabled()) {
+    logSystemDetails();
+  }
  // Leaving below to help debug temp file issues
  // displayAppTmpFilesContent();
  std::string amdGPUDeviceList = displayAllDevicePaths(devices_);
@@ -599,9 +599,19 @@ std::tuple<bool, std::string> readTmpFile(uint32_t dv_ind,
 }

 // wrapper to return string expression of a rsmi_status_t return
-std::string getRSMIStatusString(rsmi_status_t ret) {
+// rsmi_status_t ret - return value of RSMI API function
+// bool fullStatus - defaults to true, set to false to chop off description
+// Returns:
+// string - if fullStatus == true, returns full decription of return value
+//      ex. 'RSMI_STATUS_SUCCESS: The function has been executed successfully.'
+// string - if fullStatus == false, returns a minimalized return value
+//      ex. 'RSMI_STATUS_SUCCESS'
+std::string getRSMIStatusString(rsmi_status_t ret, bool fullStatus) {
  const char *err_str;
  rsmi_status_string(ret, &err_str);
+  if (!fullStatus) {
+    return splitString(std::string(err_str), ':');
+  }
  return std::string(err_str);
 }

@@ -620,9 +630,13 @@ std::string getRSMIStatusString(rsmi_status_t ret) {
 // Expressed as big endian or little endian.
 // Big Endian (BE), multi-bit symbols encoded as big endian (MSB first)
 // Little Endian (LE), multi-bit symbols encoded as little endian (LSB first)
+// string rocm_lib_path = Path to library
+// string rocm_build_type = Release or debug
+// string rocm_build_date = Creation date of library
+// string dev_gfx_versions = GPU target graphics version
 std::tuple<bool, std::string, std::string, std::string, std::string,
           std::string, std::string, std::string, std::string,
-           std::string, std::string, std::string>
+           std::string, std::string, std::string, std::string, std::string>
           getSystemDetails(void) {
  struct utsname buf;
  bool errorDetected = false;
@@ -637,7 +651,9 @@ std::tuple<bool, std::string, std::string, std::string, std::string,
  std::string endianness = "<undefined>";
  std::string rocm_lib_path = "<undefined>";
  std::string rocm_build_type = "<undefined>";
+  std::string rocm_build_date = "<undefined>";
  std::string rocm_env_variables = "<undefined>";
+  std::string dev_gfx_versions = "<undefined>";

  if (uname(&buf) < 0) {
    errorDetected = true;
@@ -674,11 +690,20 @@ std::tuple<bool, std::string, std::string, std::string, std::string,
  }
  rocm_build_type = getBuildType();
  rocm_lib_path = getMyLibPath();
+  rocm_build_date = getFileCreationDate(rocm_lib_path);
  rocm_env_variables = RocmSMI::getInstance().getRSMIEnvVarInfo();
+  std::queue<std::string> devGraphicsVersions = getAllDeviceGfxVers();
+  if (devGraphicsVersions.empty() == false) {
+    dev_gfx_versions = "";
+    while (devGraphicsVersions.empty() == false) {
+      dev_gfx_versions += "\n\t" + devGraphicsVersions.front();
+      devGraphicsVersions.pop();
+    }
+  }
  return std::make_tuple(errorDetected, sysname, nodename, release,
                         version, machine, domainName, os_distribution,
                         endianness, rocm_build_type, rocm_lib_path,
-                         rocm_env_variables);
+                         rocm_build_date, rocm_env_variables, dev_gfx_versions);
 }

 // If logging is enabled through RSMI_LOGGING environment variable.
@@ -687,10 +712,11 @@ void logSystemDetails(void) {
  std::ostringstream ss;
  bool errorDetected;
  std::string sysname, node, release, version, machine, domain, distName,
-              endianness, rocm_build_type, lib_path, rocm_env_vars;
+              endianness, rocm_build_type, lib_path, build_date, rocm_env_vars,
+              dev_gfx_versions;
  std::tie(errorDetected, sysname, node, release, version, machine, domain,
-           distName, endianness, rocm_build_type, lib_path,
-           rocm_env_vars) = getSystemDetails();
+           distName, endianness, rocm_build_type, lib_path, build_date,
+           rocm_env_vars, dev_gfx_versions) = getSystemDetails();
  if (errorDetected == false) {
    ss << "====== Gathered system details ============\n"
       << "SYSTEM NAME: " << sysname << "\n"
@@ -703,7 +729,9 @@ void logSystemDetails(void) {
       << "ENDIANNESS: " << endianness << "\n"
       << "ROCM BUILD TYPE: " << rocm_build_type << "\n"
       << "ROCM-SMI-LIB PATH: " << lib_path << "\n"
-       << "ROCM ENV VARIABLES: " << rocm_env_vars << "\n";
+       << "ROCM-SMI-LIB BUILD DATE: " << build_date << "\n"
+       << "ROCM ENV VARIABLES: " << rocm_env_vars
+       << "AMD GFX VERSIONS: " << dev_gfx_versions << "\n";
    LOG_INFO(ss);
  } else {
    ss << "====== Gathered system details ============\n"
@@ -831,6 +859,13 @@ std::string getMyLibPath(void) {
  return path;
 }

+std::string getFileCreationDate(std::string path) {
+  struct stat t_stat;
+  stat(path.c_str(), &t_stat);
+  struct tm *timeinfo = localtime(&t_stat.st_ctime); // NOLINT
+  return removeNewLines(std::string(asctime(timeinfo))); // NOLINT
+}
+
 rsmi_status_t getBDFString(uint64_t bdf_id, std::string& bfd_str)
 {
  auto result = rsmi_status_t::RSMI_STATUS_SUCCESS;
@@ -974,5 +1009,164 @@ std::string power_type_string(RSMI_POWER_TYPE type) {
  return powerTypesToString.at(type);
 }

+std::string splitString(std::string str, char delim) {
+  std::vector<std::string> tokens;
+  std::stringstream ss(str);
+  std::string token;
+
+  if (str.empty()) {
+    return "";
+  }
+
+  while (std::getline(ss, token, delim)) {
+    tokens.push_back(token);
+    return token;  // return 1st match
+  }
+}
+
+static std::string pt_rng_Mhz(std::string title, rsmi_range *r) {
+  std::ostringstream ss;
+  if (r == nullptr) {
+    ss << "pt_rng_Mhz | rsmi_range r = nullptr\n";
+    return ss.str();
+  }
+
+  ss << title;
+  ss << r->lower_bound/1000000 << " to "
+     << r->upper_bound/1000000 << " MHz" << "\n";
+  return ss.str();
+}
+
+static std::string pt_rng_mV(std::string title, rsmi_range *r) {
+  std::ostringstream ss;
+  if (r == nullptr) {
+    ss << "pt_rng_mV | rsmi_range r = nullptr\n";
+    return ss.str();
+  }
+
+  ss << title;
+  ss << r->lower_bound << " to " << r->upper_bound
+     << " mV" << "\n";
+  return ss.str();
+}
+
+static std::string print_pnt(rsmi_od_vddc_point_t *pt) {
+  std::ostringstream ss;
+  ss << "\t\t** Frequency: " << pt->frequency/1000000 << " MHz\n";
+  ss << "\t\t** Voltage: " << pt->voltage << " mV\n";
+  return ss.str();
+}
+static std::string pt_vddc_curve(rsmi_od_volt_curve *c) {
+  std::ostringstream ss;
+  if (c == nullptr) {
+    ss << "pt_vddc_curve | rsmi_od_volt_curve c = nullptr\n";
+    return ss.str();
+  }
+
+  for (uint32_t i = 0; i < RSMI_NUM_VOLTAGE_CURVE_POINTS; ++i) {
+    ss << print_pnt(&c->vc_points[i]);
+  }
+  return ss.str();
+}
+
+std::string print_rsmi_od_volt_freq_data_t(rsmi_od_volt_freq_data_t *odv) {
+  std::ostringstream ss;
+  if (odv == nullptr) {
+    ss << "rsmi_od_volt_freq_data_t odv = nullptr\n";
+    return ss.str();
+  }
+
+  ss << pt_rng_Mhz("\t**Current SCLK frequency range: ", &odv->curr_sclk_range);
+  ss << pt_rng_Mhz("\t**Current MCLK frequency range: ", &odv->curr_mclk_range);
+  ss << pt_rng_Mhz("\t**Min/Max Possible SCLK frequency range: ",
+                   &odv->sclk_freq_limits);
+  ss << pt_rng_Mhz("\t**Min/Max Possible MCLK frequency range: ",
+                   &odv->mclk_freq_limits);
+
+  ss << "\t**Current Freq/Volt. curve: " << "\n";
+  ss << pt_vddc_curve(&odv->curve);
+
+  ss << "\t**Number of Freq./Volt. regions: " << odv->num_regions << "\n\n";
+  return ss.str();
+}
+
+std::string print_odv_region(rsmi_freq_volt_region_t *region) {
+  std::ostringstream ss;
+  ss << pt_rng_Mhz("\t\tFrequency range: ", &region->freq_range);
+  ss << pt_rng_mV("\t\tVoltage range: ", &region->volt_range);
+  return ss.str();
+}
+
+std::string print_rsmi_od_volt_freq_regions(uint32_t num_regions,
+                                            rsmi_freq_volt_region_t *regions) {
+  std::ostringstream ss;
+  if (regions == nullptr) {
+    ss << "rsmi_freq_volt_region_t regions = nullptr\n";
+    return ss.str();
+  }
+  for (uint32_t i = 0; i < num_regions; ++i) {
+    ss << "\tRegion " << i << ": " << "\n";
+    ss << print_odv_region(&regions[i]);
+  }
+  return ss.str();
+}
+
+bool is_sudo_user() {
+  std::ostringstream ss;
+  bool isRunningWithSudo = false;
+  auto myUID = getuid();
+  auto myPrivledges = geteuid();
+  if ((myUID == myPrivledges) && (myPrivledges == 0)) {
+    isRunningWithSudo = true;
+  }
+  ss << __PRETTY_FUNCTION__ << (isRunningWithSudo ? " | running as sudoer" :
+     " | NOT running as sudoer");
+  LOG_DEBUG(ss);
+  return isRunningWithSudo;
+}
+
+rsmi_status_t rsmi_get_gfx_target_version(uint32_t dv_ind,
+  std::string *gfx_version) {
+  std::ostringstream ss;
+  uint64_t kfd_gfx_version = 0;
+  GET_DEV_AND_KFDNODE_FROM_INDX
+
+  int ret = kfd_node->get_gfx_target_version(&kfd_gfx_version);
+  if (ret == 0) {
+    ss << "gfx" << kfd_gfx_version;
+    *gfx_version = ss.str();
+    return RSMI_STATUS_SUCCESS;
+  } else {
+    *gfx_version = "Unknown";
+    return RSMI_STATUS_NOT_SUPPORTED;
+  }
+}
+
+std::queue<std::string> getAllDeviceGfxVers() {
+  uint32_t num_monitor_devs = 0;
+  rsmi_status_t ret;
+  std::queue<std::string> deviceGfxVersions;
+  std::string response = "";
+  std::string dev_gfx_ver = "";
+
+  ret = rsmi_num_monitor_devices(&num_monitor_devs);
+  if (ret != RSMI_STATUS_SUCCESS || num_monitor_devs == 0) {
+    response = "N/A - No AMD devices detected";
+    deviceGfxVersions.push(response);
+    return deviceGfxVersions;
+  }
+
+  for (uint32_t i = 0; i < num_monitor_devs; ++i) {
+    ret = amd::smi::rsmi_get_gfx_target_version(i , &dev_gfx_ver);
+    response = "Device[" + std::to_string(i) + "]: ";
+    if (ret != RSMI_STATUS_SUCCESS) {
+      deviceGfxVersions.push(response + getRSMIStatusString(ret, false));
+    } else {
+      deviceGfxVersions.push(response + std::string(dev_gfx_ver));
+    }
+  }
+  return deviceGfxVersions;
+}
+
 }  // namespace smi
 }  // namespace amd