[SWDEV-488276/SWDEV-497613] Update memory partition set functionality

Changes: - [CLI] Added warning screen to AMD SMI users setting memory partition - [CLI] Added a progress bar time-bar for CLI sets display to 40 seconds - [API] Updated to wait until the driver reloads with SYSFS files active - [CLI] Now users can set or reset without providing: amd-smi set -g all <set arguments> or amd-smi reset -g all <set arguments> now can directly call -> sudo amd-smi set <set arguments> or sudo amd-smi reset <set arguments> - [SWDEV-475712][CLI/API] Fixed target_graphics_version field not properly displaying for older MI or Navi ASICs. - [All APIs] Added a catch for the driver to report invalid arguments now these APIs will show AMDSMI_STATUS_INVAL (ex. changing to NPS8 if the device does not support it) - [Install] Modified paths for Python install commands to support multi-ROCm installs Change-Id: Id11f25d68a82d23c6b2d77ccb30b51e860dd0ca7 Signed-off-by: Charis Poag <Charis.Poag@amd.com>
2024-11-08 17:31:25 -06:00
parent 19cc4718c0
commit 3ea4a42a6e
24 changed files with 1711 additions and 726 deletions
@@ -5729,12 +5729,22 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS
  DEVICE_MUTEX
+  const uint32_t kMaxBoardLength = 128;
  bool isCorrectDevice = false;
-  char boardName[128];
+  char boardName[kMaxBoardLength];
  boardName[0] = '\0';
+
+  const uint32_t kMaxMemoryCapabilitiesSize = 30;
+  char available_memory_capabilities[kMaxMemoryCapabilitiesSize];
+  available_memory_capabilities[0] = '\0';
+
+  const uint32_t kMaxCurrentMemoryMode = 5;
+  char current_memory_mode[kMaxCurrentMemoryMode];
+  current_memory_mode[0] = '\0';
+
  // rsmi_dev_memory_partition_set is only available for for discrete variant,
  // others are required to update through bios settings
-  rsmi_dev_name_get(dv_ind, boardName, 128);
+  rsmi_dev_name_get(dv_ind, boardName, static_cast<size_t>(kMaxBoardLength));
  std::string myBoardName = boardName;
  if (!myBoardName.empty()) {
    std::transform(myBoardName.begin(), myBoardName.end(), myBoardName.begin(),
@@ -5747,18 +5757,19 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,

  if (!isCorrectDevice) {
    ss << __PRETTY_FUNCTION__
-         << " | ======= end ======= "
-         << " | Fail "
-         << " | Device #: " << dv_ind
-         << " | Type: "
-         << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
-         << " | Cause: device board name does not support this action"
-         << " | Returning = "
-         << getRSMIStatusString(RSMI_STATUS_NOT_SUPPORTED) << " |";
+       << " | ======= end ======= "
+       << " | Fail "
+       << " | Device #: " << dv_ind
+       << " | Type: "
+       << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
+       << " | Cause: device board name does not support this action"
+       << " | Returning = "
+       << getRSMIStatusString(RSMI_STATUS_NOT_SUPPORTED, false);
      LOG_ERROR(ss);
    return RSMI_STATUS_NOT_SUPPORTED;
  }

+  // Is the current mode already what user requested?
  switch (memory_partition) {
    case RSMI_MEMORY_PARTITION_NPS1:
    case RSMI_MEMORY_PARTITION_NPS2:
@@ -5775,7 +5786,7 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
         << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
         << " | Cause: requested setting was invalid"
         << " | Returning = "
-         << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS) << " |";
+         << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS, false);
      LOG_ERROR(ss);
      return RSMI_STATUS_INVALID_ARGS;
  }
@@ -5797,7 +5808,7 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
       << " | Cause: could retrieve current memory partition or retrieved"
       << " unexpected data"
       << " | Returning = "
-       << getRSMIStatusString(ret_get) << " |";
+       << getRSMIStatusString(ret_get, false);
    LOG_ERROR(ss);
    return ret_get;
  }
@@ -5813,11 +5824,52 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
     << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
     << " | Data: " << newMemoryPartition
     << " | Returning = "
-     << getRSMIStatusString(RSMI_STATUS_SUCCESS) << " |";
+     << getRSMIStatusString(RSMI_STATUS_SUCCESS, false);
    LOG_TRACE(ss);
    return RSMI_STATUS_SUCCESS;
  }

+  // is this an available mode to set to?
+  std::string memory_capabilities_str = "unknown";
+  std::string user_requested_memory_partition = newMemoryPartition;
+  std::transform(user_requested_memory_partition.begin(), user_requested_memory_partition.end(),
+                  user_requested_memory_partition.begin(), ::toupper);
+  rsmi_status_t caps_ret = rsmi_dev_memory_partition_capabilities_get(dv_ind,
+                              available_memory_capabilities, kMaxMemoryCapabilitiesSize);
+  memory_capabilities_str = available_memory_capabilities;
+  std::transform(memory_capabilities_str.begin(), memory_capabilities_str.end(),
+                  memory_capabilities_str.begin(), ::toupper);
+  ss << __PRETTY_FUNCTION__ << " | user_requested_memory_partition: "
+     << user_requested_memory_partition
+     << "; memory_capabilities_str: " << memory_capabilities_str
+     << "; rsmi_dev_memory_partition_capabilities_get(" << dv_ind
+     << ", " << user_requested_memory_partition << "): return = "
+     << amd::smi::getRSMIStatusString(caps_ret, false);
+  LOG_DEBUG(ss);
+  if ((caps_ret == RSMI_STATUS_SUCCESS)
+      && (!memory_capabilities_str.empty())
+      && (!user_requested_memory_partition.empty())) {
+    bool is_available_mode = amd::smi::containsString(memory_capabilities_str,
+                                user_requested_memory_partition, true);
+    ss << __PRETTY_FUNCTION__
+       << " | is_available_mode: " << (is_available_mode ? "True": "False");
+    LOG_DEBUG(ss);
+    if (is_available_mode == false) {  // report RSMI_STATUS_INVALID_ARGS
+      ss << __PRETTY_FUNCTION__
+         << " | ======= Check if available mode ======= "
+         << " | WARNING: detected invalid mode to set to, will try to set anyways"
+         << " | Device #: " << dv_ind
+         << " | Type: "
+         << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
+         << " | Data (user requested mode): " << user_requested_memory_partition
+         << " | Available Memory Partition Modes: " << memory_capabilities_str
+         << " | Cause: requested setting was not an available mode"
+         << " | Returning = "
+         << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS, false);
+      LOG_INFO(ss);
+    }
+  }
+
  GET_DEV_FROM_INDX
  int ret = dev->writeDevInfo(amd::smi::kDevMemoryPartition,
                              newMemoryPartition);
@@ -5835,7 +5887,7 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
       << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
       << " | Cause: issue writing reqested setting of " + newMemoryPartition
       << " | Returning = "
-       << getRSMIStatusString(err) << " |";
+       << getRSMIStatusString(err, false);
    LOG_ERROR(ss);
    return err;
  }
@@ -5849,8 +5901,76 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
     << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
     << " | Data: " << newMemoryPartition
     << " | Returning = "
-     << getRSMIStatusString(restartRet) << " |";
+     << getRSMIStatusString(restartRet, false);
  LOG_TRACE(ss);
+  if (restartRet != RSMI_STATUS_SUCCESS) {
+    ss << __PRETTY_FUNCTION__
+       << " | ======= end ======= "
+       << " | Fail - restart AMD GPU detected"
+       << " | Device #: " << dv_ind
+       << " | Type: "
+       << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
+       << " | Cause: issue writing reqested setting of " + newMemoryPartition
+       << " | Returning = "
+       << getRSMIStatusString(restartRet, false);
+    LOG_ERROR(ss);
+    return restartRet;
+  }
+
+  std::string current_memory_mode_str = "unknown";
+  rsmi_status_t can_read_sysfs_again = RSMI_STATUS_AMDGPU_RESTART_ERR;
+  int maxWaitSeconds = 10;
+  const int k1000_MS_WAIT = 1000;
+  // wait until we can read SYSFS again
+  if (restartRet == RSMI_STATUS_SUCCESS) {
+    while (current_memory_mode_str != user_requested_memory_partition) {
+      maxWaitSeconds -= 1;
+      can_read_sysfs_again =
+        rsmi_dev_memory_partition_get(dv_ind, current_memory_mode, kMaxCurrentMemoryMode);
+      if (can_read_sysfs_again == RSMI_STATUS_SUCCESS) {
+        current_memory_mode_str.clear();
+        current_memory_mode_str = current_memory_mode;
+        ss << __PRETTY_FUNCTION__
+           << " | ======= rsmi_dev_memory_partition_get ======= "
+           << " | Success - can read SYSFS"
+           << " | Device #: " << dv_ind
+           << " | Type: "
+           << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
+           << " | Data (user requested mode): " << user_requested_memory_partition
+           << " | Current Memory Partition Mode: " << current_memory_mode_str
+           << " | Available Memory Partition Modes: " << memory_capabilities_str
+           << " | total wait time (sec): " << (10 - maxWaitSeconds)
+           << " | Returning = "
+           << getRSMIStatusString(can_read_sysfs_again, false);
+        LOG_TRACE(ss);
+        if (!current_memory_mode_str.empty()
+            && (current_memory_mode_str == user_requested_memory_partition)) {
+          break;
+        }
+      }
+      amd::smi::system_wait(k1000_MS_WAIT);
+    }
+  }
+
+  if (current_memory_mode_str == user_requested_memory_partition) {
+    restartRet = RSMI_STATUS_SUCCESS;
+  } else {
+    restartRet = RSMI_STATUS_AMDGPU_RESTART_ERR;
+  }
+
+  ss << __PRETTY_FUNCTION__
+     << " | ======= end ======= "
+     << " | Success - completed driver restart and all SYSFS are active"
+     << " | Device #: " << dv_ind
+     << " | Type: "
+     << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
+     << " | Data: " << user_requested_memory_partition
+     << " | Current Memory Partition Mode: " << current_memory_mode_str
+     << " | Available Memory Partition Modes: " << memory_capabilities_str
+     << " | Returning = "
+     << getRSMIStatusString(restartRet, false);
+  LOG_TRACE(ss);
+
  return restartRet;
  CATCH
 }
@@ -5927,79 +6047,73 @@ rsmi_dev_memory_partition_get(uint32_t dv_ind, char *memory_partition,
  CATCH
 }

-rsmi_status_t rsmi_dev_compute_partition_reset(uint32_t dv_ind) {
+rsmi_status_t rsmi_dev_memory_partition_capabilities_get(
+      uint32_t dv_ind, char *memory_partition_caps, uint32_t len) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << " | ======= start =======, " << dv_ind;
  LOG_TRACE(ss);
-  REQUIRE_ROOT_ACCESS
+
+  if ((len == 0) || (memory_partition_caps == nullptr)) {
+    ss << __PRETTY_FUNCTION__
+       << " | ======= end ======= "
+       << " | Fail "
+       << " | Device #: " << dv_ind
+       << " | Type: "
+       << amd::smi::Device::get_type_string(amd::smi::kDevAvailableMemoryPartition)
+       << " | Cause: user sent invalid arguments, len = 0 or memory_partition_caps"
+       << " was a null ptr"
+       << " | Returning = "
+       << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS, false);
+    LOG_ERROR(ss);
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+  CHK_SUPPORT_NAME_ONLY(memory_partition_caps)
  DEVICE_MUTEX
-  GET_DEV_FROM_INDX
-  rsmi_status_t ret = RSMI_STATUS_NOT_SUPPORTED;

-  // Only use 1st index, rest are there in-case of future issues
-  // NOTE: Partitions sets cause rocm-smi indexes to fluctuate
-  // since the nodes are grouped in respect to primary node - why we only use
-  // 1st node/device id to reset
-  std::string bootState =
-          dev->readBootPartitionState<rsmi_compute_partition_type_t>(0);
+  std::string availableMemoryPartitions;
+  rsmi_status_t ret =
+      get_dev_value_line(amd::smi::kDevAvailableMemoryPartition,
+                         dv_ind, &availableMemoryPartitions);
+  if (ret != RSMI_STATUS_SUCCESS) {
+    ss << __PRETTY_FUNCTION__
+       << " | ======= end ======= "
+       << " | FAIL "
+       << " | Device #: " << dv_ind
+       << " | Type: "
+       << amd::smi::Device::get_type_string(amd::smi::kDevAvailableMemoryPartition)
+       << " | Data: could not retrieve requested data"
+       << " | Returning = "
+       << getRSMIStatusString(ret, false);
+    LOG_ERROR(ss);
+    return ret;
+  }

-  // Initiate reset
-  // If bootState is UNKNOWN, we cannot reset - return RSMI_STATUS_NOT_SUPPORTED
-  // Likely due to device not supporting it
-  if (bootState != "UNKNOWN") {
-    rsmi_compute_partition_type_t compute_partition =
-      mapStringToRSMIComputePartitionTypes.at(bootState);
-    ret = rsmi_dev_compute_partition_set(dv_ind, compute_partition);
+  std::size_t length = availableMemoryPartitions.copy(memory_partition_caps, len-1);
+  memory_partition_caps[length]='\0';
+
+  if (len < (availableMemoryPartitions.size() + 1)) {
+    ss << __PRETTY_FUNCTION__
+       << " | ======= end ======= "
+       << " | Fail "
+       << " | Device #: " << dv_ind
+       << " | Type: "
+       << amd::smi::Device::get_type_string(amd::smi::kDevAvailableMemoryPartition)
+       << " | Cause: requested size was insufficient"
+       << " | Returning = "
+       << getRSMIStatusString(RSMI_STATUS_INSUFFICIENT_SIZE, false);
+    LOG_ERROR(ss);
+    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }
  ss << __PRETTY_FUNCTION__
     << " | ======= end ======= "
-     << " | Success - if original boot state was not unknown or valid setting"
+     << " | Success "
     << " | Device #: " << dv_ind
     << " | Type: "
-     << amd::smi::Device::get_type_string(amd::smi::kDevComputePartition)
-     << " | Data: " << bootState
+     << amd::smi::Device::get_type_string(amd::smi::kDevAvailableMemoryPartition)
+     << " | Data: " << memory_partition_caps
     << " | Returning = "
-     << getRSMIStatusString(ret) << " |";
-  LOG_TRACE(ss);
-  return ret;
-  CATCH
-}
-
-rsmi_status_t rsmi_dev_memory_partition_reset(uint32_t dv_ind) {
-  TRY
-  std::ostringstream ss;
-  ss << __PRETTY_FUNCTION__ << "| ======= start =======, " << dv_ind;
-  LOG_TRACE(ss);
-  REQUIRE_ROOT_ACCESS
-  DEVICE_MUTEX
-  GET_DEV_FROM_INDX
-  rsmi_status_t ret = RSMI_STATUS_NOT_SUPPORTED;
-
-  // Only use 1st index, rest are there in-case of future issues
-  // NOTE: Partitions sets cause rocm-smi indexes to fluctuate.
-  // Since the nodes are grouped in respect to primary node - why we only use
-  // 1st node/device id to reset
-  std::string bootState =
-          dev->readBootPartitionState<rsmi_memory_partition_type_t>(0);
-
-  // Initiate reset
-  // If bootState is UNKNOWN, we cannot reset - return RSMI_STATUS_NOT_SUPPORTED
-  // Likely due to device not supporting it
-  if (bootState != "UNKNOWN") {
-    rsmi_memory_partition_type_t memory_partition =
-      mapStringToMemoryPartitionTypes.at(bootState);
-    ret = rsmi_dev_memory_partition_set(dv_ind, memory_partition);
-  }
-  ss << __PRETTY_FUNCTION__
-     << " | ======= end ======= "
-     << " | Success - if original boot state was not unknown or valid setting"
-     << " | Device #: " << dv_ind
-     << " | Type: "
-     << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
-     << " | Data: " << bootState
-     << " | Returning = "
-     << getRSMIStatusString(ret) << " |";
+     << getRSMIStatusString(ret, false);
  LOG_TRACE(ss);
  return ret;
  CATCH
@@ -140,6 +140,7 @@ static const char *kDevAvailableComputePartitionFName =
                  "available_compute_partition";
 static const char *kDevComputePartitionFName = "current_compute_partition";
 static const char *kDevMemoryPartitionFName = "current_memory_partition";
+static const char *kDevAvailableMemoryPartitionFName = "available_memory_partition";

 // Firmware version files
 static const char *kDevFwVersionAsdFName = "fw_version/asd_fw_version";
@@ -328,6 +329,7 @@ static const std::map<DevInfoTypes, const char *> kDevAttribNameMap = {
    {kDevAvailableComputePartition, kDevAvailableComputePartitionFName},
    {kDevComputePartition, kDevComputePartitionFName},
    {kDevMemoryPartition, kDevMemoryPartitionFName},
+    {kDevAvailableMemoryPartition, kDevAvailableMemoryPartitionFName},
 };

 static const std::map<rsmi_dev_perf_level, const char *> kDevPerfLvlMap = {
@@ -479,6 +481,7 @@ Device::devInfoTypesStrings = {
  {kDevAvailableComputePartition, "kDevAvailableComputePartition"},
  {kDevComputePartition, "kDevComputePartition"},
  {kDevMemoryPartition, "kDevMemoryPartition"},
+  {kDevAvailableMemoryPartition, "kDevAvailableMemoryPartition"},
  {kDevPCieVendorID, "kDevPCieVendorID"},
  {kDevSocPstate, "kDevSocPstate"},
  {kDevXgmiPlpd, "kDevXgmiPlpd"},
@@ -1308,6 +1311,7 @@ int Device::readDevInfo(DevInfoTypes type, std::string *val) {
    case kDevMemoryPartition:
    case kDevNumaNode:
    case kDevXGMIPhysicalID:
+    case kDevAvailableMemoryPartition:
    case kDevProcessIsolation:
      return readDevInfoStr(type, val);
      break;
@@ -1486,10 +1490,15 @@ bool Device::DeviceAPISupported(std::string name, uint64_t variant,

 rsmi_status_t Device::restartAMDGpuDriver(void) {
  REQUIRE_ROOT_ACCESS
+  std::ostringstream ss;
  bool restartSuccessful = true;
  bool success = false;
  std::string out;
  bool wasGdmServiceActive = false;
+  bool restartInProgress = true;
+  bool isRestartInProgress = true;
+  bool isAMDGPUModuleLive = false;
+  std::string captureRestartErr;

  // sudo systemctl is-active gdm
  // we do not care about the success of checking if gdm is active
@@ -1498,8 +1507,8 @@ rsmi_status_t Device::restartAMDGpuDriver(void) {
                         (restartSuccessful = true);

  // if gdm is active -> sudo systemctl stop gdm
-  // TODO: are are there other display manager's we need to take into account?
-  // see https://en.wikipedia.org/wiki/GNOME_Display_Manager
+  // TODO(AMD_SMI_team): are are there other display manager's we need to take into account?
+  // see https://help.gnome.org/admin/gdm/stable/overview.html.en_GB
  if (success && (out == "active")) {
    wasGdmServiceActive = true;
    std::tie(success, out) = executeCommand("systemctl stop gdm&", false);
@@ -1509,8 +1518,13 @@ rsmi_status_t Device::restartAMDGpuDriver(void) {
  // sudo modprobe -r amdgpu
  // sudo modprobe amdgpu
  std::tie(success, out) =
-    executeCommand("modprobe -r amdgpu && modprobe amdgpu&", false);
+    executeCommand("modprobe -r amdgpu && modprobe amdgpu&", true);
  restartSuccessful &= success;
+  captureRestartErr = out;
+
+  if (success) {
+    restartSuccessful = false;
+  }

  // if gdm was active -> sudo systemctl start gdm
  if (wasGdmServiceActive) {
@@ -1518,7 +1532,61 @@ rsmi_status_t Device::restartAMDGpuDriver(void) {
    restartSuccessful &= success;
  }

-  return (restartSuccessful ? RSMI_STATUS_SUCCESS :
+  // Return early if there was an issue restarting amdgpu
+  if (!restartSuccessful) {
+    ss << __PRETTY_FUNCTION__ << " | [WARNING] Issue found during amdgpu restart: "
+    << captureRestartErr << "; retartSuccessful: " << (restartSuccessful ? "True" : "False");
+    LOG_INFO(ss);
+    return RSMI_STATUS_AMDGPU_RESTART_ERR;
+  }
+
+  // wait for amdgpu module to come back up
+  rsmi_status_t status = Device::isRestartInProgress(&isRestartInProgress,
+                                                    &isAMDGPUModuleLive);
+  const int kTimeToWaitForDriverMSec = 1000;
+  int maxLoops = 10;  // wait a max of 10 sec
+  while (status != RSMI_STATUS_SUCCESS) {
+    maxLoops -= 1;
+    if (maxLoops == 0) {
+      break;
+    }
+    amd::smi::system_wait(kTimeToWaitForDriverMSec);
+    status = Device::isRestartInProgress(&isRestartInProgress,
+                                         &isAMDGPUModuleLive);
+  }
+
+  return ((restartSuccessful && (!isRestartInProgress && isAMDGPUModuleLive)) ?
+          RSMI_STATUS_SUCCESS :
+          RSMI_STATUS_AMDGPU_RESTART_ERR);
+}
+
+rsmi_status_t Device::isRestartInProgress(bool *isRestartInProgress,
+                                          bool *isAMDGPUModuleLive) {
+  REQUIRE_ROOT_ACCESS
+  std::ostringstream ss;
+  bool restartSuccessful = true;
+  bool success = false;
+  std::string out;
+  bool deviceRestartInProgress = true;    // Assume in progress, we intend to disprove
+  bool isSystemAMDGPUModuleLive = false;  // Assume AMD GPU module is not live,
+                                          //  we intend to disprove
+
+  // wait for amdgpu module to come back up
+  std::tie(success, out) = executeCommand("cat /sys/module/amdgpu/initstate", true);
+  ss << __PRETTY_FUNCTION__
+     << " | success = " << success
+     << " | out = " << out;
+  LOG_DEBUG(ss);
+  if ((success == true) && (!out.empty())) {
+    isSystemAMDGPUModuleLive = containsString(out, "live");
+  }
+  if (isAMDGPUModuleLive) {
+    deviceRestartInProgress = false;
+  }
+  *isRestartInProgress = deviceRestartInProgress;
+  *isAMDGPUModuleLive = isSystemAMDGPUModuleLive;
+
+  return ((*isAMDGPUModuleLive && !*isRestartInProgress) ? RSMI_STATUS_SUCCESS :
          RSMI_STATUS_AMDGPU_RESTART_ERR);
 }

@@ -63,6 +63,7 @@
 #include <sstream>
 #include <string>
 #include <vector>
+#include <cmath>

 #include "rocm_smi/rocm_smi.h"
 #include "rocm_smi/rocm_smi_kfd.h"
@@ -357,6 +358,7 @@ rsmi_status_t ErrnoToRsmiStatus(int err) {
    case EIO:    return RSMI_STATUS_UNEXPECTED_SIZE;
    case ENXIO:  return RSMI_STATUS_UNEXPECTED_DATA;
    case EBUSY:  return RSMI_STATUS_BUSY;
+    case EINVAL: return RSMI_STATUS_INVALID_ARGS;
    default:     return RSMI_STATUS_UNKNOWN_ERROR;
  }
 }
@@ -429,14 +431,14 @@ std::pair<bool, std::string> executeCommand(std::string command, bool stdOut) {
  char buffer[128];
  std::string stdoutAndErr;
  bool successfulRun = true;
-  command = "stdbuf -i0 -o0 -e0 " + command; // remove stdOut and err buffering
+  command = "stdbuf -i0 -o0 -e0 " + command;  // remove stdOut and err buffering

  FILE *pipe = popen(command.c_str(), "r");
  if (!pipe) {
    stdoutAndErr = "[ERROR] popen failed to call " + command;
    successfulRun = false;
  } else {
-    //read until end of process
+    // read until end of process
    while (!feof(pipe)) {
      // use buffer to read and add to stdoutAndErr
      if (fgets(buffer, sizeof(buffer), pipe) != nullptr) {
@@ -459,8 +461,19 @@ std::pair<bool, std::string> executeCommand(std::string command, bool stdOut) {

 // originalString - string to search for substring
 // substring - string looking to find
-bool containsString(std::string originalString, std::string substring) {
-  return (originalString.find(substring) != std::string::npos);
+// displayComparisons = defaults to false, set to true to see debug prints
+bool containsString(std::string originalString, std::string substring,
+                  bool displayComparisons) {
+  std::ostringstream ss;
+  bool found = originalString.find(substring) != std::string::npos;
+  if (displayComparisons) {
+    ss << __PRETTY_FUNCTION__
+       << " | originalString: " << originalString
+       << " | substring: " << substring
+       << " | found: " << (found ? "True": "False");
+    LOG_TRACE(ss);
+  }
+  return found;
 }

 // Creates and stores supplied data into a temporary file (within /tmp/).
@@ -1217,7 +1230,9 @@ rsmi_status_t rsmi_get_gfx_target_version(uint32_t dv_ind, std::string *gfx_vers
    // separate out parts -> put back into normal graphics version format
    major = static_cast<uint64_t>((orig_target_version / 10000) * 100);
    minor = static_cast<uint64_t>((orig_target_version % 10000 / 100) * 10);
-    if (minor == 0) major *= 10;  // 0 as a minor is correct, but bump up by 10
+    if ((minor == 0) && (countDigit(major) < 4)) {
+      major *= 10;  // 0 as a minor is correct, but bump up by 10
+    }
    rev = static_cast<uint64_t>(orig_target_version % 100);
    *gfx_version = "gfx" + std::to_string(major + minor + rev);
    ss << __PRETTY_FUNCTION__
@@ -1278,6 +1293,31 @@ std::queue<std::string> getAllDeviceGfxVers() {
  return deviceGfxVersions;
 }

+// milli_seconds: time to wait, in milliseconds
+// 1 sec = 1000ms
+// .5 sec = 500ms
+void system_wait(int milli_seconds) {
+  std::ostringstream ss;
+  auto start = std::chrono::high_resolution_clock::now();
+  // 1 ms = 1000 us
+  int waitTime = milli_seconds * 1000;
+  ss << __PRETTY_FUNCTION__ << " | "
+     << "** Waiting for " << std::dec << waitTime
+     << " us (" << waitTime/1000 << " milli-seconds) **";
+  LOG_DEBUG(ss);
+  usleep(waitTime);
+  auto stop = std::chrono::high_resolution_clock::now();
+  auto duration =
+      std::chrono::duration_cast<std::chrono::microseconds>(stop - start);
+  ss << __PRETTY_FUNCTION__ << " | "
+     << "** Waiting took " << duration.count() / 1000
+     << " milli-seconds **";
+  LOG_DEBUG(ss);
+}
+
+int countDigit(uint64_t n) {
+  return static_cast<int>(std::floor(log10(n) + 1));
+}

 }  // namespace smi
 }  // namespace amd