SWDEV-342812- Add NPS support

Updates:
    * Added rsmi_dev_nps_mode_set and rsmi_dev_nps_mode_get
    * Added ability to set multiple SYSFS files in debug build
    * Added ability to see user's env variables set for debug build
    * Added tests for rsmi_dev_nps_mode_set and rsmi_dev_nps_mode_get
    * Added ability to restart AMD GPU driver, used in nps_mode_set
    * Updated ROCm_SMI_Manual.pdf to include new APIs
    * Added progress bar for long running python_smi_tools, used
      in setting nps_mode if runs longer than .1 seconds

Change-Id: I6d61bedd28d7cba6aff432ad2d127ba741b7d15a
Signed-off-by: Charis Poag <Charis.Poag@amd.com>

src/rocm_smi.cc

@@ -62,6 +62,7 @@
 #include <map>
 #include <fstream>
 #include <iostream>
+#include <tuple>
 
 #include "rocm_smi/rocm_smi_common.h"  // Should go before rocm_smi.h
 #include "rocm_smi/rocm_smi.h"
@@ -1696,6 +1697,7 @@ mapStringToRSMIComputePartitionTypes {
 
 std::map<rsmi_compute_partition_type_t, std::string>
 mapRSMIToStringComputePartitionTypes {
+  {RSMI_COMPUTE_PARTITION_INVALID, "UNKNOWN"},
   {RSMI_COMPUTE_PARTITION_CPX, "CPX"},
   {RSMI_COMPUTE_PARTITION_SPX, "SPX"},
   {RSMI_COMPUTE_PARTITION_DPX, "DPX"},
@@ -1703,6 +1705,23 @@ mapRSMIToStringComputePartitionTypes {
   {RSMI_COMPUTE_PARTITION_QPX, "QPX"}
 };
 
+std::map<rsmi_nps_mode_type_t, std::string>
+mapRSMIToStringNPSModeTypes {
+  {RSMI_MEMORY_PARTITION_UNKNOWN, "UNKNOWN"},
+  {RSMI_MEMORY_PARTITION_NPS1, "NPS1"},
+  {RSMI_MEMORY_PARTITION_NPS2, "NPS2"},
+  {RSMI_MEMORY_PARTITION_NPS4, "NPS4"},
+  {RSMI_MEMORY_PARTITION_NPS8, "NPS8"}
+};
+
+std::map<std::string, rsmi_nps_mode_type_t>
+mapStringToNPSModeTypes {
+  {"NPS1", RSMI_MEMORY_PARTITION_NPS1},
+  {"NPS2", RSMI_MEMORY_PARTITION_NPS2},
+  {"NPS4", RSMI_MEMORY_PARTITION_NPS4},
+  {"NPS8", RSMI_MEMORY_PARTITION_NPS8}
+};
+
 static std::string
 get_id_name_str_from_line(uint64_t id, std::string ln,
                                                  std::istringstream *ln_str) {
@@ -2780,71 +2799,84 @@ rsmi_status_string(rsmi_status_t status, const char **status_string) {
       break;
 
     case RSMI_STATUS_OUT_OF_RESOURCES:
-      *status_string = "Unable to acquire memory or other resource";
+      *status_string = "RSMI_STATUS_OUT_OF_RESOURCES: Unable to acquire memory "
+                       "or other resource";
       break;
 
     case RSMI_STATUS_INTERNAL_EXCEPTION:
-      *status_string = "An internal exception was caught";
+      *status_string = "RSMI_STATUS_INTERNAL_EXCEPTION: An internal exception "
+                       "was caught";
       break;
 
     case RSMI_STATUS_INPUT_OUT_OF_BOUNDS:
-      *status_string = "The provided input is out of allowable or safe range";
+      *status_string = "RSMI_STATUS_INPUT_OUT_OF_BOUNDS: The provided input is "
+                       "out of allowable or safe range";
       break;
 
     case RSMI_STATUS_INIT_ERROR:
-      *status_string = "An error occurred during initialization, during "
-       "monitor discovery or when when initializing internal data structures";
+      *status_string = "RSMI_STATUS_INIT_ERROR: An error occurred during "
+                       "initialization, during monitor discovery or when when "
+                       "initializing internal data structures";
       break;
 
     case RSMI_STATUS_NOT_YET_IMPLEMENTED:
-      *status_string = "The called function has not been implemented in this "
-        "system for this device type";
+      *status_string = "RSMI_STATUS_NOT_YET_IMPLEMENTED: The called function "
+                        "has not been implemented in this system for this "
+                        "device type";
       break;
 
     case RSMI_STATUS_NOT_FOUND:
-      *status_string = "An item required to complete the call was not found";
+      *status_string = "RSMI_STATUS_NOT_FOUND: An item required to complete "
+                       "the call was not found";
       break;
 
     case RSMI_STATUS_INSUFFICIENT_SIZE:
-      *status_string = "Not enough resources were available to fully execute"
-                             " the call";
+      *status_string = "RSMI_STATUS_INSUFFICIENT_SIZE: Not enough resources "
+                       "were available to fully execute the call";
       break;
 
     case RSMI_STATUS_INTERRUPT:
-      *status_string = "An interrupt occurred while executing the function";
+      *status_string = "RSMI_STATUS_INTERRUPT: An interrupt occurred while "
+                       "executing the function";
       break;
 
     case RSMI_STATUS_UNEXPECTED_SIZE:
-      *status_string = "Data (usually from reading a file) was out of"
-                                              " range from what was expected";
+      *status_string = "RSMI_STATUS_UNEXPECTED_SIZE: Data (usually from reading"
+                       " a file) was out of range from what was expected";
       break;
 
     case RSMI_STATUS_NO_DATA:
-      *status_string = "No data was found (usually from reading a file) "
-                                                    "where data was expected";
+      *status_string = "RSMI_STATUS_NO_DATA: No data was found (usually from "
+                       "reading a file) where data was expected";
       break;
 
     case RSMI_STATUS_UNEXPECTED_DATA:
-      *status_string = "Data (usually from reading a file) was not of the "
-                                                     "type that was expected";
+      *status_string = "RSMI_STATUS_UNEXPECTED_DATA: Data (usually from reading"
+                       " a file) was not of the type that was expected";
       break;
 
     case RSMI_STATUS_BUSY:
-      *status_string = "A resource or mutex could not be acquired "
-                                           "because it is already being used";
+      *status_string = "RSMI_STATUS_BUSY: A resource or mutex could not be "
+                        "acquired because it is already being used";
     break;
 
     case RSMI_STATUS_REFCOUNT_OVERFLOW:
-      *status_string = "An internal reference counter exceeded INT32_MAX";
+      *status_string = "RSMI_STATUS_REFCOUNT_OVERFLOW: An internal reference "
+                       "counter exceeded INT32_MAX";
+      break;
+
+    case RSMI_STATUS_AMDGPU_RESTART_ERR:
+      *status_string = "RSMI_STATUS_AMDGPU_RESTART_ERR: Could not successfully "
+                        "restart the amdgpu driver";
       break;
 
     case RSMI_STATUS_UNKNOWN_ERROR:
-      *status_string = "An unknown error prevented the call from completing"
-                          " successfully";
+      *status_string = "RSMI_STATUS_UNKNOWN_ERROR: An unknown error prevented "
+                       "the call from completing successfully";
       break;
 
     default:
-      *status_string = "An unknown error occurred";
+      *status_string = "RSMI_STATUS_UNKNOWN_ERROR: An unknown error occurred";
       return RSMI_STATUS_UNKNOWN_ERROR;
   }
   return RSMI_STATUS_SUCCESS;
@@ -3718,12 +3750,8 @@ rsmi_is_P2P_accessible(uint32_t dv_ind_src, uint32_t dv_ind_dst,
 static rsmi_status_t
 get_compute_partition(uint32_t dv_ind, std::string &compute_partition) {
   TRY
-  std::string val_str;
-
-  if (compute_partition.c_str() == nullptr) {
-    return RSMI_STATUS_INVALID_ARGS;
-  }
   CHK_SUPPORT_NAME_ONLY(compute_partition.c_str())
+  std::string val_str;
 
   DEVICE_MUTEX
   rsmi_status_t ret = get_dev_value_str(amd::smi::kDevComputePartition,
@@ -3811,14 +3839,18 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
   }
 
   // do nothing if compute_partition is the current compute partition
-  get_compute_partition(dv_ind, currentComputePartition);
+  rsmi_status_t ret_get = get_compute_partition(dv_ind, currentComputePartition);
+  // we can try to set, even if we get unexpected data
+  if (ret_get != RSMI_STATUS_SUCCESS
+      && ret_get != RSMI_STATUS_UNEXPECTED_DATA) {
+    return ret_get;
+  }
   rsmi_compute_partition_type_t currRSMIComputePartition
     = mapStringToRSMIComputePartitionTypes[currentComputePartition];
   if (currRSMIComputePartition == compute_partition) {
     return RSMI_STATUS_SUCCESS;
   }
 
-  newComputePartitionStr = mapRSMIToStringComputePartitionTypes[compute_partition];
   GET_DEV_FROM_INDX
   int ret = dev->writeDevInfo(amd::smi::kDevComputePartition,
                               newComputePartitionStr);
@@ -3826,6 +3858,134 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
   CATCH
 }
 
+static rsmi_status_t get_nps_mode(uint32_t dv_ind, std::string &nps_mode) {
+  TRY
+  CHK_SUPPORT_NAME_ONLY(nps_mode.c_str())
+  std::string val_str;
+
+  DEVICE_MUTEX
+  rsmi_status_t ret = get_dev_value_str(amd::smi::kDevMemoryPartition,
+                                        dv_ind, &val_str);
+
+  if (ret != RSMI_STATUS_SUCCESS) {
+    return ret;
+  }
+
+  switch (mapStringToNPSModeTypes[val_str]) {
+    case RSMI_MEMORY_PARTITION_UNKNOWN:
+      // Retrieved an unknown NPS mode
+      return RSMI_STATUS_UNEXPECTED_DATA;
+    case RSMI_MEMORY_PARTITION_NPS1:
+      break;
+    case RSMI_MEMORY_PARTITION_NPS2:
+      break;
+    case RSMI_MEMORY_PARTITION_NPS4:
+      break;
+    case RSMI_MEMORY_PARTITION_NPS8:
+      break;
+    default:
+      // Retrieved an unknown NPS mode
+      return RSMI_STATUS_UNEXPECTED_DATA;
+  }
+  nps_mode = val_str;
+  return RSMI_STATUS_SUCCESS;
+  CATCH
+}
+
+rsmi_status_t
+rsmi_dev_nps_mode_set(uint32_t dv_ind, rsmi_nps_mode_type_t nps_mode) {
+  TRY
+  REQUIRE_ROOT_ACCESS
+  DEVICE_MUTEX
+  bool isCorrectDevice = false;
+  char boardName[128];
+  boardName[0] = '\0';
+  // rsmi_dev_nps_mode_set is only available for for discrete variant,
+  // others are required to update through bios settings
+  rsmi_dev_name_get(dv_ind, boardName, 128);
+  std::string myBoardName = boardName;
+  if (!myBoardName.empty()) {
+    std::transform(myBoardName.begin(), myBoardName.end(), myBoardName.begin(),
+                   ::tolower);
+    if (myBoardName.find("mi") != std::string::npos &&
+        myBoardName.find("00x") != std::string::npos) {
+      isCorrectDevice = true;
+    }
+  }
+
+  if (isCorrectDevice == false) {
+    return RSMI_STATUS_NOT_SUPPORTED;
+  }
+
+  std::string newNPSMode
+              = mapRSMIToStringNPSModeTypes[nps_mode];
+  std::string currentNPSMode;
+
+  switch (nps_mode) {
+    case RSMI_MEMORY_PARTITION_UNKNOWN:
+      // Retrieved an unknown NPS mode
+      return RSMI_STATUS_INVALID_ARGS;
+    case RSMI_MEMORY_PARTITION_NPS1:
+      break;
+    case RSMI_MEMORY_PARTITION_NPS2:
+      break;
+    case RSMI_MEMORY_PARTITION_NPS4:
+      break;
+    case RSMI_MEMORY_PARTITION_NPS8:
+      break;
+    default:
+      return RSMI_STATUS_INVALID_ARGS;
+  }
+
+  // do nothing if nps_mode is the current NPS mode
+  rsmi_status_t ret_get = get_nps_mode(dv_ind, currentNPSMode);
+  // we can try to set, even if we get unexpected data
+  if (ret_get != RSMI_STATUS_SUCCESS
+      && ret_get != RSMI_STATUS_UNEXPECTED_DATA) {
+    return ret_get;
+  }
+  rsmi_nps_mode_type_t currRSMINpsMode
+    = mapStringToNPSModeTypes[currentNPSMode];
+  if (currRSMINpsMode == nps_mode) {
+    return RSMI_STATUS_SUCCESS;
+  }
+
+  GET_DEV_FROM_INDX
+  int ret = dev->writeDevInfo(amd::smi::kDevMemoryPartition, newNPSMode);
+
+  if (amd::smi::ErrnoToRsmiStatus(ret) != RSMI_STATUS_SUCCESS) {
+    return amd::smi::ErrnoToRsmiStatus(ret);
+  }
+
+  return dev->restartAMDGpuDriver();
+  CATCH
+}
+
+rsmi_status_t
+rsmi_dev_nps_mode_get(uint32_t dv_ind, char *nps_mode,
+                               uint32_t len) {
+  CHK_SUPPORT_NAME_ONLY(nps_mode)
+  if ((len == 0) || (nps_mode == nullptr)) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+
+  TRY
+  std::string returning_nps_mode;
+  rsmi_status_t ret = get_nps_mode(dv_ind,
+                               returning_nps_mode);
+
+  if (ret != RSMI_STATUS_SUCCESS) { return ret; }
+
+  std::size_t length = returning_nps_mode.copy(nps_mode, len);
+  nps_mode[length]='\0';
+
+  if (len < (returning_nps_mode.size() + 1)) {
+    return RSMI_STATUS_INSUFFICIENT_SIZE;
+  }
+  return ret;
+  CATCH
+}
+
 enum iterator_handle_type {
   FUNC_ITER = 0,
   VARIANT_ITER,

src/rocm_smi_device.cc

@@ -122,6 +122,7 @@ static const char *kDevSerialNumberFName = "serial_number";
 static const char *kDevNumaNodeFName = "numa_node";
 static const char *kDevGpuMetricsFName = "gpu_metrics";
 static const char *kDevComputePartitionFName = "current_compute_partition";
+static const char *kDevMemoryPartitionFName = "current_memory_partition";
 
 // Firmware version files
 static const char *kDevFwVersionAsdFName = "fw_version/asd_fw_version";
@@ -292,6 +293,7 @@ static const std::map<DevInfoTypes, const char *> kDevAttribNameMap = {
     {kDevGpuMetrics, kDevGpuMetricsFName},
     {kDevGpuReset, kDevGpuResetFName},
     {kDevComputePartition, kDevComputePartitionFName},
+    {kDevMemoryPartition, kDevMemoryPartitionFName},
 };
 
 static const std::map<rsmi_dev_perf_level, const char *> kDevPerfLvlMap = {
@@ -417,6 +419,8 @@ static const std::map<const char *, dev_depends_t> kDevFuncDependsMap = {
   {"rsmi_dev_gpu_reset",                 {{kDevGpuResetFName}, {}}},
   {"rsmi_dev_compute_partition_get",     {{kDevComputePartitionFName}, {}}},
   {"rsmi_dev_compute_partition_set",     {{kDevComputePartitionFName}, {}}},
+  {"rsmi_dev_memory_partition_get",      {{kDevMemoryPartitionFName}, {}}},
+  {"rsmi_dev_memory_partition_set",      {{kDevMemoryPartitionFName}, {}}},
 
   // These functions with variants, but no sensors/units. (May or may not
   // have mandatory dependencies.)
@@ -564,9 +568,9 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) {
   auto sysfs_path = path_;
 
 #ifdef DEBUG
-  if (env_->path_DRM_root_override && type == env_->enum_override) {
+  if (env_->path_DRM_root_override
+      && (env_->enum_overrides.find(type) != env_->enum_overrides.end())) {
     sysfs_path = env_->path_DRM_root_override;
-
   }
 #endif
 
@@ -698,6 +702,7 @@ int Device::writeDevInfo(DevInfoTypes type, std::string val) {
     case kDevPowerODVoltage:
     case kDevSOCClk:
     case kDevComputePartition:
+    case kDevMemoryPartition:
       return writeDevInfoStr(type, val);
 
     default:
@@ -925,6 +930,7 @@ int Device::readDevInfo(DevInfoTypes type, std::string *val) {
     case kDevPCIEThruPut:
     case kDevSerialNumber:
     case kDevComputePartition:
+    case kDevMemoryPartition:
       return readDevInfoStr(type, val);
       break;
 
@@ -1102,6 +1108,44 @@ bool Device::DeviceAPISupported(std::string name, uint64_t variant,
   return false;
 }
 
+rsmi_status_t Device::restartAMDGpuDriver(void) {
+  REQUIRE_ROOT_ACCESS
+  bool restartSuccessful = true;
+  bool success = false;
+  std::string out = "";
+  bool wasGdmServiceActive = false;
+
+  // sudo systemctl is-active gdm
+  // we do not care about the success of checking if gdm is active
+  std::tie(success, out) = executeCommand("systemctl is-active gdm");
+  (out == "active") ? (restartSuccessful &= success) :
+                         (restartSuccessful = true);
+
+  // if gdm is active -> sudo systemctl stop gdm
+  // TODO: are are there other display manager's we need to take into account?
+  // see https://en.wikipedia.org/wiki/GNOME_Display_Manager
+  if (success && (out == "active")) {
+    wasGdmServiceActive = true;
+    std::tie(success, out) = executeCommand("systemctl stop gdm&", false);
+    restartSuccessful &= success;
+  }
+
+  // sudo modprobe -r amdgpu
+  // sudo modprobe amdgpu
+  std::tie(success, out) =
+    executeCommand("modprobe -r amdgpu && modprobe amdgpu&", false);
+  restartSuccessful &= success;
+
+  // if gdm was active -> sudo systemctl start gdm
+  if (wasGdmServiceActive) {
+    std::tie(success, out) = executeCommand("systemctl start gdm&", false);
+    restartSuccessful &= success;
+  }
+
+  return (restartSuccessful ? RSMI_STATUS_SUCCESS :
+          RSMI_STATUS_AMDGPU_RESTART_ERR);
+}
+
 #undef RET_IF_NONZERO
 }  // namespace smi
 }  // namespace amd

src/rocm_smi_main.cc

@@ -57,6 +57,7 @@
 #include <cerrno>
 #include <unordered_map>
 #include <iostream>
+#include <sstream>
 
 #include "rocm_smi/rocm_smi.h"
 #include "rocm_smi/rocm_smi_device.h"
@@ -141,7 +142,8 @@ const std::map<amd::smi::DevInfoTypes, std::string> amd::smi::RocmSMI::devInfoTy
   {amd::smi::kDevNumaNode, amdSMI + "kDevNumaNode"},
   {amd::smi::kDevGpuMetrics, amdSMI + "kDevGpuMetrics"},
   {amd::smi::kDevGpuReset, amdSMI + "kDevGpuReset"},
-  {amd::smi::kDevComputePartition, amdSMI + "kDevComputePartition"}
+  {amd::smi::kDevComputePartition, amdSMI + "kDevComputePartition"},
+  {amd::smi::kDevMemoryPartition, amdSMI + "kDevMemoryPartition"}
 };
 
 namespace amd {
@@ -305,6 +307,8 @@ RocmSMI::Initialize(uint64_t flags) {
   euid_ = geteuid();
 
   GetEnvVariables();
+  // To help debug env variable issues
+  // printEnvVarInfo();
 
   while (env_vars_.debug_inf_loop) {}
 
@@ -429,6 +433,31 @@ static uint32_t GetEnvVarUInteger(const char *ev_str) {
   return 0;
 }
 
+static std::unordered_set<uint32_t> GetEnvVarUIntegerSets(const char *ev_str) {
+  std::unordered_set<uint32_t> returnSet;
+#ifndef DEBUG
+  (void)ev_str;
+#else
+  ev_str = getenv(ev_str);
+  if(ev_str == nullptr) { return returnSet; }
+  std::string stringEnv = ev_str;
+
+  if (stringEnv.empty() == false) {
+    // parse out values by commas
+    std::string parsedVal;
+    std::istringstream ev_str_ss(stringEnv);
+
+    while (std::getline(ev_str_ss, parsedVal, ',')) {
+      int parsedInt = std::stoi(parsedVal);
+      assert(parsedInt >= 0);
+      uint32_t parsedUInt = static_cast<uint32_t>(parsedInt);
+      returnSet.insert(parsedUInt);
+    }
+  }
+#endif
+  return returnSet;
+}
+
 // Get and store env. variables in this method
 void RocmSMI::GetEnvVariables(void) {
 #ifndef DEBUG
@@ -437,15 +466,15 @@ void RocmSMI::GetEnvVariables(void) {
   env_vars_.path_DRM_root_override = nullptr;
   env_vars_.path_HWMon_root_override = nullptr;
   env_vars_.path_power_root_override = nullptr;
-  env_vars_.enum_override = 0;
   env_vars_.debug_inf_loop = 0;
+  env_vars_.enum_overrides.clear();
 #else
   env_vars_.debug_output_bitfield = GetEnvVarUInteger("RSMI_DEBUG_BITFIELD");
   env_vars_.path_DRM_root_override   = getenv("RSMI_DEBUG_DRM_ROOT_OVERRIDE");
   env_vars_.path_HWMon_root_override = getenv("RSMI_DEBUG_HWMON_ROOT_OVERRIDE");
   env_vars_.path_power_root_override = getenv("RSMI_DEBUG_PP_ROOT_OVERRIDE");
-  env_vars_.enum_override = GetEnvVarUInteger("RSMI_DEBUG_ENUM_OVERRIDE");
   env_vars_.debug_inf_loop = GetEnvVarUInteger("RSMI_DEBUG_INFINITE_LOOP");
+  env_vars_.enum_overrides = GetEnvVarUIntegerSets("RSMI_DEBUG_ENUM_OVERRIDE");
 #endif
 }
 
@@ -453,6 +482,43 @@ const RocmSMI_env_vars& RocmSMI::getEnv(void) {
   return env_vars_;
 }
 
+void RocmSMI::printEnvVarInfo(void) {
+  std::cout << __PRETTY_FUNCTION__ << " | env_vars_.debug_output_bitfield = "
+            << ((env_vars_.debug_output_bitfield == 0) ? "<undefined>"
+                : std::to_string(env_vars_.debug_output_bitfield))
+            << std::endl;
+  std::cout << __PRETTY_FUNCTION__ << " | env_vars_.path_DRM_root_override = "
+            << ((env_vars_.path_DRM_root_override == nullptr)
+                ? "<undefined>" : env_vars_.path_DRM_root_override)
+            << std::endl;
+  std::cout << __PRETTY_FUNCTION__ << " | env_vars_.path_HWMon_root_override = "
+            << ((env_vars_.path_HWMon_root_override == nullptr)
+                ? "<undefined>" : env_vars_.path_HWMon_root_override)
+            << std::endl;
+  std::cout << __PRETTY_FUNCTION__ << " | env_vars_.path_power_root_override = "
+            << ((env_vars_.path_power_root_override == nullptr)
+                ? "<undefined>" : env_vars_.path_power_root_override)
+            << std::endl;
+  std::cout << __PRETTY_FUNCTION__ << " | env_vars_.debug_inf_loop = "
+            << ((env_vars_.debug_inf_loop == 0) ? "<undefined>"
+                : std::to_string(env_vars_.debug_output_bitfield))
+            << std::endl;
+  std::cout << __PRETTY_FUNCTION__ << " | env_vars_.enum_overrides = {";
+  if (env_vars_.enum_overrides.empty()) {
+    std::cout << "}" << std::endl;
+    return;
+  }
+  for (auto it=env_vars_.enum_overrides.begin();
+       it != env_vars_.enum_overrides.end(); ++it) {
+    std::cout << *it;
+    auto temp_it = it;
+    if(++temp_it != env_vars_.enum_overrides.end()) {
+      std::cout << ",";
+    }
+  }
+  std::cout << "}" << std::endl;
+}
+
 std::shared_ptr<Monitor>
 RocmSMI::FindMonitor(std::string monitor_path) {
   std::string tmp;

src/rocm_smi_utils.cc

@@ -3,7 +3,7 @@
  * The University of Illinois/NCSA
  * Open Source License (NCSA)
  *
- * Copyright (c) 2018, Advanced Micro Devices, Inc.
+ * Copyright (c) 2018-2023, Advanced Micro Devices, Inc.
  * All rights reserved.
  *
  * Developed by:
@@ -51,6 +51,7 @@
 #include <sstream>
 #include <algorithm>
 #include <vector>
+#include <regex>
 
 #include "rocm_smi/rocm_smi.h"
 #include "rocm_smi/rocm_smi_utils.h"
@@ -234,5 +235,68 @@ rsmi_status_t ErrnoToRsmiStatus(int err) {
   }
 }
 
+std::string leftTrim(const std::string &s) {
+  if (!s.empty()) {
+    return std::regex_replace(s, std::regex("^\\s+"), "");
+  }
+  return s;
+}
+
+std::string rightTrim(const std::string &s) {
+  if (!s.empty()) {
+    return std::regex_replace(s, std::regex("\\s+$"), "");
+  }
+  return s;
+}
+
+std::string removeNewLines(const std::string &s) {
+  if (!s.empty()) {
+    return std::regex_replace(s, std::regex("\n+"), "");
+  }
+  return s;
+}
+
+std::string trim(const std::string &s) {
+  if (!s.empty()) {
+    // remove new lines -> trim white space at ends
+    std::string noNewLines = removeNewLines(s);
+    return leftTrim(rightTrim(noNewLines));
+  }
+  return s;
+}
+
+// defaults to trim stdOut
+std::pair<bool, std::string> executeCommand(std::string command, bool stdOut) {
+  char buffer[128];
+  std::string stdoutAndErr = "";
+  bool successfulRun = true;
+  command = "stdbuf -i0 -o0 -e0 " + command; // remove stdOut and err buffering
+
+  FILE *pipe = popen(command.c_str(), "r");
+  if (!pipe) {
+    stdoutAndErr = "[ERROR] popen failed to call " + command;
+    successfulRun = false;
+  } else {
+    //read until end of process
+    while (!feof(pipe)) {
+      // use buffer to read and add to stdoutAndErr
+      if (fgets(buffer, sizeof(buffer), pipe) != nullptr) {
+        stdoutAndErr += buffer;
+      }
+    }
+  }
+
+  // any return code other than 0, is a failed execution
+  if (pclose(pipe) != 0) {
+    successfulRun = false;
+  }
+
+  if (stdOut) {
+    // remove leading and trailing spaces of output and new lines
+    stdoutAndErr = trim(stdoutAndErr);
+  }
+  return std::make_pair(successfulRun, stdoutAndErr);
+}
+
 }  // namespace smi
 }  // namespace amd