[SWDEV-488276/SWDEV-497613] Update memory partition set functionality

Changes: - Added warning screen to ROCm SMI users setting memory partition - Added new API (rsmi_dev_memory_partition_capabilities_get) to retrieve memory partition capabilities (What users can set memory partition modes to) - Increased time-bar for CLI sets display to 40 seconds - API now waits until the driver reloads with SYSFS files active - [SWDEV-475712] [CLI/API] Fixed target_graphics_version field not properly displaying for MI2x or Navi 3x ASICs. - Updated tests Change-Id: Iaf89d1b7ad9ceb449b289bc82ea198fe3b23992e Signed-off-by: Charis Poag <Charis.Poag@amd.com>
2024-11-06 15:13:32 -06:00
parent 568cc6e7c7
commit 46902274b6
10 changed files with 595 additions and 123 deletions
@@ -3273,7 +3273,7 @@ rsmi_dev_memory_total_get(uint32_t dv_ind, rsmi_memory_type_t mem_type,
      ss << __PRETTY_FUNCTION__
         << " | inside success fallback... "
         << " | Device #: " << std::to_string(dv_ind)
-         << " | Type = " << devInfoTypesStrings.at(mem_type_file)
+         << " | Type = " << amd::smi::Device::get_type_string(mem_type_file)
         << " | Data: total = " << std::to_string(*total)
         << " | ret = " << getRSMIStatusString(RSMI_STATUS_SUCCESS);
      LOG_DEBUG(ss);
@@ -3284,7 +3284,7 @@ rsmi_dev_memory_total_get(uint32_t dv_ind, rsmi_memory_type_t mem_type,
  ss << __PRETTY_FUNCTION__
     << " | after fallback... "
     << " | Device #: " << std::to_string(dv_ind)
-     << " | Type = " << devInfoTypesStrings.at(mem_type_file)
+     << " | Type = " << amd::smi::Device::get_type_string(mem_type_file)
     << " | Data: total = " << std::to_string(*total)
     << " | ret = " << getRSMIStatusString(ret);
  LOG_DEBUG(ss);
@@ -3333,7 +3333,7 @@ rsmi_dev_memory_usage_get(uint32_t dv_ind, rsmi_memory_type_t mem_type,
      ss << __PRETTY_FUNCTION__
         << " no fallback needed! - "
         << " | Device #: " << std::to_string(dv_ind)
-         << " | Type = " << devInfoTypesStrings.at(mem_type_file)
+         << " | Type = " << amd::smi::Device::get_type_string(mem_type_file)
         << " | Data: Used = " << std::to_string(*used)
         << " | Data: total = " << std::to_string(total)
         << " | ret = " << getRSMIStatusString(ret);
@@ -3344,7 +3344,7 @@ rsmi_dev_memory_usage_get(uint32_t dv_ind, rsmi_memory_type_t mem_type,
      ss << __PRETTY_FUNCTION__
         << " | in fallback == success ..."
         << " | Device #: " << std::to_string(dv_ind)
-         << " | Type = " << devInfoTypesStrings.at(mem_type_file)
+         << " | Type = " << amd::smi::Device::get_type_string(mem_type_file)
         << " | Data: Used = " << std::to_string(*used)
         << " | Data: total = " << std::to_string(total)
         << " | ret = " << getRSMIStatusString(RSMI_STATUS_SUCCESS);
@@ -3355,7 +3355,7 @@ rsmi_dev_memory_usage_get(uint32_t dv_ind, rsmi_memory_type_t mem_type,
  ss << __PRETTY_FUNCTION__
     << " | at end!!!! after fallback ..."
     << " | Device #: " << std::to_string(dv_ind)
-     << " | Type = " << devInfoTypesStrings.at(mem_type_file)
+     << " | Type = " << amd::smi::Device::get_type_string(mem_type_file)
     << " | Data: Used = " << std::to_string(*used)
     << " | ret = " << getRSMIStatusString(ret);
  LOG_DEBUG(ss);
@@ -4640,7 +4640,7 @@ rsmi_dev_compute_partition_get(uint32_t dv_ind, char *compute_partition,
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
-       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
+       << amd::smi::Device::get_type_string(amd::smi::kDevComputePartition)
       << " | Cause: len was 0 or compute_partition variable was null"
       << " | Returning = "
       << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS) << " |";
@@ -4659,7 +4659,7 @@ rsmi_dev_compute_partition_get(uint32_t dv_ind, char *compute_partition,
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
-       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
+       << amd::smi::Device::get_type_string(amd::smi::kDevComputePartition)
       << " | Cause: could not retrieve current compute partition"
       << " | Returning = "
       << getRSMIStatusString(ret) << " |";
@@ -4676,7 +4676,7 @@ rsmi_dev_compute_partition_get(uint32_t dv_ind, char *compute_partition,
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
-       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
+       << amd::smi::Device::get_type_string(amd::smi::kDevComputePartition)
       << " | Cause: requested size was insufficient"
       << " | Returning = "
       << getRSMIStatusString(RSMI_STATUS_INSUFFICIENT_SIZE) << " |";
@@ -4688,7 +4688,7 @@ rsmi_dev_compute_partition_get(uint32_t dv_ind, char *compute_partition,
     << " | Success "
     << " | Device #: " << dv_ind
     << " | Type: "
-     << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
+     << amd::smi::Device::get_type_string(amd::smi::kDevComputePartition)
     << " | Data: " << compute_partition
     << " | Returning = "
     << getRSMIStatusString(ret) << " |";
@@ -4715,7 +4715,7 @@ is_available_compute_partition(uint32_t dv_ind,
       << " | FAIL "
       << " | Device #: " << dv_ind
       << " | Type: "
-       << devInfoTypesStrings.at(amd::smi::kDevAvailableComputePartition)
+       << amd::smi::Device::get_type_string(amd::smi::kDevAvailableComputePartition)
       << " | Data: could not retrieve requested data"
       << " | Returning = "
       << getRSMIStatusString(ret) << " |";
@@ -4734,7 +4734,7 @@ is_available_compute_partition(uint32_t dv_ind,
     << " | Success "
     << " | Device #: " << dv_ind
     << " | Type: "
-     << devInfoTypesStrings.at(amd::smi::kDevAvailableComputePartition)
+     << amd::smi::Device::get_type_string(amd::smi::kDevAvailableComputePartition)
     << " | Data: available_partitions = " << availableComputePartitions
     << " | Data: isComputePartitionAvailable = "
     << (isComputePartitionAvailable ? "True" : "False")
@@ -4777,7 +4777,7 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
         << " | Fail "
         << " | Device #: " << dv_ind
         << " | Type: "
-         << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
+         << amd::smi::Device::get_type_string(amd::smi::kDevComputePartition)
         << " | Data: " << newComputePartitionStr
         << " | Cause: requested setting was invalid"
         << " | Returning = "
@@ -4796,7 +4796,7 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
-       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
+       << amd::smi::Device::get_type_string(amd::smi::kDevComputePartition)
       << " | Data: " << newComputePartitionStr
       << " | Cause: not an available compute partition setting"
       << " | Returning = "
@@ -4816,7 +4816,7 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
-       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
+       << amd::smi::Device::get_type_string(amd::smi::kDevComputePartition)
       << " | Cause: could retrieve current compute partition or retrieved"
       << " unexpected data"
       << " | Returning = "
@@ -4832,7 +4832,7 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
       << " | Success - compute partition was already set at requested value"
       << " | Device #: " << dv_ind
       << " | Type: "
-       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
+       << amd::smi::Device::get_type_string(amd::smi::kDevComputePartition)
       << " | Data: " << newComputePartitionStr
       << " | Returning = "
       << getRSMIStatusString(RSMI_STATUS_SUCCESS) << " |";
@@ -4858,7 +4858,7 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
     << " | Success "
     << " | Device #: " << dv_ind
     << " | Type: "
-     << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
+     << amd::smi::Device::get_type_string(amd::smi::kDevComputePartition)
     << " | Data: " << newComputePartitionStr
     << " | Returning = "
     << getRSMIStatusString(returnResponse) << " |";
@@ -4912,12 +4912,22 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS
  DEVICE_MUTEX
+  const uint32_t kMaxBoardLength = 128;
  bool isCorrectDevice = false;
-  char boardName[128];
+  char boardName[kMaxBoardLength];
  boardName[0] = '\0';
+
+  const uint32_t kMaxMemoryCapabilitiesSize = 30;
+  char available_memory_capabilities[kMaxMemoryCapabilitiesSize];
+  available_memory_capabilities[0] = '\0';
+
+  const uint32_t kMaxCurrentMemoryMode = 5;
+  char current_memory_mode[kMaxCurrentMemoryMode];
+  current_memory_mode[0] = '\0';
+
  // rsmi_dev_memory_partition_set is only available for for discrete variant,
  // others are required to update through bios settings
-  rsmi_dev_name_get(dv_ind, boardName, 128);
+  rsmi_dev_name_get(dv_ind, boardName, static_cast<size_t>(kMaxBoardLength));
  std::string myBoardName = boardName;
  if (!myBoardName.empty()) {
    std::transform(myBoardName.begin(), myBoardName.end(), myBoardName.begin(),
@@ -4930,18 +4940,19 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,

  if (!isCorrectDevice) {
    ss << __PRETTY_FUNCTION__
-         << " | ======= end ======= "
-         << " | Fail "
-         << " | Device #: " << dv_ind
-         << " | Type: "
-         << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
-         << " | Cause: device board name does not support this action"
-         << " | Returning = "
-         << getRSMIStatusString(RSMI_STATUS_NOT_SUPPORTED) << " |";
+       << " | ======= end ======= "
+       << " | Fail "
+       << " | Device #: " << dv_ind
+       << " | Type: "
+       << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
+       << " | Cause: device board name does not support this action"
+       << " | Returning = "
+       << getRSMIStatusString(RSMI_STATUS_NOT_SUPPORTED, false);
      LOG_ERROR(ss);
    return RSMI_STATUS_NOT_SUPPORTED;
  }

+  // Is the current mode already what user requested?
  switch (memory_partition) {
    case RSMI_MEMORY_PARTITION_NPS1:
    case RSMI_MEMORY_PARTITION_NPS2:
@@ -4955,10 +4966,10 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
         << " | Fail "
         << " | Device #: " << dv_ind
         << " | Type: "
-         << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
+         << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
         << " | Cause: requested setting was invalid"
         << " | Returning = "
-         << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS) << " |";
+         << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS, false);
      LOG_ERROR(ss);
      return RSMI_STATUS_INVALID_ARGS;
  }
@@ -4976,11 +4987,11 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
-       << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
+       << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
       << " | Cause: could retrieve current memory partition or retrieved"
       << " unexpected data"
       << " | Returning = "
-       << getRSMIStatusString(ret_get) << " |";
+       << getRSMIStatusString(ret_get, false);
    LOG_ERROR(ss);
    return ret_get;
  }
@@ -4993,14 +5004,55 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
     << " setting"
     << " | Device #: " << dv_ind
     << " | Type: "
-     << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
+     << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
     << " | Data: " << newMemoryPartition
     << " | Returning = "
-     << getRSMIStatusString(RSMI_STATUS_SUCCESS) << " |";
+     << getRSMIStatusString(RSMI_STATUS_SUCCESS, false);
    LOG_TRACE(ss);
    return RSMI_STATUS_SUCCESS;
  }

+  // is this an available mode to set to?
+  std::string memory_capabilities_str = "unknown";
+  std::string user_requested_memory_partition = newMemoryPartition;
+  std::transform(user_requested_memory_partition.begin(), user_requested_memory_partition.end(),
+                  user_requested_memory_partition.begin(), ::toupper);
+  rsmi_status_t caps_ret = rsmi_dev_memory_partition_capabilities_get(dv_ind,
+                              available_memory_capabilities, kMaxMemoryCapabilitiesSize);
+  memory_capabilities_str = available_memory_capabilities;
+  std::transform(memory_capabilities_str.begin(), memory_capabilities_str.end(),
+                  memory_capabilities_str.begin(), ::toupper);
+  ss << __PRETTY_FUNCTION__ << " | user_requested_memory_partition: "
+     << user_requested_memory_partition
+     << "; memory_capabilities_str: " << memory_capabilities_str
+     << "; rsmi_dev_memory_partition_capabilities_get(" << dv_ind
+     << ", " << user_requested_memory_partition << "): return = "
+     << amd::smi::getRSMIStatusString(caps_ret, false);
+  LOG_DEBUG(ss);
+  if ((caps_ret == RSMI_STATUS_SUCCESS)
+      && (!memory_capabilities_str.empty())
+      && (!user_requested_memory_partition.empty())) {
+    bool is_available_mode = amd::smi::containsString(memory_capabilities_str,
+                                user_requested_memory_partition, true);
+    ss << __PRETTY_FUNCTION__
+       << " | is_available_mode: " << (is_available_mode ? "True": "False");
+    LOG_DEBUG(ss);
+    if (is_available_mode == false) {  // report RSMI_STATUS_INVALID_ARGS
+      ss << __PRETTY_FUNCTION__
+         << " | ======= Check if available mode ======= "
+         << " | WARNING: detected invalid mode to set to, will try to set anyways"
+         << " | Device #: " << dv_ind
+         << " | Type: "
+         << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
+         << " | Data (user requested mode): " << user_requested_memory_partition
+         << " | Available Memory Partition Modes: " << memory_capabilities_str
+         << " | Cause: requested setting was not an available mode"
+         << " | Returning = "
+         << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS, false);
+      LOG_INFO(ss);
+    }
+  }
+
  GET_DEV_FROM_INDX
  int ret = dev->writeDevInfo(amd::smi::kDevMemoryPartition,
                              newMemoryPartition);
@@ -5015,10 +5067,10 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
-       << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
+       << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
       << " | Cause: issue writing reqested setting of " + newMemoryPartition
       << " | Returning = "
-       << getRSMIStatusString(err) << " |";
+       << getRSMIStatusString(err, false);
    LOG_ERROR(ss);
    return err;
  }
@@ -5029,11 +5081,79 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
     << " | Success - if restart completed successfully"
     << " | Device #: " << dv_ind
     << " | Type: "
-     << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
+     << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
     << " | Data: " << newMemoryPartition
     << " | Returning = "
-     << getRSMIStatusString(restartRet) << " |";
+     << getRSMIStatusString(restartRet, false);
  LOG_TRACE(ss);
+  if (restartRet != RSMI_STATUS_SUCCESS) {
+    ss << __PRETTY_FUNCTION__
+       << " | ======= end ======= "
+       << " | Fail - restart AMD GPU detected"
+       << " | Device #: " << dv_ind
+       << " | Type: "
+       << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
+       << " | Cause: issue writing reqested setting of " + newMemoryPartition
+       << " | Returning = "
+       << getRSMIStatusString(restartRet, false);
+    LOG_ERROR(ss);
+    return restartRet;
+  }
+
+  std::string current_memory_mode_str = "unknown";
+  rsmi_status_t can_read_sysfs_again = RSMI_STATUS_AMDGPU_RESTART_ERR;
+  int maxWaitSeconds = 10;
+  const int k1000_MS_WAIT = 1000;
+  // wait until we can read SYSFS again
+  if (restartRet == RSMI_STATUS_SUCCESS) {
+    while (current_memory_mode_str != user_requested_memory_partition) {
+      maxWaitSeconds -= 1;
+      can_read_sysfs_again =
+        rsmi_dev_memory_partition_get(dv_ind, current_memory_mode, kMaxCurrentMemoryMode);
+      if (can_read_sysfs_again == RSMI_STATUS_SUCCESS) {
+        current_memory_mode_str.clear();
+        current_memory_mode_str = current_memory_mode;
+        ss << __PRETTY_FUNCTION__
+           << " | ======= rsmi_dev_memory_partition_get ======= "
+           << " | Success - can read SYSFS"
+           << " | Device #: " << dv_ind
+           << " | Type: "
+           << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
+           << " | Data (user requested mode): " << user_requested_memory_partition
+           << " | Current Memory Partition Mode: " << current_memory_mode_str
+           << " | Available Memory Partition Modes: " << memory_capabilities_str
+           << " | total wait time (sec): " << (10 - maxWaitSeconds)
+           << " | Returning = "
+           << getRSMIStatusString(can_read_sysfs_again, false);
+        LOG_TRACE(ss);
+        if (!current_memory_mode_str.empty()
+            && (current_memory_mode_str == user_requested_memory_partition)) {
+          break;
+        }
+      }
+      amd::smi::system_wait(k1000_MS_WAIT);
+    }
+  }
+
+  if (current_memory_mode_str == user_requested_memory_partition) {
+    restartRet = RSMI_STATUS_SUCCESS;
+  } else {
+    restartRet = RSMI_STATUS_AMDGPU_RESTART_ERR;
+  }
+
+  ss << __PRETTY_FUNCTION__
+     << " | ======= end ======= "
+     << " | Success - completed driver restart and all SYSFS are active"
+     << " | Device #: " << dv_ind
+     << " | Type: "
+     << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
+     << " | Data: " << user_requested_memory_partition
+     << " | Current Memory Partition Mode: " << current_memory_mode_str
+     << " | Available Memory Partition Modes: " << memory_capabilities_str
+     << " | Returning = "
+     << getRSMIStatusString(restartRet, false);
+  LOG_TRACE(ss);
+
  return restartRet;
  CATCH
 }
@@ -5051,7 +5171,7 @@ rsmi_dev_memory_partition_get(uint32_t dv_ind, char *memory_partition,
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
-       << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
+       << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
       << " | Cause: user sent invalid arguments, len = 0 or memory partition"
       << " was a null ptr"
       << " | Returning = "
@@ -5071,7 +5191,7 @@ rsmi_dev_memory_partition_get(uint32_t dv_ind, char *memory_partition,
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
-       << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
+       << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
       << " | Cause: could not successfully retrieve current memory partition "
       << " | Returning = "
       << getRSMIStatusString(ret) << " |";
@@ -5089,7 +5209,7 @@ rsmi_dev_memory_partition_get(uint32_t dv_ind, char *memory_partition,
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
-       << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
+       << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
       << " | Cause: could not successfully retrieve current memory partition "
       << " | Returning = "
       << getRSMIStatusString(ret) << " |";
@@ -5101,7 +5221,7 @@ rsmi_dev_memory_partition_get(uint32_t dv_ind, char *memory_partition,
     << " | Success "
     << " | Device #: " << dv_ind
     << " | Type: "
-     << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
+     << amd::smi::Device::get_type_string(amd::smi::kDevMemoryPartition)
     << " | Data: " << memory_partition
     << " | Returning = "
     << getRSMIStatusString(ret) << " |";
@@ -5110,6 +5230,78 @@ rsmi_dev_memory_partition_get(uint32_t dv_ind, char *memory_partition,
  CATCH
 }

+rsmi_status_t rsmi_dev_memory_partition_capabilities_get(
+      uint32_t dv_ind, char *memory_partition_caps, uint32_t len) {
+  TRY
+  std::ostringstream ss;
+  ss << __PRETTY_FUNCTION__ << " | ======= start =======, " << dv_ind;
+  LOG_TRACE(ss);
+
+  if ((len == 0) || (memory_partition_caps == nullptr)) {
+    ss << __PRETTY_FUNCTION__
+       << " | ======= end ======= "
+       << " | Fail "
+       << " | Device #: " << dv_ind
+       << " | Type: "
+       << amd::smi::Device::get_type_string(amd::smi::kDevAvailableMemoryPartition)
+       << " | Cause: user sent invalid arguments, len = 0 or memory_partition_caps"
+       << " was a null ptr"
+       << " | Returning = "
+       << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS, false);
+    LOG_ERROR(ss);
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+  CHK_SUPPORT_NAME_ONLY(memory_partition_caps)
+  DEVICE_MUTEX
+
+  std::string availableMemoryPartitions;
+  rsmi_status_t ret =
+      get_dev_value_line(amd::smi::kDevAvailableMemoryPartition,
+                         dv_ind, &availableMemoryPartitions);
+  if (ret != RSMI_STATUS_SUCCESS) {
+    ss << __PRETTY_FUNCTION__
+       << " | ======= end ======= "
+       << " | FAIL "
+       << " | Device #: " << dv_ind
+       << " | Type: "
+       << amd::smi::Device::get_type_string(amd::smi::kDevAvailableMemoryPartition)
+       << " | Data: could not retrieve requested data"
+       << " | Returning = "
+       << getRSMIStatusString(ret, false);
+    LOG_ERROR(ss);
+    return ret;
+  }
+
+  std::size_t length = availableMemoryPartitions.copy(memory_partition_caps, len-1);
+  memory_partition_caps[length]='\0';
+
+  if (len < (availableMemoryPartitions.size() + 1)) {
+    ss << __PRETTY_FUNCTION__
+       << " | ======= end ======= "
+       << " | Fail "
+       << " | Device #: " << dv_ind
+       << " | Type: "
+       << amd::smi::Device::get_type_string(amd::smi::kDevAvailableMemoryPartition)
+       << " | Cause: requested size was insufficient"
+       << " | Returning = "
+       << getRSMIStatusString(RSMI_STATUS_INSUFFICIENT_SIZE, false);
+    LOG_ERROR(ss);
+    return RSMI_STATUS_INSUFFICIENT_SIZE;
+  }
+  ss << __PRETTY_FUNCTION__
+     << " | ======= end ======= "
+     << " | Success "
+     << " | Device #: " << dv_ind
+     << " | Type: "
+     << amd::smi::Device::get_type_string(amd::smi::kDevAvailableMemoryPartition)
+     << " | Data: " << memory_partition_caps
+     << " | Returning = "
+     << getRSMIStatusString(ret, false);
+  LOG_TRACE(ss);
+  return ret;
+  CATCH
+}
+
 rsmi_status_t
 rsmi_dev_partition_id_get(uint32_t dv_ind, uint32_t *partition_id) {
  TRY
@@ -128,6 +128,7 @@ static const char *kDevAvailableComputePartitionFName =
                  "available_compute_partition";
 static const char *kDevComputePartitionFName = "current_compute_partition";
 static const char *kDevMemoryPartitionFName = "current_memory_partition";
+static const char *kDevAvailableMemoryPartitionFName = "available_memory_partition";

 // Firmware version files
 static const char *kDevFwVersionAsdFName = "fw_version/asd_fw_version";
@@ -306,6 +307,7 @@ static const std::map<DevInfoTypes, const char *> kDevAttribNameMap = {
    {kDevAvailableComputePartition, kDevAvailableComputePartitionFName},
    {kDevComputePartition, kDevComputePartitionFName},
    {kDevMemoryPartition, kDevMemoryPartitionFName},
+    {kDevAvailableMemoryPartition, kDevAvailableMemoryPartitionFName},
 };

 static const std::map<rsmi_dev_perf_level, const char *> kDevPerfLvlMap = {
@@ -452,6 +454,7 @@ Device::devInfoTypesStrings = {
  {kDevAvailableComputePartition, "kDevAvailableComputePartition"},
  {kDevComputePartition, "kDevComputePartition"},
  {kDevMemoryPartition, "kDevMemoryPartition"},
+  {kDevAvailableMemoryPartition, "kDevAvailableMemoryPartition"},
 };

 static const std::map<const char *, dev_depends_t> kDevFuncDependsMap = {
@@ -680,7 +683,7 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) {
  if (ret != 0) {
    ss << __PRETTY_FUNCTION__ << " | Issue: File did not exist - SYSFS file ("
       << sysfs_path
-       << ") for DevInfoInfoType (" << devInfoTypesStrings.at(type)
+       << ") for DevInfoInfoType (" << get_type_string(type)
       << "), returning " << std::to_string(ret);
    LOG_ERROR(ss);
    return ret;
@@ -689,7 +692,7 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) {
    ss << __PRETTY_FUNCTION__
       << " | Issue: File is not a regular file - SYSFS file ("
       << sysfs_path << ") for "
-       << "DevInfoInfoType (" << devInfoTypesStrings.at(type) << "),"
+       << "DevInfoInfoType (" << get_type_string(type) << "),"
       << " returning ENOENT (" << std::strerror(ENOENT) << ")";
    LOG_ERROR(ss);
    return ENOENT;
@@ -700,7 +703,7 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) {
  if (!fs->is_open()) {
    ss << __PRETTY_FUNCTION__
       << " | Issue: Could not open - SYSFS file (" << sysfs_path << ") for "
-       << "DevInfoInfoType (" << devInfoTypesStrings.at(type) << "), "
+       << "DevInfoInfoType (" << get_type_string(type) << "), "
       << ", returning " << std::to_string(errno) << " ("
       << std::strerror(errno) << ")";
    LOG_ERROR(ss);
@@ -709,7 +712,7 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) {

  ss << __PRETTY_FUNCTION__ << " | Successfully opened SYSFS file ("
     << sysfs_path
-     << ") for DevInfoInfoType (" << devInfoTypesStrings.at(type)
+     << ") for DevInfoInfoType (" << get_type_string(type)
     << ")";
  LOG_INFO(ss);
  return 0;
@@ -726,7 +729,7 @@ int Device::readDebugInfoStr(DevInfoTypes type, std::string *retStr) {
  ret = openDebugFileStream(type, &fs);
  if (ret != 0) {
    ss << "Could not read debugInfoStr for DevInfoType ("
-     << devInfoTypesStrings.at(type)<< "), returning "
+     << get_type_string(type) << "), returning "
     << std::to_string(ret);
    LOG_ERROR(ss);
    return ret;
@@ -740,7 +743,7 @@ int Device::readDebugInfoStr(DevInfoTypes type, std::string *retStr) {
  fs.close();

  ss << "Successfully read debugInfoStr for DevInfoType ("
-     << devInfoTypesStrings.at(type)<< "), retString= " << *retStr;
+     << get_type_string(type) << "), retString= " << *retStr;
  LOG_INFO(ss);

  return 0;
@@ -756,7 +759,7 @@ int Device::readDevInfoStr(DevInfoTypes type, std::string *retStr) {
  ret = openSysfsFileStream(type, &fs);
  if (ret != 0) {
    ss << "Could not read device info string for DevInfoType ("
-     << devInfoTypesStrings.at(type) << "), returning "
+     << get_type_string(type) << "), returning "
     << std::to_string(ret);
    LOG_ERROR(ss);
    return ret;
@@ -765,8 +768,8 @@ int Device::readDevInfoStr(DevInfoTypes type, std::string *retStr) {
  fs >> *retStr;
  fs.close();
  ss << __PRETTY_FUNCTION__
-     << "Successfully read device info string for DevInfoType (" <<
-            devInfoTypesStrings.at(type) << "): " + *retStr
+     << "Successfully read device info string for DevInfoType ("
+     << get_type_string(type) << "): " + *retStr
     << " | "
     << (fs.is_open() ? " File stream is opened" : " File stream is closed")
     << " | " << (fs.bad() ? "[ERROR] Bad read operation" :
@@ -801,7 +804,7 @@ int Device::writeDevInfoStr(DevInfoTypes type, std::string valStr,
    fs.close();
    ss << __PRETTY_FUNCTION__ << " | Issue: Could not open fileStream; "
       << "Could not write device info string (" << valStr
-       << ") for DevInfoType (" << devInfoTypesStrings.at(type)
+       << ") for DevInfoType (" << get_type_string(type)
       << "), returning " << std::to_string(ret);
    LOG_ERROR(ss);
    return ret;
@@ -812,7 +815,7 @@ int Device::writeDevInfoStr(DevInfoTypes type, std::string valStr,
    fs.flush();
    fs.close();
    ss << "Successfully wrote device info string (" << valStr
-       << ") for DevInfoType (" << devInfoTypesStrings.at(type)
+       << ") for DevInfoType (" << get_type_string(type)
       << "), returning RSMI_STATUS_SUCCESS";
    LOG_INFO(ss);
    ret = RSMI_STATUS_SUCCESS;
@@ -826,7 +829,7 @@ int Device::writeDevInfoStr(DevInfoTypes type, std::string valStr,
    fs.close();
    ss << __PRETTY_FUNCTION__ << " | Issue: Could not write to file; "
       << "Could not write device info string (" << valStr
-       << ") for DevInfoType (" << devInfoTypesStrings.at(type)
+       << ") for DevInfoType (" << get_type_string(type)
       << "), returning " << getRSMIStatusString(ErrnoToRsmiStatus(ret));
    ss << " | "
       << (fs.is_open() ? "[ERROR] File stream open" :
@@ -913,20 +916,29 @@ int Device::readDevInfoLine(DevInfoTypes type, std::string *line) {
  ret = openSysfsFileStream(type, &fs);
  if (ret != 0) {
    ss << "Could not read DevInfoLine for DevInfoType ("
-       << devInfoTypesStrings.at(type) << ")";
+       << get_type_string(type) << ")";
    LOG_ERROR(ss);
    return ret;
  }

  std::getline(fs, *line);
  ss << "Successfully read DevInfoLine for DevInfoType ("
-     << devInfoTypesStrings.at(type) << "), returning *line = "
+     << get_type_string(type) << "), returning *line = "
     << *line;
  LOG_INFO(ss);

  return 0;
 }

+const char* Device::get_type_string(DevInfoTypes type) {
+  auto ite = devInfoTypesStrings.find(type);
+  if (ite != devInfoTypesStrings.end()) {
+    return ite->second;
+  }
+
+  return "Unknown";
+}
+
 int Device::readDevInfoBinary(DevInfoTypes type, std::size_t b_size,
                                void *p_binary_data) {
  auto sysfs_path = path_;
@@ -939,7 +951,7 @@ int Device::readDevInfoBinary(DevInfoTypes type, std::size_t b_size,
  ptr = fopen(sysfs_path.c_str(), "rb");
  if (!ptr) {
    ss << "Could not read DevInfoBinary for DevInfoType ("
-       << devInfoTypesStrings.at(type) << ")"
+       << get_type_string(type) << ")"
       << " - SYSFS (" << sysfs_path << ")"
       << ", returning " << std::to_string(errno) << " ("
       << std::strerror(errno) << ")";
@@ -951,7 +963,7 @@ int Device::readDevInfoBinary(DevInfoTypes type, std::size_t b_size,
  fclose(ptr);
  if ((num*b_size) != b_size) {
    ss << "Could not read DevInfoBinary for DevInfoType ("
-       << devInfoTypesStrings.at(type) << ") - SYSFS ("
+       << get_type_string(type) << ") - SYSFS ("
       << sysfs_path << "), binary size error; "
       << "[buff: "
       << p_binary_data
@@ -966,7 +978,7 @@ int Device::readDevInfoBinary(DevInfoTypes type, std::size_t b_size,
  }
  if (ROCmLogging::Logger::getInstance()->isLoggerEnabled()) {
    ss << "Successfully read DevInfoBinary for DevInfoType ("
-       << devInfoTypesStrings.at(type) << ") - SYSFS ("
+       << get_type_string(type) << ") - SYSFS ("
       << sysfs_path << "), returning binaryData = " << p_binary_data
       << "; byte_size = " << std::dec << static_cast<int>(b_size);

@@ -999,7 +1011,7 @@ int Device::readDevInfoMultiLineStr(DevInfoTypes type,

  if (retVec->empty()) {
    ss << "Read devInfoMultiLineStr for DevInfoType ("
-       << devInfoTypesStrings.at(type) << ")"
+       << get_type_string(type) << ")"
       << ", but contained no string lines";
    LOG_ERROR(ss);
    return ENXIO;
@@ -1017,12 +1029,12 @@ int Device::readDevInfoMultiLineStr(DevInfoTypes type,

  if (!allLines.empty()) {
    ss << "Successfully read devInfoMultiLineStr for DevInfoType ("
-       << devInfoTypesStrings.at(type) << ") "
+       << get_type_string(type) << ") "
       << ", returning lines read = " << allLines;
    LOG_INFO(ss);
  } else {
    ss << "Read devInfoMultiLineStr for DevInfoType ("
-       << devInfoTypesStrings.at(type) << ")"
+       << get_type_string(type) << ")"
       << ", but lines were empty";
    LOG_INFO(ss);
    return ENXIO;
@@ -1193,6 +1205,7 @@ int Device::readDevInfo(DevInfoTypes type, std::string *val) {
    case kDevMemoryPartition:
    case kDevNumaNode:
    case kDevXGMIPhysicalID:
+    case kDevAvailableMemoryPartition:
      return readDevInfoStr(type, val);
      break;

@@ -1370,10 +1383,15 @@ bool Device::DeviceAPISupported(std::string name, uint64_t variant,

 rsmi_status_t Device::restartAMDGpuDriver(void) {
  REQUIRE_ROOT_ACCESS
+  std::ostringstream ss;
  bool restartSuccessful = true;
  bool success = false;
  std::string out;
  bool wasGdmServiceActive = false;
+  bool restartInProgress = true;
+  bool isRestartInProgress = true;
+  bool isAMDGPUModuleLive = false;
+  std::string captureRestartErr;

  // sudo systemctl is-active gdm
  // we do not care about the success of checking if gdm is active
@@ -1382,8 +1400,8 @@ rsmi_status_t Device::restartAMDGpuDriver(void) {
                         (restartSuccessful = true);

  // if gdm is active -> sudo systemctl stop gdm
-  // TODO: are are there other display manager's we need to take into account?
-  // see https://en.wikipedia.org/wiki/GNOME_Display_Manager
+  // TODO(AMD_SMI_team): are are there other display manager's we need to take into account?
+  // see https://help.gnome.org/admin/gdm/stable/overview.html.en_GB
  if (success && (out == "active")) {
    wasGdmServiceActive = true;
    std::tie(success, out) = executeCommand("systemctl stop gdm&", false);
@@ -1393,8 +1411,13 @@ rsmi_status_t Device::restartAMDGpuDriver(void) {
  // sudo modprobe -r amdgpu
  // sudo modprobe amdgpu
  std::tie(success, out) =
-    executeCommand("modprobe -r amdgpu && modprobe amdgpu&", false);
+    executeCommand("modprobe -r amdgpu && modprobe amdgpu&", true);
  restartSuccessful &= success;
+  captureRestartErr = out;
+
+  if (success) {
+    restartSuccessful = false;
+  }

  // if gdm was active -> sudo systemctl start gdm
  if (wasGdmServiceActive) {
@@ -1402,7 +1425,61 @@ rsmi_status_t Device::restartAMDGpuDriver(void) {
    restartSuccessful &= success;
  }

-  return (restartSuccessful ? RSMI_STATUS_SUCCESS :
+  // Return early if there was an issue restarting amdgpu
+  if (!restartSuccessful) {
+    ss << __PRETTY_FUNCTION__ << " | [WARNING] Issue found during amdgpu restart: "
+    << captureRestartErr << "; retartSuccessful: " << (restartSuccessful ? "True" : "False");
+    LOG_INFO(ss);
+    return RSMI_STATUS_AMDGPU_RESTART_ERR;
+  }
+
+  // wait for amdgpu module to come back up
+  rsmi_status_t status = Device::isRestartInProgress(&isRestartInProgress,
+                                                    &isAMDGPUModuleLive);
+  const int kTimeToWaitForDriverMSec = 1000;
+  int maxLoops = 10;  // wait a max of 10 sec
+  while (status != RSMI_STATUS_SUCCESS) {
+    maxLoops -= 1;
+    if (maxLoops == 0) {
+      break;
+    }
+    amd::smi::system_wait(kTimeToWaitForDriverMSec);
+    status = Device::isRestartInProgress(&isRestartInProgress,
+                                         &isAMDGPUModuleLive);
+  }
+
+  return ((restartSuccessful && (!isRestartInProgress && isAMDGPUModuleLive)) ?
+          RSMI_STATUS_SUCCESS :
+          RSMI_STATUS_AMDGPU_RESTART_ERR);
+}
+
+rsmi_status_t Device::isRestartInProgress(bool *isRestartInProgress,
+                                          bool *isAMDGPUModuleLive) {
+  REQUIRE_ROOT_ACCESS
+  std::ostringstream ss;
+  bool restartSuccessful = true;
+  bool success = false;
+  std::string out;
+  bool deviceRestartInProgress = true;    // Assume in progress, we intend to disprove
+  bool isSystemAMDGPUModuleLive = false;  // Assume AMD GPU module is not live,
+                                          //  we intend to disprove
+
+  // wait for amdgpu module to come back up
+  std::tie(success, out) = executeCommand("cat /sys/module/amdgpu/initstate", true);
+  ss << __PRETTY_FUNCTION__
+     << " | success = " << success
+     << " | out = " << out;
+  LOG_DEBUG(ss);
+  if ((success == true) && (!out.empty())) {
+    isSystemAMDGPUModuleLive = containsString(out, "live");
+  }
+  if (isAMDGPUModuleLive) {
+    deviceRestartInProgress = false;
+  }
+  *isRestartInProgress = deviceRestartInProgress;
+  *isAMDGPUModuleLive = isSystemAMDGPUModuleLive;
+
+  return ((*isAMDGPUModuleLive && !*isRestartInProgress) ? RSMI_STATUS_SUCCESS :
          RSMI_STATUS_AMDGPU_RESTART_ERR);
 }

@@ -567,9 +567,9 @@ std::string RocmSMI::getRSMIEnvVarInfo(void) {
  for (auto it=env_vars_.enum_overrides.begin();
       it != env_vars_.enum_overrides.end(); ++it) {
    DevInfoTypes type = static_cast<DevInfoTypes>(*it);
-    ss << (std::to_string(*it) + " (" + Device::devInfoTypesStrings.at(type) + ")");
+    ss << (std::to_string(*it) + " (" + Device::get_type_string(type) + ")");
    auto temp_it = it;
-    if(++temp_it != env_vars_.enum_overrides.end()) {
+    if (++temp_it != env_vars_.enum_overrides.end()) {
      ss << ", ";
    }
  }
@@ -63,6 +63,7 @@
 #include <sstream>
 #include <string>
 #include <vector>
+#include <cmath>

 #include "rocm_smi/rocm_smi.h"
 #include "rocm_smi/rocm_smi_utils.h"
@@ -343,6 +344,7 @@ rsmi_status_t ErrnoToRsmiStatus(int err) {
    case EIO:    return RSMI_STATUS_UNEXPECTED_SIZE;
    case ENXIO:  return RSMI_STATUS_UNEXPECTED_DATA;
    case EBUSY:  return RSMI_STATUS_BUSY;
+    case EINVAL: return RSMI_STATUS_INVALID_ARGS;
    default:     return RSMI_STATUS_UNKNOWN_ERROR;
  }
 }
@@ -415,14 +417,14 @@ std::pair<bool, std::string> executeCommand(std::string command, bool stdOut) {
  char buffer[128];
  std::string stdoutAndErr;
  bool successfulRun = true;
-  command = "stdbuf -i0 -o0 -e0 " + command; // remove stdOut and err buffering
+  command = "stdbuf -i0 -o0 -e0 " + command;  // remove stdOut and err buffering

  FILE *pipe = popen(command.c_str(), "r");
  if (!pipe) {
    stdoutAndErr = "[ERROR] popen failed to call " + command;
    successfulRun = false;
  } else {
-    //read until end of process
+    // read until end of process
    while (!feof(pipe)) {
      // use buffer to read and add to stdoutAndErr
      if (fgets(buffer, sizeof(buffer), pipe) != nullptr) {
@@ -445,8 +447,19 @@ std::pair<bool, std::string> executeCommand(std::string command, bool stdOut) {

 // originalString - string to search for substring
 // substring - string looking to find
-bool containsString(std::string originalString, std::string substring) {
-  return (originalString.find(substring) != std::string::npos);
+// displayComparisons = defaults to false, set to true to see debug prints
+bool containsString(std::string originalString, std::string substring,
+                  bool displayComparisons) {
+  std::ostringstream ss;
+  bool found = originalString.find(substring) != std::string::npos;
+  if (displayComparisons) {
+    ss << __PRETTY_FUNCTION__
+       << " | originalString: " << originalString
+       << " | substring: " << substring
+       << " | found: " << (found ? "True": "False");
+    LOG_TRACE(ss);
+  }
+  return found;
 }

 // Creates and stores supplied data into a temporary file (within /tmp/).
@@ -1180,7 +1193,9 @@ rsmi_status_t rsmi_get_gfx_target_version(uint32_t dv_ind, std::string *gfx_vers
    // separate out parts -> put back into normal graphics version format
    major = static_cast<uint64_t>((orig_target_version / 10000) * 100);
    minor = static_cast<uint64_t>((orig_target_version % 10000 / 100) * 10);
-    if (minor == 0) major *= 10;  // 0 as a minor is correct, but bump up by 10
+    if ((minor == 0) && (countDigit(major) < 4)) {
+      major *= 10;  // 0 as a minor is correct, but bump up by 10
+    }
    rev = static_cast<uint64_t>(orig_target_version % 100);
    *gfx_version = "gfx" + std::to_string(major + minor + rev);
    ss << __PRETTY_FUNCTION__
@@ -1222,6 +1237,31 @@ std::queue<std::string> getAllDeviceGfxVers() {
  return deviceGfxVersions;
 }

+// milli_seconds: time to wait, in milliseconds
+// 1 sec = 1000ms
+// .5 sec = 500ms
+void system_wait(int milli_seconds) {
+  std::ostringstream ss;
+  auto start = std::chrono::high_resolution_clock::now();
+  // 1 ms = 1000 us
+  int waitTime = milli_seconds * 1000;
+  ss << __PRETTY_FUNCTION__ << " | "
+     << "** Waiting for " << std::dec << waitTime
+     << " us (" << waitTime/1000 << " milli-seconds) **";
+  LOG_DEBUG(ss);
+  usleep(waitTime);
+  auto stop = std::chrono::high_resolution_clock::now();
+  auto duration =
+      std::chrono::duration_cast<std::chrono::microseconds>(stop - start);
+  ss << __PRETTY_FUNCTION__ << " | "
+     << "** Waiting took " << duration.count() / 1000
+     << " milli-seconds **";
+  LOG_DEBUG(ss);
+}
+
+int countDigit(uint64_t n) {
+  return static_cast<int>(std::floor(log10(n) + 1));
+}

 }  // namespace smi
 }  // namespace amd