[SWDEV-381630] Add reset partition functionality

Updates:
    * Added rsmi_dev_compute_partition_reset & rsmi_dev_nps_mode_reset
    * Added --resetcomputepartition and --resetnpsmode python smi calls
    * Added temp data files rocmsmi_boot_compute_partition_<device num>
      & rocmsmi_boot_nps_mode_partition_<device num>, writes UNKNOWN
      if data cannot be read or device does not support
    * Cleaned up NPS & compute API documentation
    * Added creation and reading of API temp files (used in reset
      functionality)
    * Cleaned up output of rocm_smi_example
    * Updated rocm_smi_example to check if running with sudo permission
      before executing write API calls (cleans up erroneous output)
    * Added template specialization for storing temp data, requires
      specific rsmi_type_t enums (restrics what data can be stored)
    * Added storage of temp data, if temp files do not exist
    * Updated google tests for NPS & compute to include reset API calls

Change-Id: I69895a466b97107617e6dbb355737b84499a76c9
Signed-off-by: Charis Poag <Charis.Poag@amd.com>


[ROCm/rocm_smi_lib commit: 77c950a4bf]

projects/rocm-smi-lib/src/rocm_smi.cc

@@ -3986,6 +3986,47 @@ rsmi_dev_nps_mode_get(uint32_t dv_ind, char *nps_mode,
   CATCH
 }
 
+rsmi_status_t rsmi_dev_compute_partition_reset(uint32_t dv_ind) {
+  TRY
+  REQUIRE_ROOT_ACCESS
+  DEVICE_MUTEX
+  GET_DEV_FROM_INDX
+  rsmi_status_t ret = RSMI_STATUS_NOT_SUPPORTED;
+  // read temp file
+  std::string bootState =
+          dev->readBootPartitionState<rsmi_compute_partition_type_t>(dv_ind);
+  // Initiate reset
+  // If bootState is UNKNOWN, we cannot reset - return RSMI_STATUS_NOT_SUPPORTED
+  // Likely due to device not supporting it
+  if (bootState != "UNKNOWN") {
+    rsmi_compute_partition_type_t compute_partition =
+                              mapStringToRSMIComputePartitionTypes[bootState];
+    ret = rsmi_dev_compute_partition_set(dv_ind, compute_partition);
+  }
+  return ret;
+  CATCH
+}
+
+rsmi_status_t rsmi_dev_nps_mode_reset(uint32_t dv_ind) {
+  TRY
+  REQUIRE_ROOT_ACCESS
+  DEVICE_MUTEX
+  GET_DEV_FROM_INDX
+  rsmi_status_t ret = RSMI_STATUS_NOT_SUPPORTED;
+  // read temp file
+  std::string bootState =
+          dev->readBootPartitionState<rsmi_nps_mode_type_t>(dv_ind);
+  // Initiate reset
+  // If bootState is UNKNOWN, we cannot reset - return RSMI_STATUS_NOT_SUPPORTED
+  // Likely due to device not supporting it
+  if (bootState != "UNKNOWN") {
+    rsmi_nps_mode_type_t nps_mode = mapStringToNPSModeTypes[bootState];
+    ret = rsmi_dev_nps_mode_set(dv_ind, nps_mode);
+  }
+  return ret;
+  CATCH
+}
+
 enum iterator_handle_type {
   FUNC_ITER = 0,
   VARIANT_ITER,

projects/rocm-smi-lib/src/rocm_smi_device.cc

@@ -48,6 +48,7 @@
 #include <assert.h>
 #include <sys/stat.h>
 #include <stdint.h>
+
 #include <string>
 #include <map>
 #include <fstream>
@@ -1146,6 +1147,128 @@ rsmi_status_t Device::restartAMDGpuDriver(void) {
           RSMI_STATUS_AMDGPU_RESTART_ERR);
 }
 
+template <typename T> rsmi_status_t storeParameter(uint32_t dv_ind);
+
+// Stores parameters depending on which rsmi type is provided.
+// Uses template specialization, to restrict types to identify
+// calls needed to complete the function.
+// typename - restricted to
+// rsmi_compute_partition_type_t or rsmi_compute_partition_type_t
+// dv_ind - device index
+// tempFileName - base file name
+template <>
+rsmi_status_t storeParameter<rsmi_compute_partition_type_t>(uint32_t dv_ind) {
+  rsmi_status_t returnStatus = RSMI_STATUS_SUCCESS;
+  bool doesFileExist;
+  std::tie(doesFileExist, std::ignore) = readTmpFile(dv_ind, "boot",
+                                                     "compute_partition");
+  // if temporary file exists -> we do not need to store anything new
+  // if not, read & store the state value
+  if (doesFileExist) {
+    return returnStatus;
+  }
+  uint32_t length = 128;
+  char data[length];
+  rsmi_status_t ret = rsmi_dev_compute_partition_get(dv_ind, data, length);
+  rsmi_status_t storeRet;
+
+  if (ret == RSMI_STATUS_SUCCESS) {
+    storeRet = storeTmpFile(dv_ind, "compute_partition", "boot", data);
+  } else if (ret == RSMI_STATUS_NOT_SUPPORTED) {
+    // not supported is ok
+    storeRet = storeTmpFile(dv_ind, "compute_partition", "boot", "UNKNOWN");
+  } else {
+    storeRet = storeTmpFile(dv_ind, "compute_partition", "boot", "UNKNOWN");
+    returnStatus = ret;
+  }
+
+  if (storeRet != RSMI_STATUS_SUCCESS) {
+    // file storage err takes precedence over other errors
+    returnStatus = storeRet;
+  }
+  return returnStatus;
+}
+
+// Stores parameters depending on which rsmi type is provided.
+// Uses template specialization, to restrict types to identify
+// calls needed to complete the function.
+// typename - restricted to
+// rsmi_compute_partition_type_t or rsmi_compute_partition_type_t
+// dv_ind - device index
+// tempFileName - base file name
+template <> rsmi_status_t storeParameter<rsmi_nps_mode_type_t>(uint32_t dv_ind) {
+  rsmi_status_t returnStatus = RSMI_STATUS_SUCCESS;
+  uint32_t length = 128;
+  char data[length];
+  bool doesFileExist;
+  std::tie(doesFileExist, std::ignore) = readTmpFile(dv_ind, "boot",
+                                                     "nps_mode");
+  // if temporary file exists -> we do not need to store anything new
+  // if not, read & store the state value
+  if (doesFileExist) {
+    return returnStatus;
+  }
+  rsmi_status_t ret = rsmi_dev_nps_mode_get(dv_ind, data, length);
+  rsmi_status_t storeRet;
+
+  if (ret == RSMI_STATUS_SUCCESS) {
+    storeRet = storeTmpFile(dv_ind, "nps_mode", "boot", data);
+  } else if (ret == RSMI_STATUS_NOT_SUPPORTED) {
+    // not supported is ok
+    storeRet = storeTmpFile(dv_ind, "nps_mode", "boot", "UNKNOWN");
+  } else {
+    storeRet = storeTmpFile(dv_ind, "nps_mode", "boot", "UNKNOWN");
+    returnStatus = ret;
+  }
+
+  if (storeRet != RSMI_STATUS_SUCCESS) {
+    // file storage err takes precedence over other errors
+    returnStatus = storeRet;
+  }
+  return returnStatus;
+}
+
+rsmi_status_t Device::storeDevicePartitions(uint32_t dv_ind) {
+  rsmi_status_t returnStatus = RSMI_STATUS_SUCCESS;
+  returnStatus = storeParameter<rsmi_compute_partition_type_t>(dv_ind);
+  rsmi_status_t npsRet = storeParameter<rsmi_nps_mode_type_t>(dv_ind);
+  if (returnStatus == RSMI_STATUS_SUCCESS) { // only record earliest error
+    returnStatus = npsRet;
+  }
+  return returnStatus;
+}
+
+// Reads a device's boot partition state, depending on which rsmi type is
+// provided and device index.
+// Uses template specialization, to restrict types to identify
+// calls needed to complete the function.
+// typename - restricted to rsmi_compute_partition_type_t
+// or rsmi_compute_partition_type_t
+// dv_ind - device index
+template <>
+std::string Device::readBootPartitionState<rsmi_compute_partition_type_t>(
+    uint32_t dv_ind) {
+  std::string boot_state;
+  std::tie(std::ignore, boot_state) = readTmpFile(dv_ind, "boot",
+                                                  "compute_partition");
+  return boot_state;
+}
+
+// Reads a device's boot partition state, depending on which rsmi type is
+// provided and device index.
+// Uses template specialization, to restrict types to identify
+// calls needed to complete the function.
+// typename - restricted to rsmi_compute_partition_type_t
+// or rsmi_compute_partition_type_t
+// dv_ind - device index
+template <>
+std::string Device::readBootPartitionState<rsmi_nps_mode_type_t>(
+    uint32_t dv_ind) {
+  std::string boot_state;
+  std::tie(std::ignore, boot_state) = readTmpFile(dv_ind, "boot", "nps_mode");
+  return boot_state;
+}
+
 #undef RET_IF_NONZERO
 }  // namespace smi
 }  // namespace amd

projects/rocm-smi-lib/src/rocm_smi_main.cc

@@ -373,6 +373,7 @@ RocmSMI::Initialize(uint64_t flags) {
   // 1. construct kfd_node_map_ with gpu_id as key and *Device as value
   // 2. for each kfd node, write the corresponding dv_ind
   // 3. for each amdgpu device, write the corresponding gpu_id
+  // 4. for each amdgpu device, attempt to store it's boot partition
   for (uint32_t dv_ind = 0; dv_ind < devices_.size(); ++dv_ind) {
     dev = devices_[dv_ind];
     uint64_t bdfid = dev->bdfid();
@@ -387,7 +388,12 @@ RocmSMI::Initialize(uint64_t flags) {
     uint64_t gpu_id = tmp_map[bdfid]->gpu_id();
     dev->set_kfd_gpu_id(gpu_id);
     kfd_node_map_[gpu_id] = tmp_map[bdfid];
+
+    // store each device boot partition state, if file doesn't exist
+    dev->storeDevicePartitions(dv_ind);
   }
+  // Leaving below to help debug temp file issues
+  // displayAppTmpFilesContent();
 }
 
 void

projects/rocm-smi-lib/src/rocm_smi_utils.cc

@@ -43,6 +43,8 @@
 #include <assert.h>
 #include <errno.h>
 #include <sys/stat.h>
+#include <unistd.h>
+#include <dirent.h>
 
 #include <fstream>
 #include <string>
@@ -61,6 +63,7 @@
 
 namespace amd {
 namespace smi {
+const std::string kTmpFilePrefix = "rocmsmi_";
 
 // Return 0 if same file, 1 if not, and -1 for error
 int SameFile(const std::string fileA, const std::string fileB) {
@@ -298,5 +301,140 @@ std::pair<bool, std::string> executeCommand(std::string command, bool stdOut) {
   return std::make_pair(successfulRun, stdoutAndErr);
 }
 
+// originalstring - string to search for substring
+// substring - string looking to find
+bool containsString(std::string originalString, std::string substring) {
+  if (originalString.find(substring) != std::string::npos) {
+    return true;
+  } else {
+    return false;
+  }
+}
+
+// Creates and stores supplied data into a temporary file (within /tmp/).
+// All temporary files are removed upon reboot.
+// Allows all users/groups to read the temporary file.
+//
+// For more detail, refer to mkstemp manpage:
+// https://man7.org/linux/man-pages/man3/mkstemp.3.html
+//
+// Temporary file name format:
+// <app prefix>_<state name>_<paramenter name>_<device id>
+// <app prefix> - prefix for our application's identifier (see kTmpFilePrefix)
+// <paramenter name> - name of parameter being stored
+// <state name> - state at which the stored value captures
+// <device index> - device identifier
+//
+// dv_ind - device index
+// parameterName - name of parameter stored
+// stateName - state at which the stored value captures
+// storageData - string value of data to be stored
+rsmi_status_t storeTmpFile(uint32_t dv_ind, std::string parameterName,
+                           std::string stateName, std::string storageData) {
+  // Required tags needed to store our files
+  // Files name format:
+  // <app prefix>_<stateName>_<parameterName>_<device id>
+  std::string fullFileName = kTmpFilePrefix + stateName + "_" +
+                             parameterName + "_" + std::to_string(dv_ind);
+  bool doesFileExist;
+  std::tie(doesFileExist, std::ignore) =
+        readTmpFile(dv_ind, stateName, parameterName);
+  if (doesFileExist) {
+    // do not store, if file already exists
+    return RSMI_STATUS_SUCCESS;
+  }
+  // template for our file
+  std::string fullTempFilePath = "/tmp/" + fullFileName + ".XXXXXX";
+  char *fileName = &fullTempFilePath[0];
+  int fd = mkstemp(fileName);
+  if (fd == -1) {
+    return RSMI_STATUS_FILE_ERROR;
+  }
+
+  chmod(fileName, S_IRUSR|S_IRGRP|S_IROTH);
+  write(fd, storageData.c_str(), storageData.size());
+  close(fd);
+  return RSMI_STATUS_SUCCESS;
+}
+
+std::vector<std::string> getListOfAppTmpFiles() {
+  std::string path = "/tmp";
+  DIR *dir;
+  struct dirent *ent;
+  std::vector<std::string> tmpFiles;
+
+  if ((dir = opendir(path.c_str())) != nullptr) {
+    // captures all files & directories under specified path
+    while ((ent  = readdir(dir)) != nullptr) {
+      std::string fileDirName = ent->d_name;
+      // we only want our app specific files
+      if (containsString(fileDirName, kTmpFilePrefix)) {
+        tmpFiles.emplace_back(path + "/" + fileDirName);
+      } else {
+        continue;
+      }
+    }
+  }
+  return tmpFiles;
+}
+
+// Reads a temporary file in path provided
+// If file does not exist, returns an empty string
+// If file exists, returns content (which could be an empty string)
+std::string readTemporaryFile(std::string path) {
+  std::string fileContent;
+  std::ifstream inFileStream(path);
+  if (inFileStream.is_open()) {
+    inFileStream >> fileContent;
+  }
+  return fileContent;
+}
+
+// Used to debug application temporary files (idenified by kTmpFilePrefix)
+// and their content
+void displayAppTmpFilesContent() {
+  std::vector<std::string> tmpFiles = getListOfAppTmpFiles();
+  if (tmpFiles.empty() == false) {
+    for (auto &x: tmpFiles) {
+      std::string out = readTemporaryFile(x);
+      std::cout << __PRETTY_FUNCTION__ << " | Temporary file: " << x
+                << "; Contained content: " << out << std::endl;
+    }
+  } else {
+    std::cout << __PRETTY_FUNCTION__ << " | No temporary files were found"
+              << std::endl;
+  }
+}
+
+// Attempts to read application specific temporary file
+// This method is to be used for reading (or determing if it exists),
+// in order to keep file naming scheme consistent.
+//
+// dv_ind - device index
+// parameterName - name of parameter stored
+// stateName - state at which the stored value captures
+// Returns:
+// boolean - if temporary file exists
+// string - content of temporary file, if it exists (otherwise, an empty
+// string is returned)
+std::tuple<bool, std::string> readTmpFile(uint32_t dv_ind,
+                                          std::string stateName,
+                                          std::string parameterName) {
+  bool fileExists = false;
+  std::string tmpFileName = kTmpFilePrefix + stateName + "_" +parameterName +
+                            "_" + std::to_string(dv_ind);
+  std::string fileContent;
+  std::vector<std::string> tmpFiles = getListOfAppTmpFiles();
+  if (tmpFiles.empty() == false) {
+    for (auto &x: tmpFiles) {
+      if (containsString(x, tmpFileName)) {
+        fileContent = readTemporaryFile(x);
+        fileExists = true;
+        break;
+      }
+    }
+  }
+  return std::make_tuple(fileExists, fileContent);
+}
 }  // namespace smi
 }  // namespace amd