SWDEV-335697- Add support for dynamic partitioning

Original updates:
    * Added .gitignore to help with future commits
    * Updated/added copyrights on modified or added files
    * Updated rocm_smi.h/.cc
      - Added 3 new SMI API functions:
          rsmi_dev_compute_partition_set &
          rsmi_dev_compute_partition_get
      - Added helpful maps/enums used in
        new get/set compute_partition API calls
    * Updated rocm_smi.py
      - Added --showcomputepartition
      - Added --setcomputepartition
      - Fixed a few mistypes
    * Updated rsmiBindings.py - added helpful class/dict/list
    * Updated rocm_smi_example.cc
      - Added helpful MACRO to detect if api is not supported.
      - Added current_compute_partition set/get rocm lib calls
      - Added helpful macro to discover future RSMI errors
      - Commented out test_set_freq, was having permission issues
        on a Navi21
    * Updated rocm_smi_main.cc
      - Added helpful map to debug API calls, left in for future use
      - Added comment to better understand a non-class function returns
    * Added computepartition_read_write.cc/.h
      - Added get/set compute partition API test calls
      - Confirmed on devices that do not support the API calls, tests pass
    * Updated rocm_smi_test/main.cc
      - Calls new compute partition gtests

Added following updates from review feedback:
   * Updated rocm_smi.h/cc
       - Removed C++ API calls, adding support for both C/C++
         API calls could cause confusion and adds extra work for us
       - rsmi_dev_compute_partition_get -> Fixed an edge case where
         user gives a small buffer length size (smaller than data
         received), but does not receive the partial buffer back.
         google Tests are updated to reflect this find.
   * Updated rocm_smi_example.cc
       - Fixed test_set_freq, issue was that file was not writable.
         We now indicate this warning, so prior errors make sense.
       - General test code cleanup. Removed extra code,
         by creating loops for tests.
   * Updated rocm_smi_main.cc
     - Moved and got rid of an external reference to a map used
       for debugging RSMI enums, now is a const public reference.
   * Updated rocm_smi.py
     - Updated python code to identify NOT_SUPPORTED due to
       (currently) only a few GPU support the feature

Change-Id: I4a567acbb59d6771fb64df08d19175fe3604fd1b

src/rocm_smi.cc

@@ -3,7 +3,7 @@
  * The University of Illinois/NCSA
  * Open Source License (NCSA)
  *
- * Copyright (c) 2017, Advanced Micro Devices, Inc.
+ * Copyright (c) 2017-2023, Advanced Micro Devices, Inc.
  * All rights reserved.
  *
  * Developed by:
@@ -45,7 +45,7 @@
 #include <errno.h>
 #include <sys/utsname.h>
 #include <pthread.h>
-#include <string.h>
+#include <string>
 #include <unistd.h>
 #include <poll.h>
 #include <fcntl.h>
@@ -1678,13 +1678,30 @@ static std::vector<std::string> pci_name_files = {
   "/var/lib/pciutils/pci.ids"
 };
 
-
 enum eNameStrType {
   NAME_STR_VENDOR = 0,
   NAME_STR_DEVICE,
   NAME_STR_SUBSYS
 };
 
+std::map<std::string, rsmi_compute_partition_type_t>
+mapStringToRSMIComputePartitionTypes {
+  {"CPX", RSMI_COMPUTE_PARTITION_CPX},
+  {"SPX", RSMI_COMPUTE_PARTITION_SPX},
+  {"DPX", RSMI_COMPUTE_PARTITION_DPX},
+  {"TPX", RSMI_COMPUTE_PARTITION_TPX},
+  {"QPX", RSMI_COMPUTE_PARTITION_QPX}
+};
+
+std::map<rsmi_compute_partition_type_t, std::string>
+mapRSMIToStringComputePartitionTypes {
+  {RSMI_COMPUTE_PARTITION_CPX, "CPX"},
+  {RSMI_COMPUTE_PARTITION_SPX, "SPX"},
+  {RSMI_COMPUTE_PARTITION_DPX, "DPX"},
+  {RSMI_COMPUTE_PARTITION_TPX, "TPX"},
+  {RSMI_COMPUTE_PARTITION_QPX, "QPX"}
+};
+
 static std::string
 get_id_name_str_from_line(uint64_t id, std::string ln,
                                                  std::istringstream *ln_str) {
@@ -3697,6 +3714,117 @@ rsmi_is_P2P_accessible(uint32_t dv_ind_src, uint32_t dv_ind_dst,
   CATCH
 }
 
+static rsmi_status_t
+get_compute_partition(uint32_t dv_ind, std::string &compute_partition) {
+  TRY
+  std::string val_str;
+
+  if (compute_partition.c_str() == nullptr) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+  CHK_SUPPORT_NAME_ONLY(compute_partition.c_str())
+
+  DEVICE_MUTEX
+  rsmi_status_t ret = get_dev_value_str(amd::smi::kDevComputePartition,
+                                        dv_ind, &val_str);
+
+  if (ret != RSMI_STATUS_SUCCESS) {
+    return ret;
+  }
+
+  switch (mapStringToRSMIComputePartitionTypes[val_str]) {
+    case RSMI_COMPUTE_PARTITION_INVALID:
+      // Retrieved an unknown compute partition
+      return RSMI_STATUS_UNEXPECTED_DATA;
+    case RSMI_COMPUTE_PARTITION_CPX:
+      break;
+    case RSMI_COMPUTE_PARTITION_SPX:
+      break;
+    case RSMI_COMPUTE_PARTITION_DPX:
+      break;
+    case RSMI_COMPUTE_PARTITION_TPX:
+      break;
+    case RSMI_COMPUTE_PARTITION_QPX:
+      break;
+    default:
+      // Retrieved an unknown compute partition
+      return RSMI_STATUS_UNEXPECTED_DATA;
+  }
+  compute_partition = val_str;
+  return RSMI_STATUS_SUCCESS;
+  CATCH
+}
+
+rsmi_status_t
+rsmi_dev_compute_partition_get(uint32_t dv_ind, char *compute_partition,
+                               uint32_t len) {
+  CHK_SUPPORT_NAME_ONLY(compute_partition)
+  if ((len == 0) || (compute_partition == nullptr)) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+
+  TRY
+  std::string returning_compute_partition;
+  rsmi_status_t ret = get_compute_partition(dv_ind,
+                               returning_compute_partition);
+
+  if (ret != RSMI_STATUS_SUCCESS) { return ret; }
+
+  std::size_t length = returning_compute_partition.copy(compute_partition, len);
+  compute_partition[length]='\0';
+
+  if (len < (returning_compute_partition.size() + 1)) {
+    return RSMI_STATUS_INSUFFICIENT_SIZE;
+  }
+  return ret;
+  CATCH
+}
+
+rsmi_status_t
+rsmi_dev_compute_partition_set(uint32_t dv_ind,
+                              rsmi_compute_partition_type_t compute_partition) {
+  TRY
+  REQUIRE_ROOT_ACCESS
+  DEVICE_MUTEX
+
+  std::string newComputePartitionStr
+              = mapRSMIToStringComputePartitionTypes[compute_partition];
+  std::string currentComputePartition;
+
+  switch (compute_partition) {
+    case RSMI_COMPUTE_PARTITION_INVALID:
+      // Retrieved an unknown compute partition
+      return RSMI_STATUS_INVALID_ARGS;
+    case RSMI_COMPUTE_PARTITION_CPX:
+      break;
+    case RSMI_COMPUTE_PARTITION_SPX:
+      break;
+    case RSMI_COMPUTE_PARTITION_DPX:
+      break;
+    case RSMI_COMPUTE_PARTITION_TPX:
+      break;
+    case RSMI_COMPUTE_PARTITION_QPX:
+      break;
+    default:
+      return RSMI_STATUS_INVALID_ARGS;
+  }
+
+  // do nothing if compute_partition is the current compute partition
+  get_compute_partition(dv_ind, currentComputePartition);
+  rsmi_compute_partition_type_t currRSMIComputePartition
+    = mapStringToRSMIComputePartitionTypes[currentComputePartition];
+  if (currRSMIComputePartition == compute_partition) {
+    return RSMI_STATUS_SUCCESS;
+  }
+
+  newComputePartitionStr = mapRSMIToStringComputePartitionTypes[compute_partition];
+  GET_DEV_FROM_INDX
+  int ret = dev->writeDevInfo(amd::smi::kDevComputePartition,
+                              newComputePartitionStr);
+  return amd::smi::ErrnoToRsmiStatus(ret);
+  CATCH
+}
+
 enum iterator_handle_type {
   FUNC_ITER = 0,
   VARIANT_ITER,

src/rocm_smi_device.cc

@@ -3,7 +3,7 @@
  * The University of Illinois/NCSA
  * Open Source License (NCSA)
  *
- * Copyright (c) 2017, Advanced Micro Devices, Inc.
+ * Copyright (c) 2017-2023, Advanced Micro Devices, Inc.
  * All rights reserved.
  *
  * Developed by:
@@ -121,6 +121,7 @@ static const char *kDevXGMIErrorFName = "xgmi_error";
 static const char *kDevSerialNumberFName = "serial_number";
 static const char *kDevNumaNodeFName = "numa_node";
 static const char *kDevGpuMetricsFName = "gpu_metrics";
+static const char *kDevComputePartitionFName = "current_compute_partition";
 
 // Firmware version files
 static const char *kDevFwVersionAsdFName = "fw_version/asd_fw_version";
@@ -290,6 +291,7 @@ static const std::map<DevInfoTypes, const char *> kDevAttribNameMap = {
     {kDevNumaNode, kDevNumaNodeFName},
     {kDevGpuMetrics, kDevGpuMetricsFName},
     {kDevGpuReset, kDevGpuResetFName},
+    {kDevComputePartition, kDevComputePartitionFName},
 };
 
 static const std::map<rsmi_dev_perf_level, const char *> kDevPerfLvlMap = {
@@ -413,6 +415,8 @@ static const std::map<const char *, dev_depends_t> kDevFuncDependsMap = {
   {"rsmi_topo_numa_affinity_get",        {{kDevNumaNodeFName}, {}}},
   {"rsmi_dev_gpu_metrics_info_get",      {{kDevGpuMetricsFName}, {}}},
   {"rsmi_dev_gpu_reset",                 {{kDevGpuResetFName}, {}}},
+  {"rsmi_dev_compute_partition_get",     {{kDevComputePartitionFName}, {}}},
+  {"rsmi_dev_compute_partition_set",     {{kDevComputePartitionFName}, {}}},
 
   // These functions with variants, but no sensors/units. (May or may not
   // have mandatory dependencies.)
@@ -563,9 +567,6 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) {
   if (env_->path_DRM_root_override && type == env_->enum_override) {
     sysfs_path = env_->path_DRM_root_override;
 
-    if (str) {
-      sysfs_path += ".write";
-    }
   }
 #endif
 
@@ -587,7 +588,7 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) {
   fs->open(sysfs_path);
 
   if (!fs->is_open()) {
-      return errno;
+    return errno;
   }
 
   return 0;
@@ -696,6 +697,7 @@ int Device::writeDevInfo(DevInfoTypes type, std::string val) {
     case kDevPCIEClk:
     case kDevPowerODVoltage:
     case kDevSOCClk:
+    case kDevComputePartition:
       return writeDevInfoStr(type, val);
 
     default:
@@ -922,6 +924,7 @@ int Device::readDevInfo(DevInfoTypes type, std::string *val) {
     case kDevVBiosVer:
     case kDevPCIEThruPut:
     case kDevSerialNumber:
+    case kDevComputePartition:
       return readDevInfoStr(type, val);
       break;
 

src/rocm_smi_gpu_metrics.cc

@@ -3,7 +3,7 @@
  * The University of Illinois/NCSA
  * Open Source License (NCSA)
  *
- * Copyright (c) 2021, Advanced Micro Devices, Inc.
+ * Copyright (c) 2017-2023, Advanced Micro Devices, Inc.
  * All rights reserved.
  *
  * Developed by:
@@ -280,7 +280,7 @@ rsmi_dev_gpu_metrics_info_get(uint32_t dv_ind, rsmi_gpu_metrics_t *smu) {
     return RSMI_STATUS_NOT_SUPPORTED;
   }
 
-  // Initialize the smu fiedls to zero as some of them only valid in
+  // Initialize the smu fields to zero as some of them only valid in
   // a specific version.
   *smu = {};
 

src/rocm_smi_main.cc

@@ -2,7 +2,7 @@
  * The University of Illinois/NCSA
  * Open Source License (NCSA)
  *
- * Copyright (c) 2017, Advanced Micro Devices, Inc.
+ * Copyright (c) 2017-2023, Advanced Micro Devices, Inc.
  * All rights reserved.
  *
  * Developed by:
@@ -73,6 +73,77 @@ static const char *kDeviceNamePrefix = "card";
 
 static const char *kAMDMonitorTypes[] = {"radeon", "amdgpu", ""};
 
+static const std::string amdSMI = "amd::smi::";
+const std::map<amd::smi::DevInfoTypes, std::string> amd::smi::RocmSMI::devInfoTypesStrings = {
+  {amd::smi::kDevPerfLevel, amdSMI + "kDevPerfLevel"},
+  {amd::smi::kDevOverDriveLevel, amdSMI + "kDevOverDriveLevel"},
+  {amd::smi::kDevMemOverDriveLevel, amdSMI + "kDevMemOverDriveLevel"},
+  {amd::smi::kDevDevID, amdSMI + "kDevDevID"},
+  {amd::smi::kDevDevProdName, amdSMI + "kDevDevProdName"},
+  {amd::smi::kDevDevProdNum, amdSMI + "kDevDevProdNum"},
+  {amd::smi::kDevVendorID, amdSMI + "kDevVendorID"},
+  {amd::smi::kDevSubSysDevID, amdSMI + "kDevSubSysDevID"},
+  {amd::smi::kDevSubSysVendorID, amdSMI + "kDevSubSysVendorID"},
+  {amd::smi::kDevGPUMClk, amdSMI + "kDevGPUMClk"},
+  {amd::smi::kDevGPUSClk, amdSMI + "kDevGPUSClk"},
+  {amd::smi::kDevDCEFClk, amdSMI + "kDevDCEFClk"},
+  {amd::smi::kDevFClk, amdSMI + "kDevFClk"},
+  {amd::smi::kDevSOCClk, amdSMI + "kDevSOCClk"},
+  {amd::smi::kDevPCIEClk, amdSMI + "kDevPCIEClk"},
+  {amd::smi::kDevPowerProfileMode, amdSMI + "kDevPowerProfileMode"},
+  {amd::smi::kDevUsage, amdSMI + "kDevUsage"},
+  {amd::smi::kDevPowerODVoltage, amdSMI + "kDevPowerODVoltage"},
+  {amd::smi::kDevVBiosVer, amdSMI + "kDevVBiosVer"},
+  {amd::smi::kDevPCIEThruPut, amdSMI + "kDevPCIEThruPut"},
+  {amd::smi::kDevErrCntSDMA, amdSMI + "kDevErrCntSDMA"},
+  {amd::smi::kDevErrCntUMC, amdSMI + "kDevErrCntUMC"},
+  {amd::smi::kDevErrCntGFX, amdSMI + "kDevErrCntGFX"},
+  {amd::smi::kDevErrCntMMHUB, amdSMI + "kDevErrCntMMHUB"},
+  {amd::smi::kDevErrCntPCIEBIF, amdSMI + "kDevErrCntPCIEBIF"},
+  {amd::smi::kDevErrCntHDP, amdSMI + "kDevErrCntHDP"},
+  {amd::smi::kDevErrCntXGMIWAFL, amdSMI + "kDevErrCntXGMIWAFL"},
+  {amd::smi::kDevErrCntFeatures, amdSMI + "kDevErrCntFeatures"},
+  {amd::smi::kDevMemTotGTT, amdSMI + "kDevMemTotGTT"},
+  {amd::smi::kDevMemTotVisVRAM, amdSMI + "kDevMemTotVisVRAM"},
+  {amd::smi::kDevMemTotVRAM, amdSMI + "kDevMemTotVRAM"},
+  {amd::smi::kDevMemUsedGTT, amdSMI + "kDevMemUsedGTT"},
+  {amd::smi::kDevMemUsedVisVRAM, amdSMI + "kDevMemUsedVisVRAM"},
+  {amd::smi::kDevMemUsedVRAM, amdSMI + "kDevMemUsedVRAM"},
+  {amd::smi::kDevVramVendor, amdSMI + "kDevVramVendor"},
+  {amd::smi::kDevPCIEReplayCount, amdSMI + "kDevPCIEReplayCount"},
+  {amd::smi::kDevUniqueId, amdSMI + "kDevUniqueId"},
+  {amd::smi::kDevDFCountersAvailable, amdSMI + "kDevDFCountersAvailable"},
+  {amd::smi::kDevMemBusyPercent, amdSMI + "kDevMemBusyPercent"},
+  {amd::smi::kDevXGMIError, amdSMI + "kDevXGMIError"},
+  {amd::smi::kDevFwVersionAsd, amdSMI + "kDevFwVersionAsd"},
+  {amd::smi::kDevFwVersionCe, amdSMI + "kDevFwVersionCe"},
+  {amd::smi::kDevFwVersionDmcu, amdSMI + "kDevFwVersionDmcu"},
+  {amd::smi::kDevFwVersionMc, amdSMI + "kDevFwVersionMc"},
+  {amd::smi::kDevFwVersionMe, amdSMI + "kDevFwVersionMe"},
+  {amd::smi::kDevFwVersionMec, amdSMI + "kDevFwVersionMec"},
+  {amd::smi::kDevFwVersionMec2, amdSMI + "kDevFwVersionMec2"},
+  {amd::smi::kDevFwVersionPfp, amdSMI + "kDevFwVersionPfp"},
+  {amd::smi::kDevFwVersionRlc, amdSMI + "kDevFwVersionRlc"},
+  {amd::smi::kDevFwVersionRlcSrlc, amdSMI + "kDevFwVersionRlcSrlc"},
+  {amd::smi::kDevFwVersionRlcSrlg, amdSMI + "kDevFwVersionRlcSrlg"},
+  {amd::smi::kDevFwVersionRlcSrls, amdSMI + "kDevFwVersionRlcSrls"},
+  {amd::smi::kDevFwVersionSdma, amdSMI + "kDevFwVersionSdma"},
+  {amd::smi::kDevFwVersionSdma2, amdSMI + "kDevFwVersionSdma2"},
+  {amd::smi::kDevFwVersionSmc, amdSMI + "kDevFwVersionSmc"},
+  {amd::smi::kDevFwVersionSos, amdSMI + "kDevFwVersionSos"},
+  {amd::smi::kDevFwVersionTaRas, amdSMI + "kDevFwVersionTaRas"},
+  {amd::smi::kDevFwVersionTaXgmi, amdSMI + "kDevFwVersionTaXgmi"},
+  {amd::smi::kDevFwVersionUvd, amdSMI + "kDevFwVersionUvd"},
+  {amd::smi::kDevFwVersionVce, amdSMI + "kDevFwVersionVce"},
+  {amd::smi::kDevFwVersionVcn, amdSMI + "kDevFwVersionVcn"},
+  {amd::smi::kDevSerialNumber, amdSMI + "kDevSerialNumber"},
+  {amd::smi::kDevMemPageBad, amdSMI + "kDevMemPageBad"},
+  {amd::smi::kDevNumaNode, amdSMI + "kDevNumaNode"},
+  {amd::smi::kDevGpuMetrics, amdSMI + "kDevGpuMetrics"},
+  {amd::smi::kDevGpuReset, amdSMI + "kDevGpuReset"},
+  {amd::smi::kDevComputePartition, amdSMI + "kDevComputePartition"}
+};
+
 namespace amd {
 namespace smi {
 
@@ -179,6 +250,8 @@ static bool bdfid_from_path(const std::string in_name, uint64_t *bdfid) {
   return true;
 }
 
+// 0 = successful bdfid found
+// 1 = not a good bdfid found
 static uint32_t ConstructBDFID(std::string path, uint64_t *bdfid) {
   assert(bdfid != nullptr);
   char tpath[256] = {'\0'};