[SWDEV-498711] RDC Partition Implementation (#119)

* [SWDEV-498711] RDC Partition Implementation

Change-Id: Ibfc3709793770537e4c9d36458f34c6b4f461724
Signed-off-by: adapryor <Adam.pryor@amd.com>

rdci/CMakeLists.txt

@@ -56,7 +56,8 @@ set(INC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include")
 
 include_directories(${INC_DIR} ${PROJECT_SOURCE_DIR}/include
     "${GRPC_ROOT}/include"
-    ${PROJECT_SOURCE_DIR})
+    ${PROJECT_SOURCE_DIR}
+    ${AMD_SMI_INCLUDE_DIR})
 
 set(RDCI_SRC_LIST
     "${COMMON_DIR}/rdc_fields_supported.cc"

rdci/include/RdciDiscoverySubSystem.h

@@ -37,7 +37,9 @@ class RdciDiscoverySubSystem : public RdciSubSystem {
   bool show_help_;
   void show_help() const;
   bool is_list_;
+  bool is_partition_;
   void show_attributes();
+  void show_attributes_with_partitions();
   bool show_version_;
   void show_version();
 };

rdci/include/RdciDmonSubSystem.h

@@ -43,6 +43,9 @@ class RdciDmonSubSystem : public RdciSubSystem {
   void show_field_usage() const;
   void clean_up();
 
+  // Need to resolve gpu indexes after process is called
+  void resolve_gpu_indexes();
+
   void create_temp_group();
   void create_temp_field_group();
 
@@ -64,6 +67,7 @@ class RdciDmonSubSystem : public RdciSubSystem {
 
   std::map<OPTIONS, uint32_t> options_;
   std::vector<rdc_field_t> field_ids_;
+  std::string raw_gpu_indexes_;
   std::vector<uint32_t> gpu_indexes_;
   bool need_cleanup_;
   uint64_t latest_time_stamp_;

rdci/src/RdciDiscoverySubSystem.cc

@@ -24,6 +24,10 @@ THE SOFTWARE.
 #include <getopt.h>
 #include <unistd.h>
 
+#include <cstring>
+#include <iomanip>
+#include <set>
+
 #include "rdc/rdc.h"
 #include "rdc/rdc_private.h"
 #include "rdc_lib/RdcException.h"
@@ -33,22 +37,23 @@ namespace amd {
 namespace rdc {
 
 RdciDiscoverySubSystem::RdciDiscoverySubSystem()
-    : show_help_(false),
-      is_list_(false),
-      show_version_(false) {}
+    : show_help_(false), is_list_(false), is_partition_(false), show_version_(false) {}
 
 void RdciDiscoverySubSystem::parse_cmd_opts(int argc, char** argv) {
   const int HOST_OPTIONS = 1000;
   const int JSON_OPTIONS = 1001;
-  const struct option long_options[] = {
-      {"host", required_argument, nullptr, HOST_OPTIONS}, {"help", optional_argument, nullptr, 'h'},
-      {"unauth", optional_argument, nullptr, 'u'},        {"list", optional_argument, nullptr, 'l'},
-      {"json", optional_argument, nullptr, JSON_OPTIONS}, {"version", optional_argument, nullptr, 'v'}, {nullptr, 0, nullptr, 0}};
+  const struct option long_options[] = {{"host", required_argument, nullptr, HOST_OPTIONS},
+                                        {"help", optional_argument, nullptr, 'h'},
+                                        {"unauth", optional_argument, nullptr, 'u'},
+                                        {"list", optional_argument, nullptr, 'l'},
+                                        {"json", optional_argument, nullptr, JSON_OPTIONS},
+                                        {"version", optional_argument, nullptr, 'v'},
+                                        {nullptr, 0, nullptr, 0}};
 
   int option_index = 0;
   int opt = 0;
 
-  while ((opt = getopt_long(argc, argv, "hluv", long_options, &option_index)) != -1) {
+  while ((opt = getopt_long(argc, argv, "hliuv", long_options, &option_index)) != -1) {
     switch (opt) {
       case HOST_OPTIONS:
         ip_port_ = optarg;
@@ -65,6 +70,9 @@ void RdciDiscoverySubSystem::parse_cmd_opts(int argc, char** argv) {
       case 'l':
         is_list_ = true;
         break;
+      case 'i':
+        is_partition_ = true;
+        break;
       case 'v':
         show_version_ = true;
         break;
@@ -74,9 +82,10 @@ void RdciDiscoverySubSystem::parse_cmd_opts(int argc, char** argv) {
     }
   }
 
-  if ((!is_list_ && !show_version_) || (is_list_ && show_version_)) {
+  int opCount = (is_list_ ? 1 : 0) + (is_partition_ ? 1 : 0) + (show_version_ ? 1 : 0);
+  if (opCount != 1) {
     show_help();
-    throw RdcException(RDC_ST_BAD_PARAMETER, "Need to specify operations");
+    throw RdcException(RDC_ST_BAD_PARAMETER, "Need to specify exactly one operation");
   }
 }
 
@@ -93,6 +102,8 @@ void RdciDiscoverySubSystem::show_help() const {
             << "Output using json.\n";
   std::cout << "  -l  --list                     list GPU discovered"
             << " on the system\n";
+  std::cout << "  -i  --gpu-instance             list GPU discovered"
+            << " on the system with partitions\n";
   std::cout << "  -v  --version                  Display version information of the"
             << " the server and libraries used by the server\n";
 }
@@ -108,7 +119,7 @@ void RdciDiscoverySubSystem::show_attributes() {
     if (is_json_output()) {
       std::cout << "\"gpus\" : [], \"status\": \"ok\"";
     } else {
-      std::cout << "No GPUs find on the system\n";
+      std::cout << "No GPUs found on the system\n";
     }
     return;
   }
@@ -145,6 +156,145 @@ void RdciDiscoverySubSystem::show_attributes() {
   }
 }
 
+void RdciDiscoverySubSystem::show_attributes_with_partitions() {
+  uint32_t gpu_index_list[RDC_MAX_NUM_DEVICES];
+  uint32_t count = 0;
+  rdc_status_t result = rdc_device_get_all(rdc_handle_, gpu_index_list, &count);
+  if (result != RDC_ST_OK) {
+    throw RdcException(result, "Fail to get all devices");
+  }
+
+  if (count == 0) {
+    if (is_json_output())
+      std::cout << "\"gpus\" : [], \"status\": \"ok\"";
+    else
+      std::cout << "No GPUs found on the system\n";
+    return;
+  }
+
+  // Print header.
+  if (!is_json_output()) {
+    std::cout << count << " GPUs found." << std::endl;
+    std::cout << "---------------------------------------------------------------------"
+              << std::endl;
+    std::cout << std::setw(12) << std::left << "GPU Index" << std::setw(20) << "Instance Index"
+              << std::setw(25) << "Device Information" << std::setw(8) << "XCC" << std::setw(8)
+              << "DECODER" << std::endl;
+  } else {
+    std::cout << "\"gpus\" : [";
+  }
+
+  // Loop over each GPU.
+  for (uint32_t i = 0; i < count; i++) {
+    rdc_device_attributes_t attribute;
+    result = rdc_device_get_attributes(rdc_handle_, gpu_index_list[i], &attribute);
+    if (result != RDC_ST_OK) return;
+
+    // Build physical device entity info.
+    rdc_entity_info_t phys_info;
+    phys_info.device_index = i;
+    phys_info.instance_index = 0;
+    phys_info.entity_role = RDC_DEVICE_ROLE_PHYSICAL;
+    phys_info.device_type = RDC_DEVICE_TYPE_GPU;
+    uint32_t phys_entity_index = rdc_get_entity_index_from_info(phys_info);
+
+    rdc_resource_profile_t phys_xcc = {};
+    rdc_resource_profile_t phys_decoder_profile = {};
+    result =
+        rdc_instance_profile_get(rdc_handle_, phys_entity_index, RDC_ACCELERATOR_XCC, &phys_xcc);
+    result = rdc_instance_profile_get(rdc_handle_, phys_entity_index, RDC_ACCELERATOR_DECODER,
+                                      &phys_decoder_profile);
+
+    std::string phys_xcc_str = std::to_string(phys_xcc.partition_resource);
+    std::string phys_decoder_str = std::to_string(phys_decoder_profile.partition_resource);
+
+    if (!is_json_output()) {
+      std::cout << std::setw(12) << std::left << i << std::setw(20) << "" << std::setw(25)
+                << attribute.device_name << std::setw(8) << phys_xcc_str << std::setw(8)
+                << phys_decoder_str << std::endl;
+    } else {
+      std::cout << "{\"gpu_index\": \"" << i << "\", "
+                << "\"device_name\": \"" << attribute.device_name << "\", "
+                << "\"physical\": {"
+                << "\"xcc\": \"" << phys_xcc_str << "\", "
+                << "\"decoder\": \"" << phys_decoder_str << "\" "
+                << "}";
+    }
+
+    uint16_t num_partition = 0;
+    rdc_status_t result = rdc_get_num_partition(rdc_handle_, i, &num_partition);
+    if (result != RDC_ST_OK) {
+      return;
+    }
+
+    // A partitionable device not in partitionable mode will have metrics.num_partition=1
+    // Where as, a non-partitionable device will have metrics.num_partition = UINT16_MAX
+    if (num_partition != UINT16_MAX && num_partition > 1) {
+      if (is_json_output()) {
+        std::cout << ", \"partitions\": [";
+      }
+      for (uint32_t pid = 0; pid < num_partition; pid++) {
+        std::string instance_str = "g" + std::to_string(i) + "." + std::to_string(pid);
+
+        rdc_entity_info_t part_info;
+        part_info.device_index = i;
+        part_info.instance_index = pid;
+        part_info.entity_role = RDC_DEVICE_ROLE_PARTITION_INSTANCE;
+        part_info.device_type = RDC_DEVICE_TYPE_GPU;
+        uint32_t part_entity_index = rdc_get_entity_index_from_info(part_info);
+
+        rdc_resource_profile_t part_xcc = {};
+        rdc_resource_profile_t part_decoder = {};
+        result = rdc_instance_profile_get(rdc_handle_, part_entity_index, RDC_ACCELERATOR_XCC,
+                                          &part_xcc);
+        result = rdc_instance_profile_get(rdc_handle_, part_entity_index, RDC_ACCELERATOR_DECODER,
+                                          &part_decoder);
+
+        std::string part_decoder_str = std::to_string(part_decoder.partition_resource);
+        std::string part_xcc_str = std::to_string(part_xcc.partition_resource);
+        std::string starColumn = " ";
+        if (part_decoder.num_partitions_share_resource > 1) {
+          starColumn = "*";
+        }
+
+        if (!is_json_output()) {
+          std::cout << std::setw(12) << "" << std::setw(20) << instance_str << std::setw(25) << ""
+                    << std::setw(7) << part_xcc_str << std::setw(1) << starColumn << std::setw(8)
+                    << part_decoder_str << std::endl;
+        } else {
+          std::string decoder_shared =
+              (part_decoder.num_partitions_share_resource > 1) ? "true" : "false";
+          std::cout << "{\"instance_index\": \"" << instance_str << "\", "
+                    << "\"xcc\": \"" << part_xcc_str << "\", "
+                    << "\"decoder\": \"" << part_decoder_str << "\", "
+                    << "\"decoder_shared\": " << decoder_shared << "}";
+
+          if (pid != num_partition - 1) {
+            std::cout << ",";
+          } else {
+            std::cout << "]";
+          }
+        }
+      }
+    }
+
+    if (is_json_output()) {
+      if (i != count - 1)
+        std::cout << "},";
+      else
+        std::cout << "}";
+    }
+  }
+
+  if (!is_json_output()) {
+    std::cout << "---------------------------------------------------------------------"
+              << std::endl;
+    std::cout << "* if the resource is shared" << std::endl;
+  } else {
+    std::cout << ']';
+  }
+}
+
 void RdciDiscoverySubSystem::show_version() {
   rdc_component_version_t smiv;
   rdc_status_t result = rdc_device_get_component_version(rdc_handle_, RDC_AMDMSI_COMPONENT, &smiv);
@@ -155,18 +305,21 @@ void RdciDiscoverySubSystem::show_version() {
   mixed_component_version_t rdcdv;
   uint32_t ret = get_mixed_component_version(rdc_handle_, RDCD_COMPONENT, &rdcdv);
   if (ret) {
-    std::cout << "get rdcd version fail"<< std::endl;
+    std::cout << "get rdcd version fail" << std::endl;
     return;
   }
 
-   if (is_json_output()) {
+  if (is_json_output()) {
     std::cout << "\"version\" : ";
     std::cout << '{';
-    std::cout << "\"rdcd\": " << "\"" << rdcdv.version << "\", ";
-    std::cout << "\"amdsmi_lib\": " << "\"" << smiv.version << "\"";
+    std::cout << "\"rdcd\": "
+              << "\"" << rdcdv.version << "\", ";
+    std::cout << "\"amdsmi_lib\": "
+              << "\"" << smiv.version << "\"";
     std::cout << '}';
   } else {
-    std::cout << "RDCD : " << rdcdv.version << "  |  " << "AMDSMI Library : " << smiv.version << std::endl;
+    std::cout << "RDCD : " << rdcdv.version << "  |  "
+              << "AMDSMI Library : " << smiv.version << std::endl;
   }
 
   return;
@@ -181,6 +334,10 @@ void RdciDiscoverySubSystem::process() {
     return show_attributes();
   }
 
+  if (is_partition_) {
+    return show_attributes_with_partitions();
+  }
+
   if (show_version_) {
     return show_version();
   }

rdci/src/RdciDmonSubSystem.cc

@@ -26,6 +26,8 @@ THE SOFTWARE.
 #include <signal.h>
 #include <unistd.h>
 
+#include <algorithm>
+#include <cctype>
 #include <cmath>
 #include <ctime>
 #include <iomanip>
@@ -62,6 +64,15 @@ void RdciDmonSubSystem::set_terminating(int sig) {
   }
 }
 
+std::string entity_to_string(uint32_t entity_index) {
+  rdc_entity_info_t info = rdc_get_info_from_entity_index(entity_index);
+
+  if (info.entity_role == RDC_DEVICE_ROLE_PARTITION_INSTANCE) {
+    return "g" + std::to_string(info.device_index) + "." + std::to_string(info.instance_index);
+  }
+  return std::to_string(info.device_index);
+}
+
 void RdciDmonSubSystem::parse_cmd_opts(int argc, char** argv) {
   const int HOST_OPTIONS = 1000;
   const int LIST_ALL_FIELDS_OPT = 1001;
@@ -174,15 +185,6 @@ void RdciDmonSubSystem::parse_cmd_opts(int argc, char** argv) {
     if (gpu_indexes == "") {
       show_help();
       throw RdcException(RDC_ST_BAD_PARAMETER, "Need to specify the GPUs or group id");
-    } else {
-      std::vector<std::string> vec_ids = split_string(gpu_indexes, ',');
-      for (uint32_t i = 0; i < vec_ids.size(); i++) {
-        if (!IsNumber(vec_ids[i])) {
-          throw RdcException(RDC_ST_BAD_PARAMETER,
-                             "The GPU index " + vec_ids[i] + " needs to be a number");
-        }
-        gpu_indexes_.push_back(std::stoi(vec_ids[i]));
-      }
     }
   }
 
@@ -207,6 +209,9 @@ void RdciDmonSubSystem::parse_cmd_opts(int argc, char** argv) {
   if (options_.find(OPTIONS_COUNT) == options_.end()) {
     options_.insert({OPTIONS_COUNT, std::numeric_limits<uint32_t>::max()});
   }
+
+  // Store gpu indexes to parse later
+  raw_gpu_indexes_ = gpu_indexes;
 }
 
 void RdciDmonSubSystem::show_help() const {
@@ -272,8 +277,15 @@ void RdciDmonSubSystem::create_temp_group() {
   for (uint32_t i = 0; i < gpu_indexes_.size(); i++) {
     result = rdc_group_gpu_add(rdc_handle_, group_id, gpu_indexes_[i]);
     if (result != RDC_ST_OK) {
-      throw RdcException(result,
-                         "Fail to add " + std::to_string(gpu_indexes_[i]) + " to the dmon group.");
+      rdc_entity_info_t info = rdc_get_info_from_entity_index(gpu_indexes_[i]);
+      std::string info_str;
+      if (info.entity_role == RDC_DEVICE_ROLE_PARTITION_INSTANCE) {
+        info_str =
+            "g" + std::to_string(info.device_index) + "." + std::to_string(info.instance_index);
+      } else {
+        info_str = std::to_string(info.device_index);
+      }
+      throw RdcException(result, "Fail to add " + info_str + " to the dmon group.");
     }
   }
   options_.insert({OPTIONS_GROUP_ID, group_id});
@@ -301,6 +313,73 @@ void RdciDmonSubSystem::create_temp_field_group() {
   options_.insert({OPTIONS_FIELD_GROUP_ID, group_id});
 }
 
+void RdciDmonSubSystem::resolve_gpu_indexes() {
+  uint32_t device_list[RDC_MAX_NUM_DEVICES];
+  uint32_t count = 0;
+  rdc_status_t res = rdc_device_get_all(rdc_handle_, device_list, &count);
+  if (res != RDC_ST_OK) {
+    throw RdcException(res, "Failed to get all devices");
+  }
+
+  std::vector<std::string> vec_ids = split_string(raw_gpu_indexes_, ',');
+  for (uint32_t i = 0; i < vec_ids.size(); i++) {
+    if (rdc_is_partition_string(vec_ids[i].c_str())) {
+      uint32_t logicalPhysicalGpu;
+      uint32_t partition;
+      if (!rdc_parse_partition_string(vec_ids[i].c_str(), &logicalPhysicalGpu, &partition)) {
+        throw RdcException(RDC_ST_BAD_PARAMETER, "Invalid partition format: " + vec_ids[i]);
+      }
+
+      if (logicalPhysicalGpu >= count) {
+        throw RdcException(RDC_ST_BAD_PARAMETER,
+                           "GPU " + std::to_string(logicalPhysicalGpu) + " is out of range");
+      }
+
+      uint32_t physicalGpu = device_list[logicalPhysicalGpu];
+
+      uint16_t num_partitions = 0;
+      rdc_status_t st = rdc_get_num_partition(rdc_handle_, physicalGpu, &num_partitions);
+      if (st != RDC_ST_OK) {
+        throw RdcException(st,
+                           "Failed to get partition info for GPU " + std::to_string(physicalGpu));
+      }
+
+      if (num_partitions == UINT16_MAX || num_partitions <= 1) {
+        if (partition != 0) {
+          throw RdcException(RDC_ST_BAD_PARAMETER, "GPU " + std::to_string(physicalGpu) +
+                                                       " is not partitioned, so partition " +
+                                                       std::to_string(partition) + " is invalid");
+        }
+      } else {
+        if (partition >= num_partitions) {
+          throw RdcException(RDC_ST_BAD_PARAMETER,
+                             "GPU " + std::to_string(physicalGpu) + " supports only " +
+                                 std::to_string(num_partitions) + " partitions, partition " +
+                                 std::to_string(partition) + " is invalid");
+        }
+      }
+
+      rdc_entity_info_t phys_info;
+      phys_info.device_index = physicalGpu;
+      phys_info.instance_index = partition;
+      phys_info.entity_role = RDC_DEVICE_ROLE_PARTITION_INSTANCE;
+      phys_info.device_type = RDC_DEVICE_TYPE_GPU;
+      uint32_t phys_entity_index = rdc_get_entity_index_from_info(phys_info);
+      gpu_indexes_.push_back(phys_entity_index);
+    } else if (IsNumber(vec_ids[i])) {
+      uint32_t logicalIndex = std::stoi(vec_ids[i]);
+      if (logicalIndex >= count) {
+        throw RdcException(RDC_ST_BAD_PARAMETER,
+                           "GPU " + std::to_string(logicalIndex) + " is out of range");
+      }
+      gpu_indexes_.push_back(std::stoi(vec_ids[i]));
+    } else {
+      throw RdcException(RDC_ST_BAD_PARAMETER, "The GPU index " + vec_ids[i] +
+                                                   " needs to be a number or a valid partition");
+    }
+  }
+}
+
 void RdciDmonSubSystem::show_field_usage() const {
   std::cout << "Supported fields Ids:" << std::endl;
 
@@ -430,6 +509,8 @@ void RdciDmonSubSystem::process() {
   rdc_group_info_t group_info;
   rdc_field_group_info_t field_info;
 
+  resolve_gpu_indexes();
+
   // Create a temporary group/field if pass as GPU indexes or field ids
   create_temp_group();
   create_temp_field_group();
@@ -516,7 +597,8 @@ void RdciDmonSubSystem::process() {
     print_and_clr_notif_pq(&notif_pq, show_timpstamps_);
 
     for (uint32_t gindex = 0; gindex < group_info.count; gindex++) {
-      std::cout << group_info.entity_ids[gindex] << "\t";
+      std::cout << std::setw(12) << std::left << entity_to_string(group_info.entity_ids[gindex])
+                << "\t";
       for (uint32_t findex = 0; findex < reg_fields.size(); findex++) {
         rdc_field_value value;