Enable RDC topology feature

1.Add topology APIs
2.Add topology example for topology API usage

Change-Id: Ib79c06d0bac85119672f194ba685ebf25029979c

rdc_libs/bootstrap/src/RdcBootStrap.cc

@@ -494,3 +494,18 @@ rdc_status_t rdc_policy_unregister(rdc_handle_t p_rdc_handle, rdc_gpu_group_t gr
   return static_cast<amd::rdc::RdcHandler*>(p_rdc_handle)
       ->rdc_policy_unregister(group_id);
 }
+rdc_status_t rdc_device_topology_get(rdc_handle_t p_rdc_handle, uint32_t gpu_index,
+                                     rdc_device_topology_t* results) {
+  if (!p_rdc_handle) {
+    return RDC_ST_INVALID_HANDLER;
+  }
+  return static_cast<amd::rdc::RdcHandler*>(p_rdc_handle)
+      ->rdc_device_topology_get(gpu_index, results);
+}
+rdc_status_t rdc_link_status_get(rdc_handle_t p_rdc_handle, rdc_link_status_t* results) {
+  if (!p_rdc_handle) {
+    return RDC_ST_INVALID_HANDLER;
+  }
+  return static_cast<amd::rdc::RdcHandler*>(p_rdc_handle)
+      ->rdc_link_status_get(results);
+}

rdc_libs/rdc/CMakeLists.txt

@@ -19,6 +19,7 @@ set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST}
     "${SRC_DIR}/RdcNotificationImpl.cc"
     "${SRC_DIR}/RdcPerfTimer.cc"
     "${SRC_DIR}/RdcPolicyImpl.cc"
+    "${SRC_DIR}/RdcTopologyLinkImpl.cc"
     "${SRC_DIR}/RdcRocpLib.cc"
     "${SRC_DIR}/RdcRocrLib.cc"
     "${SRC_DIR}/RdcRVSLib.cc"

rdc_libs/rdc/src/RdcEmbeddedHandler.cc

@@ -36,6 +36,7 @@ THE SOFTWARE.
 #include "rdc_lib/impl/RdcModuleMgrImpl.h"
 #include "rdc_lib/impl/RdcNotificationImpl.h"
 #include "rdc_lib/impl/RdcPolicyImpl.h"
+#include "rdc_lib/impl/RdcTopologyLinkImpl.h"
 #include "rdc_lib/impl/RdcWatchTableImpl.h"
 #include "rdc_lib/rdc_common.h"
 
@@ -81,7 +82,8 @@ RdcEmbeddedHandler::RdcEmbeddedHandler(rdc_operation_mode_t mode)
       rdc_notif_(new RdcNotificationImpl()),
       watch_table_(new RdcWatchTableImpl(group_settings_, cache_mgr_, metric_fetcher_, rdc_module_mgr_, rdc_notif_)),
       metrics_updater_(new RdcMetricsUpdaterImpl(watch_table_, METIC_UPDATE_FREQUENCY)),
-      policy_(new RdcPolicyImpl(group_settings_,metric_fetcher_)) {
+      policy_(new RdcPolicyImpl(group_settings_,metric_fetcher_)),
+      topologylink_(new RdcTopologyLinkImpl(group_settings_, metric_fetcher_)) {
   if (mode == RDC_OPERATION_MODE_AUTO) {
     RDC_LOG(RDC_DEBUG, "Run RDC with RDC_OPERATION_MODE_AUTO");
     metrics_updater_->start();
@@ -493,5 +495,14 @@ rdc_status_t RdcEmbeddedHandler::rdc_health_clear(rdc_gpu_group_t group_id) {
   return watch_table_->rdc_health_clear(group_id);
 }
 
+rdc_status_t RdcEmbeddedHandler::rdc_device_topology_get(uint32_t gpu_index,
+                                                         rdc_device_topology_t* results) {
+  return topologylink_->rdc_device_topology_get(gpu_index, results);
+}
+
+rdc_status_t RdcEmbeddedHandler::rdc_link_status_get(rdc_link_status_t* results) {
+  return topologylink_->rdc_link_status_get(results);
+}
+
 }  // namespace rdc
 }  // namespace amd

rdc_libs/rdc/src/RdcTopologyLinkImpl.cc

@@ -0,0 +1,141 @@
+/*
+Copyright (c) 2024 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include "rdc_lib/impl/RdcTopologyLinkImpl.h"
+
+#include <sys/time.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <ctime>
+#include <map>
+#include <sstream>
+#include <unordered_map>
+
+#include "common/rdc_utils.h"
+#include "rdc/rdc.h"
+#include "rdc_lib/RdcLogger.h"
+#include "rdc_lib/impl/SmiUtils.h"
+#include "rdc_lib/rdc_common.h"
+
+namespace amd {
+namespace rdc {
+
+RdcTopologyLinkImpl::RdcTopologyLinkImpl(const RdcGroupSettingsPtr& group_settings,
+                                         RdcMetricFetcherPtr metric_fetcher)
+    : group_settings_(group_settings), metric_fetcher_(metric_fetcher) {}
+
+RdcTopologyLinkImpl::~RdcTopologyLinkImpl() {}
+
+rdc_status_t RdcTopologyLinkImpl::rdc_device_topology_get(uint32_t gpu_index,
+                                                          rdc_device_topology_t* results) {
+  rdc_status_t status = RDC_ST_NOT_FOUND;
+  amdsmi_status_t err = AMDSMI_STATUS_SUCCESS;
+  uint32_t gpu_index_list[RDC_MAX_NUM_DEVICES];
+
+  amdsmi_processor_handle processor_handle;
+  err = get_processor_handle_from_id(gpu_index, &processor_handle);
+  if (err != AMDSMI_STATUS_SUCCESS) {
+    RDC_LOG(RDC_INFO, "Fail to get process GPUs processor handle information: " << err);
+    return status;
+  }
+
+  uint32_t numa_node = 0;
+  err = amdsmi_topo_get_numa_node_number(processor_handle, &numa_node);
+  if (err != AMDSMI_STATUS_SUCCESS) {
+    RDC_LOG(RDC_INFO, "Fail to get process GPUs numa_node information: " << err);
+    return status;
+  }
+
+  uint32_t count = 0;
+  rdc_field_value device_count;
+  status = metric_fetcher_->fetch_smi_field(0, RDC_FI_GPU_COUNT, &device_count);
+  if (status != RDC_ST_OK) {
+    return status;
+  }
+
+  // Assign the index to the index list
+  count = device_count.value.l_int;
+  for (uint32_t i = 0; i < count; i++) {
+    gpu_index_list[i] = i;
+  }
+
+  results->num_of_gpus = count;
+  results->numa_node = numa_node;
+
+  for (uint32_t i = 0; i < count; i++) {
+    for (uint32_t j = 0; j < count; j++) {
+      if (gpu_index_list[i] == gpu_index_list[j]) continue;
+      std::pair<amdsmi_processor_handle, amdsmi_processor_handle> ph;
+      err = get_processor_handle_from_id(gpu_index_list[i], &ph.first);
+      err = get_processor_handle_from_id(gpu_index_list[i], &ph.second);
+      if (err != AMDSMI_STATUS_SUCCESS) {
+        RDC_LOG(RDC_INFO, "Fail to get process GPUs processor handle information: " << err);
+        return status;
+      }
+
+      uint64_t weight = std::numeric_limits<uint64_t>::max();
+      err = amdsmi_topo_get_link_weight(ph.first, ph.second, &weight);
+      if (err != AMDSMI_STATUS_SUCCESS) {
+        RDC_LOG(RDC_INFO, "Fail to get process GPUs weight information: " << err);
+      }
+
+      uint64_t min_bandwidth = std::numeric_limits<uint64_t>::max();
+      uint64_t max_bandwidth = std::numeric_limits<uint64_t>::max();
+      err = amdsmi_get_minmax_bandwidth_between_processors(ph.first, ph.second, &min_bandwidth,
+                                                           &max_bandwidth);
+      if (err != AMDSMI_STATUS_SUCCESS) {
+        RDC_LOG(RDC_INFO, "Fail to get process GPUs detail information: " << err);
+      }
+
+      uint64_t hops = std::numeric_limits<uint64_t>::max();
+      amdsmi_io_link_type_t type = AMDSMI_IOLINK_TYPE_UNDEFINED;
+      err = amdsmi_topo_get_link_type(ph.first, ph.second, &hops, &type);
+      if (err != AMDSMI_STATUS_SUCCESS) {
+        RDC_LOG(RDC_INFO, "Fail to get process GPUs hops and type information: " << err);
+      }
+
+      bool accessible;
+      err = amdsmi_is_P2P_accessible(ph.first, ph.second, &accessible);
+      if (err != AMDSMI_STATUS_SUCCESS) {
+        RDC_LOG(RDC_INFO, "Fail to get process GPUs P2P accessible information: " << err);
+      }
+
+      results->link_infos[i].gpu_index = gpu_index_list[i];
+      results->link_infos[i].weight = weight;
+      results->link_infos[i].min_bandwidth = min_bandwidth;
+      results->link_infos[i].max_bandwidth = max_bandwidth;
+      results->link_infos[i].hops = hops;
+      results->link_infos[i].link_type = static_cast<rdc_topology_link_type_t>(type);
+    }
+  }
+
+  return RDC_ST_OK;
+}
+
+rdc_status_t RdcTopologyLinkImpl::rdc_link_status_get(rdc_link_status_t* results) {
+  rdc_status_t status = RDC_ST_NOT_FOUND;
+  return status;
+}
+
+}  // namespace rdc
+}  // namespace amd

rdc_libs/rdc_client/src/RdcStandaloneHandler.cc

@@ -952,5 +952,40 @@ rdc_status_t RdcStandaloneHandler::rdc_health_clear(rdc_gpu_group_t group_id) {
   return RDC_ST_OK;
 }
 
+rdc_status_t RdcStandaloneHandler::rdc_device_topology_get(uint32_t gpu_index,
+                                                           rdc_device_topology_t* results) {
+  ::rdc::GetTopologyRequest request;
+  ::rdc::GetTopologyResponse reply;
+  ::grpc::ClientContext context;
+
+  request.set_gpu_index(gpu_index);
+  ::grpc::Status status = stub_->GetTopology(&context, request, &reply);
+  rdc_status_t err_status = error_handle(status, reply.status());
+  if (err_status != RDC_ST_OK) return err_status;
+
+  ::rdc::Topology Topology = reply.toppology();
+  results->num_of_gpus= Topology.num_of_gpus();
+  results->numa_node= Topology.numa_node();
+
+  for (uint32_t i = 0; i < Topology.num_of_gpus(); ++i) {
+  ::rdc::TopologyLinkInfo linkinfo = Topology.link_infos(i);
+  results->link_infos[i].gpu_index=linkinfo.gpu_index();
+  results->link_infos[i].weight=linkinfo.weight();
+  results->link_infos[i].min_bandwidth=linkinfo.min_bandwidth();
+  results->link_infos[i].max_bandwidth=linkinfo.max_bandwidth();
+  results->link_infos[i].hops=linkinfo.hops();
+  results->link_infos[i].link_type=static_cast<rdc_topology_link_type_t>(linkinfo.link_type());
+  results->link_infos[i].is_p2p_accessible=linkinfo.p2p_accessible();
+  }
+
+  return RDC_ST_OK;
+}
+
+rdc_status_t RdcStandaloneHandler::rdc_link_status_get(rdc_link_status_t* results) {
+  ::rdc::UpdateAllFieldsResponse reply;
+  ::grpc::Status status = grpc::Status::OK;
+  return error_handle(status, reply.status());
+}
+
 }  // namespace rdc
 }  // namespace amd