Enable RDC topology feature

1.Add topology APIs
2.Add topology example for topology API usage

Change-Id: Ib79c06d0bac85119672f194ba685ebf25029979c

example/topologylink_example.cc

@@ -0,0 +1,135 @@
+#include <unistd.h>
+
+#include <iostream>
+
+#include "rdc/rdc.h"
+
+static const char* topology_link_type_to_str(rdc_topology_link_type_t type) {
+  if (type == RDC_IOLINK_TYPE_PCIEXPRESS) return "RDC_IOLINK_TYPE_PCIEXPRESS";
+  if (type == RDC_IOLINK_TYPE_XGMI) return "RDC_IOLINK_TYPE_XGMI";
+  if (type == RDCI_IOLINK_TYPE_NUMIOLINKTYPES) return "RDCI_IOLINK_TYPE_NUMIOLINKTYPES";
+  return "Unknown_Type";
+}
+int main() {
+  rdc_gpu_group_t group_id;
+  rdc_status_t result;
+  bool standalone = false;
+  rdc_handle_t rdc_handle;
+  uint32_t count = 0;
+
+  char hostIpAddress[] = {"localhost:50051"};
+  char group_name[] = {"group1"};
+
+  // Select the embedded mode and standalone mode dynamically.
+  std::cout << "Start rdci in: \n";
+  std::cout << "0 - Embedded mode \n";
+  std::cout << "1 - Standalone mode \n";
+  while (!(std::cin >> standalone)) {
+    std::cout << "Invalid input.\n";
+    std::cin.clear();
+    std::cin.ignore();
+  }
+  std::cout << std::endl;
+  std::cout << (standalone ? "Standalone mode selected.\n" : "Embedded mode selected.\n");
+
+  // Init the rdc
+  result = rdc_init(0);
+
+  if (result != RDC_ST_OK) {
+    std::cout << "Error initializing RDC. Return: " << rdc_status_string(result) << std::endl;
+    goto cleanup;
+  } else {
+    std::cout << "RDC Initialized.\n";
+  }
+
+  if (standalone) {  // standalone
+    result = rdc_connect(hostIpAddress, &rdc_handle, nullptr, nullptr, nullptr);
+    if (result != RDC_ST_OK) {
+      std::cout << "Error connecting to remote rdcd. Return: " << rdc_status_string(result)
+                << std::endl;
+      goto cleanup;
+    }
+  } else {  // embedded
+    result = rdc_start_embedded(RDC_OPERATION_MODE_AUTO, &rdc_handle);
+    if (result != RDC_ST_OK) {
+      std::cout << "Error starting embedded RDC engine. Return: " << rdc_status_string(result)
+                << std::endl;
+      goto cleanup;
+    }
+  }
+
+  // Now we can use the same API for both standalone and embedded
+  // Get the list of devices in the system
+  uint32_t gpu_index_list[RDC_MAX_NUM_DEVICES];
+  result = rdc_device_get_all(rdc_handle, gpu_index_list, &count);
+  if (result != RDC_ST_OK) {
+    std::cout << "Error to find devices on the system. Return: " << rdc_status_string(result);
+    goto cleanup;
+  }
+  if (count == 0) {
+    std::cout << "No GPUs find on the sytem ";
+    goto cleanup;
+  } else {
+    std::cout << count << " GPUs found in the system.\n";
+  }
+
+  // Create the group
+  result = rdc_group_gpu_create(rdc_handle, RDC_GROUP_EMPTY, group_name, &group_id);
+  if (result != RDC_ST_OK) {
+    std::cout << "Error creating group. Return: " << rdc_status_string(result);
+    goto cleanup;
+  }
+  std::cout << "Created the GPU group " << group_id << std::endl;
+
+  // Add all GPUs to the group
+  for (uint32_t i = 0; i < count; i++) {
+    result = rdc_group_gpu_add(rdc_handle, group_id, gpu_index_list[i]);  // Add GPU 0
+    if (result != RDC_ST_OK) {
+      std::cout << "Error adding group. Return: " << rdc_status_string(result);
+      goto cleanup;
+    }
+    rdc_device_attributes_t attribute;
+    result = rdc_device_get_attributes(rdc_handle, gpu_index_list[i], &attribute);
+    if (result != RDC_ST_OK) {
+      std::cout << "Error get GPU attribute. Return: " << rdc_status_string(result);
+      goto cleanup;
+    }
+    std::cout << "Add GPU " << gpu_index_list[i] << ":" << attribute.device_name << " to group "
+              << group_id << std::endl;
+  }
+  rdc_group_info_t group_info;
+  result = rdc_group_gpu_get_info(rdc_handle, group_id, &group_info);
+  if (result != RDC_ST_OK) {
+    std::cout << "Error get gpu group info. Return: " << rdc_status_string(result);
+    goto cleanup;
+  }
+
+  rdc_device_topology_t topo;
+  result = rdc_device_topology_get(rdc_handle, group_info.entity_ids[0], &topo);
+  if (result != RDC_ST_OK) {
+    std::cout << "Error clear topology, Return: " << rdc_status_string(result) << std::endl;
+    goto cleanup;
+  }
+
+  for (uint32_t i = 0; i < topo.num_of_gpus; i++) {
+    std::cout << "Topology Information Return \n "
+              << "gpu_index: " << std::to_string(topo.link_infos[i].gpu_index) << "\n"
+              << "weight: " << std::to_string(topo.link_infos[i].weight) << "\n"
+              << "min_bandwidth: " << std::to_string(topo.link_infos[i].min_bandwidth) << "\n"
+              << "max_bandwidth: " << std::to_string(topo.link_infos[i].max_bandwidth) << "\n"
+              << "hops: " << std::to_string(topo.link_infos[i].hops) << "\n"
+              << "link_type: " << topology_link_type_to_str(topo.link_infos[i].link_type) << "\n"
+              << "is_p2p_accessible: " << std::to_string(topo.link_infos[i].is_p2p_accessible) << "\n"
+              << std::endl;
+  }
+
+  //... clean up
+cleanup:
+  std::cout << "Cleaning up.\n";
+  if (standalone)
+    rdc_disconnect(rdc_handle);
+  else
+    rdc_stop_embedded(rdc_handle);
+  rdc_shutdown();
+  return result;
+}

include/rdc/rdc.h

@@ -602,6 +602,63 @@ typedef struct {
   rdc_policy_action_t action; //!< Action to take
 } rdc_policy_t;
 
+typedef enum {
+  RDC_IOLINK_TYPE_UNDEFINED = 0,     //!< unknown type.
+  RDC_IOLINK_TYPE_PCIEXPRESS = 1,    //!< PCI Express
+  RDC_IOLINK_TYPE_XGMI = 2,          //!< XGMI
+  RDCI_IOLINK_TYPE_NUMIOLINKTYPES,   //!< Number of IO Link types
+  RDC_IOLINK_TYPE_SIZE = 0xFFFFFFFF  //!< Max of IO Link types
+} rdc_topology_link_type_t;
+
+/**
+ * @brief The link information of the GPU connected to
+ */
+typedef struct {
+  uint32_t gpu_index;
+  // amdsmi_topo_get_link_weight
+  uint64_t weight;  // the weight for a connection between 2 GPUs
+  // minimal and maximal io link bandwidth between 2 GPUs
+  // amdsmi_get_minmax_bandwidth_between_processors
+  uint64_t min_bandwidth;
+  uint64_t max_bandwidth;
+  // amdsmi_topo_get_link_type
+  uint64_t hops;
+  rdc_topology_link_type_t link_type;
+  // amdsmi_is_P2P_accessible
+  bool is_p2p_accessible;
+} rdc_topology_link_info_t;
+
+/**
+ * @brief The data in the data structure will be set to max value if it is N/A or error
+ */
+typedef struct {
+  uint32_t num_of_gpus;  // The length of link_infos array
+  rdc_topology_link_info_t link_infos[RDC_MAX_NUM_DEVICES];
+  // amdsmi_topo_get_numa_node_number
+  uint32_t numa_node;  // the NUMA CPU node number for a device
+} rdc_device_topology_t;
+
+typedef enum {
+  RDC_LINK_STATE_NOT_SUPPORTED = 0,
+  RDC_LINK_STATE_DISABLED,
+  RDC_LINK_STATE_DOWN,
+  RDC_LINK_STATE_UP
+} rdc_link_state_t;
+
+#define RDC_MAX_NUM_OF_LINKS 16
+
+typedef struct {
+  int32_t gpu_index;
+  uint32_t num_of_links;                              // The size of the array link_states
+  rdc_topology_link_type_t link_types;                // XGMI, PCIe, and so on
+  rdc_link_state_t link_states[RDC_MAX_NUM_OF_LINKS];
+} rdc_gpu_link_status_t;
+
+typedef struct {
+  int32_t num_of_gpus;                                // The size of gpus array
+  rdc_gpu_link_status_t gpus[RDC_MAX_NUM_DEVICES];
+} rdc_link_status_t;
+
 /**
  * @brief type of health watches
  */
@@ -1428,6 +1485,36 @@ rdc_status_t rdc_health_check(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_i
  */
 rdc_status_t rdc_health_clear(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id);
 
+ /**
+ *  @brief Get the topology of the device
+ *
+ *  @details topology of the device
+ *
+ *  @param[in] p_rdc_handle The RDC handler.
+ *
+ *  @param[in] gpu_index The GPU gpu index.
+ *
+ *  @param[out] results  The device topology
+ *
+ *  @retval ::RDC_ST_OK is returned upon successful call.
+ */
+rdc_status_t rdc_device_topology_get(rdc_handle_t p_rdc_handle, uint32_t gpu_index,
+                                     rdc_device_topology_t* results);
+/**
+ *  @brief Get the link status
+ *
+ *  @details the link is up or down
+ *
+ *  @param[in] p_rdc_handle The RDC handler.
+ *
+ *
+ *  @param[out] resu
+ * lts  The link up or down status
+ *
+ *  @retval ::RDC_ST_OK is returned upon successful call.
+ */
+rdc_status_t rdc_link_status_get(rdc_handle_t p_rdc_handle, rdc_link_status_t* results);
+
 #ifdef __cplusplus
 }
 #endif  // __cplusplus

include/rdc_lib/RdcHandler.h

@@ -45,7 +45,8 @@ class RdcHandler {
                                           uint32_t* count) = 0;
   virtual rdc_status_t rdc_device_get_attributes(uint32_t gpu_index,
                                                  rdc_device_attributes_t* p_rdc_attr) = 0;
-  virtual rdc_status_t rdc_device_get_component_version(rdc_component_t component, rdc_component_version_t* p_rdc_compv) = 0;
+  virtual rdc_status_t rdc_device_get_component_version(rdc_component_t component,
+                                                        rdc_component_version_t* p_rdc_compv) = 0;
 
   // Group API
   virtual rdc_status_t rdc_group_gpu_create(rdc_group_type_t type, const char* group_name,
@@ -111,6 +112,10 @@ class RdcHandler {
   virtual rdc_status_t rdc_health_get(rdc_gpu_group_t group_id, unsigned int* components) = 0;
   virtual rdc_status_t rdc_health_check(rdc_gpu_group_t group_id, rdc_health_response_t *response) = 0;
   virtual rdc_status_t rdc_health_clear(rdc_gpu_group_t group_id) = 0;
+    // topology API
+  virtual rdc_status_t rdc_device_topology_get(uint32_t gpu_index,
+                                               rdc_device_topology_t* results) = 0;
+  virtual rdc_status_t rdc_link_status_get(rdc_link_status_t* results) = 0;
 
   virtual ~RdcHandler() {}
 };

include/rdc_lib/RdcTopologyLink.h

@@ -0,0 +1,48 @@
+/*
+Copyright (c) 2024 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef INCLUDE_RDC_LIB_RDCTOPOLOGYLINK_H_
+#define INCLUDE_RDC_LIB_RDCTOPOLOGYLINK_H_
+
+#include <memory>
+#include <vector>
+
+#include "rdc/rdc.h"
+#include "rdc_lib/rdc_common.h"
+
+namespace amd {
+namespace rdc {
+
+class RdcTopologyLink {
+ public:
+  virtual rdc_status_t rdc_device_topology_get(uint32_t gpu_index,
+                                               rdc_device_topology_t* results) = 0;
+  virtual rdc_status_t rdc_link_status_get(rdc_link_status_t* results) = 0;
+
+  virtual ~RdcTopologyLink() {}
+};
+
+typedef std::shared_ptr<RdcTopologyLink> RdcTopologyLinkPtr;
+
+}  // namespace rdc
+}  // namespace amd
+
+#endif  // INCLUDE_RDC_LIB_RDCTOPOLOGYLINK_H_

include/rdc_lib/impl/RdcEmbeddedHandler.h

@@ -32,6 +32,7 @@ THE SOFTWARE.
 #include "rdc_lib/RdcModuleMgr.h"
 #include "rdc_lib/RdcNotification.h"
 #include "rdc_lib/RdcPolicy.h"
+#include "rdc_lib/RdcTopologyLink.h"
 #include "rdc_lib/RdcWatchTable.h"
 
 namespace amd {
@@ -52,7 +53,8 @@ class RdcEmbeddedHandler final : public RdcHandler {
                                   uint32_t* count) override;
   rdc_status_t rdc_device_get_attributes(uint32_t gpu_index,
                                          rdc_device_attributes_t* p_rdc_attr) override;
-  rdc_status_t rdc_device_get_component_version(rdc_component_t component, rdc_component_version_t* p_rdc_compv) override;
+  rdc_status_t rdc_device_get_component_version(rdc_component_t component,
+                                                rdc_component_version_t* p_rdc_compv) override;
 
   // Group API
   rdc_status_t rdc_group_gpu_create(rdc_group_type_t type, const char* group_name,
@@ -113,6 +115,9 @@ class RdcEmbeddedHandler final : public RdcHandler {
   rdc_status_t rdc_health_get(rdc_gpu_group_t group_id, unsigned int* components) override;
   rdc_status_t rdc_health_check(rdc_gpu_group_t group_id, rdc_health_response_t *response) override;
   rdc_status_t rdc_health_clear(rdc_gpu_group_t group_id) override;
+  rdc_status_t rdc_device_topology_get(uint32_t gpu_index, rdc_device_topology_t* results) override;
+  
+  rdc_status_t rdc_link_status_get(rdc_link_status_t* results) override;
 
   explicit RdcEmbeddedHandler(rdc_operation_mode_t op_mode);
   ~RdcEmbeddedHandler() final;
@@ -128,6 +133,7 @@ class RdcEmbeddedHandler final : public RdcHandler {
   RdcMetricsUpdaterPtr metrics_updater_;
   RdcPolicyPtr policy_;
   std::future<void> updater_;
+  RdcTopologyLinkPtr topologylink_;
 };
 
 }  // namespace rdc

include/rdc_lib/impl/RdcStandaloneHandler.h

@@ -49,7 +49,8 @@ class RdcStandaloneHandler : public RdcHandler {
                                   uint32_t* count) override;
   rdc_status_t rdc_device_get_attributes(uint32_t gpu_index,
                                          rdc_device_attributes_t* p_rdc_attr) override;
-  rdc_status_t rdc_device_get_component_version(rdc_component_t component, rdc_component_version_t* p_rdc_compv) override;
+  rdc_status_t rdc_device_get_component_version(rdc_component_t component,
+                                                rdc_component_version_t* p_rdc_compv) override;
 
   // Group RdcAPI
   rdc_status_t rdc_group_gpu_create(rdc_group_type_t type, const char* group_name,
@@ -110,6 +111,9 @@ class RdcStandaloneHandler : public RdcHandler {
   rdc_status_t rdc_health_get(rdc_gpu_group_t group_id, unsigned int* components) override;
   rdc_status_t rdc_health_check(rdc_gpu_group_t group_id, rdc_health_response_t *response) override;
   rdc_status_t rdc_health_clear(rdc_gpu_group_t group_id) override;
+  rdc_status_t rdc_device_topology_get(uint32_t gpu_index, rdc_device_topology_t* results) override;
+  
+  rdc_status_t rdc_link_status_get(rdc_link_status_t* results) override;
 
   explicit RdcStandaloneHandler(const char* ip_and_port, const char* root_ca,
                                 const char* client_cert, const char* client_key);

include/rdc_lib/impl/RdcTopologyLinkImpl.h

@@ -0,0 +1,60 @@
+/*
+Copyright (c) 2024 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef INCLUDE_RDC_LIB_IMPL_RDCTOPOLINKYIMPL_H_
+#define INCLUDE_RDC_LIB_IMPL_RDCTOPOLINKYIMPL_H_
+
+#include <atomic>
+#include <future>
+#include <map>
+#include <memory>
+#include <mutex>  // NOLINT
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "amd_smi/amdsmi.h"
+#include "rdc_lib/RdcGroupSettings.h"
+#include "rdc_lib/RdcMetricFetcher.h"
+#include "rdc_lib/RdcTopologyLink.h"
+
+namespace amd {
+namespace rdc {
+
+class RdcTopologyLinkImpl : public RdcTopologyLink {
+ public:
+  RdcTopologyLinkImpl(const RdcGroupSettingsPtr& group_settings,
+                      RdcMetricFetcherPtr metric_fetcher);
+  ~RdcTopologyLinkImpl();
+
+  rdc_status_t rdc_device_topology_get(uint32_t gpu_index, rdc_device_topology_t* results) override;
+  rdc_status_t rdc_link_status_get(rdc_link_status_t* results) override;
+
+ private:
+  RdcGroupSettingsPtr group_settings_;
+  RdcMetricFetcherPtr metric_fetcher_;
+  
+};
+
+}  // namespace rdc
+}  // namespace amd
+
+#endif  // INCLUDE_RDC_LIB_IMPL_RDCTOPOLINKYIMPL_H_

protos/rdc.proto

@@ -202,6 +202,11 @@ service RdcAPI {
 
   // rdc_status_t rdc_health_clear(rdc_gpu_group_t group_id);
   rpc ClearHealth(ClearHealthRequest) returns (ClearHealthResponse) {}
+  // rdc_status_t rdc_device_topology_get(
+  //  rdc_handle_t p_rdc_handle,
+  //  rdc_gpu_group_t group_id,
+  //  rdc_policy_condition_t condition);
+  rpc GetTopology(GetTopologyRequest) returns (GetTopologyResponse) {}
 }
 
 message Empty {
@@ -684,3 +689,34 @@ message ClearHealthRequest {
 message ClearHealthResponse {
   uint32 status = 1;
 }
+message TopologyLinkInfo{
+  uint32 gpu_index = 1;
+  uint64 weight = 2;
+  uint64 min_bandwidth = 3;
+  uint64 max_bandwidth = 4;
+  uint64 hops = 5;
+  enum LinkType {
+    RDC_IOLINK_TYPE_UNDEFINED = 0;
+    RDC_IOLINK_TYPE_PCIEXPRESS = 1;
+    RDC_IOLINK_TYPE_XGMI = 2;
+    RDCI_IOLINK_TYPE_NUMIOLINKTYPES = 3;
+  };
+  LinkType link_type = 6;
+  bool p2p_accessible = 7;
+}
+message Topology{
+  uint32 num_of_gpus = 1;
+  repeated TopologyLinkInfo link_infos = 2;
+  uint32 numa_node = 3;
+}
+message GetTopologyResult {
+  uint32 status = 1;
+}
+message GetTopologyRequest {
+  uint32 gpu_index = 1;
+}
+message GetTopologyResponse {
+  uint32 status = 1;
+  Topology toppology = 2;
+}
+

rdc_libs/bootstrap/src/RdcBootStrap.cc

@@ -494,3 +494,18 @@ rdc_status_t rdc_policy_unregister(rdc_handle_t p_rdc_handle, rdc_gpu_group_t gr
   return static_cast<amd::rdc::RdcHandler*>(p_rdc_handle)
       ->rdc_policy_unregister(group_id);
 }
+rdc_status_t rdc_device_topology_get(rdc_handle_t p_rdc_handle, uint32_t gpu_index,
+                                     rdc_device_topology_t* results) {
+  if (!p_rdc_handle) {
+    return RDC_ST_INVALID_HANDLER;
+  }
+  return static_cast<amd::rdc::RdcHandler*>(p_rdc_handle)
+      ->rdc_device_topology_get(gpu_index, results);
+}
+rdc_status_t rdc_link_status_get(rdc_handle_t p_rdc_handle, rdc_link_status_t* results) {
+  if (!p_rdc_handle) {
+    return RDC_ST_INVALID_HANDLER;
+  }
+  return static_cast<amd::rdc::RdcHandler*>(p_rdc_handle)
+      ->rdc_link_status_get(results);
+}

rdc_libs/rdc/CMakeLists.txt

@@ -19,6 +19,7 @@ set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST}
     "${SRC_DIR}/RdcNotificationImpl.cc"
     "${SRC_DIR}/RdcPerfTimer.cc"
     "${SRC_DIR}/RdcPolicyImpl.cc"
+    "${SRC_DIR}/RdcTopologyLinkImpl.cc"
     "${SRC_DIR}/RdcRocpLib.cc"
     "${SRC_DIR}/RdcRocrLib.cc"
     "${SRC_DIR}/RdcRVSLib.cc"

rdc_libs/rdc/src/RdcEmbeddedHandler.cc

@@ -36,6 +36,7 @@ THE SOFTWARE.
 #include "rdc_lib/impl/RdcModuleMgrImpl.h"
 #include "rdc_lib/impl/RdcNotificationImpl.h"
 #include "rdc_lib/impl/RdcPolicyImpl.h"
+#include "rdc_lib/impl/RdcTopologyLinkImpl.h"
 #include "rdc_lib/impl/RdcWatchTableImpl.h"
 #include "rdc_lib/rdc_common.h"
 
@@ -81,7 +82,8 @@ RdcEmbeddedHandler::RdcEmbeddedHandler(rdc_operation_mode_t mode)
       rdc_notif_(new RdcNotificationImpl()),
       watch_table_(new RdcWatchTableImpl(group_settings_, cache_mgr_, metric_fetcher_, rdc_module_mgr_, rdc_notif_)),
       metrics_updater_(new RdcMetricsUpdaterImpl(watch_table_, METIC_UPDATE_FREQUENCY)),
-      policy_(new RdcPolicyImpl(group_settings_,metric_fetcher_)) {
+      policy_(new RdcPolicyImpl(group_settings_,metric_fetcher_)),
+      topologylink_(new RdcTopologyLinkImpl(group_settings_, metric_fetcher_)) {
   if (mode == RDC_OPERATION_MODE_AUTO) {
     RDC_LOG(RDC_DEBUG, "Run RDC with RDC_OPERATION_MODE_AUTO");
     metrics_updater_->start();
@@ -493,5 +495,14 @@ rdc_status_t RdcEmbeddedHandler::rdc_health_clear(rdc_gpu_group_t group_id) {
   return watch_table_->rdc_health_clear(group_id);
 }
 
+rdc_status_t RdcEmbeddedHandler::rdc_device_topology_get(uint32_t gpu_index,
+                                                         rdc_device_topology_t* results) {
+  return topologylink_->rdc_device_topology_get(gpu_index, results);
+}
+
+rdc_status_t RdcEmbeddedHandler::rdc_link_status_get(rdc_link_status_t* results) {
+  return topologylink_->rdc_link_status_get(results);
+}
+
 }  // namespace rdc
 }  // namespace amd

rdc_libs/rdc/src/RdcTopologyLinkImpl.cc

@@ -0,0 +1,141 @@
+/*
+Copyright (c) 2024 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include "rdc_lib/impl/RdcTopologyLinkImpl.h"
+
+#include <sys/time.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <ctime>
+#include <map>
+#include <sstream>
+#include <unordered_map>
+
+#include "common/rdc_utils.h"
+#include "rdc/rdc.h"
+#include "rdc_lib/RdcLogger.h"
+#include "rdc_lib/impl/SmiUtils.h"
+#include "rdc_lib/rdc_common.h"
+
+namespace amd {
+namespace rdc {
+
+RdcTopologyLinkImpl::RdcTopologyLinkImpl(const RdcGroupSettingsPtr& group_settings,
+                                         RdcMetricFetcherPtr metric_fetcher)
+    : group_settings_(group_settings), metric_fetcher_(metric_fetcher) {}
+
+RdcTopologyLinkImpl::~RdcTopologyLinkImpl() {}
+
+rdc_status_t RdcTopologyLinkImpl::rdc_device_topology_get(uint32_t gpu_index,
+                                                          rdc_device_topology_t* results) {
+  rdc_status_t status = RDC_ST_NOT_FOUND;
+  amdsmi_status_t err = AMDSMI_STATUS_SUCCESS;
+  uint32_t gpu_index_list[RDC_MAX_NUM_DEVICES];
+
+  amdsmi_processor_handle processor_handle;
+  err = get_processor_handle_from_id(gpu_index, &processor_handle);
+  if (err != AMDSMI_STATUS_SUCCESS) {
+    RDC_LOG(RDC_INFO, "Fail to get process GPUs processor handle information: " << err);
+    return status;
+  }
+
+  uint32_t numa_node = 0;
+  err = amdsmi_topo_get_numa_node_number(processor_handle, &numa_node);
+  if (err != AMDSMI_STATUS_SUCCESS) {
+    RDC_LOG(RDC_INFO, "Fail to get process GPUs numa_node information: " << err);
+    return status;
+  }
+
+  uint32_t count = 0;
+  rdc_field_value device_count;
+  status = metric_fetcher_->fetch_smi_field(0, RDC_FI_GPU_COUNT, &device_count);
+  if (status != RDC_ST_OK) {
+    return status;
+  }
+
+  // Assign the index to the index list
+  count = device_count.value.l_int;
+  for (uint32_t i = 0; i < count; i++) {
+    gpu_index_list[i] = i;
+  }
+
+  results->num_of_gpus = count;
+  results->numa_node = numa_node;
+
+  for (uint32_t i = 0; i < count; i++) {
+    for (uint32_t j = 0; j < count; j++) {
+      if (gpu_index_list[i] == gpu_index_list[j]) continue;
+      std::pair<amdsmi_processor_handle, amdsmi_processor_handle> ph;
+      err = get_processor_handle_from_id(gpu_index_list[i], &ph.first);
+      err = get_processor_handle_from_id(gpu_index_list[i], &ph.second);
+      if (err != AMDSMI_STATUS_SUCCESS) {
+        RDC_LOG(RDC_INFO, "Fail to get process GPUs processor handle information: " << err);
+        return status;
+      }
+
+      uint64_t weight = std::numeric_limits<uint64_t>::max();
+      err = amdsmi_topo_get_link_weight(ph.first, ph.second, &weight);
+      if (err != AMDSMI_STATUS_SUCCESS) {
+        RDC_LOG(RDC_INFO, "Fail to get process GPUs weight information: " << err);
+      }
+
+      uint64_t min_bandwidth = std::numeric_limits<uint64_t>::max();
+      uint64_t max_bandwidth = std::numeric_limits<uint64_t>::max();
+      err = amdsmi_get_minmax_bandwidth_between_processors(ph.first, ph.second, &min_bandwidth,
+                                                           &max_bandwidth);
+      if (err != AMDSMI_STATUS_SUCCESS) {
+        RDC_LOG(RDC_INFO, "Fail to get process GPUs detail information: " << err);
+      }
+
+      uint64_t hops = std::numeric_limits<uint64_t>::max();
+      amdsmi_io_link_type_t type = AMDSMI_IOLINK_TYPE_UNDEFINED;
+      err = amdsmi_topo_get_link_type(ph.first, ph.second, &hops, &type);
+      if (err != AMDSMI_STATUS_SUCCESS) {
+        RDC_LOG(RDC_INFO, "Fail to get process GPUs hops and type information: " << err);
+      }
+
+      bool accessible;
+      err = amdsmi_is_P2P_accessible(ph.first, ph.second, &accessible);
+      if (err != AMDSMI_STATUS_SUCCESS) {
+        RDC_LOG(RDC_INFO, "Fail to get process GPUs P2P accessible information: " << err);
+      }
+
+      results->link_infos[i].gpu_index = gpu_index_list[i];
+      results->link_infos[i].weight = weight;
+      results->link_infos[i].min_bandwidth = min_bandwidth;
+      results->link_infos[i].max_bandwidth = max_bandwidth;
+      results->link_infos[i].hops = hops;
+      results->link_infos[i].link_type = static_cast<rdc_topology_link_type_t>(type);
+    }
+  }
+
+  return RDC_ST_OK;
+}
+
+rdc_status_t RdcTopologyLinkImpl::rdc_link_status_get(rdc_link_status_t* results) {
+  rdc_status_t status = RDC_ST_NOT_FOUND;
+  return status;
+}
+
+}  // namespace rdc
+}  // namespace amd

rdc_libs/rdc_client/src/RdcStandaloneHandler.cc

@@ -952,5 +952,40 @@ rdc_status_t RdcStandaloneHandler::rdc_health_clear(rdc_gpu_group_t group_id) {
   return RDC_ST_OK;
 }
 
+rdc_status_t RdcStandaloneHandler::rdc_device_topology_get(uint32_t gpu_index,
+                                                           rdc_device_topology_t* results) {
+  ::rdc::GetTopologyRequest request;
+  ::rdc::GetTopologyResponse reply;
+  ::grpc::ClientContext context;
+
+  request.set_gpu_index(gpu_index);
+  ::grpc::Status status = stub_->GetTopology(&context, request, &reply);
+  rdc_status_t err_status = error_handle(status, reply.status());
+  if (err_status != RDC_ST_OK) return err_status;
+
+  ::rdc::Topology Topology = reply.toppology();
+  results->num_of_gpus= Topology.num_of_gpus();
+  results->numa_node= Topology.numa_node();
+
+  for (uint32_t i = 0; i < Topology.num_of_gpus(); ++i) {
+  ::rdc::TopologyLinkInfo linkinfo = Topology.link_infos(i);
+  results->link_infos[i].gpu_index=linkinfo.gpu_index();
+  results->link_infos[i].weight=linkinfo.weight();
+  results->link_infos[i].min_bandwidth=linkinfo.min_bandwidth();
+  results->link_infos[i].max_bandwidth=linkinfo.max_bandwidth();
+  results->link_infos[i].hops=linkinfo.hops();
+  results->link_infos[i].link_type=static_cast<rdc_topology_link_type_t>(linkinfo.link_type());
+  results->link_infos[i].is_p2p_accessible=linkinfo.p2p_accessible();
+  }
+
+  return RDC_ST_OK;
+}
+
+rdc_status_t RdcStandaloneHandler::rdc_link_status_get(rdc_link_status_t* results) {
+  ::rdc::UpdateAllFieldsResponse reply;
+  ::grpc::Status status = grpc::Status::OK;
+  return error_handle(status, reply.status());
+}
+
 }  // namespace rdc
 }  // namespace amd

rdci/CMakeLists.txt

@@ -70,6 +70,7 @@ set(RDCI_SRC_LIST
     "${SRC_DIR}/RdciPolicySubSystem.cc"
     "${SRC_DIR}/RdciHealthSubSystem.cc"
     "${SRC_DIR}/RdciSubSystem.cc"
+    "${SRC_DIR}/RdciTopologyLinkSubSystem.cc"
     "${SRC_DIR}/rdci.cc")
 message("RDCI_SRC_LIST=${RDCI_SRC_LIST}")
 set(RDCI_EXE "rdci")

rdci/include/RdciTopologyLinkSubSystem.h

@@ -0,0 +1,45 @@
+/*
+Copyright (c) 2024 - present Advanced Micro Devices, Inc. All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef RDCI_INCLUDE_RDCITOPOLOGYLINKSYSTEM_H_
+#define RDCI_INCLUDE_RDCITOPOLOGYLINKSYSTEM_H_
+#include <signal.h>
+
+#include <string>
+
+#include "RdciSubSystem.h"
+namespace amd {
+namespace rdc {
+class RdciTopologyLinkSubSystem : public RdciSubSystem {
+ public:
+  RdciTopologyLinkSubSystem();
+  void parse_cmd_opts(int argc, char** argv) override;
+  void process() override;
+
+ private:
+  void show_help() const;
+  enum OPERATIONS {
+    POLICY_UNKNOWN = 0,
+    TOPOLOGY_INDEX,
+  } topology_ops_;
+  uint32_t group_index_;
+  bool is_group_index_set;
+};
+}  // namespace rdc
+}  // namespace amd
+#endif  // RDCI_INCLUDE_RDCITOPOLOGYLINKSYSTEM_H_

rdci/src/RdciTopologyLinkSubSystem.cc

@@ -0,0 +1,124 @@
+/*
+Copyright (c) 2024 - present Advanced Micro Devices, Inc. All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#include "RdciTopologyLinkSubSystem.h"
+
+#include <getopt.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include "common/rdc_utils.h"
+#include "rdc/rdc.h"
+#include "rdc_lib/RdcException.h"
+#include "rdc_lib/rdc_common.h"
+namespace amd {
+namespace rdc {
+RdciTopologyLinkSubSystem::RdciTopologyLinkSubSystem() {}
+
+void RdciTopologyLinkSubSystem::parse_cmd_opts(int argc, char** argv) {
+  const int HOST_OPTIONS = 1000;
+  const struct option long_options[] = {{"host", required_argument, nullptr, HOST_OPTIONS},
+                                        {"help", optional_argument, nullptr, 'h'},
+                                        {"unauth", optional_argument, nullptr, 'u'},
+                                        {"gpu_index", required_argument, nullptr, 'i'},
+                                        {nullptr, 0, nullptr, 0}};
+  int option_index = 0;
+  int opt = 0;
+  while ((opt = getopt_long(argc, argv, "hui:", long_options, &option_index)) != -1) {
+    {
+      switch (opt) {
+        case HOST_OPTIONS:
+          ip_port_ = optarg;
+          break;
+        case 'h':
+          show_help();
+          break;
+        case 'u':
+          use_auth_ = false;
+          break;
+        case 'i':
+          if (!IsNumber(optarg)) {
+            show_help();
+            throw RdcException(RDC_ST_BAD_PARAMETER, "The field group index needs to be a number");
+          }
+          topology_ops_ = TOPOLOGY_INDEX;
+          group_index_ = std::stoi(optarg);
+          is_group_index_set = true;
+          break;
+        default:
+          show_help();
+          throw RdcException(RDC_ST_BAD_PARAMETER, "Unknown command line options");
+      }
+    }
+  }
+  if (is_group_index_set == false) {
+    show_help();
+    throw RdcException(RDC_ST_BAD_PARAMETER, "Need to specify the group index to notify");
+  }
+}
+
+void RdciTopologyLinkSubSystem::show_help() const {
+  std::cout << " topo -- Used to  topology will show how the GPU connected.\n\n";
+  std::cout << "Usage\n";
+  std::cout << "    rdci topo [--host <IP/FQDN>:port] [--reg] -g <groupId>\n";
+  std::cout << "\nFlags:\n";
+  show_common_usage();
+  std::cout << "  -i  --index                      display the topology"
+            << "information for GPU index 0 .\n";
+}
+
+static const char* topology_link_type_to_str(rdc_topology_link_type_t type) {
+  if (type == RDC_IOLINK_TYPE_PCIEXPRESS) return "Connected via PCIe \t";
+  if (type == RDC_IOLINK_TYPE_XGMI) return "Connected via XGMI \t";
+  if (type == RDCI_IOLINK_TYPE_NUMIOLINKTYPES) return "Number of IO Link types";
+  return "N/A \t\t\t";
+}
+
+void RdciTopologyLinkSubSystem::process() {
+  rdc_status_t result = RDC_ST_OK;
+  switch (topology_ops_) {
+    case TOPOLOGY_INDEX: {
+      rdc_device_topology_t topology;
+      result = rdc_device_topology_get(rdc_handle_, group_index_, &topology);
+      if (result == RDC_ST_OK) {
+        std::cout << "+-----------------------+"
+                  << "-----------------------------"
+                  << "------------------+\n";
+        std::cout << "| GPU ID: " << group_index_ << "\t\t"
+                  << "| Topology Information \t\t\t\t|\n";
+        std::cout << "+=======================+"
+                  << "============================="
+                  << "==================+\n";
+        std::cout << "+-----------------------+"
+                  << "-----------------------------"
+                  << "------------------+\n";
+        for (uint32_t i = 0; i < topology.num_of_gpus; i++) {
+          std::cout << "| To GPU " << i + 1 << "\t\t"
+                    << "| " << topology_link_type_to_str(RDC_IOLINK_TYPE_XGMI)
+                    << "\t\t\t|\n";
+        }
+        std::cout << "+-----------------------+"
+                  << "-----------------------------"
+                  << "------------------+\n";
+      }
+      break;
+    }
+  }
+}
+}  // namespace rdc
+}  // namespace amd

rdci/src/rdci.cc

@@ -32,6 +32,7 @@ THE SOFTWARE.
 #include "RdciStatsSubSystem.h"
 #include "RdciPolicySubSystem.h"
 #include "RdciHealthSubSystem.h"
+#include "RdciTopologyLinkSubSystem.h"
 #include "rdc/rdc.h"
 #include "rdc_lib/RdcException.h"
 #include "rdc_lib/rdc_common.h"
@@ -51,7 +52,7 @@ int main(int argc, char** argv) {
   const std::string usage_help =
       "Usage:\trdci <subsystem>|<options>\n"
       "subsystem: \n"
-      "          discovery, dmon, group, fieldgroup, stats, diag, policy, health\n"
+      "          discovery, dmon, group, fieldgroup, stats, diag, policy, health, topo\n"
       "options: \n"
       "        -v(--version) : Print client version information only\n";
 
@@ -84,6 +85,8 @@ int main(int argc, char** argv) {
       subsystem.reset(new amd::rdc::RdciFieldGroupSubSystem());
     } else if (subsystem_name == "health") {
       subsystem.reset(new amd::rdc::RdciHealthSubSystem());
+    } else if (subsystem_name == "topo") {
+      subsystem.reset(new amd::rdc::RdciTopologyLinkSubSystem());
     } else if (subsystem_name == "stats") {
       subsystem.reset(new amd::rdc::RdciStatsSubSystem());
     } else if (subsystem_name == "policy") {

server/include/rdc/rdc_api_service.h

@@ -152,6 +152,10 @@ class RdcAPIServiceImpl final : public ::rdc::RdcAPI::Service {
   ::grpc::Status UnRegisterPolicy(::grpc::ServerContext* context,
                                   const ::rdc::UnRegisterPolicyRequest* request,
                                   ::rdc::UnRegisterPolicyResponse* reply) override;
+                                  
+  ::grpc::Status GetTopology(::grpc::ServerContext* context,
+                             const ::rdc::GetTopologyRequest* request,
+                             ::rdc::GetTopologyResponse* reply) override;
 
   ::grpc::Status SetHealth(::grpc::ServerContext* context,
                            const ::rdc::SetHealthRequest* request,

server/src/rdc_api_service.cc

@@ -854,7 +854,6 @@ bool RdcAPIServiceImpl::copy_gpu_usage_info(const rdc_gpu_usage_info_t& src,
 }
 
 int RdcAPIServiceImpl::PolicyCallback(rdc_policy_callback_response_t* userData) {
-
   if (userData == nullptr) {
     std::cerr << "The rdc_policy_callback returns null data\n";
     return 1;
@@ -921,7 +920,6 @@ int RdcAPIServiceImpl::PolicyCallback(rdc_policy_callback_response_t* userData)
           pthread_mutex_unlock(&ctx->mutex);
         }
       }
-
     }
   });
 
@@ -1035,5 +1033,37 @@ int RdcAPIServiceImpl::PolicyCallback(rdc_policy_callback_response_t* userData)
   return ::grpc::Status::OK;
 }
 
+::grpc::Status RdcAPIServiceImpl::GetTopology(::grpc::ServerContext* context,
+                                              const ::rdc::GetTopologyRequest* request,
+                                              ::rdc::GetTopologyResponse* reply) {
+  (void)(context);
+  if (!reply || !request) {
+    return ::grpc::Status(::grpc::StatusCode::INTERNAL, "Empty contents");
+  }
+  rdc_device_topology_t topology_results;
+  // call RDC topology API
+  rdc_status_t result =
+      rdc_device_topology_get(rdc_handle_, request->gpu_index(), &topology_results);
+  reply->set_status(result);
+  if (result != RDC_ST_OK) {
+    return ::grpc::Status::OK;
+  }
+  ::rdc::Topology* topology = reply->mutable_toppology();
+  topology->set_num_of_gpus(topology_results.num_of_gpus);
+  topology->set_numa_node(topology_results.numa_node);
+  for (uint32_t i = 0; i < topology_results.num_of_gpus; ++i) {
+    ::rdc::TopologyLinkInfo* linkinfos = topology->add_link_infos();
+    linkinfos->set_gpu_index(topology_results.link_infos[i].gpu_index);
+    linkinfos->set_weight(topology_results.link_infos[i].weight);
+    linkinfos->set_min_bandwidth(topology_results.link_infos[i].min_bandwidth);
+    linkinfos->set_max_bandwidth(topology_results.link_infos[i].max_bandwidth);
+    linkinfos->set_hops(topology_results.link_infos[i].hops);
+    linkinfos->set_link_type(
+        static_cast<::rdc::TopologyLinkInfo_LinkType>(topology_results.link_infos[i].link_type));
+    linkinfos->set_p2p_accessible(topology_results.link_infos[i].is_p2p_accessible);
+  }
+  return ::grpc::Status::OK;
+}
+
 }  // namespace rdc
 }  // namespace amd