Add functions that are used to query Hardware topology.

Change-Id: I0f4cd02b237bde4d6dccfb0e83e65376ecb1cfaa
Signed-off-by: Mike Li <Tianxinmike.Li@amd.com>


[ROCm/rocm_smi_lib commit: c7d349183a]

projects/rocm-smi-lib/CMakeLists.txt

@@ -141,6 +141,7 @@ set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_power_mon.cc")
 set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_utils.cc")
 set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_counters.cc")
 set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_kfd.cc")
+set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_io_link.cc")
 set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/shared_mutex/shared_mutex.cc")
 
 set(SMI_INC_LIST "${INC_DIR}/rocm_smi_device.h")
@@ -152,6 +153,7 @@ set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_common.h")
 set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_exception.h")
 set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_counters.h")
 set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_kfd.h")
+set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_io_link.h")
 set(SMI_INC_LIST ${SMI_INC_LIST} "${SRC_DIR}/shared_mutex/shared_mutex.h")
 
 set(SMI_EXAMPLE_EXE "rocm_smi_ex")

projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h

@@ -522,6 +522,17 @@ typedef enum {
   RSMI_MEM_PAGE_STATUS_UNRESERVABLE   //!< Unable to reserve this page
 } rsmi_memory_page_status_t;
 
+/**
+ * @brief Types for IO Link
+ */
+typedef enum _RSMI_IO_LINK_TYPE {
+  RSMI_IOLINK_TYPE_UNDEFINED      = 0,         //!< unknown type.
+  RSMI_IOLINK_TYPE_PCIEXPRESS     = 1,         //!< PCI Express
+  RSMI_IOLINK_TYPE_XGMI           = 2,         //!< XGMI
+  RSMI_IOLINK_TYPE_NUMIOLINKTYPES,             //!< Number of IO Link types
+  RSMI_IOLINK_TYPE_SIZE           = 0xFFFFFFFF //!< Max of IO Link types
+} RSMI_IO_LINK_TYPE;
+
 /**
  * @brief Reserved Memory Page Record
  */
@@ -2615,6 +2626,85 @@ rsmi_dev_xgmi_error_reset(uint32_t dv_ind);
 
 /** @} */  // end of SysInfo
 
+/*****************************************************************************/
+/** @defgroup HWTopo Hardware Topology Functions
+ *  These functions are used to query Hardware topology.
+ *  @{
+ */
+
+/**
+ *  @brief Retrieve the NUMA CPU node number for a device
+ *
+ *  @details Given a device index @p dv_ind, and a pointer to an
+ *  uint32_t @p numa_node, this function will write the
+ *  node number of NUMA CPU for the device @p dv_ind to the memory
+ *  pointed to by @p numa_node.
+ *
+ *  @param[in] dv_ind a device index
+ *
+ *  @param[inout] numa_node A pointer to an uint32_t to which the
+ *  numa node number should be written.
+ *
+ *  @retval ::RSMI_STATUS_SUCCESS call was successful
+ *  @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid
+ *
+ */
+rsmi_status_t
+rsmi_topo_get_numa_node_number(uint32_t dv_ind, uint32_t *numa_node);
+
+/**
+ *  @brief Retrieve the weight for a connection between 2 GPUs
+ *
+ *  @details Given a source device index @p dv_ind_src and
+ *  a destination device index @p dv_ind_dst, and a pointer to an
+ *  uint64_t @p weight, this function will write the
+ *  weight for the connection between the device @p dv_ind_src
+ *  and @p dv_ind_dst to the memory pointed to by @p weight.
+ *
+ *  @param[in] dv_ind_src the source device index
+ *
+ *  @param[in] dv_ind_dst the destination device index
+ *
+ *  @param[inout] weight A pointer to an uint64_t to which the
+ *  weight for the connection should be written.
+ *
+ *  @retval ::RSMI_STATUS_SUCCESS call was successful
+ *  @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid
+ *
+ */
+rsmi_status_t
+rsmi_topo_get_link_weight(uint32_t dv_ind_src, uint32_t dv_ind_dst,
+                          uint64_t *weight);
+
+/**
+ *  @brief Retrieve the hops and the connection type between 2 GPUs
+ *
+ *  @details Given a source device index @p dv_ind_src and
+ *  a destination device index @p dv_ind_dst, and a pointer to an
+ *  uint64_t @p hops and a pointer to an RSMI_IO_LINK_TYPE @p type,
+ *  this function will write the number of hops and the connection type
+ *  between the device @p dv_ind_src and @p dv_ind_dst to the memory
+ *  pointed to by @p hops and @p type.
+ *
+ *  @param[in] dv_ind_src the source device index
+ *
+ *  @param[in] dv_ind_dst the destination device index
+ *
+ *  @param[inout] hops A pointer to an uint64_t to which the
+ *  hops for the connection should be written.
+ *
+ *  @param[inout] type A pointer to an ::RSMI_IO_LINK_TYPE to which the
+ *  type for the connection should be written.
+ *
+ *  @retval ::RSMI_STATUS_SUCCESS call was successful
+ *  @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid
+ *
+ */
+rsmi_status_t
+rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst,
+                        uint64_t *hops, RSMI_IO_LINK_TYPE *type);
+
+/** @} */  // end of HWTopo
 
 /*****************************************************************************/
 /** @defgroup APISupport Supported Functions

projects/rocm-smi-lib/include/rocm_smi/rocm_smi_io_link.h

@@ -0,0 +1,116 @@
+/*
+ * =============================================================================
+ * The University of Illinois/NCSA
+ * Open Source License (NCSA)
+ *
+ * Copyright (c) 2020, Advanced Micro Devices, Inc.
+ * All rights reserved.
+ *
+ * Developed by:
+ *
+ *                 AMD Research and AMD ROC Software Development
+ *
+ *                 Advanced Micro Devices, Inc.
+ *
+ *                 www.amd.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal with the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimers.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimers in
+ *    the documentation and/or other materials provided with the distribution.
+ *  - Neither the names of <Name of Development Group, Name of Institution>,
+ *    nor the names of its contributors may be used to endorse or promote
+ *    products derived from this Software without specific prior written
+ *    permission.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS WITH THE SOFTWARE.
+ *
+ */
+#ifndef INCLUDE_ROCM_SMI_ROCM_SMI_IO_LINK_H_
+#define INCLUDE_ROCM_SMI_ROCM_SMI_IO_LINK_H_
+
+#include <string>
+#include <vector>
+#include <unordered_set>
+#include <memory>
+#include <map>
+
+#include "rocm_smi/rocm_smi.h"
+
+namespace amd {
+namespace smi {
+
+typedef enum _IO_LINK_TYPE {
+  IOLINK_TYPE_UNDEFINED      = 0,
+  IOLINK_TYPE_HYPERTRANSPORT = 1,
+  IOLINK_TYPE_PCIEXPRESS     = 2,
+  IOLINK_TYPE_AMBA           = 3,
+  IOLINK_TYPE_MIPI           = 4,
+  IOLINK_TYPE_QPI_1_1        = 5,
+  IOLINK_TYPE_RESERVED1      = 6,
+  IOLINK_TYPE_RESERVED2      = 7,
+  IOLINK_TYPE_RAPID_IO       = 8,
+  IOLINK_TYPE_INFINIBAND     = 9,
+  IOLINK_TYPE_RESERVED3      = 10,
+  IOLINK_TYPE_XGMI           = 11,
+  IOLINK_TYPE_XGOP           = 12,
+  IOLINK_TYPE_GZ             = 13,
+  IOLINK_TYPE_ETHERNET_RDMA  = 14,
+  IOLINK_TYPE_RDMA_OTHER     = 15,
+  IOLINK_TYPE_OTHER          = 16,
+  IOLINK_TYPE_NUMIOLINKTYPES,
+  IOLINK_TYPE_SIZE           = 0xFFFFFFFF
+} IO_LINK_TYPE;
+
+class IOLink {
+ public:
+    explicit IOLink(uint32_t node_indx, uint32_t link_indx) :
+                    node_indx_(node_indx), link_indx_(link_indx) {}
+    ~IOLink();
+
+    int Initialize();
+    int ReadProperties(void);
+    int get_property_value(std::string property, uint64_t *value);
+    uint32_t get_node_indx(void) const {return node_indx_;}
+    uint32_t get_link_indx(void) const {return link_indx_;}
+    IO_LINK_TYPE type(void) const {return type_;}
+    uint32_t node_from(void) const {return node_from_;}
+    uint32_t node_to(void) const {return node_to_;}
+    uint64_t weight(void) const {return weight_;}
+
+ private:
+    uint32_t node_indx_;
+    uint32_t link_indx_;
+    IO_LINK_TYPE type_;
+    uint32_t node_from_;
+    uint32_t node_to_;
+    uint64_t weight_;
+    std::map<std::string, uint64_t> properties_;
+};
+
+int
+DiscoverIOLinksPerNode(uint32_t node_indx, std::map<uint32_t,
+                       std::shared_ptr<IOLink>> *links);
+
+int
+DiscoverIOLinks(std::map<std::pair<uint32_t, uint32_t>,
+                std::shared_ptr<IOLink>> *links);
+
+}  // namespace smi
+}  // namespace amd
+
+#endif  // INCLUDE_ROCM_SMI_ROCM_SMI_IO_LINK_H_

projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h

@@ -51,6 +51,7 @@
 
 #include "rocm_smi/rocm_smi.h"
 #include "rocm_smi/rocm_smi_device.h"
+#include "rocm_smi/rocm_smi_io_link.h"
 
 namespace amd {
 namespace smi {
@@ -65,6 +66,12 @@ class KFDNode {
     int get_property_value(std::string property, uint64_t *value);
     uint64_t gpu_id(void) const {return gpu_id_;}
     std::string name(void) const {return name_;}
+    uint32_t node_index(void) const {return node_indx_;}
+    uint32_t numa_node_number(void) const {return numa_node_number_;}
+    uint64_t numa_node_weight(void) const {return numa_node_weight_;}
+    IO_LINK_TYPE numa_node_type(void) const {return numa_node_type_;}
+    int get_io_link_type(uint32_t node_to, IO_LINK_TYPE *type);
+    int get_io_link_weight(uint32_t node_to, uint64_t *weight);
     std::shared_ptr<Device> amdgpu_device(void) const {return amdgpu_device_;}
     uint32_t amdgpu_dev_index(void) const {return amdgpu_dev_index_;}
     void set_amdgpu_dev_index(uint32_t val) {amdgpu_dev_index_ = val;}
@@ -74,6 +81,12 @@ class KFDNode {
     uint32_t amdgpu_dev_index_;
     uint64_t gpu_id_;
     std::string name_;
+    uint32_t numa_node_number_;
+    uint64_t numa_node_weight_;
+    IO_LINK_TYPE numa_node_type_;
+    std::map<uint32_t, IO_LINK_TYPE> io_link_type_;
+    std::map<uint32_t, uint64_t> io_link_weight_;
+    std::map<uint32_t, std::shared_ptr<IOLink>> io_link_map_;
     std::map<std::string, uint64_t> properties_;
     std::shared_ptr<Device> amdgpu_device_;
 };

projects/rocm-smi-lib/include/rocm_smi/rocm_smi_main.h

@@ -55,6 +55,7 @@
 #include <map>
 #include <mutex>  // NOLINT
 
+#include "rocm_smi/rocm_smi_io_link.h"
 #include "rocm_smi/rocm_smi_kfd.h"
 #include "rocm_smi/rocm_smi_device.h"
 #include "rocm_smi/rocm_smi_monitor.h"
@@ -106,6 +107,9 @@ class RocmSMI {
                                            return ++kfd_notif_evt_fh_refcnt_;}
     uint32_t kfd_notif_evt_fh_refcnt_dec(void) {
                                            return --kfd_notif_evt_fh_refcnt_;}
+    int get_io_link_weight(uint32_t node_from, uint32_t node_to,
+                           uint64_t *weight);
+    int get_node_index(uint32_t dv_ind, uint32_t *node_ind);
 
  private:
     std::vector<std::shared_ptr<Device>> devices_;
@@ -113,6 +117,9 @@ class RocmSMI {
     std::vector<std::shared_ptr<Monitor>> monitors_;
     std::vector<std::shared_ptr<PowerMon>> power_mons_;
     std::set<std::string> amd_monitor_types_;
+    std::map<std::pair<uint32_t, uint32_t>, std::shared_ptr<IOLink>>
+      io_link_map_;
+    std::map<uint32_t, uint32_t> dev_ind_to_node_ind_map_;
     void AddToDeviceList(std::string dev_name);
     void GetEnvVariables(void);
     uint32_t DiscoverAMDMonitors(void);

projects/rocm-smi-lib/src/rocm_smi.cc

@@ -70,6 +70,7 @@
 #include "rocm_smi/rocm_smi_exception.h"
 #include "rocm_smi/rocm_smi_counters.h"
 #include "rocm_smi/rocm_smi_kfd.h"
+#include "rocm_smi/rocm_smi_io_link.h"
 
 #include "rocm_smi/rocm_smi64Config.h"
 
@@ -1241,6 +1242,30 @@ static rsmi_status_t set_power_profile(uint32_t dv_ind,
   CATCH
 }
 
+static rsmi_status_t topo_get_numa_node_number(uint32_t dv_ind,
+                     uint32_t *numa_node_number) {
+  TRY
+
+  GET_DEV_AND_KFDNODE_FROM_INDX
+
+  *numa_node_number = kfd_node->numa_node_number();
+
+  return RSMI_STATUS_SUCCESS;
+  CATCH
+}
+
+static rsmi_status_t topo_get_numa_node_weight(uint32_t dv_ind,
+                     uint64_t *weight) {
+  TRY
+
+  GET_DEV_AND_KFDNODE_FROM_INDX
+
+  *weight = kfd_node->numa_node_weight();
+
+  return RSMI_STATUS_SUCCESS;
+  CATCH
+}
+
 rsmi_status_t
 rsmi_dev_gpu_clk_freq_get(uint32_t dv_ind, rsmi_clk_type_t clk_type,
                                                         rsmi_frequencies_t *f) {
@@ -2964,6 +2989,158 @@ rsmi_dev_xgmi_error_reset(uint32_t dv_ind) {
   CATCH
 }
 
+rsmi_status_t
+rsmi_topo_get_numa_node_number(uint32_t dv_ind, uint32_t *numa_node) {
+  TRY
+
+  return topo_get_numa_node_number(dv_ind, numa_node);
+  CATCH
+}
+
+rsmi_status_t
+rsmi_topo_get_link_weight(uint32_t dv_ind_src, uint32_t dv_ind_dst,
+                          uint64_t *weight) {
+  TRY
+
+  uint32_t dv_ind = dv_ind_src;
+  GET_DEV_AND_KFDNODE_FROM_INDX
+  DEVICE_MUTEX
+
+  if (weight == nullptr) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+
+  rsmi_status_t status;
+  uint32_t node_ind_dst;
+  uint32_t ret = smi.get_node_index(dv_ind_dst, &node_ind_dst);
+
+  if (!ret) {
+    amd::smi::IO_LINK_TYPE type;
+    ret = kfd_node->get_io_link_type(node_ind_dst, &type);
+    if (!ret) {
+      if (type == amd::smi::IOLINK_TYPE_XGMI) {
+        ret = kfd_node->get_io_link_weight(node_ind_dst, weight);
+        if (!ret)
+          status = RSMI_STATUS_SUCCESS;
+        else
+          status = RSMI_STATUS_INIT_ERROR;
+      } else {
+        assert(!"Unexpected IO Link type read");
+        status = RSMI_STATUS_NOT_SUPPORTED;
+      }
+    } else {
+      *weight = kfd_node->numa_node_weight();  // from src GPU to it's CPU node
+      uint64_t numa_weight_dst;
+      status = topo_get_numa_node_weight(dv_ind_dst, &numa_weight_dst);
+      // from dst GPU to it's CPU node
+      if (status == RSMI_STATUS_SUCCESS) {
+        *weight = *weight + numa_weight_dst;
+        uint32_t numa_number_src = kfd_node->numa_node_number();
+        uint32_t numa_number_dst;
+        status = topo_get_numa_node_number(dv_ind_dst, &numa_number_dst);
+        if (status == RSMI_STATUS_SUCCESS) {
+          if (numa_number_src != numa_number_dst) {
+            uint64_t io_link_weight;
+            ret = smi.get_io_link_weight(numa_number_src, numa_number_dst,
+                                         &io_link_weight);
+            if (!ret) {
+              *weight = *weight + io_link_weight;
+              // from src numa CPU node to dst numa CPU node
+            } else {
+              *weight = *weight + 10;
+              // More than one CPU hops, hard coded 10
+            }
+          }
+          status = RSMI_STATUS_SUCCESS;
+        } else {
+          assert(!"Error to read numa node number");
+          status = RSMI_STATUS_INIT_ERROR;
+        }
+      } else {
+        assert(!"Error to read numa node weight");
+        status = RSMI_STATUS_INIT_ERROR;
+      }
+    }
+  } else {
+    status = RSMI_STATUS_INVALID_ARGS;
+  }
+
+  return status;
+  CATCH
+}
+
+rsmi_status_t
+rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst,
+                        uint64_t *hops, RSMI_IO_LINK_TYPE *type) {
+  TRY
+
+  uint32_t dv_ind = dv_ind_src;
+  GET_DEV_AND_KFDNODE_FROM_INDX
+
+  if (hops == nullptr) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+
+  if (type == nullptr) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+
+  rsmi_status_t status;
+  uint32_t node_ind_dst;
+  uint32_t ret = smi.get_node_index(dv_ind_dst, &node_ind_dst);
+
+  if (!ret) {
+    amd::smi::IO_LINK_TYPE io_link_type;
+    ret = kfd_node->get_io_link_type(node_ind_dst, &io_link_type);
+    if (!ret) {
+      if (io_link_type == amd::smi::IOLINK_TYPE_XGMI) {
+        *type = RSMI_IOLINK_TYPE_XGMI;
+        *hops = 1;
+        status = RSMI_STATUS_SUCCESS;
+      } else {
+        assert(!"Unexpected IO Link type read");
+        status = RSMI_STATUS_NOT_SUPPORTED;
+      }
+    } else {
+      uint32_t numa_number_dst;
+      status = topo_get_numa_node_number(dv_ind_dst, &numa_number_dst);
+      if (status == RSMI_STATUS_SUCCESS) {
+        uint32_t numa_number_src = kfd_node->numa_node_number();
+        if (numa_number_src == numa_number_dst) {
+          *hops = 2;  // same CPU node
+        } else {
+          uint64_t io_link_weight;
+          ret = smi.get_io_link_weight(numa_number_src, numa_number_dst,
+                                       &io_link_weight);
+          if (!ret)
+            *hops = 3;  // from src CPU node to dst CPU node
+          else
+            *hops = 4;  // More than one CPU hops, hard coded as 4
+        }
+
+        amd::smi::IO_LINK_TYPE numa_node_type = kfd_node->numa_node_type();
+        if (numa_node_type == amd::smi::IOLINK_TYPE_PCIEXPRESS) {
+          *type = RSMI_IOLINK_TYPE_PCIEXPRESS;
+          status = RSMI_STATUS_SUCCESS;
+        } else if (numa_node_type == amd::smi::IOLINK_TYPE_XGMI) {
+          *type = RSMI_IOLINK_TYPE_XGMI;
+          status = RSMI_STATUS_SUCCESS;
+        } else {
+          status = RSMI_STATUS_INIT_ERROR;
+        }
+      } else {
+        assert(!"Error to get numa node number");
+        status = RSMI_STATUS_INIT_ERROR;
+      }
+    }
+  } else {
+    status = RSMI_STATUS_INVALID_ARGS;
+  }
+
+  return status;
+  CATCH
+}
+
 enum iterator_handle_type {
   FUNC_ITER = 0,
   VARIANT_ITER,

projects/rocm-smi-lib/src/rocm_smi_io_link.cc

@@ -0,0 +1,356 @@
+/*
+ * =============================================================================
+ * The University of Illinois/NCSA
+ * Open Source License (NCSA)
+ *
+ * Copyright (c) 2020, Advanced Micro Devices, Inc.
+ * All rights reserved.
+ *
+ * Developed by:
+ *
+ *                 AMD Research and AMD ROC Software Development
+ *
+ *                 Advanced Micro Devices, Inc.
+ *
+ *                 www.amd.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal with the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimers.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimers in
+ *    the documentation and/or other materials provided with the distribution.
+ *  - Neither the names of <Name of Development Group, Name of Institution>,
+ *    nor the names of its contributors may be used to endorse or promote
+ *    products derived from this Software without specific prior written
+ *    permission.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS WITH THE SOFTWARE.
+ *
+ */
+
+#include <assert.h>
+#include <sys/stat.h>
+#include <dirent.h>
+
+#include <algorithm>
+#include <string>
+#include <unordered_set>
+#include <fstream>
+#include <cstdint>
+#include <iostream>
+#include <sstream>
+
+#include "rocm_smi/rocm_smi.h"
+#include "rocm_smi/rocm_smi_exception.h"
+#include "rocm_smi/rocm_smi_utils.h"
+#include "rocm_smi/rocm_smi_io_link.h"
+
+namespace amd {
+namespace smi {
+
+static const char *kKFDNodesPathRoot = "/sys/class/kfd/kfd/topology/nodes";
+
+// IO Link Property strings
+static const char *kIOLinkPropTYPEStr =  "type";
+// static const char *kIOLinkPropVERSION_MAJORStr = "version_major";
+// static const char *kIOLinkPropVERSION_MINORStr = "version_minor";
+static const char *kIOLinkPropNODE_FROMStr = "node_from";
+static const char *kIOLinkPropNODE_TOStr = "node_to";
+static const char *kIOLinkPropWEIGHTStr = "weight";
+// static const char *kIOLinkPropMIN_LATENCYStr = "min_latency";
+// static const char *kIOLinkPropMAX_LATENCYStr = "max_latency";
+// static const char *kIOLinkPropMIN_BANDWIDTHStr = "min_bandwidth";
+// static const char *kIOLinkPropMAX_BANDWIDTHStr = "max_bandwidth";
+// static const char *kIOLinkPropRECOMMENDED_TRANSFER_SIZEStr =
+// "recommended_transfer_size";
+// static const char *kIOLinkPropFLAGSStr = "flags";
+
+static bool is_number(const std::string &s) {
+  return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
+}
+
+static std::string IOLinkPathRoot(uint32_t node_indx) {
+  std::string io_link_path = kKFDNodesPathRoot;
+  io_link_path += '/';
+  io_link_path += std::to_string(node_indx);
+  io_link_path += '/';
+  io_link_path += "io_links";
+  return io_link_path;
+}
+
+static std::string IOLinkPath(uint32_t node_indx, uint32_t link_indx) {
+  std::string io_link_path = IOLinkPathRoot(node_indx);
+  io_link_path += '/';
+  io_link_path += std::to_string(link_indx);
+  return io_link_path;
+}
+
+static int OpenIOLinkProperties(uint32_t node_indx, uint32_t link_indx,
+                                std::ifstream *fs) {
+  int ret;
+  std::string f_path;
+  bool reg_file;
+
+  assert(fs != nullptr);
+  if (fs == nullptr) {
+    return EINVAL;
+  }
+
+  f_path = IOLinkPath(node_indx, link_indx);
+  f_path += "/";
+  f_path += "properties";
+
+  ret = isRegularFile(f_path, &reg_file);
+
+  if (ret != 0) {
+    return ret;
+  }
+  if (!reg_file) {
+    return ENOENT;
+  }
+
+  fs->open(f_path);
+
+  if (!fs->is_open()) {
+    return errno;
+  }
+
+  return 0;
+}
+
+static int ReadIOLinkProperties(uint32_t node_indx, uint32_t link_indx,
+                                std::vector<std::string> *retVec) {
+  std::string line;
+  int ret;
+  std::ifstream fs;
+
+  assert(retVec != nullptr);
+  if (retVec == nullptr) {
+    return EINVAL;
+  }
+
+  ret = OpenIOLinkProperties(node_indx, link_indx, &fs);
+
+  if (ret) {
+    return ret;
+  }
+
+  while (std::getline(fs, line)) {
+    retVec->push_back(line);
+  }
+
+  if (retVec->size() == 0) {
+    fs.close();
+    return 0;
+  }
+
+  // Remove any *trailing* empty (whitespace) lines
+  while (retVec->back().find_first_not_of(" \t\n\v\f\r") == std::string::npos) {
+    retVec->pop_back();
+  }
+
+  fs.close();
+  return 0;
+}
+
+int DiscoverIOLinks(std::map<std::pair<uint32_t, uint32_t>,
+                    std::shared_ptr<IOLink>> *links) {
+  assert(links != nullptr);
+  if (links == nullptr) {
+    return EINVAL;
+  }
+  assert(links->size() == 0);
+
+  links->clear();
+
+  auto kfd_node_dir = opendir(kKFDNodesPathRoot);
+  assert(kfd_node_dir != nullptr);
+
+  auto dentry_kfd = readdir(kfd_node_dir);
+  while (dentry_kfd != nullptr) {
+    if (dentry_kfd->d_name[0] == '.') {
+      dentry_kfd = readdir(kfd_node_dir);
+      continue;
+    }
+
+    if (!is_number(dentry_kfd->d_name)) {
+      dentry_kfd = readdir(kfd_node_dir);
+      continue;
+    }
+
+    uint32_t node_indx = std::stoi(dentry_kfd->d_name);
+    std::shared_ptr<IOLink> link;
+    uint32_t link_indx;
+    std::string io_link_path_root = IOLinkPathRoot(node_indx);
+
+    auto io_link_dir = opendir(io_link_path_root.c_str());
+    assert(io_link_dir != nullptr);
+
+    auto dentry_io_link = readdir(io_link_dir);
+    while (dentry_io_link != nullptr) {
+      if (dentry_io_link->d_name[0] == '.') {
+        dentry_io_link = readdir(io_link_dir);
+        continue;
+      }
+
+      if (!is_number(dentry_io_link->d_name)) {
+        dentry_io_link = readdir(io_link_dir);
+        continue;
+      }
+
+      link_indx = std::stoi(dentry_io_link->d_name);
+      link = std::shared_ptr<IOLink>(new IOLink(node_indx, link_indx));
+
+      link->Initialize();
+
+      (*links)[std::make_pair(link->node_from(), link->node_to())] = link;
+
+      dentry_io_link = readdir(io_link_dir);
+    }
+
+    if (closedir(io_link_dir)) {
+      return 1;
+    }
+
+    dentry_kfd = readdir(kfd_node_dir);
+  }
+
+  if (closedir(kfd_node_dir)) {
+    return 1;
+  }
+  return 0;
+}
+
+int DiscoverIOLinksPerNode(uint32_t node_indx, std::map<uint32_t,
+                           std::shared_ptr<IOLink>> *links) {
+  assert(links != nullptr);
+  if (links == nullptr) {
+    return EINVAL;
+  }
+  assert(links->size() == 0);
+
+  links->clear();
+
+  std::shared_ptr<IOLink> link;
+  uint32_t link_indx;
+  std::string io_link_path_root = IOLinkPathRoot(node_indx);
+
+  auto io_link_dir = opendir(io_link_path_root.c_str());
+  assert(io_link_dir != nullptr);
+
+  auto dentry = readdir(io_link_dir);
+  while (dentry != nullptr) {
+    if (dentry->d_name[0] == '.') {
+      dentry = readdir(io_link_dir);
+      continue;
+    }
+
+    if (!is_number(dentry->d_name)) {
+      dentry = readdir(io_link_dir);
+      continue;
+    }
+
+    link_indx = std::stoi(dentry->d_name);
+    link = std::shared_ptr<IOLink>(new IOLink(node_indx, link_indx));
+
+    link->Initialize();
+
+    (*links)[link->node_to()] = link;
+
+    dentry = readdir(io_link_dir);
+  }
+
+  if (closedir(io_link_dir)) {
+    return 1;
+  }
+  return 0;
+}
+
+IOLink::~IOLink() {
+}
+
+int IOLink::ReadProperties(void) {
+  int ret;
+
+  std::vector<std::string> propVec;
+
+  assert(properties_.size() == 0);
+  if (properties_.size() > 0) {
+    return 0;
+  }
+
+  ret = ReadIOLinkProperties(node_indx_, link_indx_, &propVec);
+
+  if (ret) {
+    return ret;
+  }
+
+  std::string key_str;
+  uint64_t val_int;  // Assume all properties are unsigned integers for now
+  std::istringstream fs;
+
+  for (uint32_t i = 0; i < propVec.size(); ++i) {
+    fs.str(propVec[i]);
+    fs >> key_str;
+    fs >> val_int;
+
+    properties_[key_str] = val_int;
+
+    fs.str("");
+    fs.clear();
+  }
+
+  return 0;
+}
+
+int
+IOLink::Initialize(void) {
+  int ret = 0;
+  ret = ReadProperties();
+  if (ret) {return ret;}
+
+  ret = get_property_value(kIOLinkPropTYPEStr,
+                           reinterpret_cast<uint64_t *>(&type_));
+  if (ret) {return ret;}
+
+  ret = get_property_value(kIOLinkPropNODE_FROMStr,
+                           reinterpret_cast<uint64_t *>(&node_from_));
+  if (ret) {return ret;}
+
+  ret = get_property_value(kIOLinkPropNODE_TOStr,
+                           reinterpret_cast<uint64_t *>(&node_to_));
+  if (ret) {return ret;}
+
+  ret = get_property_value(kIOLinkPropWEIGHTStr, &weight_);
+
+  return ret;
+}
+
+int
+IOLink::get_property_value(std::string property, uint64_t *value) {
+  assert(value != nullptr);
+  if (value == nullptr) {
+    return EINVAL;
+  }
+  if (properties_.find(property) == properties_.end()) {
+    return EINVAL;
+  }
+  *value = properties_[property];
+  return 0;
+}
+
+}  // namespace smi
+}  // namespace amd

projects/rocm-smi-lib/src/rocm_smi_kfd.cc

@@ -53,6 +53,7 @@
 #include <iostream>
 #include <sstream>
 
+#include "rocm_smi/rocm_smi_io_link.h"
 #include "rocm_smi/rocm_smi_kfd.h"
 #include "rocm_smi/rocm_smi.h"
 #include "rocm_smi/rocm_smi_exception.h"
@@ -531,12 +532,49 @@ KFDNode::Initialize(void) {
   if (ret) {return ret;}
 
   ret = ReadKFDGpuId(node_indx_, &gpu_id_);
-  if (ret) {return ret;}
+  if (ret || (gpu_id_ == 0)) {return ret;}
 
   ret = ReadKFDGpuName(node_indx_, &name_);
 
+  std::map<uint32_t, std::shared_ptr<IOLink>> io_link_map_tmp;
+  ret = DiscoverIOLinksPerNode(node_indx_, &io_link_map_tmp);
+  if (ret != 0) {
+    throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR,
+    "Failed to initialize rocm_smi library (IO Links discovery per node).");
+  }
+
+  std::map<uint32_t, std::shared_ptr<IOLink>>::iterator it;
+  uint32_t node_to;
+  uint64_t node_to_gpu_id;
+  std::shared_ptr<IOLink> link;
+  bool numa_node_found = false;
+  for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++) {
+    io_link_map_[it->first] = it->second;
+    node_to = it->first;
+    link = it->second;
+    ret = ReadKFDGpuId(node_to, &node_to_gpu_id);
+    if (ret) {return ret;}
+    if (node_to_gpu_id == 0) {  //  CPU node
+      if (numa_node_found) {
+        if (numa_node_weight_ > link->weight()) {
+          numa_node_number_ = node_to;
+          numa_node_weight_ = link->weight();
+          numa_node_type_ = link->type();
+        }
+      } else {
+        numa_node_number_ = node_to;
+        numa_node_weight_ = link->weight();
+        numa_node_type_ = link->type();
+        numa_node_found = true;
+      }
+    } else {
+      io_link_type_[node_to] = link->type();
+      io_link_weight_[node_to] = link->weight();
+    }
+  }
   return ret;
 }
+
 int
 KFDNode::get_property_value(std::string property, uint64_t *value) {
   assert(value != nullptr);
@@ -550,5 +588,31 @@ KFDNode::get_property_value(std::string property, uint64_t *value) {
   return 0;
 }
 
+int
+KFDNode::get_io_link_type(uint32_t node_to, IO_LINK_TYPE *type) {
+  assert(type != nullptr);
+  if (type == nullptr) {
+    return EINVAL;
+  }
+  if (io_link_type_.find(node_to) == io_link_type_.end()) {
+    return EINVAL;
+  }
+  *type = io_link_type_[node_to];
+  return 0;
+}
+
+int
+KFDNode::get_io_link_weight(uint32_t node_to, uint64_t *weight) {
+  assert(weight != nullptr);
+  if (weight == nullptr) {
+    return EINVAL;
+  }
+  if (io_link_weight_.find(node_to) == io_link_weight_.end()) {
+    return EINVAL;
+  }
+  *weight = io_link_weight_[node_to];
+  return 0;
+}
+
 }  // namespace smi
 }  // namespace amd

projects/rocm-smi-lib/src/rocm_smi_main.cc

@@ -280,6 +280,17 @@ RocmSMI::Initialize(uint64_t flags) {
                  "Failed to initialize rocm_smi library (KFD node discovery).");
   }
 
+  std::map<std::pair<uint32_t, uint32_t>, std::shared_ptr<IOLink>>
+    io_link_map_tmp;
+  ret = DiscoverIOLinks(&io_link_map_tmp);
+  if (ret != 0) {
+    throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR,
+                 "Failed to initialize rocm_smi library (IO Links discovery).");
+  }
+  std::map<std::pair<uint32_t, uint32_t>, std::shared_ptr<IOLink>>::iterator it;
+  for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++)
+    io_link_map_[it->first] = it->second;
+
   std::shared_ptr<amd::smi::Device> dev;
 
   // 1. construct kfd_node_map_ with gpu_id as key and *Device as value
@@ -295,6 +306,7 @@ RocmSMI::Initialize(uint64_t flags) {
     }
 
     tmp_map[bdfid]->set_amdgpu_dev_index(dv_ind);
+    dev_ind_to_node_ind_map_[dv_ind] = tmp_map[bdfid]->node_index();
     uint64_t gpu_id = tmp_map[bdfid]->gpu_id();
     dev->set_kfd_gpu_id(gpu_id);
     kfd_node_map_[gpu_id] = tmp_map[bdfid];
@@ -591,5 +603,27 @@ uint32_t RocmSMI::IterateSMIDevices(
   return 0;
 }
 
+int RocmSMI::get_node_index(uint32_t dv_ind, uint32_t *node_ind) {
+  if (dev_ind_to_node_ind_map_.find(dv_ind) == dev_ind_to_node_ind_map_.end()) {
+    return EINVAL;
+  }
+  *node_ind = dev_ind_to_node_ind_map_[dv_ind];
+  return 0;
+}
+
+int RocmSMI::get_io_link_weight(uint32_t node_from, uint32_t node_to,
+                                uint64_t *weight) {
+  assert(weight != nullptr);
+  if (weight == nullptr) {
+    return EINVAL;
+  }
+  if (io_link_map_.find(std::make_pair(node_from, node_to)) ==
+      io_link_map_.end()) {
+    return EINVAL;
+  }
+  *weight = io_link_map_[std::make_pair(node_from, node_to)]->weight();
+  return 0;
+}
+
 }  // namespace smi
 }  // namespace amd

projects/rocm-smi-lib/tests/rocm_smi_test/functional/hw_topology_read.cc

@@ -0,0 +1,252 @@
+/*
+ * =============================================================================
+ *   ROC Runtime Conformance Release License
+ * =============================================================================
+ * The University of Illinois/NCSA
+ * Open Source License (NCSA)
+ *
+ * Copyright (c) 2020, Advanced Micro Devices, Inc.
+ * All rights reserved.
+ *
+ * Developed by:
+ *
+ *                 AMD Research and AMD ROC Software Development
+ *
+ *                 Advanced Micro Devices, Inc.
+ *
+ *                 www.amd.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal with the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimers.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimers in
+ *    the documentation and/or other materials provided with the distribution.
+ *  - Neither the names of <Name of Development Group, Name of Institution>,
+ *    nor the names of its contributors may be used to endorse or promote
+ *    products derived from this Software without specific prior written
+ *    permission.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS WITH THE SOFTWARE.
+ *
+ */
+
+#include <stdint.h>
+#include <stddef.h>
+
+#include <iostream>
+#include <string>
+
+#include "gtest/gtest.h"
+#include "rocm_smi/rocm_smi.h"
+#include "rocm_smi_test/functional/hw_topology_read.h"
+#include "rocm_smi_test/test_common.h"
+
+typedef struct {
+  std::string type;
+  uint64_t hops;
+  uint64_t weight;
+} gpu_link_t;
+
+TestHWTopologyRead::TestHWTopologyRead() : TestBase() {
+  set_title("RSMI Hardware Topology Read Test");
+  set_description("This test verifies that Hardware Topology can be read properly.");
+}
+
+TestHWTopologyRead::~TestHWTopologyRead(void) {
+}
+
+void TestHWTopologyRead::SetUp(void) {
+  TestBase::SetUp();
+
+  return;
+}
+
+void TestHWTopologyRead::DisplayTestInfo(void) {
+  TestBase::DisplayTestInfo();
+}
+
+void TestHWTopologyRead::DisplayResults(void) const {
+  TestBase::DisplayResults();
+  return;
+}
+
+void TestHWTopologyRead::Close() {
+  // This will close handles opened within rsmitst utility calls and call
+  // rsmi_shut_down(), so it should be done after other cleanup
+  TestBase::Close();
+}
+
+void TestHWTopologyRead::Run(void) {
+  rsmi_status_t err;
+  uint32_t i, j;
+
+  TestBase::Run();
+  if (setup_failed_) {
+    std::cout << "** SetUp Failed for this test. Skipping.**" << std::endl;
+    return;
+  }
+
+  uint32_t num_devices;
+  err = rsmi_num_monitor_devices(&num_devices);
+  CHK_ERR_ASRT(err)
+
+  gpu_link_t gpu_links[num_devices][num_devices];
+  uint32_t numa_numbers[num_devices];
+
+  for (uint32_t dv_ind=0; dv_ind<num_devices; dv_ind++) {
+    err = rsmi_topo_get_numa_node_number(dv_ind, &numa_numbers[dv_ind]);
+    if (err != RSMI_STATUS_SUCCESS) {
+      if (err == RSMI_STATUS_NOT_SUPPORTED) {
+        IF_VERB(STANDARD) {
+          std::cout << "\t**Numa Node Number. read: Not supported on this machine"
+                                                                   << std::endl;
+          return;
+        }
+      } else {
+        CHK_ERR_ASRT(err)
+      }
+    }
+  }
+
+  for (uint32_t dv_ind_src=0; dv_ind_src<num_devices; dv_ind_src++) {
+    for (uint32_t dv_ind_dst=0; dv_ind_dst<num_devices; dv_ind_dst++) {
+      if(dv_ind_src == dv_ind_dst) {
+        gpu_links[dv_ind_src][dv_ind_dst].type = "X";
+        gpu_links[dv_ind_src][dv_ind_dst].hops = 0;
+        gpu_links[dv_ind_src][dv_ind_dst].weight = 0;
+      } else {
+        RSMI_IO_LINK_TYPE type;
+        err = rsmi_topo_get_link_type(dv_ind_src, dv_ind_dst,
+                &gpu_links[dv_ind_src][dv_ind_dst].hops, &type);
+        if (err != RSMI_STATUS_SUCCESS) {
+          if (err == RSMI_STATUS_NOT_SUPPORTED) {
+            IF_VERB(STANDARD) {
+              std::cout << "\t**Link Type. read: Not supported on this machine"
+                                                                       << std::endl;
+              return;
+            }
+          } else {
+            CHK_ERR_ASRT(err)
+          }
+        } else {
+          switch (type) {
+            case RSMI_IOLINK_TYPE_PCIEXPRESS:
+              gpu_links[dv_ind_src][dv_ind_dst].type = "PCIE";
+              break;
+
+            case RSMI_IOLINK_TYPE_XGMI:
+              gpu_links[dv_ind_src][dv_ind_dst].type = "XGMI";
+              break;
+
+            default:
+              gpu_links[dv_ind_src][dv_ind_dst].type = "XXXX";
+              std::cout << "\t**Invalid IO LINK type. type=" << type << std::endl;
+          }
+        }
+        err = rsmi_topo_get_link_weight(dv_ind_src, dv_ind_dst, &gpu_links[dv_ind_src][dv_ind_dst].weight);
+        if (err != RSMI_STATUS_SUCCESS) {
+          if (err == RSMI_STATUS_NOT_SUPPORTED) {
+            IF_VERB(STANDARD) {
+              std::cout << "\t**Link Weight. read: Not supported on this machine"
+                                                                       << std::endl;
+              return;
+            }
+          } else {
+            CHK_ERR_ASRT(err)
+          }
+        }
+      }
+    }
+  }
+
+  std::cout << "**NUMA node number of GPUs**" << std::endl;
+  std::cout << std::setw(12) << std::left <<"GPU#";
+  std::cout <<"NUMA node number";
+  std::cout << std::endl;
+  for (i = 0; i < num_devices; ++i) {
+    std::cout << std::setw(12) << std::left << i;
+    std::cout << numa_numbers[i];
+    std::cout << std::endl;
+  }
+  std::cout << std::endl;
+  std::cout << std::endl;
+
+  std::string tmp;
+  std::cout << "**Type between two GPUs**" << std::endl;
+  std::cout << "      ";
+  for (i = 0; i < num_devices; ++i) {
+    tmp = "GPU" + std::to_string(i);
+    std::cout << std::setw(12) << std::left << tmp;
+  }
+  std::cout << std::endl;
+
+  for (i = 0; i < num_devices; i++) {
+    tmp = "GPU" + std::to_string(i);
+    std::cout << std::setw(6) << std::left << tmp;
+    for (j = 0; j < num_devices; j++) {
+      if(i == j)
+        std::cout << std::setw(12) << std::left << "X";
+      else
+        std::cout << std::setw(12) << std::left << gpu_links[i][j].type;
+    }
+    std::cout << std::endl;
+  }
+  std::cout << std::endl;
+
+  std::cout << "**Hops between two GPUs**" << std::endl;
+  std::cout << "      ";
+  for (i = 0; i < num_devices; ++i) {
+    tmp = "GPU" + std::to_string(i);
+    std::cout << std::setw(12) << std::left << tmp;
+  }
+  std::cout << std::endl;
+
+  for (i = 0; i < num_devices; i++) {
+    tmp = "GPU" + std::to_string(i);
+    std::cout << std::setw(6) << std::left << tmp;
+    for (j = 0; j < num_devices; j++) {
+      if(i == j)
+        std::cout << std::setw(12) << std::left << "X";
+      else {
+        std::cout << std::setw(12) << std::left << gpu_links[i][j].hops;
+      }
+    }
+    std::cout << std::endl;
+  }
+  std::cout << std::endl;
+
+  std::cout << "**Weight between two GPUs**" << std::endl;
+  std::cout << "      ";
+  for (i = 0; i < num_devices; ++i) {
+    tmp = "GPU" + std::to_string(i);
+    std::cout << std::setw(12) << std::left << tmp;
+  }
+  std::cout << std::endl;
+
+  for (i = 0; i < num_devices; i++) {
+    tmp = "GPU" + std::to_string(i);
+    std::cout << std::setw(6) << std::left << tmp;
+    for (j = 0; j < num_devices; j++) {
+      if(i == j)
+        std::cout << std::setw(12) << std::left << "X";
+      else {
+        std::cout << std::setw(12) << std::left << gpu_links[i][j].weight;
+      }
+    }
+    std::cout << std::endl;
+  }
+  std::cout << std::endl;
+}

projects/rocm-smi-lib/tests/rocm_smi_test/functional/hw_topology_read.h

@@ -0,0 +1,73 @@
+/*
+ * =============================================================================
+ *   ROC Runtime Conformance Release License
+ * =============================================================================
+ * The University of Illinois/NCSA
+ * Open Source License (NCSA)
+ *
+ * Copyright (c) 2020, Advanced Micro Devices, Inc.
+ * All rights reserved.
+ *
+ * Developed by:
+ *
+ *                 AMD Research and AMD ROC Software Development
+ *
+ *                 Advanced Micro Devices, Inc.
+ *
+ *                 www.amd.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal with the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimers.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimers in
+ *    the documentation and/or other materials provided with the distribution.
+ *  - Neither the names of <Name of Development Group, Name of Institution>,
+ *    nor the names of its contributors may be used to endorse or promote
+ *    products derived from this Software without specific prior written
+ *    permission.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS WITH THE SOFTWARE.
+ *
+ */
+#ifndef TESTS_ROCM_SMI_TEST_FUNCTIONAL_HW_TOPOLOGY_READ_H_
+#define TESTS_ROCM_SMI_TEST_FUNCTIONAL_HW_TOPOLOGY_READ_H_
+
+#include "rocm_smi_test/test_base.h"
+
+class TestHWTopologyRead : public TestBase {
+ public:
+  TestHWTopologyRead();
+
+  // @Brief: Destructor for test case of TestHWTopologyRead
+  virtual ~TestHWTopologyRead();
+
+  // @Brief: Setup the environment for measurement
+  virtual void SetUp();
+
+  // @Brief: Core measurement execution
+  virtual void Run();
+
+  // @Brief: Clean up and retrieve the resource
+  virtual void Close();
+
+  // @Brief: Display  results
+  virtual void DisplayResults() const;
+
+  // @Brief: Display information about what this test does
+  virtual void DisplayTestInfo(void);
+};
+
+#endif  // TESTS_ROCM_SMI_TEST_FUNCTIONAL_HW_TOPOLOGY_READ_H_

projects/rocm-smi-lib/tests/rocm_smi_test/main.cc

@@ -80,6 +80,7 @@
 #include "functional/mutual_exclusion.h"
 #include "functional/evt_notif_read_write.h"
 #include "functional/init_shutdown_refcount.h"
+#include "rocm_smi_test/functional/hw_topology_read.h"
 
 static RSMITstGlobals *sRSMIGlvalues = nullptr;
 
@@ -214,6 +215,10 @@ TEST(rsmitstReadOnly, TestProcInfoRead) {
   TestProcInfoRead tst;
   RunGenericTest(&tst);
 }
+TEST(rsmitstReadOnly, TestHWTopologyRead) {
+  TestHWTopologyRead tst;
+  RunGenericTest(&tst);
+}
 TEST(rsmitstReadWrite, TestXGMIReadWrite) {
   TestXGMIReadWrite tst;
   RunGenericTest(&tst);