From f7885be06e41c1bfc09fca8836f18845361974e1 Mon Sep 17 00:00:00 2001 From: Mike Li Date: Thu, 7 May 2020 11:37:14 -0700 Subject: [PATCH] Add functions that are used to query Hardware topology. Change-Id: I0f4cd02b237bde4d6dccfb0e83e65376ecb1cfaa Signed-off-by: Mike Li [ROCm/rocm_smi_lib commit: c7d349183a63529246cd394f0a3131eb7c381146] --- projects/rocm-smi-lib/CMakeLists.txt | 2 + .../rocm-smi-lib/include/rocm_smi/rocm_smi.h | 90 +++++ .../include/rocm_smi/rocm_smi_io_link.h | 116 ++++++ .../include/rocm_smi/rocm_smi_kfd.h | 13 + .../include/rocm_smi/rocm_smi_main.h | 7 + projects/rocm-smi-lib/src/rocm_smi.cc | 177 +++++++++ projects/rocm-smi-lib/src/rocm_smi_io_link.cc | 356 ++++++++++++++++++ projects/rocm-smi-lib/src/rocm_smi_kfd.cc | 66 +++- projects/rocm-smi-lib/src/rocm_smi_main.cc | 34 ++ .../functional/hw_topology_read.cc | 252 +++++++++++++ .../functional/hw_topology_read.h | 73 ++++ .../rocm-smi-lib/tests/rocm_smi_test/main.cc | 5 + 12 files changed, 1190 insertions(+), 1 deletion(-) create mode 100755 projects/rocm-smi-lib/include/rocm_smi/rocm_smi_io_link.h create mode 100755 projects/rocm-smi-lib/src/rocm_smi_io_link.cc create mode 100755 projects/rocm-smi-lib/tests/rocm_smi_test/functional/hw_topology_read.cc create mode 100755 projects/rocm-smi-lib/tests/rocm_smi_test/functional/hw_topology_read.h diff --git a/projects/rocm-smi-lib/CMakeLists.txt b/projects/rocm-smi-lib/CMakeLists.txt index daf84700d5..c52da631a5 100755 --- a/projects/rocm-smi-lib/CMakeLists.txt +++ b/projects/rocm-smi-lib/CMakeLists.txt @@ -141,6 +141,7 @@ set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_power_mon.cc") set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_utils.cc") set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_counters.cc") set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_kfd.cc") +set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_io_link.cc") set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/shared_mutex/shared_mutex.cc") set(SMI_INC_LIST "${INC_DIR}/rocm_smi_device.h") @@ -152,6 +153,7 @@ set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_common.h") set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_exception.h") set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_counters.h") set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_kfd.h") +set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_io_link.h") set(SMI_INC_LIST ${SMI_INC_LIST} "${SRC_DIR}/shared_mutex/shared_mutex.h") set(SMI_EXAMPLE_EXE "rocm_smi_ex") diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h index a6b66b30ae..f0daf74fee 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h @@ -522,6 +522,17 @@ typedef enum { RSMI_MEM_PAGE_STATUS_UNRESERVABLE //!< Unable to reserve this page } rsmi_memory_page_status_t; +/** + * @brief Types for IO Link + */ +typedef enum _RSMI_IO_LINK_TYPE { + RSMI_IOLINK_TYPE_UNDEFINED = 0, //!< unknown type. + RSMI_IOLINK_TYPE_PCIEXPRESS = 1, //!< PCI Express + RSMI_IOLINK_TYPE_XGMI = 2, //!< XGMI + RSMI_IOLINK_TYPE_NUMIOLINKTYPES, //!< Number of IO Link types + RSMI_IOLINK_TYPE_SIZE = 0xFFFFFFFF //!< Max of IO Link types +} RSMI_IO_LINK_TYPE; + /** * @brief Reserved Memory Page Record */ @@ -2615,6 +2626,85 @@ rsmi_dev_xgmi_error_reset(uint32_t dv_ind); /** @} */ // end of SysInfo +/*****************************************************************************/ +/** @defgroup HWTopo Hardware Topology Functions + * These functions are used to query Hardware topology. + * @{ + */ + +/** + * @brief Retrieve the NUMA CPU node number for a device + * + * @details Given a device index @p dv_ind, and a pointer to an + * uint32_t @p numa_node, this function will write the + * node number of NUMA CPU for the device @p dv_ind to the memory + * pointed to by @p numa_node. + * + * @param[in] dv_ind a device index + * + * @param[inout] numa_node A pointer to an uint32_t to which the + * numa node number should be written. + * + * @retval ::RSMI_STATUS_SUCCESS call was successful + * @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid + * + */ +rsmi_status_t +rsmi_topo_get_numa_node_number(uint32_t dv_ind, uint32_t *numa_node); + +/** + * @brief Retrieve the weight for a connection between 2 GPUs + * + * @details Given a source device index @p dv_ind_src and + * a destination device index @p dv_ind_dst, and a pointer to an + * uint64_t @p weight, this function will write the + * weight for the connection between the device @p dv_ind_src + * and @p dv_ind_dst to the memory pointed to by @p weight. + * + * @param[in] dv_ind_src the source device index + * + * @param[in] dv_ind_dst the destination device index + * + * @param[inout] weight A pointer to an uint64_t to which the + * weight for the connection should be written. + * + * @retval ::RSMI_STATUS_SUCCESS call was successful + * @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid + * + */ +rsmi_status_t +rsmi_topo_get_link_weight(uint32_t dv_ind_src, uint32_t dv_ind_dst, + uint64_t *weight); + +/** + * @brief Retrieve the hops and the connection type between 2 GPUs + * + * @details Given a source device index @p dv_ind_src and + * a destination device index @p dv_ind_dst, and a pointer to an + * uint64_t @p hops and a pointer to an RSMI_IO_LINK_TYPE @p type, + * this function will write the number of hops and the connection type + * between the device @p dv_ind_src and @p dv_ind_dst to the memory + * pointed to by @p hops and @p type. + * + * @param[in] dv_ind_src the source device index + * + * @param[in] dv_ind_dst the destination device index + * + * @param[inout] hops A pointer to an uint64_t to which the + * hops for the connection should be written. + * + * @param[inout] type A pointer to an ::RSMI_IO_LINK_TYPE to which the + * type for the connection should be written. + * + * @retval ::RSMI_STATUS_SUCCESS call was successful + * @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid + * + */ +rsmi_status_t +rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst, + uint64_t *hops, RSMI_IO_LINK_TYPE *type); + +/** @} */ // end of HWTopo /*****************************************************************************/ /** @defgroup APISupport Supported Functions diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_io_link.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_io_link.h new file mode 100755 index 0000000000..cd10b30ad6 --- /dev/null +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_io_link.h @@ -0,0 +1,116 @@ +/* + * ============================================================================= + * The University of Illinois/NCSA + * Open Source License (NCSA) + * + * Copyright (c) 2020, Advanced Micro Devices, Inc. + * All rights reserved. + * + * Developed by: + * + * AMD Research and AMD ROC Software Development + * + * Advanced Micro Devices, Inc. + * + * www.amd.com + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal with the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimers. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimers in + * the documentation and/or other materials provided with the distribution. + * - Neither the names of , + * nor the names of its contributors may be used to endorse or promote + * products derived from this Software without specific prior written + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS WITH THE SOFTWARE. + * + */ +#ifndef INCLUDE_ROCM_SMI_ROCM_SMI_IO_LINK_H_ +#define INCLUDE_ROCM_SMI_ROCM_SMI_IO_LINK_H_ + +#include +#include +#include +#include +#include + +#include "rocm_smi/rocm_smi.h" + +namespace amd { +namespace smi { + +typedef enum _IO_LINK_TYPE { + IOLINK_TYPE_UNDEFINED = 0, + IOLINK_TYPE_HYPERTRANSPORT = 1, + IOLINK_TYPE_PCIEXPRESS = 2, + IOLINK_TYPE_AMBA = 3, + IOLINK_TYPE_MIPI = 4, + IOLINK_TYPE_QPI_1_1 = 5, + IOLINK_TYPE_RESERVED1 = 6, + IOLINK_TYPE_RESERVED2 = 7, + IOLINK_TYPE_RAPID_IO = 8, + IOLINK_TYPE_INFINIBAND = 9, + IOLINK_TYPE_RESERVED3 = 10, + IOLINK_TYPE_XGMI = 11, + IOLINK_TYPE_XGOP = 12, + IOLINK_TYPE_GZ = 13, + IOLINK_TYPE_ETHERNET_RDMA = 14, + IOLINK_TYPE_RDMA_OTHER = 15, + IOLINK_TYPE_OTHER = 16, + IOLINK_TYPE_NUMIOLINKTYPES, + IOLINK_TYPE_SIZE = 0xFFFFFFFF +} IO_LINK_TYPE; + +class IOLink { + public: + explicit IOLink(uint32_t node_indx, uint32_t link_indx) : + node_indx_(node_indx), link_indx_(link_indx) {} + ~IOLink(); + + int Initialize(); + int ReadProperties(void); + int get_property_value(std::string property, uint64_t *value); + uint32_t get_node_indx(void) const {return node_indx_;} + uint32_t get_link_indx(void) const {return link_indx_;} + IO_LINK_TYPE type(void) const {return type_;} + uint32_t node_from(void) const {return node_from_;} + uint32_t node_to(void) const {return node_to_;} + uint64_t weight(void) const {return weight_;} + + private: + uint32_t node_indx_; + uint32_t link_indx_; + IO_LINK_TYPE type_; + uint32_t node_from_; + uint32_t node_to_; + uint64_t weight_; + std::map properties_; +}; + +int +DiscoverIOLinksPerNode(uint32_t node_indx, std::map> *links); + +int +DiscoverIOLinks(std::map, + std::shared_ptr> *links); + +} // namespace smi +} // namespace amd + +#endif // INCLUDE_ROCM_SMI_ROCM_SMI_IO_LINK_H_ diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h index 1301b6729c..cff8def68e 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h @@ -51,6 +51,7 @@ #include "rocm_smi/rocm_smi.h" #include "rocm_smi/rocm_smi_device.h" +#include "rocm_smi/rocm_smi_io_link.h" namespace amd { namespace smi { @@ -65,6 +66,12 @@ class KFDNode { int get_property_value(std::string property, uint64_t *value); uint64_t gpu_id(void) const {return gpu_id_;} std::string name(void) const {return name_;} + uint32_t node_index(void) const {return node_indx_;} + uint32_t numa_node_number(void) const {return numa_node_number_;} + uint64_t numa_node_weight(void) const {return numa_node_weight_;} + IO_LINK_TYPE numa_node_type(void) const {return numa_node_type_;} + int get_io_link_type(uint32_t node_to, IO_LINK_TYPE *type); + int get_io_link_weight(uint32_t node_to, uint64_t *weight); std::shared_ptr amdgpu_device(void) const {return amdgpu_device_;} uint32_t amdgpu_dev_index(void) const {return amdgpu_dev_index_;} void set_amdgpu_dev_index(uint32_t val) {amdgpu_dev_index_ = val;} @@ -74,6 +81,12 @@ class KFDNode { uint32_t amdgpu_dev_index_; uint64_t gpu_id_; std::string name_; + uint32_t numa_node_number_; + uint64_t numa_node_weight_; + IO_LINK_TYPE numa_node_type_; + std::map io_link_type_; + std::map io_link_weight_; + std::map> io_link_map_; std::map properties_; std::shared_ptr amdgpu_device_; }; diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_main.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_main.h index c46918811b..f59950f15d 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_main.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_main.h @@ -55,6 +55,7 @@ #include #include // NOLINT +#include "rocm_smi/rocm_smi_io_link.h" #include "rocm_smi/rocm_smi_kfd.h" #include "rocm_smi/rocm_smi_device.h" #include "rocm_smi/rocm_smi_monitor.h" @@ -106,6 +107,9 @@ class RocmSMI { return ++kfd_notif_evt_fh_refcnt_;} uint32_t kfd_notif_evt_fh_refcnt_dec(void) { return --kfd_notif_evt_fh_refcnt_;} + int get_io_link_weight(uint32_t node_from, uint32_t node_to, + uint64_t *weight); + int get_node_index(uint32_t dv_ind, uint32_t *node_ind); private: std::vector> devices_; @@ -113,6 +117,9 @@ class RocmSMI { std::vector> monitors_; std::vector> power_mons_; std::set amd_monitor_types_; + std::map, std::shared_ptr> + io_link_map_; + std::map dev_ind_to_node_ind_map_; void AddToDeviceList(std::string dev_name); void GetEnvVariables(void); uint32_t DiscoverAMDMonitors(void); diff --git a/projects/rocm-smi-lib/src/rocm_smi.cc b/projects/rocm-smi-lib/src/rocm_smi.cc index c39c2b7d82..b40063b4bd 100755 --- a/projects/rocm-smi-lib/src/rocm_smi.cc +++ b/projects/rocm-smi-lib/src/rocm_smi.cc @@ -70,6 +70,7 @@ #include "rocm_smi/rocm_smi_exception.h" #include "rocm_smi/rocm_smi_counters.h" #include "rocm_smi/rocm_smi_kfd.h" +#include "rocm_smi/rocm_smi_io_link.h" #include "rocm_smi/rocm_smi64Config.h" @@ -1241,6 +1242,30 @@ static rsmi_status_t set_power_profile(uint32_t dv_ind, CATCH } +static rsmi_status_t topo_get_numa_node_number(uint32_t dv_ind, + uint32_t *numa_node_number) { + TRY + + GET_DEV_AND_KFDNODE_FROM_INDX + + *numa_node_number = kfd_node->numa_node_number(); + + return RSMI_STATUS_SUCCESS; + CATCH +} + +static rsmi_status_t topo_get_numa_node_weight(uint32_t dv_ind, + uint64_t *weight) { + TRY + + GET_DEV_AND_KFDNODE_FROM_INDX + + *weight = kfd_node->numa_node_weight(); + + return RSMI_STATUS_SUCCESS; + CATCH +} + rsmi_status_t rsmi_dev_gpu_clk_freq_get(uint32_t dv_ind, rsmi_clk_type_t clk_type, rsmi_frequencies_t *f) { @@ -2964,6 +2989,158 @@ rsmi_dev_xgmi_error_reset(uint32_t dv_ind) { CATCH } +rsmi_status_t +rsmi_topo_get_numa_node_number(uint32_t dv_ind, uint32_t *numa_node) { + TRY + + return topo_get_numa_node_number(dv_ind, numa_node); + CATCH +} + +rsmi_status_t +rsmi_topo_get_link_weight(uint32_t dv_ind_src, uint32_t dv_ind_dst, + uint64_t *weight) { + TRY + + uint32_t dv_ind = dv_ind_src; + GET_DEV_AND_KFDNODE_FROM_INDX + DEVICE_MUTEX + + if (weight == nullptr) { + return RSMI_STATUS_INVALID_ARGS; + } + + rsmi_status_t status; + uint32_t node_ind_dst; + uint32_t ret = smi.get_node_index(dv_ind_dst, &node_ind_dst); + + if (!ret) { + amd::smi::IO_LINK_TYPE type; + ret = kfd_node->get_io_link_type(node_ind_dst, &type); + if (!ret) { + if (type == amd::smi::IOLINK_TYPE_XGMI) { + ret = kfd_node->get_io_link_weight(node_ind_dst, weight); + if (!ret) + status = RSMI_STATUS_SUCCESS; + else + status = RSMI_STATUS_INIT_ERROR; + } else { + assert(!"Unexpected IO Link type read"); + status = RSMI_STATUS_NOT_SUPPORTED; + } + } else { + *weight = kfd_node->numa_node_weight(); // from src GPU to it's CPU node + uint64_t numa_weight_dst; + status = topo_get_numa_node_weight(dv_ind_dst, &numa_weight_dst); + // from dst GPU to it's CPU node + if (status == RSMI_STATUS_SUCCESS) { + *weight = *weight + numa_weight_dst; + uint32_t numa_number_src = kfd_node->numa_node_number(); + uint32_t numa_number_dst; + status = topo_get_numa_node_number(dv_ind_dst, &numa_number_dst); + if (status == RSMI_STATUS_SUCCESS) { + if (numa_number_src != numa_number_dst) { + uint64_t io_link_weight; + ret = smi.get_io_link_weight(numa_number_src, numa_number_dst, + &io_link_weight); + if (!ret) { + *weight = *weight + io_link_weight; + // from src numa CPU node to dst numa CPU node + } else { + *weight = *weight + 10; + // More than one CPU hops, hard coded 10 + } + } + status = RSMI_STATUS_SUCCESS; + } else { + assert(!"Error to read numa node number"); + status = RSMI_STATUS_INIT_ERROR; + } + } else { + assert(!"Error to read numa node weight"); + status = RSMI_STATUS_INIT_ERROR; + } + } + } else { + status = RSMI_STATUS_INVALID_ARGS; + } + + return status; + CATCH +} + +rsmi_status_t +rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst, + uint64_t *hops, RSMI_IO_LINK_TYPE *type) { + TRY + + uint32_t dv_ind = dv_ind_src; + GET_DEV_AND_KFDNODE_FROM_INDX + + if (hops == nullptr) { + return RSMI_STATUS_INVALID_ARGS; + } + + if (type == nullptr) { + return RSMI_STATUS_INVALID_ARGS; + } + + rsmi_status_t status; + uint32_t node_ind_dst; + uint32_t ret = smi.get_node_index(dv_ind_dst, &node_ind_dst); + + if (!ret) { + amd::smi::IO_LINK_TYPE io_link_type; + ret = kfd_node->get_io_link_type(node_ind_dst, &io_link_type); + if (!ret) { + if (io_link_type == amd::smi::IOLINK_TYPE_XGMI) { + *type = RSMI_IOLINK_TYPE_XGMI; + *hops = 1; + status = RSMI_STATUS_SUCCESS; + } else { + assert(!"Unexpected IO Link type read"); + status = RSMI_STATUS_NOT_SUPPORTED; + } + } else { + uint32_t numa_number_dst; + status = topo_get_numa_node_number(dv_ind_dst, &numa_number_dst); + if (status == RSMI_STATUS_SUCCESS) { + uint32_t numa_number_src = kfd_node->numa_node_number(); + if (numa_number_src == numa_number_dst) { + *hops = 2; // same CPU node + } else { + uint64_t io_link_weight; + ret = smi.get_io_link_weight(numa_number_src, numa_number_dst, + &io_link_weight); + if (!ret) + *hops = 3; // from src CPU node to dst CPU node + else + *hops = 4; // More than one CPU hops, hard coded as 4 + } + + amd::smi::IO_LINK_TYPE numa_node_type = kfd_node->numa_node_type(); + if (numa_node_type == amd::smi::IOLINK_TYPE_PCIEXPRESS) { + *type = RSMI_IOLINK_TYPE_PCIEXPRESS; + status = RSMI_STATUS_SUCCESS; + } else if (numa_node_type == amd::smi::IOLINK_TYPE_XGMI) { + *type = RSMI_IOLINK_TYPE_XGMI; + status = RSMI_STATUS_SUCCESS; + } else { + status = RSMI_STATUS_INIT_ERROR; + } + } else { + assert(!"Error to get numa node number"); + status = RSMI_STATUS_INIT_ERROR; + } + } + } else { + status = RSMI_STATUS_INVALID_ARGS; + } + + return status; + CATCH +} + enum iterator_handle_type { FUNC_ITER = 0, VARIANT_ITER, diff --git a/projects/rocm-smi-lib/src/rocm_smi_io_link.cc b/projects/rocm-smi-lib/src/rocm_smi_io_link.cc new file mode 100755 index 0000000000..83530a2cba --- /dev/null +++ b/projects/rocm-smi-lib/src/rocm_smi_io_link.cc @@ -0,0 +1,356 @@ +/* + * ============================================================================= + * The University of Illinois/NCSA + * Open Source License (NCSA) + * + * Copyright (c) 2020, Advanced Micro Devices, Inc. + * All rights reserved. + * + * Developed by: + * + * AMD Research and AMD ROC Software Development + * + * Advanced Micro Devices, Inc. + * + * www.amd.com + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal with the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimers. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimers in + * the documentation and/or other materials provided with the distribution. + * - Neither the names of , + * nor the names of its contributors may be used to endorse or promote + * products derived from this Software without specific prior written + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS WITH THE SOFTWARE. + * + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "rocm_smi/rocm_smi.h" +#include "rocm_smi/rocm_smi_exception.h" +#include "rocm_smi/rocm_smi_utils.h" +#include "rocm_smi/rocm_smi_io_link.h" + +namespace amd { +namespace smi { + +static const char *kKFDNodesPathRoot = "/sys/class/kfd/kfd/topology/nodes"; + +// IO Link Property strings +static const char *kIOLinkPropTYPEStr = "type"; +// static const char *kIOLinkPropVERSION_MAJORStr = "version_major"; +// static const char *kIOLinkPropVERSION_MINORStr = "version_minor"; +static const char *kIOLinkPropNODE_FROMStr = "node_from"; +static const char *kIOLinkPropNODE_TOStr = "node_to"; +static const char *kIOLinkPropWEIGHTStr = "weight"; +// static const char *kIOLinkPropMIN_LATENCYStr = "min_latency"; +// static const char *kIOLinkPropMAX_LATENCYStr = "max_latency"; +// static const char *kIOLinkPropMIN_BANDWIDTHStr = "min_bandwidth"; +// static const char *kIOLinkPropMAX_BANDWIDTHStr = "max_bandwidth"; +// static const char *kIOLinkPropRECOMMENDED_TRANSFER_SIZEStr = +// "recommended_transfer_size"; +// static const char *kIOLinkPropFLAGSStr = "flags"; + +static bool is_number(const std::string &s) { + return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit); +} + +static std::string IOLinkPathRoot(uint32_t node_indx) { + std::string io_link_path = kKFDNodesPathRoot; + io_link_path += '/'; + io_link_path += std::to_string(node_indx); + io_link_path += '/'; + io_link_path += "io_links"; + return io_link_path; +} + +static std::string IOLinkPath(uint32_t node_indx, uint32_t link_indx) { + std::string io_link_path = IOLinkPathRoot(node_indx); + io_link_path += '/'; + io_link_path += std::to_string(link_indx); + return io_link_path; +} + +static int OpenIOLinkProperties(uint32_t node_indx, uint32_t link_indx, + std::ifstream *fs) { + int ret; + std::string f_path; + bool reg_file; + + assert(fs != nullptr); + if (fs == nullptr) { + return EINVAL; + } + + f_path = IOLinkPath(node_indx, link_indx); + f_path += "/"; + f_path += "properties"; + + ret = isRegularFile(f_path, ®_file); + + if (ret != 0) { + return ret; + } + if (!reg_file) { + return ENOENT; + } + + fs->open(f_path); + + if (!fs->is_open()) { + return errno; + } + + return 0; +} + +static int ReadIOLinkProperties(uint32_t node_indx, uint32_t link_indx, + std::vector *retVec) { + std::string line; + int ret; + std::ifstream fs; + + assert(retVec != nullptr); + if (retVec == nullptr) { + return EINVAL; + } + + ret = OpenIOLinkProperties(node_indx, link_indx, &fs); + + if (ret) { + return ret; + } + + while (std::getline(fs, line)) { + retVec->push_back(line); + } + + if (retVec->size() == 0) { + fs.close(); + return 0; + } + + // Remove any *trailing* empty (whitespace) lines + while (retVec->back().find_first_not_of(" \t\n\v\f\r") == std::string::npos) { + retVec->pop_back(); + } + + fs.close(); + return 0; +} + +int DiscoverIOLinks(std::map, + std::shared_ptr> *links) { + assert(links != nullptr); + if (links == nullptr) { + return EINVAL; + } + assert(links->size() == 0); + + links->clear(); + + auto kfd_node_dir = opendir(kKFDNodesPathRoot); + assert(kfd_node_dir != nullptr); + + auto dentry_kfd = readdir(kfd_node_dir); + while (dentry_kfd != nullptr) { + if (dentry_kfd->d_name[0] == '.') { + dentry_kfd = readdir(kfd_node_dir); + continue; + } + + if (!is_number(dentry_kfd->d_name)) { + dentry_kfd = readdir(kfd_node_dir); + continue; + } + + uint32_t node_indx = std::stoi(dentry_kfd->d_name); + std::shared_ptr link; + uint32_t link_indx; + std::string io_link_path_root = IOLinkPathRoot(node_indx); + + auto io_link_dir = opendir(io_link_path_root.c_str()); + assert(io_link_dir != nullptr); + + auto dentry_io_link = readdir(io_link_dir); + while (dentry_io_link != nullptr) { + if (dentry_io_link->d_name[0] == '.') { + dentry_io_link = readdir(io_link_dir); + continue; + } + + if (!is_number(dentry_io_link->d_name)) { + dentry_io_link = readdir(io_link_dir); + continue; + } + + link_indx = std::stoi(dentry_io_link->d_name); + link = std::shared_ptr(new IOLink(node_indx, link_indx)); + + link->Initialize(); + + (*links)[std::make_pair(link->node_from(), link->node_to())] = link; + + dentry_io_link = readdir(io_link_dir); + } + + if (closedir(io_link_dir)) { + return 1; + } + + dentry_kfd = readdir(kfd_node_dir); + } + + if (closedir(kfd_node_dir)) { + return 1; + } + return 0; +} + +int DiscoverIOLinksPerNode(uint32_t node_indx, std::map> *links) { + assert(links != nullptr); + if (links == nullptr) { + return EINVAL; + } + assert(links->size() == 0); + + links->clear(); + + std::shared_ptr link; + uint32_t link_indx; + std::string io_link_path_root = IOLinkPathRoot(node_indx); + + auto io_link_dir = opendir(io_link_path_root.c_str()); + assert(io_link_dir != nullptr); + + auto dentry = readdir(io_link_dir); + while (dentry != nullptr) { + if (dentry->d_name[0] == '.') { + dentry = readdir(io_link_dir); + continue; + } + + if (!is_number(dentry->d_name)) { + dentry = readdir(io_link_dir); + continue; + } + + link_indx = std::stoi(dentry->d_name); + link = std::shared_ptr(new IOLink(node_indx, link_indx)); + + link->Initialize(); + + (*links)[link->node_to()] = link; + + dentry = readdir(io_link_dir); + } + + if (closedir(io_link_dir)) { + return 1; + } + return 0; +} + +IOLink::~IOLink() { +} + +int IOLink::ReadProperties(void) { + int ret; + + std::vector propVec; + + assert(properties_.size() == 0); + if (properties_.size() > 0) { + return 0; + } + + ret = ReadIOLinkProperties(node_indx_, link_indx_, &propVec); + + if (ret) { + return ret; + } + + std::string key_str; + uint64_t val_int; // Assume all properties are unsigned integers for now + std::istringstream fs; + + for (uint32_t i = 0; i < propVec.size(); ++i) { + fs.str(propVec[i]); + fs >> key_str; + fs >> val_int; + + properties_[key_str] = val_int; + + fs.str(""); + fs.clear(); + } + + return 0; +} + +int +IOLink::Initialize(void) { + int ret = 0; + ret = ReadProperties(); + if (ret) {return ret;} + + ret = get_property_value(kIOLinkPropTYPEStr, + reinterpret_cast(&type_)); + if (ret) {return ret;} + + ret = get_property_value(kIOLinkPropNODE_FROMStr, + reinterpret_cast(&node_from_)); + if (ret) {return ret;} + + ret = get_property_value(kIOLinkPropNODE_TOStr, + reinterpret_cast(&node_to_)); + if (ret) {return ret;} + + ret = get_property_value(kIOLinkPropWEIGHTStr, &weight_); + + return ret; +} + +int +IOLink::get_property_value(std::string property, uint64_t *value) { + assert(value != nullptr); + if (value == nullptr) { + return EINVAL; + } + if (properties_.find(property) == properties_.end()) { + return EINVAL; + } + *value = properties_[property]; + return 0; +} + +} // namespace smi +} // namespace amd diff --git a/projects/rocm-smi-lib/src/rocm_smi_kfd.cc b/projects/rocm-smi-lib/src/rocm_smi_kfd.cc index 5d9a13d4f3..bbaa2c43fb 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_kfd.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_kfd.cc @@ -53,6 +53,7 @@ #include #include +#include "rocm_smi/rocm_smi_io_link.h" #include "rocm_smi/rocm_smi_kfd.h" #include "rocm_smi/rocm_smi.h" #include "rocm_smi/rocm_smi_exception.h" @@ -531,12 +532,49 @@ KFDNode::Initialize(void) { if (ret) {return ret;} ret = ReadKFDGpuId(node_indx_, &gpu_id_); - if (ret) {return ret;} + if (ret || (gpu_id_ == 0)) {return ret;} ret = ReadKFDGpuName(node_indx_, &name_); + std::map> io_link_map_tmp; + ret = DiscoverIOLinksPerNode(node_indx_, &io_link_map_tmp); + if (ret != 0) { + throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR, + "Failed to initialize rocm_smi library (IO Links discovery per node)."); + } + + std::map>::iterator it; + uint32_t node_to; + uint64_t node_to_gpu_id; + std::shared_ptr link; + bool numa_node_found = false; + for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++) { + io_link_map_[it->first] = it->second; + node_to = it->first; + link = it->second; + ret = ReadKFDGpuId(node_to, &node_to_gpu_id); + if (ret) {return ret;} + if (node_to_gpu_id == 0) { // CPU node + if (numa_node_found) { + if (numa_node_weight_ > link->weight()) { + numa_node_number_ = node_to; + numa_node_weight_ = link->weight(); + numa_node_type_ = link->type(); + } + } else { + numa_node_number_ = node_to; + numa_node_weight_ = link->weight(); + numa_node_type_ = link->type(); + numa_node_found = true; + } + } else { + io_link_type_[node_to] = link->type(); + io_link_weight_[node_to] = link->weight(); + } + } return ret; } + int KFDNode::get_property_value(std::string property, uint64_t *value) { assert(value != nullptr); @@ -550,5 +588,31 @@ KFDNode::get_property_value(std::string property, uint64_t *value) { return 0; } +int +KFDNode::get_io_link_type(uint32_t node_to, IO_LINK_TYPE *type) { + assert(type != nullptr); + if (type == nullptr) { + return EINVAL; + } + if (io_link_type_.find(node_to) == io_link_type_.end()) { + return EINVAL; + } + *type = io_link_type_[node_to]; + return 0; +} + +int +KFDNode::get_io_link_weight(uint32_t node_to, uint64_t *weight) { + assert(weight != nullptr); + if (weight == nullptr) { + return EINVAL; + } + if (io_link_weight_.find(node_to) == io_link_weight_.end()) { + return EINVAL; + } + *weight = io_link_weight_[node_to]; + return 0; +} + } // namespace smi } // namespace amd diff --git a/projects/rocm-smi-lib/src/rocm_smi_main.cc b/projects/rocm-smi-lib/src/rocm_smi_main.cc index ce5f911b8e..1619eed250 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_main.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_main.cc @@ -280,6 +280,17 @@ RocmSMI::Initialize(uint64_t flags) { "Failed to initialize rocm_smi library (KFD node discovery)."); } + std::map, std::shared_ptr> + io_link_map_tmp; + ret = DiscoverIOLinks(&io_link_map_tmp); + if (ret != 0) { + throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR, + "Failed to initialize rocm_smi library (IO Links discovery)."); + } + std::map, std::shared_ptr>::iterator it; + for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++) + io_link_map_[it->first] = it->second; + std::shared_ptr dev; // 1. construct kfd_node_map_ with gpu_id as key and *Device as value @@ -295,6 +306,7 @@ RocmSMI::Initialize(uint64_t flags) { } tmp_map[bdfid]->set_amdgpu_dev_index(dv_ind); + dev_ind_to_node_ind_map_[dv_ind] = tmp_map[bdfid]->node_index(); uint64_t gpu_id = tmp_map[bdfid]->gpu_id(); dev->set_kfd_gpu_id(gpu_id); kfd_node_map_[gpu_id] = tmp_map[bdfid]; @@ -591,5 +603,27 @@ uint32_t RocmSMI::IterateSMIDevices( return 0; } +int RocmSMI::get_node_index(uint32_t dv_ind, uint32_t *node_ind) { + if (dev_ind_to_node_ind_map_.find(dv_ind) == dev_ind_to_node_ind_map_.end()) { + return EINVAL; + } + *node_ind = dev_ind_to_node_ind_map_[dv_ind]; + return 0; +} + +int RocmSMI::get_io_link_weight(uint32_t node_from, uint32_t node_to, + uint64_t *weight) { + assert(weight != nullptr); + if (weight == nullptr) { + return EINVAL; + } + if (io_link_map_.find(std::make_pair(node_from, node_to)) == + io_link_map_.end()) { + return EINVAL; + } + *weight = io_link_map_[std::make_pair(node_from, node_to)]->weight(); + return 0; +} + } // namespace smi } // namespace amd diff --git a/projects/rocm-smi-lib/tests/rocm_smi_test/functional/hw_topology_read.cc b/projects/rocm-smi-lib/tests/rocm_smi_test/functional/hw_topology_read.cc new file mode 100755 index 0000000000..490adb6f37 --- /dev/null +++ b/projects/rocm-smi-lib/tests/rocm_smi_test/functional/hw_topology_read.cc @@ -0,0 +1,252 @@ +/* + * ============================================================================= + * ROC Runtime Conformance Release License + * ============================================================================= + * The University of Illinois/NCSA + * Open Source License (NCSA) + * + * Copyright (c) 2020, Advanced Micro Devices, Inc. + * All rights reserved. + * + * Developed by: + * + * AMD Research and AMD ROC Software Development + * + * Advanced Micro Devices, Inc. + * + * www.amd.com + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal with the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimers. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimers in + * the documentation and/or other materials provided with the distribution. + * - Neither the names of , + * nor the names of its contributors may be used to endorse or promote + * products derived from this Software without specific prior written + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS WITH THE SOFTWARE. + * + */ + +#include +#include + +#include +#include + +#include "gtest/gtest.h" +#include "rocm_smi/rocm_smi.h" +#include "rocm_smi_test/functional/hw_topology_read.h" +#include "rocm_smi_test/test_common.h" + +typedef struct { + std::string type; + uint64_t hops; + uint64_t weight; +} gpu_link_t; + +TestHWTopologyRead::TestHWTopologyRead() : TestBase() { + set_title("RSMI Hardware Topology Read Test"); + set_description("This test verifies that Hardware Topology can be read properly."); +} + +TestHWTopologyRead::~TestHWTopologyRead(void) { +} + +void TestHWTopologyRead::SetUp(void) { + TestBase::SetUp(); + + return; +} + +void TestHWTopologyRead::DisplayTestInfo(void) { + TestBase::DisplayTestInfo(); +} + +void TestHWTopologyRead::DisplayResults(void) const { + TestBase::DisplayResults(); + return; +} + +void TestHWTopologyRead::Close() { + // This will close handles opened within rsmitst utility calls and call + // rsmi_shut_down(), so it should be done after other cleanup + TestBase::Close(); +} + +void TestHWTopologyRead::Run(void) { + rsmi_status_t err; + uint32_t i, j; + + TestBase::Run(); + if (setup_failed_) { + std::cout << "** SetUp Failed for this test. Skipping.**" << std::endl; + return; + } + + uint32_t num_devices; + err = rsmi_num_monitor_devices(&num_devices); + CHK_ERR_ASRT(err) + + gpu_link_t gpu_links[num_devices][num_devices]; + uint32_t numa_numbers[num_devices]; + + for (uint32_t dv_ind=0; dv_ind, + * nor the names of its contributors may be used to endorse or promote + * products derived from this Software without specific prior written + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS WITH THE SOFTWARE. + * + */ +#ifndef TESTS_ROCM_SMI_TEST_FUNCTIONAL_HW_TOPOLOGY_READ_H_ +#define TESTS_ROCM_SMI_TEST_FUNCTIONAL_HW_TOPOLOGY_READ_H_ + +#include "rocm_smi_test/test_base.h" + +class TestHWTopologyRead : public TestBase { + public: + TestHWTopologyRead(); + + // @Brief: Destructor for test case of TestHWTopologyRead + virtual ~TestHWTopologyRead(); + + // @Brief: Setup the environment for measurement + virtual void SetUp(); + + // @Brief: Core measurement execution + virtual void Run(); + + // @Brief: Clean up and retrieve the resource + virtual void Close(); + + // @Brief: Display results + virtual void DisplayResults() const; + + // @Brief: Display information about what this test does + virtual void DisplayTestInfo(void); +}; + +#endif // TESTS_ROCM_SMI_TEST_FUNCTIONAL_HW_TOPOLOGY_READ_H_ diff --git a/projects/rocm-smi-lib/tests/rocm_smi_test/main.cc b/projects/rocm-smi-lib/tests/rocm_smi_test/main.cc index f8c50d54b0..906375dd15 100755 --- a/projects/rocm-smi-lib/tests/rocm_smi_test/main.cc +++ b/projects/rocm-smi-lib/tests/rocm_smi_test/main.cc @@ -80,6 +80,7 @@ #include "functional/mutual_exclusion.h" #include "functional/evt_notif_read_write.h" #include "functional/init_shutdown_refcount.h" +#include "rocm_smi_test/functional/hw_topology_read.h" static RSMITstGlobals *sRSMIGlvalues = nullptr; @@ -214,6 +215,10 @@ TEST(rsmitstReadOnly, TestProcInfoRead) { TestProcInfoRead tst; RunGenericTest(&tst); } +TEST(rsmitstReadOnly, TestHWTopologyRead) { + TestHWTopologyRead tst; + RunGenericTest(&tst); +} TEST(rsmitstReadWrite, TestXGMIReadWrite) { TestXGMIReadWrite tst; RunGenericTest(&tst);