diff --git a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_io_link.h b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_io_link.h index e7bc35ebc2..f57987e8fd 100644 --- a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_io_link.h +++ b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_io_link.h @@ -82,6 +82,14 @@ typedef enum _LINK_DIRECTORY_TYPE { P2P_LINK_DIRECTORY = 1 } LINK_DIRECTORY_TYPE; +enum class IOLinkDirectionType_t +{ + kNonDirectional = 0, + kUniDirectional = 1, + kBiDirectional = 2, +}; + + class IOLink { public: explicit IOLink(uint32_t node_indx, uint32_t link_indx, LINK_DIRECTORY_TYPE link_dir_type) : @@ -121,6 +129,9 @@ class IOLink { rsmi_p2p_capability_t link_cap_; }; +using IOLinksPerNodeList_t = std::map>; +using IOLinksPerNodeListPtr_t = IOLinksPerNodeList_t*; + int DiscoverIOLinksPerNode(uint32_t node_indx, std::map> *links); @@ -137,6 +148,10 @@ int DiscoverP2PLinks(std::map, std::shared_ptr> *links); +IOLinkDirectionType_t DiscoverIOLinkPerNodeDirection(uint32_t src_node_idx, + uint32_t dst_node_idx); + + } // namespace smi } // namespace amd diff --git a/projects/amdsmi/rocm_smi/src/rocm_smi.cc b/projects/amdsmi/rocm_smi/src/rocm_smi.cc index 9a03cb8917..90593b0523 100644 --- a/projects/amdsmi/rocm_smi/src/rocm_smi.cc +++ b/projects/amdsmi/rocm_smi/src/rocm_smi.cc @@ -810,7 +810,7 @@ rsmi_dev_pci_id_get(uint32_t dv_ind, uint64_t *bdfid) { * Add domain to full pci_id: * BDFID = ((DOMAIN & 0xFFFFFFFF) << 32) | ((PARTITION_ID & 0xF) << 28) | * ((BUS & 0xFF) << 8) | ((DEVICE & 0x1F) <<3 ) | (FUNCTION & 0x7) - * + * * bits [63:32] = domain * bits [31:28] or bits [2:0] = partition id * bits [27:16] = reserved @@ -5370,7 +5370,25 @@ rsmi_topo_get_p2p_status(uint32_t dv_ind_src, uint32_t dv_ind_dst, // Unexpected IO Link type read return RSMI_STATUS_NOT_SUPPORTED; } - *cap = it->second->get_link_capability(); + + /* + * Note: Adjust tmp_capability for the returned capabilities. + * Todo: We need to fix it directy as part of the KFD Nodes 'KFDNode::Initialize(void)' + * However, it involves a more complex change, so we will discuss it and fix in in the future. + * Ideally, due to the fact we would need to check every IO link (for each KFD node), and + * considering the topology could change (ie; new GPUs added, partioning changed, etc), we + * are looking into O(N^2) time complexity, no to mention the fact that the IO links then + * need to be changed/updated too. I have some ideas about how to do this, but it will take + * some time to implement and test it, should we consider it is *really necessary*. + * + */ + auto tmp_capability = it->second->get_link_capability(); + if (auto link_direction_result = amd::smi::DiscoverIOLinkPerNodeDirection(node_ind_src, node_ind_dst); + link_direction_result == amd::smi::IOLinkDirectionType_t::kBiDirectional) { + // 1 = true, 0 = false + tmp_capability.is_iolink_bi_directional = 1; + } + *cap = tmp_capability; return RSMI_STATUS_SUCCESS; } diff --git a/projects/amdsmi/rocm_smi/src/rocm_smi_io_link.cc b/projects/amdsmi/rocm_smi/src/rocm_smi_io_link.cc index 93b4ae238a..6ff48fe9b7 100644 --- a/projects/amdsmi/rocm_smi/src/rocm_smi_io_link.cc +++ b/projects/amdsmi/rocm_smi/src/rocm_smi_io_link.cc @@ -442,5 +442,38 @@ int IOLink::UpdateP2pCapability(void) { return 0; } +IOLinkDirectionType_t DiscoverIOLinkPerNodeDirection(uint32_t src_node_idx, uint32_t dst_node_idx) +{ + /* Note: Lets look at the IOLinks of the source node and see if there is a link to the target node + * Then we do the same inverting the actors (source and target) and see if there is a link + */ + auto direction_type(IOLinkDirectionType_t::kNonDirectional); + auto src_links_list = IOLinksPerNodeList_t(); + auto dst_links_list = IOLinksPerNodeList_t(); + if (auto src_discover_result = DiscoverLinksPerNode(src_node_idx, &src_links_list, IO_LINK_DIRECTORY); + src_discover_result == 0) { + for (const auto& [key, value] : src_links_list) { + if (key == dst_node_idx) { + direction_type = IOLinkDirectionType_t::kUniDirectional; + break; + } + } + } + + if (auto dst_discover_result = DiscoverLinksPerNode(dst_node_idx, &dst_links_list, IO_LINK_DIRECTORY); + dst_discover_result == 0) { + for (const auto& [key, value] : dst_links_list) { + if (key == src_node_idx) { + direction_type = (direction_type == IOLinkDirectionType_t::kUniDirectional ? + IOLinkDirectionType_t::kBiDirectional : IOLinkDirectionType_t::kUniDirectional); + break; + } + } + } + + return direction_type; +} + + } // namespace smi } // namespace amd