From 4658630d8d1947a449792eb853e2cabcc584db1c Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Fri, 26 Aug 2022 20:29:04 +0000 Subject: [PATCH 1/2] Avoid report PCIe peer devices with CPU XGMI iolinks Devices with CPU XGMI iolink do not support PCIe peer access. Therefore, they should not be reported as accessible links in the topology. Signed-off-by: Alex Sierra Change-Id: I3ee51796945dc0966200dee03886510e8f1846b7 --- src/rocm_smi.cc | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index 6c71343f8b..c2cbb238c1 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -3490,7 +3490,7 @@ rsmi_topo_get_link_weight(uint32_t dv_ind_src, uint32_t dv_ind_dst, assert(false); // Unexpected IO Link type read status = RSMI_STATUS_NOT_SUPPORTED; } - } else { + } else if (kfd_node->numa_node_type() == amd::smi::IOLINK_TYPE_PCIEXPRESS) { *weight = kfd_node->numa_node_weight(); // from src GPU to it's CPU node uint64_t numa_weight_dst = 0; status = topo_get_numa_node_weight(dv_ind_dst, &numa_weight_dst); @@ -3522,6 +3522,8 @@ rsmi_topo_get_link_weight(uint32_t dv_ind_src, uint32_t dv_ind_dst, assert(false); // Error to read numa node weight status = RSMI_STATUS_INIT_ERROR; } + } else { + status = RSMI_STATUS_NOT_SUPPORTED; } } else { status = RSMI_STATUS_INVALID_ARGS; @@ -3597,7 +3599,7 @@ rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst, if (ret == 0) { amd::smi::IO_LINK_TYPE io_link_type; ret = kfd_node->get_io_link_type(node_ind_dst, &io_link_type); - if (!ret) { + if (ret == 0) { if (io_link_type == amd::smi::IOLINK_TYPE_XGMI) { *type = RSMI_IOLINK_TYPE_XGMI; *hops = 1; @@ -3606,7 +3608,7 @@ rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst, assert(false); // Unexpected IO Link type read status = RSMI_STATUS_NOT_SUPPORTED; } - } else { + } else if (kfd_node->numa_node_type() == amd::smi::IOLINK_TYPE_PCIEXPRESS) { uint32_t numa_number_dst; status = topo_get_numa_node_number(dv_ind_dst, &numa_number_dst); if (status == RSMI_STATUS_SUCCESS) { @@ -3622,21 +3624,14 @@ rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst, else *hops = 4; // More than one CPU hops, hard coded as 4 } - - amd::smi::IO_LINK_TYPE numa_node_type = kfd_node->numa_node_type(); - if (numa_node_type == amd::smi::IOLINK_TYPE_PCIEXPRESS) { - *type = RSMI_IOLINK_TYPE_PCIEXPRESS; - status = RSMI_STATUS_SUCCESS; - } else if (numa_node_type == amd::smi::IOLINK_TYPE_XGMI) { - *type = RSMI_IOLINK_TYPE_XGMI; - status = RSMI_STATUS_SUCCESS; - } else { - status = RSMI_STATUS_INIT_ERROR; - } + *type = RSMI_IOLINK_TYPE_PCIEXPRESS; + status = RSMI_STATUS_SUCCESS; } else { assert(false); // Error to get numa node number status = RSMI_STATUS_INIT_ERROR; } + } else { + status = RSMI_STATUS_NOT_SUPPORTED; } } else { status = RSMI_STATUS_INVALID_ARGS; From 03fab6b2b63a74e55659ed3a92d987c68a9d4bd2 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Fri, 26 Aug 2022 20:51:24 +0000 Subject: [PATCH 2/2] Consider invalid peer link type during topology report Invalid peer links are labeled as N/A during topology report creation. This invalid link type could be triggered by having a configuration with CPU XGMI iolinks and disable XGMI peer to peer access. This can be done by setting the driver parameter 'use_xgmi_p2p = 0'. Signed-off-by: Alex Sierra Change-Id: Ifb09a8f3266a3f07686615dfb45781d6cfe55e83 --- python_smi_tools/rocm_smi.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index bd9fc76edf..73016599e0 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -2425,6 +2425,8 @@ def showWeightTopology(deviceList): gpu_links_weight[srcdevice][destdevice] = weight else: printErrLog(srcdevice, 'Cannot read Link Weight: Not supported on this machine') + gpu_links_weight[srcdevice][destdevice] = None + if PRINT_JSON: formatMatrixToJSON(deviceList, gpu_links_weight, "(Topology) Weight between DRM devices {} and {}") @@ -2441,6 +2443,8 @@ def showWeightTopology(deviceList): for gpu2 in deviceList: if (gpu1 == gpu2): printTableRow('%-12s', '0') + elif (gpu_links_weight[gpu1][gpu2] == None): + printTableRow('%-12s', 'N/A') else: printTableRow('%-12s', gpu_links_weight[gpu1][gpu2].value) printEmptyLine() @@ -2468,6 +2472,7 @@ def showHopsTopology(deviceList): gpu_links_hops[srcdevice][destdevice] = hops else: printErrLog(srcdevice, 'Cannot read Link Hops: Not supported on this machine') + gpu_links_hops[srcdevice][destdevice] = None if PRINT_JSON: formatMatrixToJSON(deviceList, gpu_links_hops, "(Topology) Hops between DRM devices {} and {}") @@ -2484,6 +2489,8 @@ def showHopsTopology(deviceList): for gpu2 in deviceList: if (gpu1 == gpu2): printTableRow('%-12s', '0') + elif (gpu_links_hops[gpu1][gpu2] == None): + printTableRow('%-12s', 'N/A') else: printTableRow('%-12s', gpu_links_hops[gpu1][gpu2].value) printEmptyLine() @@ -2516,6 +2523,8 @@ def showTypeTopology(deviceList): gpu_links_type[srcdevice][destdevice] = "XXXX" else: printErrLog(srcdevice, 'Cannot read Link Type: Not supported on this machine') + gpu_links_type[srcdevice][destdevice] = "XXXX" + if PRINT_JSON: formatMatrixToJSON(deviceList, gpu_links_type, "(Topology) Link type between DRM devices {} and {}") return