AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[SWDEV-463402] - Support retrieving connection type and P2P capabilities between two GPUs

Browse Source 
1. Add a API interface amdsmi_topo_get_p2p_status to retrieve
connection type and P2P capabilities between 2 GPUs.

2. Add getting p2p status test in hw_topology_read
to print P2P capability information.

3. Add below tables for cli topology sub commands:
  - CACHE COHERANCY TABLE
  - ATOMICS TABLE
  - DMA TABLE
  - BI-DIRECTIONAL TABLE

Change-Id: I199173030d4170115cea27c472958a4826e4e1bf
Signed-off-by: Tim Huang <tim.huang@amd.com>


[ROCm/amdsmi commit: 260edaa752]
This commit is contained in:
Tim Huang
2024-08-21 11:26:36 +08:00
committed by Charis Poag
parent 104bf22265
commit 202ddc01aa
13 changed files with 842 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/amdsmi/tests/amd_smi_test/functional/hw_topology_read.cc
+154 -1
View File
@@ -60,6 +60,7 @@ typedef struct {
uint64_t hops;
uint64_t weight;
bool accessible;
amdsmi_p2p_capability_t cap;
} gpu_link_t;
TestHWTopologyRead::TestHWTopologyRead() : TestBase() {
@@ -136,9 +137,11 @@ void TestHWTopologyRead::Run(void) {
gpu_links[dv_ind_src][dv_ind_dst].hops = 0;
gpu_links[dv_ind_src][dv_ind_dst].weight = 0;
gpu_links[dv_ind_src][dv_ind_dst].accessible = true;
gpu_links[dv_ind_src][dv_ind_dst].cap =
{UINT8_MAX, UINT8_MAX, UINT8_MAX, UINT8_MAX, UINT8_MAX};
} else {
amdsmi_io_link_type_t type;
err = amdsmi_topo_get_link_type(processor_handles_[dv_ind_src],
err = amdsmi_topo_get_link_type(processor_handles_[dv_ind_src],
processor_handles_[dv_ind_dst],
&gpu_links[dv_ind_src][dv_ind_dst].hops, &type);
if (err != AMDSMI_STATUS_SUCCESS) {
@@ -170,6 +173,34 @@ void TestHWTopologyRead::Run(void) {
}
}
}
err = amdsmi_topo_get_p2p_status(processor_handles_[dv_ind_src],
processor_handles_[dv_ind_dst],
&type, &gpu_links[dv_ind_src][dv_ind_dst].cap);
if (err != AMDSMI_STATUS_SUCCESS) {
if (err == AMDSMI_STATUS_NOT_SUPPORTED) {
IF_VERB(STANDARD) {
std::cout <<
"\t**Link Type. read: Not supported on this machine"
<< std::endl;
return;
}
} else {
CHK_ERR_ASRT(err)
}
} else {
switch (type) {
case AMDSMI_IOLINK_TYPE_PCIEXPRESS:
case AMDSMI_IOLINK_TYPE_XGMI:
// Do nothing, the type is printed by the previous test for amdsmi_topo_get_link_type
break;
default:
gpu_links[dv_ind_src][dv_ind_dst].type = "XXXX";
IF_VERB(STANDARD) {
std::cout << "\t**Invalid IO LINK type. type=" << type <<
std::endl;
}
}
}
err = amdsmi_topo_get_link_weight(processor_handles_[dv_ind_src],
processor_handles_[dv_ind_dst],
&gpu_links[dv_ind_src][dv_ind_dst].weight);
@@ -286,6 +317,7 @@ void TestHWTopologyRead::Run(void) {
std::cout << std::endl;
}
std::cout << std::endl;
std::cout << "**Access between two GPUs**" << std::endl;
std::cout << " ";
for (i = 0; i < num_devices; ++i) {
@@ -303,4 +335,125 @@ void TestHWTopologyRead::Run(void) {
std::cout << std::endl;
}
std::cout << std::endl;
std::cout << "**Cache coherency between two GPUs**" << std::endl;
std::cout << " ";
for (i = 0; i < num_devices; ++i) {
tmp = "GPU" + std::to_string(i);
std::cout << std::setw(12) << std::left << tmp;
}
std::cout << std::endl;
for (i = 0; i < num_devices; i++) {
tmp = "GPU" + std::to_string(i);
std::cout << std::setw(6) << std::left << tmp;
for (j = 0; j < num_devices; j++) {
if (i == j) {
std::cout << std::setw(12) << std::left << "X";
continue;
}
if (gpu_links[i][j].cap.is_iolink_coherent == UINT8_MAX) {
std::cout << std::setw(12) << std::left << "N/A";
continue;
}
std::cout << std::setw(12) << std::left
<< (gpu_links[i][j].cap.is_iolink_coherent ? "C" : "NC");
}
std::cout << std::endl;
}
std::cout << std::endl;
std::cout << "**Atomics between two GPUs**" << std::endl;
std::cout << " ";
for (i = 0; i < num_devices; ++i) {
tmp = "GPU" + std::to_string(i);
std::cout << std::setw(12) << std::left << tmp;
}
std::cout << std::endl;
for (i = 0; i < num_devices; i++) {
tmp = "GPU" + std::to_string(i);
std::cout << std::setw(6) << std::left << tmp;
for (j = 0; j < num_devices; j++) {
if (i == j) {
std::cout << std::setw(12) << std::left << "X";
continue;
}
if (gpu_links[i][j].cap.is_iolink_atomics_64bit == UINT8_MAX ||
gpu_links[i][j].cap.is_iolink_atomics_32bit == UINT8_MAX) {
std::cout << std::setw(12) << std::left << "N/A";
continue;
}
tmp = gpu_links[i][j].cap.is_iolink_atomics_64bit ? "64" : "";
if (gpu_links[i][j].cap.is_iolink_atomics_32bit) {
if (!tmp.empty()) {
tmp += ",";
}
tmp += "32";
}
std::cout << std::setw(12) << std::left << (tmp.empty() ? "N/A" : tmp);
}
std::cout << std::endl;
}
std::cout << std::endl;
std::cout << "**DMA between two GPUs**" << std::endl;
std::cout << " ";
for (i = 0; i < num_devices; ++i) {
tmp = "GPU" + std::to_string(i);
std::cout << std::setw(12) << std::left << tmp;
}
std::cout << std::endl;
for (i = 0; i < num_devices; i++) {
tmp = "GPU" + std::to_string(i);
std::cout << std::setw(6) << std::left << tmp;
for (j = 0; j < num_devices; j++) {
if (i == j) {
std::cout << std::setw(12) << std::left << "X";
continue;
}
if (gpu_links[i][j].cap.is_iolink_dma == UINT8_MAX) {
std::cout << std::setw(12) << std::left << "N/A";
continue;
}
std::cout << std::boolalpha;
std::cout << std::setw(12) << std::left
<< static_cast<bool>(gpu_links[i][j].cap.is_iolink_dma);
}
std::cout << std::endl;
}
std::cout << std::endl;
std::cout << "**BI-Directional between two GPUs**" << std::endl;
std::cout << " ";
for (i = 0; i < num_devices; ++i) {
tmp = "GPU" + std::to_string(i);
std::cout << std::setw(12) << std::left << tmp;
}
std::cout << std::endl;
for (i = 0; i < num_devices; i++) {
tmp = "GPU" + std::to_string(i);
std::cout << std::setw(6) << std::left << tmp;
for (j = 0; j < num_devices; j++) {
if (i == j) {
std::cout << std::setw(12) << std::left << "X";
continue;
}
if (gpu_links[i][j].cap.is_iolink_dma == UINT8_MAX) {
std::cout << std::setw(12) << std::left << "N/A";
continue;
}
std::cout << std::boolalpha;
std::cout << std::setw(12) << std::left
<< static_cast<bool>(gpu_links[i][j].cap.is_iolink_bi_directional);
}
std::cout << std::endl;
}
std::cout << std::endl;
}