AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add QPI or HT io_links

Browse Source 
Create QPI or HT links among all NUMA nodes. For now, assume all the
NUMA nodes are interconnected with same Weight (=1).

Change-Id: Id48ba95b9d75515a186f7dc5006b19bd92743ae3


[ROCm/ROCR-Runtime commit: f1fbacca15]
This commit is contained in:
Harish Kasiviswanathan
2016-03-11 18:54:46 -05:00
committed by Andres Rodriguez
parent 14e60b6ab3
commit 718e3600b8
1 changed files with 53 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocr-runtime/src/topology.c
+53 -3
View File
@@ -908,6 +908,20 @@ err1:
return ret;
}
/* topology_get_numa_node_link_tye - Return NUMA node interconnect based
* on processor vendor
*/
static HSA_IOLINKTYPE topology_get_numa_node_link_tye(void)
{
if (processor_vendor == GENUINE_INTEL)
return HSA_IOLINK_TYPE_QPI_1_1;
else if (processor_vendor == AUTHENTIC_AMD)
return HSA_IOLINKTYPE_HYPERTRANSPORT;
else
return HSA_IOLINKTYPE_UNDEFINED;
}
/* topology_get_free_io_link_slot_for_node - For the given node_id, find the next
* available free slot to add an io_link
*/
@@ -959,6 +973,39 @@ static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_id,
return HSAKMT_STATUS_SUCCESS;
}
/* topology_create_qpi_links - Create QPI or HT links among all NUMA nodes
* For now, assume all the nodes are interconnected with same Weight (=1)
*/
static void topology_create_qpi_links(const HsaSystemProperties *sys_props,
node_t *temp_nodes)
{
unsigned int i, j;
HSAKMT_STATUS ret;
/* Find all CPU Nodes and connect each other via HT or QPI io_link */
for (i = 0; i < sys_props->NumNodes - 1; i++) {
for (j = i + 1; j < sys_props->NumNodes; j++) {
if (temp_nodes[i].gpu_id == 0 &&
temp_nodes[j].gpu_id == 0) {
ret = topology_add_io_link_for_node(i,
sys_props, temp_nodes, topology_get_numa_node_link_tye(),
j, 1);
if (ret != HSAKMT_STATUS_SUCCESS)
printf("Error [%d]. Failed to add QPI link from Node [%d]->[%d]\n",
ret, i, j);
ret = topology_add_io_link_for_node(j,
sys_props, temp_nodes, topology_get_numa_node_link_tye(),
i, 1);
if (ret != HSAKMT_STATUS_SUCCESS)
printf("Error [%d]. Failed to add QPI link from Node [%d]->[%d]\n",
ret, j, i);
}
}
}
}
/* topology_create_reverse_io_link - Create io_links from the given CPU
* NUMA node to all the GPUs attached to that node
*/
@@ -1024,10 +1071,10 @@ static void topology_create_indirect_gpu_links(uint32_t cpu_node,
continue;
/* The link is from GPU to non-parent NUMA node. So set link type
* to HT */
* to HT or QPI */
if (temp_nodes[props[i].NodeTo].gpu_id == 0 ||
temp_nodes[props[j].NodeTo].gpu_id == 0)
IoLinkType = HSA_IOLINKTYPE_HYPERTRANSPORT;
IoLinkType = topology_get_numa_node_link_tye();
else
IoLinkType = HSA_IOLINKTYPE_PCIEXPRESS;
@@ -1159,6 +1206,10 @@ retry:
}
}
/* Create QPI or HT links among CPU (NUMA) nodes. For now assume
* all nodes are interconnected with same weight */
topology_create_qpi_links(&sys_props, temp_nodes);
/* Create In-direct links for GPUs. Connect all the (Peer-to-Peer) GPUs
* that belong to same NUMA node.
* For each CPU (NUMA) node, interconnect all the GPUs. */
@@ -1168,7 +1219,6 @@ retry:
}
}
ret = topology_sysfs_get_generation(&gen_end);
if (ret != HSAKMT_STATUS_SUCCESS) {
free_nodes(temp_nodes, sys_props.NumNodes);