libhsakmt: use the closest NUMA node to allocate queue ctx area

On NUMA system, allocate queue ctx save restore area on the closest NUMA
node to the GPU which the queue is going to run. This will improve
performance on NUMA system generally by reducing schedule latency and
fix the multi-node rccl-tests unstable performance issue.

If the closest NUMA node has no memory available, set flags NoNUMABind=1
to bypass mbind, to use default NUMA memory policy to allocate system
memory.



Change-Id: Ic62bfa5bb2efbf4f6ae79ff403e9610ddf18d45c
Signed-off-by: Philip Yang <Philip.Yang@amd.com>


[ROCm/ROCR-Runtime commit: 59c857476f]

projects/rocr-runtime/src/fmm.c

@@ -1462,6 +1462,9 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem,
 	int num_node;
 	long r;
 
+	pr_debug("%s mem %p flags 0x%x size 0x%lx node_id %d\n", __func__,
+		mem, flags.Value, SizeInBytes, node_id);
+
 	if (flags.ui32.NoNUMABind)
 		return 0;
 

projects/rocr-runtime/src/libhsakmt.h

@@ -147,6 +147,7 @@ bool prefer_ats(HSAuint32 node_id);
 uint16_t get_device_id_by_node_id(HSAuint32 node_id);
 bool is_kaveri(HSAuint32 node_id);
 uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id);
+uint32_t get_direct_link_cpu(uint32_t gpu_node);
 int get_drm_render_fd_by_gpu_id(HSAuint32 gpu_id);
 HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array,
 		uint32_t NumberOfNodes, uint32_t *NodeArray);

projects/rocr-runtime/src/queues.c

@@ -424,6 +424,7 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align,
 	HSAuint64 gpu_va;
 	HsaMemFlags flags;
 	HSAKMT_STATUS ret;
+	HSAuint32 cpu_id = 0;
 
 	flags.Value = 0;
 	flags.ui32.HostAccess = !DeviceLocal;
@@ -431,11 +432,21 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align,
 	flags.ui32.NonPaged = nonPaged;
 	flags.ui32.PageSize = HSA_PAGE_SIZE_4KB;
 	flags.ui32.CoarseGrain = DeviceLocal;
-	flags.ui32.NoNUMABind = 1;
+
+	/* Get the closest cpu_id to GPU NodeId for system memory allocation
+	 * nonPaged=1 system memory allocation uses GTT path
+	 */
+	if (!DeviceLocal && !nonPaged) {
+		cpu_id = get_direct_link_cpu(NodeId);
+		if (cpu_id == INVALID_NODEID) {
+			flags.ui32.NoNUMABind = 1;
+			cpu_id = 0;
+		}
+	}
 
 	size = ALIGN_UP(size, align);
 
-	ret = hsaKmtAllocMemory(DeviceLocal ? NodeId : 0, size, flags, &mem);
+	ret = hsaKmtAllocMemory(DeviceLocal ? NodeId : cpu_id, size, flags, &mem);
 	if (ret != HSAKMT_STATUS_SUCCESS)
 		return NULL;
 

projects/rocr-runtime/src/topology.c

@@ -2224,6 +2224,25 @@ uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id)
 	return 0;
 }
 
+uint32_t get_direct_link_cpu(uint32_t gpu_node)
+{
+	HSAuint64 size = 0;
+	int32_t cpu_id;
+	HSAuint32 i;
+
+	cpu_id = gpu_get_direct_link_cpu(gpu_node, g_props);
+	if (cpu_id == -1)
+		return INVALID_NODEID;
+
+	assert(g_props[cpu_id].mem);
+
+	for (i = 0; i < g_props[cpu_id].node.NumMemoryBanks; i++)
+		size += g_props[cpu_id].mem[i].SizeInBytes;
+
+	return size ? (uint32_t)cpu_id : INVALID_NODEID;
+}
+
+
 HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array,
 		uint32_t NumberOfNodes, uint32_t *NodeArray)
 {