From bdb519bd35162f8cb34a0329565fe5664695c3dc Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 6 Nov 2019 14:32:25 -0500 Subject: [PATCH] libhsakmt: use the closest NUMA node to allocate queue ctx area On NUMA system, allocate queue ctx save restore area on the closest NUMA node to the GPU which the queue is going to run. This will improve performance on NUMA system generally by reducing schedule latency and fix the multi-node rccl-tests unstable performance issue. If the closest NUMA node has no memory available, set flags NoNUMABind=1 to bypass mbind, to use default NUMA memory policy to allocate system memory. Change-Id: Ic62bfa5bb2efbf4f6ae79ff403e9610ddf18d45c Signed-off-by: Philip Yang [ROCm/ROCR-Runtime commit: 59c857476fa39447e176d32c6f19c66a446795d2] --- projects/rocr-runtime/src/fmm.c | 3 +++ projects/rocr-runtime/src/libhsakmt.h | 1 + projects/rocr-runtime/src/queues.c | 15 +++++++++++++-- projects/rocr-runtime/src/topology.c | 19 +++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/projects/rocr-runtime/src/fmm.c b/projects/rocr-runtime/src/fmm.c index fd69522aa0..37e543a099 100644 --- a/projects/rocr-runtime/src/fmm.c +++ b/projects/rocr-runtime/src/fmm.c @@ -1462,6 +1462,9 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, int num_node; long r; + pr_debug("%s mem %p flags 0x%x size 0x%lx node_id %d\n", __func__, + mem, flags.Value, SizeInBytes, node_id); + if (flags.ui32.NoNUMABind) return 0; diff --git a/projects/rocr-runtime/src/libhsakmt.h b/projects/rocr-runtime/src/libhsakmt.h index d0e95ae170..40f5e97762 100644 --- a/projects/rocr-runtime/src/libhsakmt.h +++ b/projects/rocr-runtime/src/libhsakmt.h @@ -147,6 +147,7 @@ bool prefer_ats(HSAuint32 node_id); uint16_t get_device_id_by_node_id(HSAuint32 node_id); bool is_kaveri(HSAuint32 node_id); uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id); +uint32_t get_direct_link_cpu(uint32_t gpu_node); int get_drm_render_fd_by_gpu_id(HSAuint32 gpu_id); HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray); diff --git a/projects/rocr-runtime/src/queues.c b/projects/rocr-runtime/src/queues.c index 8ee165cf0c..7d0e67d0a1 100644 --- a/projects/rocr-runtime/src/queues.c +++ b/projects/rocr-runtime/src/queues.c @@ -424,6 +424,7 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, HSAuint64 gpu_va; HsaMemFlags flags; HSAKMT_STATUS ret; + HSAuint32 cpu_id = 0; flags.Value = 0; flags.ui32.HostAccess = !DeviceLocal; @@ -431,11 +432,21 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, flags.ui32.NonPaged = nonPaged; flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; flags.ui32.CoarseGrain = DeviceLocal; - flags.ui32.NoNUMABind = 1; + + /* Get the closest cpu_id to GPU NodeId for system memory allocation + * nonPaged=1 system memory allocation uses GTT path + */ + if (!DeviceLocal && !nonPaged) { + cpu_id = get_direct_link_cpu(NodeId); + if (cpu_id == INVALID_NODEID) { + flags.ui32.NoNUMABind = 1; + cpu_id = 0; + } + } size = ALIGN_UP(size, align); - ret = hsaKmtAllocMemory(DeviceLocal ? NodeId : 0, size, flags, &mem); + ret = hsaKmtAllocMemory(DeviceLocal ? NodeId : cpu_id, size, flags, &mem); if (ret != HSAKMT_STATUS_SUCCESS) return NULL; diff --git a/projects/rocr-runtime/src/topology.c b/projects/rocr-runtime/src/topology.c index b73753ff6e..742252d5df 100644 --- a/projects/rocr-runtime/src/topology.c +++ b/projects/rocr-runtime/src/topology.c @@ -2224,6 +2224,25 @@ uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) return 0; } +uint32_t get_direct_link_cpu(uint32_t gpu_node) +{ + HSAuint64 size = 0; + int32_t cpu_id; + HSAuint32 i; + + cpu_id = gpu_get_direct_link_cpu(gpu_node, g_props); + if (cpu_id == -1) + return INVALID_NODEID; + + assert(g_props[cpu_id].mem); + + for (i = 0; i < g_props[cpu_id].node.NumMemoryBanks; i++) + size += g_props[cpu_id].mem[i].SizeInBytes; + + return size ? (uint32_t)cpu_id : INVALID_NODEID; +} + + HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray) {