From 9b451c41e8cfd73a0d9c449989b9338af90c8d39 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 9 Jul 2019 14:54:49 -0400 Subject: [PATCH] fix mbind on NUMA system mbind walks through pages to setup vma memory policy. So we need do mmap to create vma mappings first, then call mbind. mbind will do nothing if vma does not exist. And add numa available check before executing mbind, and return NULL to hsaKmtAllocMemory if mbind failed. Change-Id: I28ab661885d807ca51ef90e87230669dc80f10ec Signed-off-by: Philip Yang [ROCm/ROCR-Runtime commit: 67f366243d8f47adcc1c7fd038a59ec533c3e4ca] --- projects/rocr-runtime/src/fmm.c | 76 ++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/projects/rocr-runtime/src/fmm.c b/projects/rocr-runtime/src/fmm.c index aaad772101..4aced94956 100644 --- a/projects/rocr-runtime/src/fmm.c +++ b/projects/rocr-runtime/src/fmm.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include "rbtree.h" #ifndef MPOL_F_STATIC_NODES @@ -1456,6 +1457,39 @@ static void *fmm_allocate_host_cpu(void *address, uint64_t MemorySizeInBytes, return mem; } +static int bind_mem_to_numa(uint32_t node_id, void *mem, + uint64_t MemorySizeInBytes, HsaMemFlags flags) +{ + int mode = MPOL_F_STATIC_NODES; + struct bitmask *node_mask; + int num_node; + + if (numa_available() == -1) + return 0; + + num_node = numa_num_task_nodes(); + + if (num_node > 1) { + node_mask = numa_bitmask_alloc(num_node); + if (!node_mask) + return -ENOMEM; + + numa_bitmask_setbit(node_mask, node_id); + mode |= flags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; + if (mbind(mem, MemorySizeInBytes, mode, node_mask->maskp, + num_node + 1, 0)) { + pr_warn("Failed to set NUMA policy for %p\n", mem); + + numa_bitmask_free(node_mask); + return -EFAULT; + } + + numa_bitmask_free(node_mask); + } + + return 0; +} + static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags flags) { @@ -1492,10 +1526,6 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, * memory is allocated from KFD */ if (!flags.ui32.NonPaged && svm.userptr_for_paged_mem) { - const unsigned int bits_per_long = sizeof(unsigned long) * 8; - unsigned long node_mask[node_id / bits_per_long + 1]; - int mode = MPOL_F_STATIC_NODES; - /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, address, size); @@ -1503,24 +1533,15 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, if (!mem) return NULL; - /* Bind to NUMA node */ - memset(node_mask, 0, sizeof(node_mask)); - node_mask[node_id / bits_per_long] = 1UL << (node_id % bits_per_long); - mode |= flags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; - if (mbind(mem, MemorySizeInBytes, mode, node_mask, node_id+1, 0)) - pr_warn("Failed to set NUMA policy for %lu pages at %p\n", - MemorySizeInBytes >> 12, mem); - /* Map anonymous pages */ if (mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0) - == MAP_FAILED) { - /* Release address space */ - pthread_mutex_lock(&aperture->fmm_mutex); - aperture_release_area(aperture, mem, size); - pthread_mutex_unlock(&aperture->fmm_mutex); - return NULL; - } + == MAP_FAILED) + goto out_release_area; + + /* Bind to NUMA node */ + if (bind_mem_to_numa(node_id, mem, MemorySizeInBytes, flags)) + goto out_release_area; /* Mappings in the DGPU aperture don't need to be copied on * fork. This avoids MMU notifiers and evictions due to user @@ -1534,13 +1555,8 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, vm_obj = fmm_allocate_memory_object(gpu_id, mem, size, aperture, &mmap_offset, ioc_flags); - if (!vm_obj) { - /* Release address space */ - pthread_mutex_lock(&aperture->fmm_mutex); - aperture_release_area(aperture, mem, size); - pthread_mutex_unlock(&aperture->fmm_mutex); - return NULL; - } + if (!vm_obj) + goto out_release_area; } else { ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_GTT; mem = __fmm_allocate_device(gpu_id, address, size, aperture, @@ -1576,6 +1592,14 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, } return mem; + +out_release_area: + /* Release address space */ + pthread_mutex_lock(&aperture->fmm_mutex); + aperture_release_area(aperture, mem, size); + pthread_mutex_unlock(&aperture->fmm_mutex); + + return NULL; } void *fmm_allocate_host(uint32_t node_id, void *address,