diff --git a/CMakeLists.txt b/CMakeLists.txt index bb283f1e27..bd8f32e447 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,7 +91,7 @@ set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${BUILD_VERSION_MAJOR}" ) target_link_libraries ( ${HSAKMT_TARGET} - pthread rt pci + pthread rt pci numa ) ## If the library is a release, strip the target library diff --git a/src/fmm.c b/src/fmm.c index cc5c4cb1a6..97ece30411 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -34,6 +34,13 @@ #include #include #include +#include +#ifndef MPOL_F_STATIC_NODES +/* Bug in numaif.h, this should be defined in there. Definition copied + * from linux/mempolicy.h. + */ +#define MPOL_F_STATIC_NODES (1 << 15) +#endif #define NON_VALID_GPU_ID 0 @@ -1170,6 +1177,10 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, * memory is allocated from KFD */ if (!flags.ui32.NonPaged && svm.userptr_for_paged_mem) { + const unsigned int bits_per_long = sizeof(unsigned long) * 8; + unsigned long node_mask[node_id / bits_per_long + 1]; + int mode = MPOL_F_STATIC_NODES; + /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, size, 0); @@ -1177,6 +1188,14 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, if (!mem) return NULL; + /* Bind to NUMA node */ + memset(node_mask, 0, sizeof(node_mask)); + node_mask[node_id / bits_per_long] = 1UL << (node_id % bits_per_long); + mode |= flags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; + if (mbind(mem, MemorySizeInBytes, mode, node_mask, node_id+1, 0)) + pr_warn("Failed to set NUMA policy for %lu pages at %p\n", + MemorySizeInBytes >> 12, mem); + /* Map anonymous pages */ if (mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0) @@ -1286,6 +1305,9 @@ static void __fmm_release(void *address, manageable_aperture_t *aperture) if (address >= dgpu_shared_aperture_base && address <= dgpu_shared_aperture_limit) { + /* Reset NUMA policy */ + mbind(address, object->size, MPOL_DEFAULT, NULL, 0, 0); + /* Remove any CPU mapping, but keep the address range reserved */ mmap_ret = mmap(address, object->size, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED,