From c89d3124d96584b3fc50f9039cf12fd27e850723 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 7 Jan 2016 16:28:20 -0500 Subject: [PATCH] Implement hsaKmtRegisterMemoryToNodes Fix hsaKmtRegisterMemory to be a no-op for now and move the multi-GPU implementation to hsaKmtRegisterMemoryToNodes. Make GPU memory mappings of host memory visible to all GPUs by default. Device memory is still visible to the allocating GPU only by default (but can be overridden with hsaKmtRegisterMemoryToNodes for experimenting with P2P). Change-Id: I73408afbe3b10c8dad2ab3a780f58413249692e6 [ROCm/ROCR-Runtime commit: 063ad3ad9e120c49f03918cc82f1da6a1493ac7b] --- projects/rocr-runtime/include/hsakmt.h | 16 ++- projects/rocr-runtime/src/fmm.c | 142 +++++++++++++----------- projects/rocr-runtime/src/fmm.h | 8 +- projects/rocr-runtime/src/libhsakmt.ver | 1 + projects/rocr-runtime/src/memory.c | 54 ++++++--- projects/rocr-runtime/src/queues.c | 8 +- 6 files changed, 137 insertions(+), 92 deletions(-) diff --git a/projects/rocr-runtime/include/hsakmt.h b/projects/rocr-runtime/include/hsakmt.h index bdb8f2bdd3..5fadf78577 100644 --- a/projects/rocr-runtime/include/hsakmt.h +++ b/projects/rocr-runtime/include/hsakmt.h @@ -369,7 +369,6 @@ hsaKmtFreeMemory( /** Registers with KFD a memory buffer that may be accessed by the GPU - This function will never be required for Linux */ HSAKMT_STATUS @@ -380,9 +379,22 @@ hsaKmtRegisterMemory( ); +/** + Registers with KFD a memory buffer that may be accessed by specific GPUs +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterMemoryToNodes( + void *MemoryAddress, // IN (page-aligned) + HSAuint64 MemorySizeInBytes, // IN (page-aligned) + HSAuint64 NumberOfNodes, // IN + HSAuint32* NodeArray // IN + ); + + /** Unregisters with KFD a memory buffer - This function will never be required for Linux */ HSAKMT_STATUS diff --git a/projects/rocr-runtime/src/fmm.c b/projects/rocr-runtime/src/fmm.c index 2e593de2a2..c8cb4c62bd 100644 --- a/projects/rocr-runtime/src/fmm.c +++ b/projects/rocr-runtime/src/fmm.c @@ -66,6 +66,7 @@ struct vm_object { uint64_t handle; /* opaque */ struct vm_object *next; struct vm_object *prev; + uint32_t flags; /* memory allocation flags */ /* * Nodes to map on SVM mGPU */ @@ -137,6 +138,10 @@ static svm_t svm = { INIT_MANAGEBLE_APERTURE(0, 0) }; +/* GPU node array for default mappings */ +static uint32_t all_gpu_id_array_size = 0; +static uint32_t *all_gpu_id_array = NULL; + extern int debug_get_reg_status(uint32_t node_id, bool* is_debugged); static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); @@ -161,7 +166,7 @@ static vm_area_t *vm_create_and_init_area(void *start, void *end) } static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, - uint64_t handle) + uint64_t handle, uint32_t flags) { vm_object_t *object = (vm_object_t *) malloc(sizeof(vm_object_t)); @@ -171,6 +176,7 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->handle = handle; object->next = object->prev = NULL; object->device_ids_array_size = 0; + object->flags = flags; } return object; @@ -399,7 +405,8 @@ static void *aperture_allocate_area(manageble_aperture_t *app, static int aperture_allocate_object(manageble_aperture_t *app, void *new_address, uint64_t handle, - uint64_t MemorySizeInBytes) + uint64_t MemorySizeInBytes, + uint32_t flags) { vm_object_t *new_object; @@ -407,8 +414,8 @@ static int aperture_allocate_object(manageble_aperture_t *app, /* Allocate new object */ new_object = vm_create_and_init_object(new_address, - MemorySizeInBytes, - handle); + MemorySizeInBytes, + handle, flags); if (!new_object) return -1; @@ -460,7 +467,7 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, /* Allocate object */ pthread_mutex_lock(&aperture->fmm_mutex); if (aperture_allocate_object(aperture, mem, args.handle, - MemorySizeInBytes)) + MemorySizeInBytes, flags)) goto err_object_allocation_failed; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -848,7 +855,7 @@ void *fmm_open_graphic_handle(uint32_t gpu_id, /* Allocate object */ if (aperture_allocate_object(&gpu_mem[i].gpuvm_aperture, mem, open_graphic_handle_args.handle, - MemorySizeInBytes)) + MemorySizeInBytes, 0)) goto release_mem; pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); @@ -888,6 +895,9 @@ static void __fmm_release(void *address, return; } + if (object->device_ids_array_size > 0) + free(object->device_ids_array); + args.handle = object->handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); @@ -1003,6 +1013,13 @@ HSAKMT_STATUS fmm_init_process_apertures(void) if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void *) &args)) return HSAKMT_STATUS_ERROR; + all_gpu_id_array_size = 0; + if (args.num_of_nodes > 0) { + all_gpu_id_array = malloc(sizeof(uint32_t) * args.num_of_nodes); + if (all_gpu_id_array == NULL) + return HSAKMT_STATUS_NO_MEMORY; + } + for (i = 0 ; i < args.num_of_nodes ; i++) { /* Map Kernel process device data node i <--> gpu_mem_id which indexes into gpu_mem[] * based on gpu_id */ @@ -1010,6 +1027,9 @@ HSAKMT_STATUS fmm_init_process_apertures(void) if (gpu_mem_id < 0) return HSAKMT_STATUS_ERROR; + all_gpu_id_array[i] = args.process_apertures[i].gpu_id; + all_gpu_id_array_size += sizeof(uint32_t); + gpu_mem[gpu_mem_id].lds_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[i].lds_base); @@ -1135,8 +1155,17 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, } args.handle = object->handle; - args.device_ids_array = object->device_ids_array; - args.device_ids_array_size = object->device_ids_array_size; + if (object->device_ids_array_size > 0) { + args.device_ids_array = object->device_ids_array; + args.device_ids_array_size = object->device_ids_array_size; + } else if (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) { + /* Only enable multi-GPU mapping on host memory for now */ + args.device_ids_array = all_gpu_id_array; + args.device_ids_array_size = all_gpu_id_array_size; + } else { + args.device_ids_array = NULL; + args.device_ids_array_size = 0; + } if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU_NEW, &args)) goto err_map_ioctl_failed; @@ -1315,8 +1344,17 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address) goto err; args.handle = object->handle; - args.device_ids_array = object->device_ids_array; - args.device_ids_array_size = object->device_ids_array_size; + if (object->device_ids_array_size > 0) { + args.device_ids_array = object->device_ids_array; + args.device_ids_array_size = object->device_ids_array_size; + } else if (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) { + /* Only enable multi-GPU mapping on host memory for now */ + args.device_ids_array = all_gpu_id_array; + args.device_ids_array_size = all_gpu_id_array_size; + } else { + args.device_ids_array = NULL; + args.device_ids_array_size = 0; + } kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &args); pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1607,14 +1645,22 @@ void fmm_release_global_resources(void) } dgpu_shared_aperture_base = NULL; dgpu_shared_aperture_limit = NULL; + + if (all_gpu_id_array != NULL) + free(all_gpu_id_array); + all_gpu_id_array = NULL; + all_gpu_id_array_size = 0; } -int fmm_register_memory(void *address, uint32_t size_in_bytes, - uint32_t *nodes_arr, uint32_t nodes_arr_size) +HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes, + uint32_t *gpu_id_array, + uint32_t gpu_id_array_size) { - bool found = false; manageble_aperture_t *aperture; - vm_object_t *object; + vm_object_t *object = NULL; + + if (gpu_id_array_size > 0 && gpu_id_array == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; /* * Object can be found only on SVM aperture as you can't map @@ -1622,37 +1668,33 @@ int fmm_register_memory(void *address, uint32_t size_in_bytes, */ aperture = &svm.dgpu_aperture; pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (object) - found = true; pthread_mutex_unlock(&aperture->fmm_mutex); - if (!found) { + if (!object) { aperture = &svm.dgpu_alt_aperture; - pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (object) - found = true; pthread_mutex_unlock(&aperture->fmm_mutex); } if (!object) - return 1; + return HSAKMT_STATUS_NOT_SUPPORTED; + if (object->device_ids_array_size > 0) + return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; - object->device_ids_array = nodes_arr; - object->device_ids_array_size = nodes_arr_size * sizeof(uint32_t); + if (gpu_id_array_size > 0) { + object->device_ids_array = gpu_id_array; + object->device_ids_array_size = gpu_id_array_size; + } - return 0; + return HSAKMT_STATUS_SUCCESS; } -void fmm_deregister_memory(void *address) +HSAKMT_STATUS fmm_deregister_memory(void *address) { - bool found = false; manageble_aperture_t *aperture; - vm_object_t *object; + vm_object_t *object = NULL; /* * Object can be found only on SVM aperture as you can't map @@ -1660,56 +1702,22 @@ void fmm_deregister_memory(void *address) */ aperture = &svm.dgpu_aperture; pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (object) - found = true; pthread_mutex_unlock(&aperture->fmm_mutex); - if (!found) { + if (!object) { aperture = &svm.dgpu_alt_aperture; pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (object) - found = true; pthread_mutex_unlock(&aperture->fmm_mutex); } if (!object || object->device_ids_array_size <= 0) - return; + return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; free(object->device_ids_array); object->device_ids_array = NULL; object->device_ids_array_size = 0; -} - -int fmm_build_nodes_array(uint32_t **array, uint32_t *nodes, uint32_t nodes_num) -{ - uint32_t i, *arr; - if (!nodes) { - nodes_num = 0; - for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS; i++) { - if (gpu_mem[i].gpu_id == 0) - continue; - nodes_num++; - } - } - - arr = (uint32_t *)malloc(sizeof(uint32_t) * nodes_num); - if (!array) - return 1; - - memset(arr, 0, sizeof(uint32_t) * nodes_num); - - nodes_num = 0; - for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS; i++) { - if (gpu_mem[i].gpu_id == 0) - continue; - arr[nodes_num] = gpu_mem[i].gpu_id; - nodes_num++; - } - - *array = arr; - return nodes_num; + + return HSAKMT_STATUS_SUCCESS; } diff --git a/projects/rocr-runtime/src/fmm.h b/projects/rocr-runtime/src/fmm.h index 37bdcb3b65..5bdbf5bbba 100644 --- a/projects/rocr-runtime/src/fmm.h +++ b/projects/rocr-runtime/src/fmm.h @@ -68,8 +68,8 @@ HSAKMT_STATUS fmm_node_removed(HSAuint32 gpu_id); HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id, HSAuint64 *aperture_base, HSAuint64 *aperture_limit); -int fmm_register_memory(void *address, uint32_t size_in_bytes, - uint32_t *nodes_arr, uint32_t nodes_arr_size); -void fmm_deregister_memory(void *address); -int fmm_build_nodes_array(uint32_t **array, uint32_t *nodes, uint32_t nodes_num); +HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes, + uint32_t *gpu_id_array, + uint32_t gpu_id_array_size); +HSAKMT_STATUS fmm_deregister_memory(void *address); #endif /* FMM_H_ */ diff --git a/projects/rocr-runtime/src/libhsakmt.ver b/projects/rocr-runtime/src/libhsakmt.ver index 3c8064d43c..9a8a14835b 100644 --- a/projects/rocr-runtime/src/libhsakmt.ver +++ b/projects/rocr-runtime/src/libhsakmt.ver @@ -25,6 +25,7 @@ hsaKmtSetMemoryPolicy; hsaKmtAllocMemory; hsaKmtFreeMemory; hsaKmtRegisterMemory; +hsaKmtRegisterMemoryToNodes; hsaKmtDeregisterMemory; hsaKmtMapMemoryToGPU; hsaKmtUnmapMemoryToGPU; diff --git a/projects/rocr-runtime/src/memory.c b/projects/rocr-runtime/src/memory.c index 1b043b60c2..c0161b36dc 100644 --- a/projects/rocr-runtime/src/memory.c +++ b/projects/rocr-runtime/src/memory.c @@ -195,22 +195,48 @@ hsaKmtRegisterMemory( HSAuint64 MemorySizeInBytes /* IN (page-aligned) */ ) { - uint32_t *NodesArray; - uint32_t NodesArraySize; - CHECK_KFD_OPEN(); - /* - * Build NodesArray from all dGPU nodes. - */ - NodesArraySize = fmm_build_nodes_array(&NodesArray, NULL, 0); - if (!NodesArray) + /* No-op for APU, TODO for dGPU */ + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterMemoryToNodes( + void *MemoryAddress, /* IN (page-aligned) */ + HSAuint64 MemorySizeInBytes, /* IN (page-aligned) */ + HSAuint64 NumberOfNodes, /* IN */ + HSAuint32* NodeArray /* IN */ +) +{ + CHECK_KFD_OPEN(); + uint32_t *gpu_id_array; + unsigned i; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + if (NumberOfNodes == 0 || NodeArray == NULL) return HSAKMT_STATUS_INVALID_PARAMETER; - if (fmm_register_memory(MemoryAddress, MemorySizeInBytes, - NodesArray, NodesArraySize) != 0) - return HSAKMT_STATUS_ERROR; - return HSAKMT_STATUS_SUCCESS; + /* Translate Node IDs to gpu_ids */ + gpu_id_array = malloc(NumberOfNodes * sizeof(uint32_t)); + if (gpu_id_array == NULL) + return HSAKMT_STATUS_NO_MEMORY; + for (i = 0; i < NumberOfNodes; i++) { + ret = validate_nodeid(NodeArray[i], &gpu_id_array[i]); + if (ret != HSAKMT_STATUS_SUCCESS) + break; + } + + if (ret == HSAKMT_STATUS_SUCCESS) + ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, + gpu_id_array, + NumberOfNodes*sizeof(uint32_t)); + + if (ret != HSAKMT_STATUS_SUCCESS) + free(gpu_id_array); + + return ret; } HSAKMT_STATUS @@ -221,9 +247,7 @@ hsaKmtDeregisterMemory( { CHECK_KFD_OPEN(); - fmm_deregister_memory(MemoryAddress); - - return HSAKMT_STATUS_SUCCESS; + return fmm_deregister_memory(MemoryAddress); } HSAKMT_STATUS diff --git a/projects/rocr-runtime/src/queues.c b/projects/rocr-runtime/src/queues.c index 10604befa7..247172ef4f 100644 --- a/projects/rocr-runtime/src/queues.c +++ b/projects/rocr-runtime/src/queues.c @@ -207,8 +207,10 @@ void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, return NULL; } - if (peer_to_peer) { - if (hsaKmtRegisterMemory(mem, size) != HSAKMT_STATUS_SUCCESS) { + if (!peer_to_peer) { + uint32_t nodes_array[1] = {NodeId}; + if (hsaKmtRegisterMemoryToNodes(mem, size, 1, nodes_array) + != HSAKMT_STATUS_SUCCESS) { hsaKmtFreeMemory(mem, size); return NULL; } @@ -229,7 +231,6 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) { hsaKmtFreeMemory(addr, size); } - hsaKmtDeregisterMemory(addr); } static void* allocate_exec_aligned_memory(uint32_t size, @@ -246,7 +247,6 @@ static void release_exec_aligned_memory_gpu(void *addr, uint32_t size) { if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) hsaKmtFreeMemory(addr, (HSAuint64)size); - hsaKmtDeregisterMemory(addr); } static void release_exec_aligned_memory(void *addr, uint32_t size, enum asic_family_type type)