diff --git a/projects/rocr-runtime/include/hsakmt.h b/projects/rocr-runtime/include/hsakmt.h index bdb8f2bdd3..5fadf78577 100644 --- a/projects/rocr-runtime/include/hsakmt.h +++ b/projects/rocr-runtime/include/hsakmt.h @@ -369,7 +369,6 @@ hsaKmtFreeMemory( /** Registers with KFD a memory buffer that may be accessed by the GPU - This function will never be required for Linux */ HSAKMT_STATUS @@ -380,9 +379,22 @@ hsaKmtRegisterMemory( ); +/** + Registers with KFD a memory buffer that may be accessed by specific GPUs +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterMemoryToNodes( + void *MemoryAddress, // IN (page-aligned) + HSAuint64 MemorySizeInBytes, // IN (page-aligned) + HSAuint64 NumberOfNodes, // IN + HSAuint32* NodeArray // IN + ); + + /** Unregisters with KFD a memory buffer - This function will never be required for Linux */ HSAKMT_STATUS diff --git a/projects/rocr-runtime/src/fmm.c b/projects/rocr-runtime/src/fmm.c index 2e593de2a2..c8cb4c62bd 100644 --- a/projects/rocr-runtime/src/fmm.c +++ b/projects/rocr-runtime/src/fmm.c @@ -66,6 +66,7 @@ struct vm_object { uint64_t handle; /* opaque */ struct vm_object *next; struct vm_object *prev; + uint32_t flags; /* memory allocation flags */ /* * Nodes to map on SVM mGPU */ @@ -137,6 +138,10 @@ static svm_t svm = { INIT_MANAGEBLE_APERTURE(0, 0) }; +/* GPU node array for default mappings */ +static uint32_t all_gpu_id_array_size = 0; +static uint32_t *all_gpu_id_array = NULL; + extern int debug_get_reg_status(uint32_t node_id, bool* is_debugged); static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); @@ -161,7 +166,7 @@ static vm_area_t *vm_create_and_init_area(void *start, void *end) } static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, - uint64_t handle) + uint64_t handle, uint32_t flags) { vm_object_t *object = (vm_object_t *) malloc(sizeof(vm_object_t)); @@ -171,6 +176,7 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->handle = handle; object->next = object->prev = NULL; object->device_ids_array_size = 0; + object->flags = flags; } return object; @@ -399,7 +405,8 @@ static void *aperture_allocate_area(manageble_aperture_t *app, static int aperture_allocate_object(manageble_aperture_t *app, void *new_address, uint64_t handle, - uint64_t MemorySizeInBytes) + uint64_t MemorySizeInBytes, + uint32_t flags) { vm_object_t *new_object; @@ -407,8 +414,8 @@ static int aperture_allocate_object(manageble_aperture_t *app, /* Allocate new object */ new_object = vm_create_and_init_object(new_address, - MemorySizeInBytes, - handle); + MemorySizeInBytes, + handle, flags); if (!new_object) return -1; @@ -460,7 +467,7 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, /* Allocate object */ pthread_mutex_lock(&aperture->fmm_mutex); if (aperture_allocate_object(aperture, mem, args.handle, - MemorySizeInBytes)) + MemorySizeInBytes, flags)) goto err_object_allocation_failed; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -848,7 +855,7 @@ void *fmm_open_graphic_handle(uint32_t gpu_id, /* Allocate object */ if (aperture_allocate_object(&gpu_mem[i].gpuvm_aperture, mem, open_graphic_handle_args.handle, - MemorySizeInBytes)) + MemorySizeInBytes, 0)) goto release_mem; pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); @@ -888,6 +895,9 @@ static void __fmm_release(void *address, return; } + if (object->device_ids_array_size > 0) + free(object->device_ids_array); + args.handle = object->handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); @@ -1003,6 +1013,13 @@ HSAKMT_STATUS fmm_init_process_apertures(void) if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void *) &args)) return HSAKMT_STATUS_ERROR; + all_gpu_id_array_size = 0; + if (args.num_of_nodes > 0) { + all_gpu_id_array = malloc(sizeof(uint32_t) * args.num_of_nodes); + if (all_gpu_id_array == NULL) + return HSAKMT_STATUS_NO_MEMORY; + } + for (i = 0 ; i < args.num_of_nodes ; i++) { /* Map Kernel process device data node i <--> gpu_mem_id which indexes into gpu_mem[] * based on gpu_id */ @@ -1010,6 +1027,9 @@ HSAKMT_STATUS fmm_init_process_apertures(void) if (gpu_mem_id < 0) return HSAKMT_STATUS_ERROR; + all_gpu_id_array[i] = args.process_apertures[i].gpu_id; + all_gpu_id_array_size += sizeof(uint32_t); + gpu_mem[gpu_mem_id].lds_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[i].lds_base); @@ -1135,8 +1155,17 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, } args.handle = object->handle; - args.device_ids_array = object->device_ids_array; - args.device_ids_array_size = object->device_ids_array_size; + if (object->device_ids_array_size > 0) { + args.device_ids_array = object->device_ids_array; + args.device_ids_array_size = object->device_ids_array_size; + } else if (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) { + /* Only enable multi-GPU mapping on host memory for now */ + args.device_ids_array = all_gpu_id_array; + args.device_ids_array_size = all_gpu_id_array_size; + } else { + args.device_ids_array = NULL; + args.device_ids_array_size = 0; + } if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU_NEW, &args)) goto err_map_ioctl_failed; @@ -1315,8 +1344,17 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address) goto err; args.handle = object->handle; - args.device_ids_array = object->device_ids_array; - args.device_ids_array_size = object->device_ids_array_size; + if (object->device_ids_array_size > 0) { + args.device_ids_array = object->device_ids_array; + args.device_ids_array_size = object->device_ids_array_size; + } else if (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) { + /* Only enable multi-GPU mapping on host memory for now */ + args.device_ids_array = all_gpu_id_array; + args.device_ids_array_size = all_gpu_id_array_size; + } else { + args.device_ids_array = NULL; + args.device_ids_array_size = 0; + } kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &args); pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1607,14 +1645,22 @@ void fmm_release_global_resources(void) } dgpu_shared_aperture_base = NULL; dgpu_shared_aperture_limit = NULL; + + if (all_gpu_id_array != NULL) + free(all_gpu_id_array); + all_gpu_id_array = NULL; + all_gpu_id_array_size = 0; } -int fmm_register_memory(void *address, uint32_t size_in_bytes, - uint32_t *nodes_arr, uint32_t nodes_arr_size) +HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes, + uint32_t *gpu_id_array, + uint32_t gpu_id_array_size) { - bool found = false; manageble_aperture_t *aperture; - vm_object_t *object; + vm_object_t *object = NULL; + + if (gpu_id_array_size > 0 && gpu_id_array == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; /* * Object can be found only on SVM aperture as you can't map @@ -1622,37 +1668,33 @@ int fmm_register_memory(void *address, uint32_t size_in_bytes, */ aperture = &svm.dgpu_aperture; pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (object) - found = true; pthread_mutex_unlock(&aperture->fmm_mutex); - if (!found) { + if (!object) { aperture = &svm.dgpu_alt_aperture; - pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (object) - found = true; pthread_mutex_unlock(&aperture->fmm_mutex); } if (!object) - return 1; + return HSAKMT_STATUS_NOT_SUPPORTED; + if (object->device_ids_array_size > 0) + return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; - object->device_ids_array = nodes_arr; - object->device_ids_array_size = nodes_arr_size * sizeof(uint32_t); + if (gpu_id_array_size > 0) { + object->device_ids_array = gpu_id_array; + object->device_ids_array_size = gpu_id_array_size; + } - return 0; + return HSAKMT_STATUS_SUCCESS; } -void fmm_deregister_memory(void *address) +HSAKMT_STATUS fmm_deregister_memory(void *address) { - bool found = false; manageble_aperture_t *aperture; - vm_object_t *object; + vm_object_t *object = NULL; /* * Object can be found only on SVM aperture as you can't map @@ -1660,56 +1702,22 @@ void fmm_deregister_memory(void *address) */ aperture = &svm.dgpu_aperture; pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (object) - found = true; pthread_mutex_unlock(&aperture->fmm_mutex); - if (!found) { + if (!object) { aperture = &svm.dgpu_alt_aperture; pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (object) - found = true; pthread_mutex_unlock(&aperture->fmm_mutex); } if (!object || object->device_ids_array_size <= 0) - return; + return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; free(object->device_ids_array); object->device_ids_array = NULL; object->device_ids_array_size = 0; -} - -int fmm_build_nodes_array(uint32_t **array, uint32_t *nodes, uint32_t nodes_num) -{ - uint32_t i, *arr; - if (!nodes) { - nodes_num = 0; - for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS; i++) { - if (gpu_mem[i].gpu_id == 0) - continue; - nodes_num++; - } - } - - arr = (uint32_t *)malloc(sizeof(uint32_t) * nodes_num); - if (!array) - return 1; - - memset(arr, 0, sizeof(uint32_t) * nodes_num); - - nodes_num = 0; - for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS; i++) { - if (gpu_mem[i].gpu_id == 0) - continue; - arr[nodes_num] = gpu_mem[i].gpu_id; - nodes_num++; - } - - *array = arr; - return nodes_num; + + return HSAKMT_STATUS_SUCCESS; } diff --git a/projects/rocr-runtime/src/fmm.h b/projects/rocr-runtime/src/fmm.h index 37bdcb3b65..5bdbf5bbba 100644 --- a/projects/rocr-runtime/src/fmm.h +++ b/projects/rocr-runtime/src/fmm.h @@ -68,8 +68,8 @@ HSAKMT_STATUS fmm_node_removed(HSAuint32 gpu_id); HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id, HSAuint64 *aperture_base, HSAuint64 *aperture_limit); -int fmm_register_memory(void *address, uint32_t size_in_bytes, - uint32_t *nodes_arr, uint32_t nodes_arr_size); -void fmm_deregister_memory(void *address); -int fmm_build_nodes_array(uint32_t **array, uint32_t *nodes, uint32_t nodes_num); +HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes, + uint32_t *gpu_id_array, + uint32_t gpu_id_array_size); +HSAKMT_STATUS fmm_deregister_memory(void *address); #endif /* FMM_H_ */ diff --git a/projects/rocr-runtime/src/libhsakmt.ver b/projects/rocr-runtime/src/libhsakmt.ver index 3c8064d43c..9a8a14835b 100644 --- a/projects/rocr-runtime/src/libhsakmt.ver +++ b/projects/rocr-runtime/src/libhsakmt.ver @@ -25,6 +25,7 @@ hsaKmtSetMemoryPolicy; hsaKmtAllocMemory; hsaKmtFreeMemory; hsaKmtRegisterMemory; +hsaKmtRegisterMemoryToNodes; hsaKmtDeregisterMemory; hsaKmtMapMemoryToGPU; hsaKmtUnmapMemoryToGPU; diff --git a/projects/rocr-runtime/src/memory.c b/projects/rocr-runtime/src/memory.c index 1b043b60c2..c0161b36dc 100644 --- a/projects/rocr-runtime/src/memory.c +++ b/projects/rocr-runtime/src/memory.c @@ -195,22 +195,48 @@ hsaKmtRegisterMemory( HSAuint64 MemorySizeInBytes /* IN (page-aligned) */ ) { - uint32_t *NodesArray; - uint32_t NodesArraySize; - CHECK_KFD_OPEN(); - /* - * Build NodesArray from all dGPU nodes. - */ - NodesArraySize = fmm_build_nodes_array(&NodesArray, NULL, 0); - if (!NodesArray) + /* No-op for APU, TODO for dGPU */ + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterMemoryToNodes( + void *MemoryAddress, /* IN (page-aligned) */ + HSAuint64 MemorySizeInBytes, /* IN (page-aligned) */ + HSAuint64 NumberOfNodes, /* IN */ + HSAuint32* NodeArray /* IN */ +) +{ + CHECK_KFD_OPEN(); + uint32_t *gpu_id_array; + unsigned i; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + if (NumberOfNodes == 0 || NodeArray == NULL) return HSAKMT_STATUS_INVALID_PARAMETER; - if (fmm_register_memory(MemoryAddress, MemorySizeInBytes, - NodesArray, NodesArraySize) != 0) - return HSAKMT_STATUS_ERROR; - return HSAKMT_STATUS_SUCCESS; + /* Translate Node IDs to gpu_ids */ + gpu_id_array = malloc(NumberOfNodes * sizeof(uint32_t)); + if (gpu_id_array == NULL) + return HSAKMT_STATUS_NO_MEMORY; + for (i = 0; i < NumberOfNodes; i++) { + ret = validate_nodeid(NodeArray[i], &gpu_id_array[i]); + if (ret != HSAKMT_STATUS_SUCCESS) + break; + } + + if (ret == HSAKMT_STATUS_SUCCESS) + ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, + gpu_id_array, + NumberOfNodes*sizeof(uint32_t)); + + if (ret != HSAKMT_STATUS_SUCCESS) + free(gpu_id_array); + + return ret; } HSAKMT_STATUS @@ -221,9 +247,7 @@ hsaKmtDeregisterMemory( { CHECK_KFD_OPEN(); - fmm_deregister_memory(MemoryAddress); - - return HSAKMT_STATUS_SUCCESS; + return fmm_deregister_memory(MemoryAddress); } HSAKMT_STATUS diff --git a/projects/rocr-runtime/src/queues.c b/projects/rocr-runtime/src/queues.c index 10604befa7..247172ef4f 100644 --- a/projects/rocr-runtime/src/queues.c +++ b/projects/rocr-runtime/src/queues.c @@ -207,8 +207,10 @@ void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, return NULL; } - if (peer_to_peer) { - if (hsaKmtRegisterMemory(mem, size) != HSAKMT_STATUS_SUCCESS) { + if (!peer_to_peer) { + uint32_t nodes_array[1] = {NodeId}; + if (hsaKmtRegisterMemoryToNodes(mem, size, 1, nodes_array) + != HSAKMT_STATUS_SUCCESS) { hsaKmtFreeMemory(mem, size); return NULL; } @@ -229,7 +231,6 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) { hsaKmtFreeMemory(addr, size); } - hsaKmtDeregisterMemory(addr); } static void* allocate_exec_aligned_memory(uint32_t size, @@ -246,7 +247,6 @@ static void release_exec_aligned_memory_gpu(void *addr, uint32_t size) { if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) hsaKmtFreeMemory(addr, (HSAuint64)size); - hsaKmtDeregisterMemory(addr); } static void release_exec_aligned_memory(void *addr, uint32_t size, enum asic_family_type type)