Add support for register/deregister memory for dGPU

Allocate SVM address space for the registered memory and use new
userptr support in KFD to create a system memory BO associated with
the given user pointer. Map this BO at the SVM address for CPU
access.

MapMemoryToGPU can be used with the registered user pointer and
will return the SVM address as alternate GPUVA.

Change-Id: I4886e193c51fb6870a567878870c36bf8b5c3748


[ROCm/ROCR-Runtime commit: 85f9efb1a0]

projects/rocr-runtime/include/hsakmt.h

@@ -374,8 +374,8 @@ hsaKmtFreeMemory(
 HSAKMT_STATUS
 HSAKMTAPI
 hsaKmtRegisterMemory(
-    void*       MemoryAddress,      //IN (page-aligned)
-    HSAuint64   MemorySizeInBytes   //IN (page-aligned)
+    void*       MemoryAddress,      //IN (cache-aligned)
+    HSAuint64   MemorySizeInBytes   //IN (cache-aligned)
     );
 
 
@@ -386,8 +386,8 @@ hsaKmtRegisterMemory(
 HSAKMT_STATUS
 HSAKMTAPI
 hsaKmtRegisterMemoryToNodes(
-    void        *MemoryAddress,     // IN (page-aligned)
-    HSAuint64   MemorySizeInBytes,  // IN (page-aligned)
+    void        *MemoryAddress,     // IN (cache-aligned)
+    HSAuint64   MemorySizeInBytes,  // IN (cache-aligned)
     HSAuint64   NumberOfNodes,      // IN
     HSAuint32*  NodeArray           // IN
     );

projects/rocr-runtime/include/linux/kfd_ioctl.h

@@ -312,12 +312,14 @@ struct kfd_ioctl_set_process_dgpu_aperture_args {
 
 #define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM		(1 << 5)
 
+#define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR		(1 << 6)
+
 struct kfd_ioctl_alloc_memory_of_gpu_new_args {
 	uint64_t va_addr;	/* to KFD */
 	uint64_t size;		/* to KFD */
 	uint64_t handle;	/* from KFD */
 	uint32_t gpu_id;	/* to KFD */
-	uint64_t mmap_offset;   /* from KFD */
+	uint64_t mmap_offset;   /* to KFD (userptr), from KFD (mmap offset) */
 	uint32_t flags;
 };
 

projects/rocr-runtime/src/fmm.c

@@ -47,6 +47,7 @@
 
 struct vm_object {
 	void *start;
+	void *userptr;
 	uint64_t size;
 	uint64_t handle; /* opaque */
 	struct vm_object *next;
@@ -174,6 +175,7 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size,
 
 	if (object) {
 		object->start = start;
+		object->userptr = NULL;
 		object->size = size;
 		object->handle = handle;
 		object->next = object->prev = NULL;
@@ -283,6 +285,21 @@ static vm_object_t *vm_find_object_by_address(manageble_aperture_t *app,
 	return cur; /* NULL if not found */
 }
 
+static vm_object_t *vm_find_object_by_userptr(manageble_aperture_t *app,
+						void *address)
+{
+	vm_object_t *cur = app->vm_objects;
+
+	/* Look up the appropriate address range containing the given address */
+	while (cur) {
+		if (cur->userptr == address)
+			break;
+		cur = cur->next;
+	};
+
+	return cur; /* NULL if not found */
+}
+
 static vm_area_t *vm_find(manageble_aperture_t *app, void *address)
 {
 	vm_area_t *cur = app->vm_ranges;
@@ -462,6 +479,8 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem,
 	args.va_addr = (uint64_t)mem;
 	if (flags == KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE)
 		args.va_addr = VOID_PTRS_SUB(mem, aperture->base);
+	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)
+		args.mmap_offset = *mmap_offset;
 
 	if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU_NEW, &args))
 		return -1;
@@ -1217,7 +1236,8 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture,
 	if (object->device_ids_array_size > 0) {
 		args.device_ids_array = object->device_ids_array;
 		args.device_ids_array_size = object->device_ids_array_size;
-	} else if (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) {
+	} else if ((object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) ||
+		   (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)) {
 		/* Only enable multi-GPU mapping on host memory for now */
 		args.device_ids_array = all_gpu_id_array;
 		args.device_ids_array_size = all_gpu_id_array_size;
@@ -1344,6 +1364,38 @@ err_object_not_found:
 	return -1;
 }
 
+static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size,
+				   uint64_t *gpuvm_addr)
+{
+	manageble_aperture_t *aperture;
+	vm_object_t *obj;
+	void *svm_addr;
+	HSAuint64 svm_size;
+	HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1);
+	int ret;
+
+	aperture = &svm.dgpu_aperture;
+
+	/* Find the start address in SVM space for GPU mapping */
+	pthread_mutex_lock(&aperture->fmm_mutex);
+	obj = vm_find_object_by_userptr(aperture, addr);
+	if (obj == NULL) {
+		pthread_mutex_unlock(&aperture->fmm_mutex);
+		return HSAKMT_STATUS_ERROR;
+	}
+	svm_addr = obj->start;
+	svm_size = obj->size;
+	pthread_mutex_unlock(&aperture->fmm_mutex);
+
+	/* Map and return the GPUVM address adjusted by the offset
+	 * from the start of the page */
+	ret = _fmm_map_to_gpu_gtt(aperture, svm_addr, svm_size);
+	if (ret == 0 && gpuvm_addr)
+		*gpuvm_addr = (uint64_t)svm_addr + page_offset;
+
+	return ret;
+}
+
 int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address)
 {
 	uint32_t i;
@@ -1379,10 +1431,18 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address)
 		return _fmm_map_to_gpu_gtt(&svm.dgpu_alt_aperture,
 						address, size);
 
+	/*
+	 * If address isn't an SVM memory address, we assume that this
+	 * is system memory address. On dGPU we need to map it,
+	 * assuming it was previously registered.
+	 */
+	if (is_dgpu)
+		/* TODO: support mixed APU and dGPU configurations */
+		return _fmm_map_to_gpu_userptr(address, size, gpuvm_address);
 
 	/*
-	 * If address isn't Local memory address, we assume that this is
-	 * system memory address accessed through IOMMU. Thus we "prefetch" it
+	 * On an APU a system memory address is accessed through
+	 * IOMMU. Thus we "prefetch" it.
 	 */
 	for (pi = 0; pi < size / PAGE_SIZE; pi++)
 		((char *) address)[pi * PAGE_SIZE] = 0;
@@ -1406,7 +1466,8 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address)
 	if (object->device_ids_array_size > 0) {
 		args.device_ids_array = object->device_ids_array;
 		args.device_ids_array_size = object->device_ids_array_size;
-	} else if (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) {
+	} else if ((object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) ||
+		   (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)) {
 		/* Only enable multi-GPU mapping on host memory for now */
 		args.device_ids_array = all_gpu_id_array;
 		args.device_ids_array_size = all_gpu_id_array_size;
@@ -1468,6 +1529,28 @@ err:
 	return -1;
 }
 
+static int _fmm_unmap_from_gpu_userptr(void *addr)
+{
+	manageble_aperture_t *aperture;
+	vm_object_t *obj;
+	void *svm_addr;
+
+	aperture = &svm.dgpu_aperture;
+
+	/* Find the start address in SVM space for GPU unmapping */
+	pthread_mutex_lock(&aperture->fmm_mutex);
+	obj = vm_find_object_by_userptr(aperture, addr);
+	if (obj == NULL) {
+		pthread_mutex_unlock(&aperture->fmm_mutex);
+		return HSAKMT_STATUS_ERROR;
+	}
+	svm_addr = obj->start;
+	pthread_mutex_unlock(&aperture->fmm_mutex);
+
+	/* Unmap */
+	return _fmm_unmap_from_gpu(aperture, svm_addr);
+}
+
 int fmm_unmap_from_gpu(void *address)
 {
 	uint32_t i;
@@ -1501,6 +1584,13 @@ int fmm_unmap_from_gpu(void *address)
 		return _fmm_unmap_from_gpu(&svm.dgpu_alt_aperture,
 							address);
 
+	/*
+	 * If address isn't an SVM address, we assume that this is
+	 * system memory address.
+	 */
+	if (is_dgpu)
+		/* TODO: support mixed APU and dGPU configurations */
+		return _fmm_unmap_from_gpu_userptr(address);
 
 	return 0;
 }
@@ -1693,27 +1783,83 @@ bool fmm_get_handle(void *address, uint64_t *handle)
 	return found;
 }
 
-HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes,
+static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_object_t **obj_ret)
+{
+	int32_t i;
+	HSAuint32 gpu_id;
+	manageble_aperture_t *aperture;
+	void *svm_addr = NULL;
+	vm_object_t *obj;
+	HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1);
+	HSAuint64 aligned_addr = (HSAuint64)addr - page_offset;
+	HSAuint64 aligned_size = PAGE_ALIGN_UP(page_offset + size);
+
+	/* Find first dGPU for creating the userptr BO */
+	i = find_first_dgpu(&gpu_id);
+	if (i < 0)
+		return HSAKMT_STATUS_ERROR;
+	aperture = &svm.dgpu_aperture;
+
+	/* Check if this address was already registered */
+	pthread_mutex_lock(&aperture->fmm_mutex);
+	obj = vm_find_object_by_userptr(aperture, addr);
+	pthread_mutex_unlock(&aperture->fmm_mutex);
+	if (obj != NULL)
+		return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED;
+
+	/* Allocate BO, userptr address is passed in mmap_offset */
+	svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, 0,
+					 &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
+	if (svm_addr == NULL)
+		return HSAKMT_STATUS_ERROR;
+
+	/* Find the object and set its userptr address */
+	pthread_mutex_lock(&aperture->fmm_mutex);
+	obj = vm_find_object_by_address(aperture, svm_addr, size);
+	if (obj == NULL) {
+		pthread_mutex_unlock(&aperture->fmm_mutex);
+		return HSAKMT_STATUS_ERROR;
+	}
+	obj->userptr = addr;
+	pthread_mutex_unlock(&aperture->fmm_mutex);
+
+	if (obj_ret)
+		*obj_ret = obj;
+	return HSAKMT_STATUS_SUCCESS;
+}
+
+HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes,
 				  uint32_t *gpu_id_array,
 				  uint32_t gpu_id_array_size)
 {
 	manageble_aperture_t *aperture;
 	vm_object_t *object = NULL;
+	HSAKMT_STATUS ret;
 
 	if (gpu_id_array_size > 0 && gpu_id_array == NULL)
 		return HSAKMT_STATUS_INVALID_PARAMETER;
 
-	/*
-	 * Object can be found only on SVM aperture as you can't map
-	 * non SVM object on different device.
-	 */
-	aperture = &svm.dgpu_aperture;
-	pthread_mutex_lock(&aperture->fmm_mutex);
-	object = vm_find_object_by_address(aperture, address, 0);
-	pthread_mutex_unlock(&aperture->fmm_mutex);
+	if ((address >= svm.dgpu_aperture.base) &&
+	    (address <= svm.dgpu_aperture.limit))
+		aperture = &svm.dgpu_aperture;
+	else if ((address >= svm.dgpu_alt_aperture.base) &&
+		 (address <= svm.dgpu_alt_aperture.limit))
+		aperture = &svm.dgpu_alt_aperture;
+	else {
+		/*
+		 * If address isn't SVM address, we assume that this
+		 * is system memory address.
+		 */
+		ret = fmm_register_user_memory(address, size_in_bytes, &object);
+		if (ret != HSAKMT_STATUS_SUCCESS)
+			return ret;
+		if (gpu_id_array_size == 0)
+			return HSAKMT_STATUS_SUCCESS;
+		aperture = &svm.dgpu_aperture;
+		/* fall through */
+	}
 
 	if (!object) {
-		aperture = &svm.dgpu_alt_aperture;
 		pthread_mutex_lock(&aperture->fmm_mutex);
 		object = vm_find_object_by_address(aperture, address, 0);
 		pthread_mutex_unlock(&aperture->fmm_mutex);
@@ -1732,30 +1878,65 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes,
 	return HSAKMT_STATUS_SUCCESS;
 }
 
+static HSAKMT_STATUS fmm_deregister_user_memory(void *addr)
+{
+	manageble_aperture_t *aperture;
+	vm_object_t *obj;
+	void *svm_addr;
+	HSAuint64 size;
+
+	aperture = &svm.dgpu_aperture;
+
+	/* Find the size and start address in SVM space */
+	pthread_mutex_lock(&aperture->fmm_mutex);
+	obj = vm_find_object_by_userptr(aperture, addr);
+	if (obj == NULL) {
+		pthread_mutex_unlock(&aperture->fmm_mutex);
+		return HSAKMT_STATUS_MEMORY_NOT_REGISTERED;
+	}
+	svm_addr = obj->start;
+	size = obj->size;
+	pthread_mutex_unlock(&aperture->fmm_mutex);
+
+	/* Destroy BO */
+	__fmm_release(svm_addr, size, aperture);
+
+	return HSAKMT_STATUS_SUCCESS;
+}
+
 HSAKMT_STATUS fmm_deregister_memory(void *address)
 {
 	manageble_aperture_t *aperture;
 	vm_object_t *object = NULL;
 
-	/*
-	 * Object can be found only on SVM aperture as you can't map
-	 * non SVM object on different device.
-	 */
-	aperture = &svm.dgpu_aperture;
+	if ((address >= svm.dgpu_aperture.base) &&
+	    (address <= svm.dgpu_aperture.limit))
+		aperture = &svm.dgpu_aperture;
+	else if ((address >= svm.dgpu_alt_aperture.base) &&
+		 (address <= svm.dgpu_alt_aperture.limit))
+		aperture = &svm.dgpu_alt_aperture;
+	else {
+		/*
+		 * If address isn't SVM address, we assume that this
+		 * is system memory address. If the userptr object had
+		 * a device_ids_array, it will be freed by
+		 * __fmm_release. Also the object will be
+		 * removed. Therefore we can short-circuit the rest of
+		 * the function below.
+		 */
+		return fmm_deregister_user_memory(address);
+	}
+
 	pthread_mutex_lock(&aperture->fmm_mutex);
 	object = vm_find_object_by_address(aperture, address, 0);
 	pthread_mutex_unlock(&aperture->fmm_mutex);
 
-	if (!object) {
-		aperture = &svm.dgpu_alt_aperture;
-		pthread_mutex_lock(&aperture->fmm_mutex);
-		object = vm_find_object_by_address(aperture, address, 0);
-		pthread_mutex_unlock(&aperture->fmm_mutex);
-	}
-
 	if (!object || object->device_ids_array_size <= 0)
 		return HSAKMT_STATUS_MEMORY_NOT_REGISTERED;
 
+	if (object->userptr)
+		return fmm_deregister_user_memory(object->userptr);
+
 	free(object->device_ids_array);
 	object->device_ids_array = NULL;
 	object->device_ids_array_size = 0;

projects/rocr-runtime/src/fmm.h

@@ -69,7 +69,7 @@ HSAKMT_STATUS fmm_node_removed(HSAuint32 gpu_id);
 HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id,
 		HSAuint64 *aperture_base, HSAuint64 *aperture_limit);
 
-HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes,
+HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes,
                                   uint32_t *gpu_id_array,
                                   uint32_t gpu_id_array_size);
 HSAKMT_STATUS fmm_deregister_memory(void *address);

projects/rocr-runtime/src/memory.c

@@ -188,21 +188,25 @@ hsaKmtFreeMemory(
 HSAKMT_STATUS
 HSAKMTAPI
 hsaKmtRegisterMemory(
-	void		*MemoryAddress,		/* IN (page-aligned) */
-	HSAuint64	MemorySizeInBytes	/* IN (page-aligned) */
+	void		*MemoryAddress,		/* IN (cache-aligned) */
+	HSAuint64	MemorySizeInBytes	/* IN (cache-aligned) */
 )
 {
 	CHECK_KFD_OPEN();
 
-        /* No-op for APU, TODO for dGPU */
-	return HSAKMT_STATUS_SUCCESS;
+	if (!is_dgpu)
+		/* TODO: support mixed APU and dGPU configurations */
+		return HSAKMT_STATUS_SUCCESS;
+
+	return fmm_register_memory(MemoryAddress, MemorySizeInBytes,
+                                   NULL, 0);
 }
 
 HSAKMT_STATUS
 HSAKMTAPI
 hsaKmtRegisterMemoryToNodes(
-	void		*MemoryAddress,		/* IN (page-aligned) */
-	HSAuint64	MemorySizeInBytes,	/* IN (page-aligned) */
+	void		*MemoryAddress,		/* IN (cache-aligned) */
+	HSAuint64	MemorySizeInBytes,	/* IN (cache-aligned) */
 	HSAuint64	NumberOfNodes,		/* IN */
 	HSAuint32*	NodeArray		/* IN */
 )
@@ -211,16 +215,20 @@ hsaKmtRegisterMemoryToNodes(
 	uint32_t *gpu_id_array;
 	HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS;
 
+	if (!is_dgpu)
+		/* TODO: support mixed APU and dGPU configurations */
+		return HSAKMT_STATUS_NOT_SUPPORTED;
+
 	ret = validate_nodeid_array(&gpu_id_array,
 			NumberOfNodes, NodeArray);
 
-	if (ret == HSAKMT_STATUS_SUCCESS)
+	if (ret == HSAKMT_STATUS_SUCCESS) {
 		ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes,
 					  gpu_id_array,
 					  NumberOfNodes*sizeof(uint32_t));
-
-	if (ret != HSAKMT_STATUS_SUCCESS)
-		free(gpu_id_array);
+		if (ret != HSAKMT_STATUS_SUCCESS)
+			free(gpu_id_array);
+	}
 
 	return ret;
 }