Add support for register/deregister memory for dGPU

Allocate SVM address space for the registered memory and use new
userptr support in KFD to create a system memory BO associated with
the given user pointer. Map this BO at the SVM address for CPU
access.

MapMemoryToGPU can be used with the registered user pointer and
will return the SVM address as alternate GPUVA.

Change-Id: I4886e193c51fb6870a567878870c36bf8b5c3748


[ROCm/ROCR-Runtime commit: 85f9efb1a0]

projects/rocr-runtime/include/hsakmt.h

@@ -374,8 +374,8 @@ hsaKmtFreeMemory(
 HSAKMT_STATUS
 HSAKMTAPI
 hsaKmtRegisterMemory(
-    void*       MemoryAddress,      //IN (page-aligned)
-    HSAuint64   MemorySizeInBytes   //IN (page-aligned)
+    void*       MemoryAddress,      //IN (cache-aligned)
+    HSAuint64   MemorySizeInBytes   //IN (cache-aligned)
     );
 
 
@@ -386,8 +386,8 @@ hsaKmtRegisterMemory(
 HSAKMT_STATUS
 HSAKMTAPI
 hsaKmtRegisterMemoryToNodes(
-    void        *MemoryAddress,     // IN (page-aligned)
-    HSAuint64   MemorySizeInBytes,  // IN (page-aligned)
+    void        *MemoryAddress,     // IN (cache-aligned)
+    HSAuint64   MemorySizeInBytes,  // IN (cache-aligned)
     HSAuint64   NumberOfNodes,      // IN
     HSAuint32*  NodeArray           // IN
     );

projects/rocr-runtime/include/linux/kfd_ioctl.h

@@ -312,12 +312,14 @@ struct kfd_ioctl_set_process_dgpu_aperture_args {
 
 #define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM		(1 << 5)
 
+#define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR		(1 << 6)
+
 struct kfd_ioctl_alloc_memory_of_gpu_new_args {
 	uint64_t va_addr;	/* to KFD */
 	uint64_t size;		/* to KFD */
 	uint64_t handle;	/* from KFD */
 	uint32_t gpu_id;	/* to KFD */
-	uint64_t mmap_offset;   /* from KFD */
+	uint64_t mmap_offset;   /* to KFD (userptr), from KFD (mmap offset) */
 	uint32_t flags;
 };