Add API entrypoints for Cross Memory Attach

Implement two new API for cross memory read and write operation.
 - hsaKmtProcessVMRead
 - hsaKmtProcessVMWrite

Add new ioclts necessary for the above APIs.

Change-Id: I0c153e3b4e1f32b7a8b102ad5c774d9ae9bfc2fa
Signed-off-by: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>


[ROCm/ROCR-Runtime commit: e79521b556]

projects/rocr-runtime/include/hsakmt.h

@@ -444,6 +444,37 @@ hsaKmtRegisterSharedHandleToNodes(
 	HSAuint32*                  NodeArray            // OUT
 );
 
+/**
+ Copy data from the GPU address space of the process identified
+ by Pid. Size Copied will return actual amount of data copied.
+ If return is not SUCCESS, partial copies could have happened.
+ */
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtProcessVMRead(
+	HSAuint32                 Pid,                     // IN
+	HsaMemoryRange            *LocalMemoryArray,       // IN
+	HSAuint64                 LocalMemoryArrayCount,   // IN
+	HsaMemoryRange            *RemoteMemoryArray,      // IN
+	HSAuint64                 RemoteMemoryArrayCount,  // IN
+	HSAuint64                 *SizeCopied              // OUT
+);
+
+/**
+ Write data to the GPU address space of the process identified
+ by Pid. See also hsaKmtProcessVMRead.
+*/
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtProcessVMWrite(
+	HSAuint32                 Pid,                     // IN
+	HsaMemoryRange            *LocalMemoryArray,       // IN
+	HSAuint64                 LocalMemoryArrayCount,   // IN
+	HsaMemoryRange            *RemoteMemoryArray,      // IN
+	HSAuint64                 RemoteMemoryArrayCount,  // IN
+	HSAuint64                 *SizeCopied              // OUT
+);
+
 /**
   Unregisters with KFD a memory buffer
 */

projects/rocr-runtime/include/hsakmttypes.h

@@ -1008,6 +1008,11 @@ typedef struct _HsaPointerInfo {
 
 typedef HSAuint32 HsaSharedMemoryHandle[8];
 
+typedef struct _HsaMemoryRange {
+	void               *MemoryAddress;   // Pointer to GPU memory
+	HSAuint64          SizeInBytes;      // Size of above memory
+} HsaMemoryRange;
+
 #pragma pack(pop, hsakmttypes_h)
 
 

projects/rocr-runtime/include/linux/kfd_ioctl.h

@@ -395,6 +395,37 @@ struct kfd_ioctl_get_tile_config_args {
 	 */
 };
 
+struct kfd_memory_range {
+	uint64_t va_addr;
+	uint64_t size;
+};
+
+/* flags definitions
+ * BIT0: 0: read operation, 1: write operation.
+ * This also identifies if the src or dst array belongs to remote process
+ */
+#define KFD_CROSS_MEMORY_RW_BIT (1 << 0)
+#define KFD_SET_CROSS_MEMORY_READ(flags) (flags &= ~KFD_CROSS_MEMORY_RW_BIT)
+#define KFD_SET_CROSS_MEMORY_WRITE(flags) (flags |= KFD_CROSS_MEMORY_RW_BIT)
+#define KFD_IS_CROSS_MEMORY_WRITE(flags) (flags & KFD_CROSS_MEMORY_RW_BIT)
+
+struct kfd_ioctl_cross_memory_copy_args {
+	/* to KFD: Process ID of the remote process */
+	uint32_t pid;
+	/* to KFD: See above definition */
+	uint32_t flags;
+	/* to KFD: Source GPU VM range */
+	uint64_t src_mem_range_array;
+	/* to KFD: Size of above array */
+	uint64_t src_mem_array_size;
+	/* to KFD: Destination GPU VM range */
+	uint64_t dst_mem_range_array;
+	/* to KFD: Size of above array */
+	uint64_t dst_mem_array_size;
+	/* from KFD: Total amount of bytes copied */
+	uint64_t bytes_copied;
+};
+
 #define AMDKFD_IOCTL_BASE 'K'
 #define AMDKFD_IO(nr)			_IO(AMDKFD_IOCTL_BASE, nr)
 #define AMDKFD_IOR(nr, type)		_IOR(AMDKFD_IOCTL_BASE, nr, type)
@@ -504,7 +535,10 @@ struct kfd_ioctl_get_tile_config_args {
 #define AMDKFD_IOC_IPC_EXPORT_HANDLE		\
 	AMDKFD_IOWR(0x23, struct kfd_ioctl_ipc_export_handle_args)
 
+#define AMDKFD_IOC_CROSS_MEMORY_COPY		\
+	AMDKFD_IOWR(0x24, struct kfd_ioctl_cross_memory_copy_args)
+
 #define AMDKFD_COMMAND_START		0x01
-#define AMDKFD_COMMAND_END		0x24
+#define AMDKFD_COMMAND_END		0x25
 
 #endif

projects/rocr-runtime/src/libhsakmt.ver

@@ -30,6 +30,8 @@ hsaKmtRegisterGraphicsHandleToNodes;
 hsaKmtShareMemory;
 hsaKmtRegisterSharedHandle;
 hsaKmtRegisterSharedHandleToNodes;
+hsaKmtProcessVMRead;
+hsaKmtProcessVMWrite;
 hsaKmtDeregisterMemory;
 hsaKmtMapMemoryToGPU;
 hsaKmtMapMemoryToGPUNodes;

projects/rocr-runtime/src/memory.c

@@ -340,6 +340,91 @@ error:
 	return ret;
 }
 
+static uint64_t convertHsaToKfdRange(
+	HsaMemoryRange *HsaRange)
+{
+	if (sizeof(struct kfd_memory_range) !=
+		sizeof(HsaMemoryRange)) {
+		fprintf(stderr, "Struct size mismatch in thunk."
+			"Cannot cast Hsa Range to KFD IOCTL range\n");
+		return 0;
+	}
+	return (uint64_t) HsaRange;
+}
+
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtProcessVMRead(
+	HSAuint32            Pid,                   /* IN */
+	HsaMemoryRange       *LocalMemoryArray,     /* IN */
+	HSAuint64            LocalMemoryArrayCount, /* IN */
+	HsaMemoryRange       *RemoteMemoryArray,    /* IN */
+	HSAuint64            RemoteMemoryArrayCount,/* IN */
+	HSAuint64            *SizeCopied            /* OUT */
+)
+{
+	struct kfd_ioctl_cross_memory_copy_args args;
+
+	if (LocalMemoryArray == NULL || RemoteMemoryArray == NULL ||
+		LocalMemoryArrayCount == 0 || RemoteMemoryArrayCount == 0)
+		return HSAKMT_STATUS_ERROR;
+
+	args.flags = 0;
+	KFD_SET_CROSS_MEMORY_READ(args.flags);
+	args.pid = Pid;
+	args.src_mem_range_array = convertHsaToKfdRange(RemoteMemoryArray);
+	args.src_mem_array_size = RemoteMemoryArrayCount;
+	args.dst_mem_range_array = convertHsaToKfdRange(LocalMemoryArray);
+	args.dst_mem_array_size = LocalMemoryArrayCount;
+	args.bytes_copied = 0;
+
+	int err = kmtIoctl(kfd_fd, AMDKFD_IOC_CROSS_MEMORY_COPY, &args);
+	if (err)
+		return HSAKMT_STATUS_ERROR;
+
+	if (SizeCopied)
+		*SizeCopied = args.bytes_copied;
+
+	return HSAKMT_STATUS_SUCCESS;
+}
+
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtProcessVMWrite(
+	HSAuint32             Pid,                       /* IN */
+	HsaMemoryRange        *LocalMemoryArray,         /* IN */
+	HSAuint64             LocalMemoryArrayCount,	 /* IN */
+	HsaMemoryRange        *RemoteMemoryArray,        /* IN */
+	HSAuint64             RemoteMemoryArrayCount,    /* IN */
+	HSAuint64             *SizeCopied                /* OUT */
+)
+{
+	struct kfd_ioctl_cross_memory_copy_args args;
+
+	if (LocalMemoryArray == NULL || RemoteMemoryArray == NULL ||
+		LocalMemoryArrayCount == 0 || RemoteMemoryArrayCount == 0)
+		return HSAKMT_STATUS_ERROR;
+
+	args.flags = 0;
+	KFD_SET_CROSS_MEMORY_WRITE(args.flags);
+	args.pid = Pid;
+	args.src_mem_range_array = convertHsaToKfdRange(LocalMemoryArray);
+	args.src_mem_array_size = LocalMemoryArrayCount;
+	args.dst_mem_range_array = convertHsaToKfdRange(RemoteMemoryArray);
+	args.dst_mem_array_size = RemoteMemoryArrayCount;
+	args.bytes_copied = 0;
+
+	int err = kmtIoctl(kfd_fd, AMDKFD_IOC_CROSS_MEMORY_COPY, &args);
+	if (err)
+		return HSAKMT_STATUS_ERROR;
+
+	if (SizeCopied)
+		*SizeCopied = args.bytes_copied;
+
+	return HSAKMT_STATUS_SUCCESS;
+}
+
+
 HSAKMT_STATUS
 HSAKMTAPI
 hsaKmtDeregisterMemory(