SWDEV-558848 - Move DRM calls to thunk for better abstraction (#1912)

* SWDEV-558848 - Move DRM calls to thunk for better abstraction

* Use thunk device handle instead of drm inside agent

* Update IPC functions with new thunk calls

* create hsaKmtHandleImport interface to support ipc

* Reset metadata inside hsaKmtMemHandleFree

* remove whitespaces and NULL usage

* Add thunk apis to libhsakmt.ver

* Add comments to new structs in thunk

* Minor fixes to declarations

* resolve merge conflicts in amd_kfd_driver

---------

Co-authored-by: Rahul Manocha <rmanocha@amd.com>

projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h

@@ -434,6 +434,7 @@ class GpuAgent : public GpuAgentInt {
 
   // @brief returns the libdrm device handle
   __forceinline amdgpu_device_handle libDrmDev() const { return ldrm_dev_; }
+  __forceinline HsaAMDGPUDeviceHandle libThunkDev() const { return libthunk_dev_; }
 
   __forceinline void CheckClockTicks() {
     // If we did not update t1 since agent initialization, force a SyncClock. Otherwise computing
@@ -831,6 +832,7 @@ class GpuAgent : public GpuAgentInt {
 
   // @brief device handle
   amdgpu_device_handle ldrm_dev_;
+  HsaAMDGPUDeviceHandle libthunk_dev_;
 
   DISALLOW_COPY_AND_ASSIGN(GpuAgent);
 

projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h

@@ -540,7 +540,8 @@ class Runtime {
           size_requested(0),
           alloc_flags(core::MemoryRegion::AllocateNoFlags),
           user_ptr(nullptr),
-          ldrm_bo(NULL) {}
+          ldrm_bo(nullptr),
+          thunk_bo(nullptr) {}
     AllocationRegion(const MemoryRegion* region_arg, size_t size_arg, size_t size_requested,
                      MemoryRegion::AllocateFlags alloc_flags)
         : region(region_arg),
@@ -548,7 +549,8 @@ class Runtime {
           size_requested(size_requested),
           alloc_flags(alloc_flags),
           user_ptr(nullptr),
-          ldrm_bo(NULL) {}
+          ldrm_bo(nullptr),
+          thunk_bo(nullptr) {}
 
     struct notifier_t {
       void* ptr;
@@ -563,6 +565,7 @@ class Runtime {
     void* user_ptr;
     std::unique_ptr<std::vector<notifier_t>> notifiers;
     amdgpu_bo_handle ldrm_bo;
+    HsaMemoryObjectHandle thunk_bo;
   };
 
   struct AsyncEventsInfo;
@@ -1012,7 +1015,8 @@ class Runtime {
   bool ipc_dmabuf_supported_;
   int  IPCClientImport(uint32_t conn_handle, uint64_t dmabuf_fd_handle,
                        unsigned int numNodes, HSAuint32 *nodes,
-                       void **importAddress, HSAuint64 *importSize, bool isdmabufSysmem);
+                       void **importAddress, HSAuint64 *importSize,
+                       bool isdmabufSysmem, uint32_t shared_handle);
 };
 
 }  // namespace core

projects/rocr-runtime/runtime/hsa-runtime/core/inc/thunk_loader.h

@@ -336,6 +336,25 @@ class ThunkLoader {
                                       void* MemoryAddress, \
                                       HSAuint64 SizeInBytes, \
                                       uint64_t* SharedMemoryHandle);
+    typedef HSAKMT_STATUS (HSAKMT_DEF(hsaKmtHandleImport))(const HsaExternalHandleDesc* ImportDesc, \
+                                      HsaHandleImportResult* ImportResult, \
+                                      HsaHandleImportFlags* flags);
+    typedef HSAKMT_STATUS (HSAKMT_DEF(hsaKmtMemoryVaMap))(HsaMemoryObjectHandle Handle, \
+                                      HSAuint64 offset, \
+                                      HSAuint64 size, \
+                                      HSAuint64 addr, \
+                                      HsaMemoryMapFlags flags);
+    typedef HSAKMT_STATUS (HSAKMT_DEF(hsaKmtMemoryVaUnmap))(HsaMemoryObjectHandle Handle, \
+                                      HSAuint64 offset, \
+                                      HSAuint64 size, \
+                                      HSAuint64 addr);
+    typedef HSAKMT_STATUS (HSAKMT_DEF(hsaKmtMemHandleFree))(HsaMemoryObjectHandle Handle);
+    typedef HSAKMT_STATUS (HSAKMT_DEF(hsaKmtMemoryGetCpuAddr))(HsaAMDGPUDeviceHandle DeviceHandle, \
+                                      HsaMemoryObjectHandle MemoryHandle, \
+                                      HSAint32* fd, \
+                                      HSAuint64* cpu_addr);
+    typedef HSAKMT_STATUS (HSAKMT_DEF(hsaKmtMemoryCpuMap))(HsaMemoryObjectHandle Handle, \
+                                      void** out_cpu_ptr);
     /* drm API */
     typedef int (DRM_DEF(amdgpu_device_initialize))(int fd, \
                                       uint32_t *major_version, \
@@ -484,6 +503,12 @@ class ThunkLoader {
     HSAKMT_DEF(hsaKmtQueueRingDoorbell)* HSAKMT_PFN(hsaKmtQueueRingDoorbell);
     HSAKMT_DEF(hsaKmtAisReadWriteFile)* HSAKMT_PFN(hsaKmtAisReadWriteFile);
     HSAKMT_DEF(hsaKmtGetMemoryHandle)* HSAKMT_PFN(hsaKmtGetMemoryHandle);
+    HSAKMT_DEF(hsaKmtHandleImport)* HSAKMT_PFN(hsaKmtHandleImport);
+    HSAKMT_DEF(hsaKmtMemoryVaMap)* HSAKMT_PFN(hsaKmtMemoryVaMap);
+    HSAKMT_DEF(hsaKmtMemoryVaUnmap)* HSAKMT_PFN(hsaKmtMemoryVaUnmap);
+    HSAKMT_DEF(hsaKmtMemHandleFree)* HSAKMT_PFN(hsaKmtMemHandleFree);
+    HSAKMT_DEF(hsaKmtMemoryGetCpuAddr)* HSAKMT_PFN(hsaKmtMemoryGetCpuAddr);
+    HSAKMT_DEF(hsaKmtMemoryCpuMap)* HSAKMT_PFN(hsaKmtMemoryCpuMap);
 
     DRM_DEF(amdgpu_device_initialize)* DRM_PFN(amdgpu_device_initialize);
     DRM_DEF(amdgpu_device_deinitialize)* DRM_PFN(amdgpu_device_deinitialize);