From 667ed434fbda0e54b3c15faeccbee2f72b209e20 Mon Sep 17 00:00:00 2001
From: David Yat Sin <David.YatSin@amd.com>
Date: Sun, 18 Dec 2022 23:54:22 +0000
Subject: [PATCH] Change libdrm initialization

Change initialize libdrm device and file descriptor initialization
to use new APIs from Thunk. Libdrm recommends that we re-use the same
file descriptor thoughout the life of a process instead of re-creating
new one each time.

This is part of patch series for Virtual Memory API.

Change-Id: I1c0b8d1bd660cd25478b5f94c84071b90d93fc6c


[ROCm/ROCR-Runtime commit: a55f11025b5592eb7264f8300f17ea22d70fc64b]
---
 .../hsa-runtime/core/inc/amd_gpu_agent.h      |  9 ++++++
 .../runtime/hsa-runtime/core/inc/runtime.h    |  5 ++++
 .../core/runtime/amd_gpu_agent.cpp            | 30 ++++++++++---------
 3 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h
index 905472af42..ab48ca989c 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h
@@ -343,6 +343,9 @@ class GpuAgent : public GpuAgentInt {
   // @brief returns true if agent uses MES scheduler
   __forceinline const bool isMES() const { return (isa_->GetMajorVersion() >= 11) ? true : false; };
 
+  // @brief returns the libdrm device handle
+  __forceinline amdgpu_device_handle libDrmDev() const { return ldrm_dev_; }
+
   void ReserveScratch();
 
   void Trim() override;
@@ -554,6 +557,9 @@ class GpuAgent : public GpuAgentInt {
   // Bind the Blit object that will drive the copy operation by engine ID
   lazy_ptr<core::Blit>& GetBlitObject(uint32_t engine_id);
 
+  // @brief initialize libdrm handle
+  void InitLibDrm();
+
   // @brief Alternative aperture base address. Only on KV.
   uintptr_t ape1_base_;
 
@@ -584,6 +590,9 @@ class GpuAgent : public GpuAgentInt {
 
   std::function<void(void*)> system_deallocator_;
 
+  // @brief device handle
+  amdgpu_device_handle ldrm_dev_;
+
   DISALLOW_COPY_AND_ASSIGN(GpuAgent);
 
   // Check if SDMA engine by ID is free
diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h
index cc61166959..0bafa7b90f 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h
@@ -52,6 +52,11 @@
 #include <utility>
 #include <thread>
 
+#if defined(__linux__)
+#include <xf86drm.h>
+#include <amdgpu.h>
+#endif
+
 #include "core/inc/hsa_ext_interface.h"
 #include "core/inc/hsa_internal.h"
 #include "core/inc/hsa_ext_amd_impl.h"
diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
index 88b2708486..e2025c6daa 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
@@ -178,25 +178,15 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xna
   max_queues_ = std::min(128U, max_queues_);
 #endif
 
+  // Initialize libdrm device handle
+  InitLibDrm();
+
 #if !defined(__linux__)
   wallclock_frequency_ = 0;
 #else
   // Get wallclock freq from libdrm.
-  int drm_fd = drmOpenRender(node_props.DrmRenderMinor);
-  if (drm_fd < 0)
-    throw AMD::hsa_exception(HSA_STATUS_ERROR, "Agent creation failed.\nlibdrm open failed.\n");
-  MAKE_SCOPE_GUARD([&]() { drmClose(drm_fd); });
-
-  amdgpu_device_handle device_handle;
-  uint32_t major_version;
-  uint32_t minor_version;
-  if (amdgpu_device_initialize(drm_fd, &major_version, &minor_version, &device_handle) < 0) {
-    throw AMD::hsa_exception(HSA_STATUS_ERROR, "Agent creation failed.\nlibdrm error.\n");
-  }
-  MAKE_SCOPE_GUARD([&]() { amdgpu_device_deinitialize(device_handle); });
-
   amdgpu_gpu_info info;
-  if (amdgpu_query_gpu_info(device_handle, &info) < 0)
+  if (amdgpu_query_gpu_info(ldrm_dev_, &info) < 0)
     throw AMD::hsa_exception(HSA_STATUS_ERROR, "Agent creation failed.\nlibdrm query failed.\n");
 
   // Reported by libdrm in KHz.
@@ -574,6 +564,18 @@ void GpuAgent::InitCacheList() {
                                      cache_props_[i].CacheLevel, cache_props_[i].CacheSize));
 }
 
+void GpuAgent::InitLibDrm() {
+  HSAKMT_STATUS status;
+
+  HsaAMDGPUDeviceHandle device_handle;
+  status = hsaKmtGetAMDGPUDeviceHandle(node_id(), &device_handle);
+  if (status != HSAKMT_STATUS_SUCCESS)
+    throw AMD::hsa_exception(HSA_STATUS_ERROR,
+                             "Agent creation failed.\nlibdrm get device handle failed.\n");
+
+  ldrm_dev_ = (amdgpu_device_handle)device_handle;
+}
+
 hsa_status_t GpuAgent::IterateRegion(
     hsa_status_t (*callback)(hsa_region_t region, void* data),
     void* data) const {