From e3514895c6d3b7246ad173f55aa85fa1daed3b24 Mon Sep 17 00:00:00 2001
From: Evgeny <evgeny.shcherbakov@amd.com>
Date: Tue, 28 Apr 2020 03:16:15 -0500
Subject: [PATCH] toll destruction fix

Change-Id: If069c820526e21a0a4b80ac516f9669a81f34cab


[ROCm/rocprofiler commit: fe70682184dbec88a0801648a7c12d9331be9aa5]
---
 projects/rocprofiler/src/core/rocprofiler.cpp |  7 +++++
 projects/rocprofiler/test/tool/tool.cpp       | 31 +++++++++++++++++--
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/projects/rocprofiler/src/core/rocprofiler.cpp b/projects/rocprofiler/src/core/rocprofiler.cpp
index 7258772e66..00990e1058 100644
--- a/projects/rocprofiler/src/core/rocprofiler.cpp
+++ b/projects/rocprofiler/src/core/rocprofiler.cpp
@@ -217,11 +217,13 @@ uint32_t LoadTool() {
     if (settings.hsa_intercepting) intercept_mode |= HSA_INTERCEPT_MODE;
   }
 
+  ONLOAD_TRACE("end intercept_mode(" << intercept_mode << ")");
   return intercept_mode;
 }
 
 // Unload profiling tool librray
 void UnloadTool() {
+  ONLOAD_TRACE("tool handle(" << tool_handle << ")");
   if (tool_handle) {
     tool_handler_t handler = reinterpret_cast<tool_handler_t>(dlsym(tool_handle, "OnUnloadTool"));
     if (handler == NULL) {
@@ -232,16 +234,21 @@ void UnloadTool() {
     handler();
     dlclose(tool_handle);
   }
+  ONLOAD_TRACE_END();
 }
 
 CONSTRUCTOR_API void constructor() {
+  ONLOAD_TRACE_BEG();
   util::Logger::Create();
+  ONLOAD_TRACE_END();
 }
 
 DESTRUCTOR_API void destructor() {
+  ONLOAD_TRACE_BEG();
   rocprofiler::MetricsDict::Destroy();
   util::HsaRsrcFactory::Destroy();
   util::Logger::Destroy();
+  ONLOAD_TRACE_END();
 }
 
 const MetricsDict* GetMetrics(const hsa_agent_t& agent) {
diff --git a/projects/rocprofiler/test/tool/tool.cpp b/projects/rocprofiler/test/tool/tool.cpp
index d0be4d064c..f0a94063aa 100644
--- a/projects/rocprofiler/test/tool/tool.cpp
+++ b/projects/rocprofiler/test/tool/tool.cpp
@@ -56,6 +56,13 @@ THE SOFTWARE.
 #define DESTRUCTOR_API __attribute__((destructor))
 #define KERNEL_NAME_LEN_MAX 128
 
+#define ONLOAD_TRACE(str) \
+  if (getenv("ROCP_ONLOAD_TRACE")) do { \
+    std::cout << "PID(" << GetPid() << "): PROF_TOOL_LIB::" << __FUNCTION__ << " " << str << std::endl << std::flush; \
+  } while(0);
+#define ONLOAD_TRACE_BEG() ONLOAD_TRACE("begin")
+#define ONLOAD_TRACE_END() ONLOAD_TRACE("end")
+
 // Disoatch callback data type
 struct callbacks_data_t {
   rocprofiler_feature_t* features;
@@ -885,6 +892,8 @@ rocprofiler_hsa_callbacks_t hsa_callbacks {
 // Tool constructor
 extern "C" PUBLIC_API void OnLoadToolProp(rocprofiler_settings_t* settings)
 {
+  ONLOAD_TRACE_BEG();
+
   if (pthread_mutex_lock(&mutex) != 0) {
     perror("pthread_mutex_lock");
     abort();
@@ -1189,10 +1198,14 @@ extern "C" PUBLIC_API void OnLoadToolProp(rocprofiler_settings_t* settings)
     if (err) { errno = err; perror("pthread_attr_init"); abort(); }
     err = pthread_create(&thread, &attr, monitor_thr_fun, NULL);
   }
+
+  ONLOAD_TRACE_END();
 }
 
 // Tool destructor
-extern "C" PUBLIC_API void OnUnloadTool() {
+void rocprofiler_unload(bool is_destr) {
+  ONLOAD_TRACE("begin loaded(" << is_loaded << ") destr(" << is_destr << ")");
+
   if (pthread_mutex_lock(&mutex) != 0) {
     perror("pthread_mutex_lock");
     abort();
@@ -1204,6 +1217,8 @@ extern "C" PUBLIC_API void OnUnloadTool() {
     abort();
   }
 
+  if (is_destr) CTX_OUTSTANDING_WAIT = 0;
+
   // Unregister dispatch callback
   rocprofiler_remove_queue_callbacks();
 
@@ -1237,10 +1252,20 @@ extern "C" PUBLIC_API void OnUnloadTool() {
   kernel_string_vec = NULL;
   delete range_vec;
   range_vec = NULL;
-  delete context_array;
+  if (!is_destr) delete context_array;
   context_array = NULL;
+
+  ONLOAD_TRACE_END();
+}
+
+extern "C" PUBLIC_API void OnUnloadTool() {
+  ONLOAD_TRACE("begin loaded(" << is_loaded << ")");
+  if (is_loaded == true) rocprofiler_unload(false);
+  ONLOAD_TRACE_END();
 }
 
 extern "C" DESTRUCTOR_API void destructor() {
-  if (is_loaded == true) OnUnloadTool();
+  ONLOAD_TRACE("begin loaded(" << is_loaded << ")");
+  if (is_loaded == true) rocprofiler_unload(true);
+  ONLOAD_TRACE_END();
 }