unloading and flush fix

src/core/loader.h

@@ -19,8 +19,12 @@ class Loader {
   public:
   typedef std::mutex mutex_t;
 
+  template <class fun_t>
+  fun_t* GetFun(const char* fun_name) { return (fun_t*) dlsym(handle_, fun_name); }
+
+  protected:
   Loader(const char* lib_name) {
-    handle_ = dlopen(lib_name, RTLD_LAZY|RTLD_NODELETE);
+    handle_ = dlopen(lib_name, RTLD_LAZY|RTLD_NOLOAD);
     if (handle_ == NULL) {
       fprintf(stderr, "roctracer: Loading '%s' failed, %s\n", lib_name, dlerror());
       abort();
@@ -31,9 +35,6 @@ class Loader {
     if (handle_ != NULL) dlclose(handle_);
   }
 
-  template <class fun_t>
-  fun_t* GetFun(const char* fun_name) { return (fun_t*) dlsym(handle_, fun_name); }
-
   protected:
   static mutex_t mutex_;
 
@@ -63,6 +64,14 @@ class HipLoader : protected Loader {
     return *instance_;
   }
 
+  RegisterApiCallback_t* RegisterApiCallback;
+  RemoveApiCallback_t* RemoveApiCallback;
+  RegisterActivityCallback_t* RegisterActivityCallback;
+  RemoveActivityCallback_t* RemoveActivityCallback;
+  KernelNameRef_t* KernelNameRef;
+  ApiName_t* ApiName;
+
+  protected:
   HipLoader() : Loader("libhip_hcc.so") {
     RegisterApiCallback = GetFun<RegisterApiCallback_t>("hipRegisterApiCallback");
     RemoveApiCallback = GetFun<RemoveApiCallback_t>("hipRemoveApiCallback");
@@ -72,13 +81,6 @@ class HipLoader : protected Loader {
     ApiName = GetFun<ApiName_t>("hipApiName");
   }
 
-  RegisterApiCallback_t* RegisterApiCallback;
-  RemoveApiCallback_t* RemoveApiCallback;
-  RegisterActivityCallback_t* RegisterActivityCallback;
-  RemoveActivityCallback_t* RemoveActivityCallback;
-  KernelNameRef_t* KernelNameRef;
-  ApiName_t* ApiName;
-
   private:
   static std::atomic<HipLoader*> instance_;
 };
@@ -106,6 +108,11 @@ class HccLoader : protected Loader {
     return *obj;
   }
 
+  InitActivityCallback_t* InitActivityCallback;
+  EnableActivityCallback_t* EnableActivityCallback;
+  GetCmdName_t* GetCmdName;
+
+  protected:
   HccLoader() : Loader("libmcwamp_hsa.so") {
     // Kalmar::CLAMP::InitActivityCallback
     InitActivityCallback = GetFun<InitActivityCallback_t>("InitActivityCallbackImpl");
@@ -115,10 +122,6 @@ class HccLoader : protected Loader {
     GetCmdName = GetFun<GetCmdName_t>("GetCmdNameImpl");
   }
 
-  InitActivityCallback_t* InitActivityCallback;
-  EnableActivityCallback_t* EnableActivityCallback;
-  GetCmdName_t* GetCmdName;
-
   private:
   static std::atomic<HccLoader*> instance_;
 };
@@ -141,14 +144,15 @@ class KfdLoader : protected Loader {
     return *instance_;
   }
 
+  RegisterApiCallback_t* RegisterApiCallback;
+  RemoveApiCallback_t* RemoveApiCallback;
+
+  protected:
   KfdLoader() : Loader("libkfdwrapper64.so") {
     RegisterApiCallback = GetFun<RegisterApiCallback_t>("RegisterApiCallback");
     RemoveApiCallback = GetFun<RemoveApiCallback_t>("RemoveApiCallback");
   }
 
-  RegisterApiCallback_t* RegisterApiCallback;
-  RemoveApiCallback_t* RemoveApiCallback;
-
   private:
   static std::atomic<KfdLoader*> instance_;
 };
@@ -171,14 +175,15 @@ class RocTxLoader : protected Loader {
     return *instance_;
   }
 
+  RegisterApiCallback_t* RegisterApiCallback;
+  RemoveApiCallback_t* RemoveApiCallback;
+
+  protected:
   RocTxLoader() : Loader("libroctx64.so") {
     RegisterApiCallback = GetFun<RegisterApiCallback_t>("RegisterApiCallback");
     RemoveApiCallback = GetFun<RemoveApiCallback_t>("RemoveApiCallback");
   }
 
-  RegisterApiCallback_t* RegisterApiCallback;
-  RemoveApiCallback_t* RemoveApiCallback;
-
   private:
   static std::atomic<RocTxLoader*> instance_;
 };

src/core/roctracer.cpp

@@ -88,7 +88,9 @@ THE SOFTWARE.
   (void)err;                                                                                       \
   return X;
 
+#ifndef onload_debug
 #define onload_debug false
+#endif
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 // Mark callback
@@ -464,6 +466,10 @@ FILE* open_output_file(const char* prefix, const char* name) {
   return file_handle;
 }
 
+void close_output_file(FILE* file_handle) {
+  if ((file_handle != NULL) && (file_handle != stdout)) fclose(file_handle);
+}
+
 FILE* kernel_file_handle = NULL;
 void hsa_kernel_handler(::proxy::Tracker::entry_t* entry) {
   static uint64_t index = 0;
@@ -998,14 +1004,16 @@ PUBLIC_API bool roctracer_load(HsaApiTable* table, uint64_t runtime_version, uin
 }
 
 PUBLIC_API void roctracer_unload(bool destruct) {
-  if (onload_debug) printf("LIB roctracer_unload (%d)\n", (int)destruct); fflush(stdout);
   static bool is_unloaded = false;
-  if (is_unloaded) return;
+
+  if (onload_debug) printf("LIB roctracer_unload (%d, %d)\n", (int)destruct, (int)is_unloaded); fflush(stdout);
+  if (destruct == false) return;
+  if (is_unloaded == true) return;
   is_unloaded = true;
 
-  //if (destruct == false) roctracer::trace_buffer.Flush();
-  if ((roctracer::hsa_support::output_prefix != NULL) && (roctracer::kernel_file_handle != NULL)) fclose(roctracer::kernel_file_handle);
-  if (onload_debug) printf("LIB roctracer_unload (%d) end\n", (int)destruct); fflush(stdout);
+  roctracer::trace_buffer.Flush();
+  roctracer::close_output_file(roctracer::kernel_file_handle);
+  if (onload_debug) printf("LIB roctracer_unload end\n"); fflush(stdout);
 }
 
 PUBLIC_API bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,

src/core/trace_buffer.h

@@ -90,7 +90,7 @@ class TraceBuffer {
     PTHREAD_CALL(pthread_join(work_thread_, &res));
     if (res != PTHREAD_CANCELED) abort_run("~TraceBuffer: consumer thread wasn't stopped correctly");
 
-    flush_buf();
+    Flush();
   }
 
 
@@ -102,13 +102,14 @@ class TraceBuffer {
   }
 
   void Flush() {
-    std::lock_guard<mutex_t> lck(mutex_);
     flush_buf();
   }
 
   private:
   void flush_buf() {
+    std::lock_guard<mutex_t> lck(mutex_);
     const bool is_flushed = atomic_flag_test_and_set_explicit(&is_flushed_, std::memory_order_acquire);
+
     if (is_flushed == false) {
       for (flush_prm_t* prm = flush_prm_arr_; prm < flush_prm_arr_ + flush_prm_count_; prm++) {
         uint32_t type = prm->type;
@@ -131,7 +132,7 @@ class TraceBuffer {
   }
 
   inline Entry* allocate_fun() {
-    Entry* ptr = (Entry*) calloc(size_, sizeof(Entry));
+    Entry* ptr = (Entry*) malloc(size_ * sizeof(Entry));
     if (ptr == NULL) abort_run("TraceBuffer::allocate_fun: calloc failed");
     //memset(ptr, 0, size_ * sizeof(Entry));
     return ptr;