multiple processes tracing support

src/core/loader.h

@@ -5,46 +5,59 @@
 #include <mutex>
 #include <dlfcn.h>
 
-#define LOADER_INSTANTIATE() \
-  std::atomic<roctracer::HipLoader*> roctracer::HipLoader::instance_{}; \
-  std::atomic<roctracer::HccLoader*> roctracer::HccLoader::instance_{}; \
-  std::atomic<roctracer::KfdLoader*> roctracer::KfdLoader::instance_{}; \
-  std::atomic<roctracer::RocTxLoader*> roctracer::RocTxLoader::instance_{}; \
-  roctracer::Loader::mutex_t roctracer::Loader::mutex_;
-
 namespace roctracer {
 
 // Base runtime loader class
-class Loader {
+template <class T>
+class BaseLoader : public T {
   public:
   typedef std::mutex mutex_t;
+  typedef BaseLoader<T> loader_t;
 
   template <class fun_t>
   fun_t* GetFun(const char* fun_name) { return (fun_t*) dlsym(handle_, fun_name); }
 
-  protected:
-  Loader(const char* lib_name) {
-    handle_ = dlopen(lib_name, RTLD_LAZY|RTLD_NOLOAD);
-    if (handle_ == NULL) {
-      fprintf(stderr, "roctracer: Loading '%s' failed, %s\n", lib_name, dlerror());
-      abort();
+  static inline loader_t& Instance(const bool& preload = false) {
+    loader_t* obj = instance_.load(std::memory_order_acquire);
+    if (obj == NULL) {
+      std::lock_guard<mutex_t> lck(mutex_);
+      if (instance_.load(std::memory_order_relaxed) == NULL) {
+        obj = new loader_t(preload);
+        instance_.store(obj, std::memory_order_release);
+      }
     }
+    return *instance_;
   }
 
-  ~Loader() {
+  static loader_t* GetRef() { return instance_; }
+
+  private:
+  BaseLoader(bool preload) {
+    const int flags = (preload) ? RTLD_LAZY : RTLD_LAZY|RTLD_NOLOAD;
+    handle_ = dlopen(lib_name_, flags);
+    if (handle_ == NULL) {
+      fprintf(stderr, "roctracer: Loading '%s' failed, preload(%d), %s\n", lib_name_, (int)preload, dlerror());
+      abort();
+    }
+
+    T::init(this);
+  }
+
+  ~BaseLoader() {
     if (handle_ != NULL) dlclose(handle_);
   }
 
-  protected:
   static mutex_t mutex_;
-
-  private:
+  static const char* lib_name_;
+  static std::atomic<loader_t*> instance_;
   void* handle_;
 };
 
 // HIP runtime library loader class
-class HipLoader : protected Loader {
+class HipApi {
   public:
+  typedef BaseLoader<HipApi> Loader;
+
   typedef decltype(hipRegisterApiCallback) RegisterApiCallback_t;
   typedef decltype(hipRemoveApiCallback) RemoveApiCallback_t;
   typedef decltype(hipRegisterActivityCallback) RegisterActivityCallback_t;
@@ -52,18 +65,6 @@ class HipLoader : protected Loader {
   typedef decltype(hipKernelNameRef) KernelNameRef_t;
   typedef decltype(hipApiName) ApiName_t;
 
-  static HipLoader& Instance() {
-    HipLoader* obj = instance_.load(std::memory_order_acquire);
-    if (obj == NULL) {
-      std::lock_guard<mutex_t> lck(mutex_);
-      if (instance_.load(std::memory_order_relaxed) == NULL) {
-        obj = new HipLoader();
-        instance_.store(obj, std::memory_order_release);
-      }
-    }
-    return *instance_;
-  }
-
   RegisterApiCallback_t* RegisterApiCallback;
   RemoveApiCallback_t* RemoveApiCallback;
   RegisterActivityCallback_t* RegisterActivityCallback;
@@ -72,122 +73,89 @@ class HipLoader : protected Loader {
   ApiName_t* ApiName;
 
   protected:
-  HipLoader() : Loader("libhip_hcc.so") {
-    RegisterApiCallback = GetFun<RegisterApiCallback_t>("hipRegisterApiCallback");
-    RemoveApiCallback = GetFun<RemoveApiCallback_t>("hipRemoveApiCallback");
-    RegisterActivityCallback = GetFun<RegisterActivityCallback_t>("hipRegisterActivityCallback");
-    RemoveActivityCallback = GetFun<RemoveActivityCallback_t>("hipRemoveActivityCallback");
-    KernelNameRef = GetFun<KernelNameRef_t>("hipKernelNameRef");
-    ApiName = GetFun<ApiName_t>("hipApiName");
+  void init(Loader* loader) {
+    RegisterApiCallback = loader->GetFun<RegisterApiCallback_t>("hipRegisterApiCallback");
+    RemoveApiCallback = loader->GetFun<RemoveApiCallback_t>("hipRemoveApiCallback");
+    RegisterActivityCallback = loader->GetFun<RegisterActivityCallback_t>("hipRegisterActivityCallback");
+    RemoveActivityCallback = loader->GetFun<RemoveActivityCallback_t>("hipRemoveActivityCallback");
+    KernelNameRef = loader->GetFun<KernelNameRef_t>("hipKernelNameRef");
+    ApiName = loader->GetFun<ApiName_t>("hipApiName");
   }
-
-  private:
-  static std::atomic<HipLoader*> instance_;
 };
 
 // HCC runtime library loader class
-class HccLoader : protected Loader {
+class HccApi {
   public:
-  typedef std::mutex mutex_t;
+  typedef BaseLoader<HccApi> Loader;
 
   typedef decltype(Kalmar::CLAMP::InitActivityCallback) InitActivityCallback_t;
   typedef decltype(Kalmar::CLAMP::EnableActivityCallback) EnableActivityCallback_t;
   typedef decltype(Kalmar::CLAMP::GetCmdName) GetCmdName_t;
 
-  static HccLoader* GetRef() { return instance_; }
-
-  static HccLoader& Instance() {
-    HccLoader* obj = instance_.load(std::memory_order_acquire);
-    if (obj == NULL) {
-      std::lock_guard<mutex_t> lck(mutex_);
-      if (instance_.load(std::memory_order_relaxed) == NULL) {
-        obj = new HccLoader();
-        instance_.store(obj, std::memory_order_release);
-      }
-    }
-    return *obj;
-  }
-
   InitActivityCallback_t* InitActivityCallback;
   EnableActivityCallback_t* EnableActivityCallback;
   GetCmdName_t* GetCmdName;
 
   protected:
-  HccLoader() : Loader("libmcwamp_hsa.so") {
+  void init(Loader* loader) {
     // Kalmar::CLAMP::InitActivityCallback
-    InitActivityCallback = GetFun<InitActivityCallback_t>("InitActivityCallbackImpl");
+    InitActivityCallback = loader->GetFun<InitActivityCallback_t>("InitActivityCallbackImpl");
     // Kalmar::CLAMP::EnableActivityIdCallback
-    EnableActivityCallback = GetFun<EnableActivityCallback_t>("EnableActivityCallbackImpl");
+    EnableActivityCallback = loader->GetFun<EnableActivityCallback_t>("EnableActivityCallbackImpl");
     // Kalmar::CLAMP::GetCmdName
-    GetCmdName = GetFun<GetCmdName_t>("GetCmdNameImpl");
+    GetCmdName = loader->GetFun<GetCmdName_t>("GetCmdNameImpl");
   }
-
-  private:
-  static std::atomic<HccLoader*> instance_;
 };
 
 // KFD runtime library loader class
-class KfdLoader : protected Loader {
+class KfdApi {
   public:
+  typedef BaseLoader<KfdApi> Loader;
+
   typedef bool (RegisterApiCallback_t)(uint32_t op, void* callback, void* arg);
   typedef bool (RemoveApiCallback_t)(uint32_t op);
 
-  static KfdLoader& Instance() {
-    KfdLoader* obj = instance_.load(std::memory_order_acquire);
-    if (obj == NULL) {
-      std::lock_guard<mutex_t> lck(mutex_);
-      if (instance_.load(std::memory_order_relaxed) == NULL) {
-        obj = new KfdLoader();
-        instance_.store(obj, std::memory_order_release);
-      }
-    }
-    return *instance_;
-  }
-
   RegisterApiCallback_t* RegisterApiCallback;
   RemoveApiCallback_t* RemoveApiCallback;
 
   protected:
-  KfdLoader() : Loader("libkfdwrapper64.so") {
-    RegisterApiCallback = GetFun<RegisterApiCallback_t>("RegisterApiCallback");
-    RemoveApiCallback = GetFun<RemoveApiCallback_t>("RemoveApiCallback");
+  void init(Loader* loader) {
+    RegisterApiCallback = loader->GetFun<RegisterApiCallback_t>("RegisterApiCallback");
+    RemoveApiCallback = loader->GetFun<RemoveApiCallback_t>("RemoveApiCallback");
   }
-
-  private:
-  static std::atomic<KfdLoader*> instance_;
 };
 
 // rocTX runtime library loader class
-class RocTxLoader : protected Loader {
+class RocTxApi {
   public:
+  typedef BaseLoader<RocTxApi> Loader;
+
   typedef bool (RegisterApiCallback_t)(uint32_t op, void* callback, void* arg);
   typedef bool (RemoveApiCallback_t)(uint32_t op);
 
-  static RocTxLoader& Instance() {
-    RocTxLoader* obj = instance_.load(std::memory_order_acquire);
-    if (obj == NULL) {
-      std::lock_guard<mutex_t> lck(mutex_);
-      if (instance_.load(std::memory_order_relaxed) == NULL) {
-        obj = new RocTxLoader();
-        instance_.store(obj, std::memory_order_release);
-      }
-    }
-    return *instance_;
-  }
-
   RegisterApiCallback_t* RegisterApiCallback;
   RemoveApiCallback_t* RemoveApiCallback;
 
   protected:
-  RocTxLoader() : Loader("libroctx64.so") {
-    RegisterApiCallback = GetFun<RegisterApiCallback_t>("RegisterApiCallback");
-    RemoveApiCallback = GetFun<RemoveApiCallback_t>("RemoveApiCallback");
+  void init(Loader* loader) {
+    RegisterApiCallback = loader->GetFun<RegisterApiCallback_t>("RegisterApiCallback");
+    RemoveApiCallback = loader->GetFun<RemoveApiCallback_t>("RemoveApiCallback");
   }
-
-  private:
-  static std::atomic<RocTxLoader*> instance_;
 };
 
+typedef BaseLoader<HipApi> HipLoader;
+typedef BaseLoader<HccApi> HccLoader;
+typedef BaseLoader<KfdApi> KfdLoader;
+typedef BaseLoader<RocTxApi> RocTxLoader;
+
 } // namespace roctracer
 
+#define LOADER_INSTANTIATE() \
+  template<class T> typename roctracer::BaseLoader<T>::mutex_t roctracer::BaseLoader<T>::mutex_; \
+  template<class T> std::atomic<roctracer::BaseLoader<T>*> roctracer::BaseLoader<T>::instance_{}; \
+  template<> const char* roctracer::HipLoader::lib_name_ = "libhip_hcc.so"; \
+  template<> const char* roctracer::HccLoader::lib_name_ = "libmcwamp_hsa.so"; \
+  template<> const char* roctracer::KfdLoader::lib_name_ = "libkfdwrapper64.so"; \
+  template<> const char* roctracer::RocTxLoader::lib_name_ = "libroctx64.so";
+
 #endif // SRC_CORE_LOADER_H_

src/core/roctracer.cpp

@@ -92,6 +92,9 @@ THE SOFTWARE.
 #define onload_debug false
 #endif
 
+static inline uint32_t GetPid() { return syscall(__NR_getpid); }
+static inline uint32_t GetTid() { return syscall(__NR_gettid); }
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 // Mark callback
 //
@@ -454,7 +457,7 @@ FILE* open_output_file(const char* prefix, const char* name) {
   FILE* file_handle = NULL;
   if (prefix != NULL) {
     std::ostringstream oss;
-    oss << prefix << "/" << name;
+    oss << prefix << "/" << GetPid() << "_" << name;
     file_handle = fopen(oss.str().c_str(), "w");
     if (file_handle == NULL) {
       std::ostringstream errmsg;

test/tool/tracer_tool.cpp

@@ -136,7 +136,7 @@ void hsa_api_callback(
 void hsa_api_flush_cb(hsa_api_trace_entry_t* entry) {
   std::ostringstream os;
   os << entry->begin << ":" << entry->end << " " << entry->pid << ":" << entry->tid << " " << hsa_api_data_pair_t(entry->cid, entry->data);
-  fprintf(hsa_api_file_handle, "%s\n", os.str().c_str());
+  fprintf(hsa_api_file_handle, "%s\n", os.str().c_str()); fflush(hsa_api_file_handle);
 }
 
 void hsa_activity_callback(
@@ -145,7 +145,7 @@ void hsa_activity_callback(
   void* arg)
 {
   static uint64_t index = 0;
-  fprintf(hsa_async_copy_file_handle, "%lu:%lu async-copy%lu\n", record->begin_ns, record->end_ns, index);
+  fprintf(hsa_async_copy_file_handle, "%lu:%lu async-copy%lu\n", record->begin_ns, record->end_ns, index); fflush(hsa_async_copy_file_handle);
   index++;
 }
 
@@ -356,7 +356,7 @@ FILE* open_output_file(const char* prefix, const char* name) {
   FILE* file_handle = NULL;
   if (prefix != NULL) {
     std::ostringstream oss;
-    oss << prefix << "/" << name;
+    oss << prefix << "/" << GetPid() << "_" << name;
     file_handle = fopen(oss.str().c_str(), "w");
     if (file_handle == NULL) {
       std::ostringstream errmsg;
@@ -409,7 +409,7 @@ extern "C" PUBLIC_API bool OnLoad(HsaApiTable* table, uint64_t runtime_version,
   // API trace vector
   std::vector<std::string> hsa_api_vec;
 
-  printf("ROCTracer: "); fflush(stdout);
+  printf("ROCTracer (pid=%d): ", (int)GetPid()); fflush(stdout);
   // XML input
   const char* xml_name = getenv("ROCP_INPUT");
   if (xml_name != NULL) {