From c3f2a22a63bb41be70fe2ddf644c3855deb346b1 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Wed, 21 Aug 2019 11:56:33 -0500 Subject: [PATCH] multiple processes tracing support [ROCm/roctracer commit: 8cfde554b184c399cebcf9e0cc96365d19a2f4db] --- projects/roctracer/src/core/loader.h | 172 ++++++++----------- projects/roctracer/src/core/roctracer.cpp | 5 +- projects/roctracer/test/tool/tracer_tool.cpp | 8 +- 3 files changed, 78 insertions(+), 107 deletions(-) diff --git a/projects/roctracer/src/core/loader.h b/projects/roctracer/src/core/loader.h index 3c48ad4448..d15d2f5580 100644 --- a/projects/roctracer/src/core/loader.h +++ b/projects/roctracer/src/core/loader.h @@ -5,46 +5,59 @@ #include #include -#define LOADER_INSTANTIATE() \ - std::atomic roctracer::HipLoader::instance_{}; \ - std::atomic roctracer::HccLoader::instance_{}; \ - std::atomic roctracer::KfdLoader::instance_{}; \ - std::atomic roctracer::RocTxLoader::instance_{}; \ - roctracer::Loader::mutex_t roctracer::Loader::mutex_; - namespace roctracer { // Base runtime loader class -class Loader { +template +class BaseLoader : public T { public: typedef std::mutex mutex_t; + typedef BaseLoader loader_t; template fun_t* GetFun(const char* fun_name) { return (fun_t*) dlsym(handle_, fun_name); } - protected: - Loader(const char* lib_name) { - handle_ = dlopen(lib_name, RTLD_LAZY|RTLD_NOLOAD); - if (handle_ == NULL) { - fprintf(stderr, "roctracer: Loading '%s' failed, %s\n", lib_name, dlerror()); - abort(); + static inline loader_t& Instance(const bool& preload = false) { + loader_t* obj = instance_.load(std::memory_order_acquire); + if (obj == NULL) { + std::lock_guard lck(mutex_); + if (instance_.load(std::memory_order_relaxed) == NULL) { + obj = new loader_t(preload); + instance_.store(obj, std::memory_order_release); + } } + return *instance_; } - ~Loader() { + static loader_t* GetRef() { return instance_; } + + private: + BaseLoader(bool preload) { + const int flags = (preload) ? RTLD_LAZY : RTLD_LAZY|RTLD_NOLOAD; + handle_ = dlopen(lib_name_, flags); + if (handle_ == NULL) { + fprintf(stderr, "roctracer: Loading '%s' failed, preload(%d), %s\n", lib_name_, (int)preload, dlerror()); + abort(); + } + + T::init(this); + } + + ~BaseLoader() { if (handle_ != NULL) dlclose(handle_); } - protected: static mutex_t mutex_; - - private: + static const char* lib_name_; + static std::atomic instance_; void* handle_; }; // HIP runtime library loader class -class HipLoader : protected Loader { +class HipApi { public: + typedef BaseLoader Loader; + typedef decltype(hipRegisterApiCallback) RegisterApiCallback_t; typedef decltype(hipRemoveApiCallback) RemoveApiCallback_t; typedef decltype(hipRegisterActivityCallback) RegisterActivityCallback_t; @@ -52,18 +65,6 @@ class HipLoader : protected Loader { typedef decltype(hipKernelNameRef) KernelNameRef_t; typedef decltype(hipApiName) ApiName_t; - static HipLoader& Instance() { - HipLoader* obj = instance_.load(std::memory_order_acquire); - if (obj == NULL) { - std::lock_guard lck(mutex_); - if (instance_.load(std::memory_order_relaxed) == NULL) { - obj = new HipLoader(); - instance_.store(obj, std::memory_order_release); - } - } - return *instance_; - } - RegisterApiCallback_t* RegisterApiCallback; RemoveApiCallback_t* RemoveApiCallback; RegisterActivityCallback_t* RegisterActivityCallback; @@ -72,122 +73,89 @@ class HipLoader : protected Loader { ApiName_t* ApiName; protected: - HipLoader() : Loader("libhip_hcc.so") { - RegisterApiCallback = GetFun("hipRegisterApiCallback"); - RemoveApiCallback = GetFun("hipRemoveApiCallback"); - RegisterActivityCallback = GetFun("hipRegisterActivityCallback"); - RemoveActivityCallback = GetFun("hipRemoveActivityCallback"); - KernelNameRef = GetFun("hipKernelNameRef"); - ApiName = GetFun("hipApiName"); + void init(Loader* loader) { + RegisterApiCallback = loader->GetFun("hipRegisterApiCallback"); + RemoveApiCallback = loader->GetFun("hipRemoveApiCallback"); + RegisterActivityCallback = loader->GetFun("hipRegisterActivityCallback"); + RemoveActivityCallback = loader->GetFun("hipRemoveActivityCallback"); + KernelNameRef = loader->GetFun("hipKernelNameRef"); + ApiName = loader->GetFun("hipApiName"); } - - private: - static std::atomic instance_; }; // HCC runtime library loader class -class HccLoader : protected Loader { +class HccApi { public: - typedef std::mutex mutex_t; + typedef BaseLoader Loader; typedef decltype(Kalmar::CLAMP::InitActivityCallback) InitActivityCallback_t; typedef decltype(Kalmar::CLAMP::EnableActivityCallback) EnableActivityCallback_t; typedef decltype(Kalmar::CLAMP::GetCmdName) GetCmdName_t; - static HccLoader* GetRef() { return instance_; } - - static HccLoader& Instance() { - HccLoader* obj = instance_.load(std::memory_order_acquire); - if (obj == NULL) { - std::lock_guard lck(mutex_); - if (instance_.load(std::memory_order_relaxed) == NULL) { - obj = new HccLoader(); - instance_.store(obj, std::memory_order_release); - } - } - return *obj; - } - InitActivityCallback_t* InitActivityCallback; EnableActivityCallback_t* EnableActivityCallback; GetCmdName_t* GetCmdName; protected: - HccLoader() : Loader("libmcwamp_hsa.so") { + void init(Loader* loader) { // Kalmar::CLAMP::InitActivityCallback - InitActivityCallback = GetFun("InitActivityCallbackImpl"); + InitActivityCallback = loader->GetFun("InitActivityCallbackImpl"); // Kalmar::CLAMP::EnableActivityIdCallback - EnableActivityCallback = GetFun("EnableActivityCallbackImpl"); + EnableActivityCallback = loader->GetFun("EnableActivityCallbackImpl"); // Kalmar::CLAMP::GetCmdName - GetCmdName = GetFun("GetCmdNameImpl"); + GetCmdName = loader->GetFun("GetCmdNameImpl"); } - - private: - static std::atomic instance_; }; // KFD runtime library loader class -class KfdLoader : protected Loader { +class KfdApi { public: + typedef BaseLoader Loader; + typedef bool (RegisterApiCallback_t)(uint32_t op, void* callback, void* arg); typedef bool (RemoveApiCallback_t)(uint32_t op); - static KfdLoader& Instance() { - KfdLoader* obj = instance_.load(std::memory_order_acquire); - if (obj == NULL) { - std::lock_guard lck(mutex_); - if (instance_.load(std::memory_order_relaxed) == NULL) { - obj = new KfdLoader(); - instance_.store(obj, std::memory_order_release); - } - } - return *instance_; - } - RegisterApiCallback_t* RegisterApiCallback; RemoveApiCallback_t* RemoveApiCallback; protected: - KfdLoader() : Loader("libkfdwrapper64.so") { - RegisterApiCallback = GetFun("RegisterApiCallback"); - RemoveApiCallback = GetFun("RemoveApiCallback"); + void init(Loader* loader) { + RegisterApiCallback = loader->GetFun("RegisterApiCallback"); + RemoveApiCallback = loader->GetFun("RemoveApiCallback"); } - - private: - static std::atomic instance_; }; // rocTX runtime library loader class -class RocTxLoader : protected Loader { +class RocTxApi { public: + typedef BaseLoader Loader; + typedef bool (RegisterApiCallback_t)(uint32_t op, void* callback, void* arg); typedef bool (RemoveApiCallback_t)(uint32_t op); - static RocTxLoader& Instance() { - RocTxLoader* obj = instance_.load(std::memory_order_acquire); - if (obj == NULL) { - std::lock_guard lck(mutex_); - if (instance_.load(std::memory_order_relaxed) == NULL) { - obj = new RocTxLoader(); - instance_.store(obj, std::memory_order_release); - } - } - return *instance_; - } - RegisterApiCallback_t* RegisterApiCallback; RemoveApiCallback_t* RemoveApiCallback; protected: - RocTxLoader() : Loader("libroctx64.so") { - RegisterApiCallback = GetFun("RegisterApiCallback"); - RemoveApiCallback = GetFun("RemoveApiCallback"); + void init(Loader* loader) { + RegisterApiCallback = loader->GetFun("RegisterApiCallback"); + RemoveApiCallback = loader->GetFun("RemoveApiCallback"); } - - private: - static std::atomic instance_; }; +typedef BaseLoader HipLoader; +typedef BaseLoader HccLoader; +typedef BaseLoader KfdLoader; +typedef BaseLoader RocTxLoader; + } // namespace roctracer +#define LOADER_INSTANTIATE() \ + template typename roctracer::BaseLoader::mutex_t roctracer::BaseLoader::mutex_; \ + template std::atomic*> roctracer::BaseLoader::instance_{}; \ + template<> const char* roctracer::HipLoader::lib_name_ = "libhip_hcc.so"; \ + template<> const char* roctracer::HccLoader::lib_name_ = "libmcwamp_hsa.so"; \ + template<> const char* roctracer::KfdLoader::lib_name_ = "libkfdwrapper64.so"; \ + template<> const char* roctracer::RocTxLoader::lib_name_ = "libroctx64.so"; + #endif // SRC_CORE_LOADER_H_ diff --git a/projects/roctracer/src/core/roctracer.cpp b/projects/roctracer/src/core/roctracer.cpp index f309bf1fb5..4781e8c9f8 100644 --- a/projects/roctracer/src/core/roctracer.cpp +++ b/projects/roctracer/src/core/roctracer.cpp @@ -92,6 +92,9 @@ THE SOFTWARE. #define onload_debug false #endif +static inline uint32_t GetPid() { return syscall(__NR_getpid); } +static inline uint32_t GetTid() { return syscall(__NR_gettid); } + /////////////////////////////////////////////////////////////////////////////////////////////////// // Mark callback // @@ -454,7 +457,7 @@ FILE* open_output_file(const char* prefix, const char* name) { FILE* file_handle = NULL; if (prefix != NULL) { std::ostringstream oss; - oss << prefix << "/" << name; + oss << prefix << "/" << GetPid() << "_" << name; file_handle = fopen(oss.str().c_str(), "w"); if (file_handle == NULL) { std::ostringstream errmsg; diff --git a/projects/roctracer/test/tool/tracer_tool.cpp b/projects/roctracer/test/tool/tracer_tool.cpp index 45bb57d19c..6be899e520 100644 --- a/projects/roctracer/test/tool/tracer_tool.cpp +++ b/projects/roctracer/test/tool/tracer_tool.cpp @@ -136,7 +136,7 @@ void hsa_api_callback( void hsa_api_flush_cb(hsa_api_trace_entry_t* entry) { std::ostringstream os; os << entry->begin << ":" << entry->end << " " << entry->pid << ":" << entry->tid << " " << hsa_api_data_pair_t(entry->cid, entry->data); - fprintf(hsa_api_file_handle, "%s\n", os.str().c_str()); + fprintf(hsa_api_file_handle, "%s\n", os.str().c_str()); fflush(hsa_api_file_handle); } void hsa_activity_callback( @@ -145,7 +145,7 @@ void hsa_activity_callback( void* arg) { static uint64_t index = 0; - fprintf(hsa_async_copy_file_handle, "%lu:%lu async-copy%lu\n", record->begin_ns, record->end_ns, index); + fprintf(hsa_async_copy_file_handle, "%lu:%lu async-copy%lu\n", record->begin_ns, record->end_ns, index); fflush(hsa_async_copy_file_handle); index++; } @@ -356,7 +356,7 @@ FILE* open_output_file(const char* prefix, const char* name) { FILE* file_handle = NULL; if (prefix != NULL) { std::ostringstream oss; - oss << prefix << "/" << name; + oss << prefix << "/" << GetPid() << "_" << name; file_handle = fopen(oss.str().c_str(), "w"); if (file_handle == NULL) { std::ostringstream errmsg; @@ -409,7 +409,7 @@ extern "C" PUBLIC_API bool OnLoad(HsaApiTable* table, uint64_t runtime_version, // API trace vector std::vector hsa_api_vec; - printf("ROCTracer: "); fflush(stdout); + printf("ROCTracer (pid=%d): ", (int)GetPid()); fflush(stdout); // XML input const char* xml_name = getenv("ROCP_INPUT"); if (xml_name != NULL) {