Minor updates for transpose, timemory submodule, roctracer, and omnitrace exe (#4)

* transpose usage message

* timemory submodule update

* roctracer updates

- Changes to verbosity of roctracer::shutdown
- protect_flush_activity prevents deadlock when error in callback

* Removed linking to timemory-cxx in omnitrace

- omnitrace exe does not link to `timemory-cxx` target

examples/transpose/transpose.cpp

@@ -187,6 +187,18 @@ main(int argc, char** argv)
     int    nthreads = 2;
     int    nitr     = 5000;
     size_t nsync    = 10;
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{ argv[i] };
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            fprintf(stderr,
+                    "usage: transpose [NUM_THREADS (%i)] [NUM_ITERATION (%i)] "
+                    "[SYNC_EVERY_N_ITERATIONS (%zu)]\n",
+                    nthreads, nitr, nsync);
+            exit(EXIT_SUCCESS);
+        }
+    }
     if(argc > 1) nthreads = atoi(argv[1]);
     if(argc > 2) nitr = atoi(argv[2]);
     if(argc > 3) nsync = atoll(argv[3]);

source/bin/omnitrace/CMakeLists.txt

@@ -22,9 +22,9 @@ target_link_libraries(
     omnitrace-exe
     PRIVATE omnitrace::omnitrace-headers
             omnitrace::omnitrace-dyninst
-            omnitrace::omnitrace-timemory
             omnitrace::omnitrace-compile-options
             omnitrace::omnitrace-compile-definitions
+            timemory::timemory-headers
             $<IF:$<BOOL:${OMNITRACE_USE_SANITIZER}>,omnitrace::omnitrace-sanitizer,>)
 
 set_target_properties(

source/bin/omnitrace/omnitrace.cpp

@@ -24,6 +24,8 @@
 #include "fwd.hpp"
 
 #include <timemory/config.hpp>
+#include <timemory/hash.hpp>
+#include <timemory/manager.hpp>
 #include <timemory/settings.hpp>
 
 #include <algorithm>

source/lib/omnitrace/library/components/roctracer.cpp

@@ -24,6 +24,7 @@
 #include "library/components/pthread_gotcha.hpp"
 #include "library/components/roctracer_callbacks.hpp"
 #include "library/config.hpp"
+#include "library/debug.hpp"
 #include "library/defines.hpp"
 #include "library/redirect.hpp"
 #include "library/sampling.hpp"
@@ -35,6 +36,16 @@ namespace tim
 {
 namespace component
 {
+namespace
+{
+auto&
+roctracer_activity_count()
+{
+    static std::atomic<int64_t> _v{ 0 };
+    return _v;
+}
+}  // namespace
+
 void
 roctracer::preinit()
 {
@@ -157,13 +168,15 @@ roctracer::shutdown()
 
     OMNITRACE_VERBOSE_F(1, "shutting down roctracer...\n");
 
-    OMNITRACE_DEBUG_F("executing hip_exec_activity_callbacks\n");
+    OMNITRACE_VERBOSE_F(2, "executing hip_exec_activity_callbacks(0..%zu)\n",
+                        max_supported_threads);
     // make sure all async operations are executed
     for(size_t i = 0; i < max_supported_threads; ++i)
         hip_exec_activity_callbacks(i);
 
     // callback for hsa
-    OMNITRACE_DEBUG_F("executing roctracer_shutdown_routines...\n");
+    OMNITRACE_VERBOSE_F(2, "executing %zu roctracer_shutdown_routines...\n",
+                        roctracer_shutdown_routines().size());
     for(auto& itr : roctracer_shutdown_routines())
         itr.second();
 
@@ -178,12 +191,36 @@ roctracer::shutdown()
 #endif
 
     // ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_ROCTX));
+    OMNITRACE_VERBOSE_F(
+        2, "executing roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API)...\n");
     ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API));
+
+    OMNITRACE_VERBOSE_F(
+        2, "executing roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS)...\n");
     ROCTRACER_CALL(roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS));
-    ROCTRACER_CALL(roctracer_flush_activity());
+
+    if(roctracer_activity_count() == 0)
+    {
+        OMNITRACE_VERBOSE_F(2, "executing roctracer_flush_activity()...\n");
+        ROCTRACER_CALL(roctracer_flush_activity());
+    }
+    else
+    {
+        OMNITRACE_CI_FAIL(true,
+                          "roctracer_activity_count() != 0 (== %li). "
+                          "roctracer::shutdown() most likely called during abort",
+                          roctracer_activity_count().load());
+    }
 
     OMNITRACE_VERBOSE_F(1, "roctracer is shutdown\n");
 }
+
+scope::transient_destructor
+roctracer::protect_flush_activity()
+{
+    return scope::transient_destructor([]() { --roctracer_activity_count(); },
+                                       []() { ++roctracer_activity_count(); });
+}
 }  // namespace component
 }  // namespace tim
 

source/lib/omnitrace/library/components/roctracer.hpp

@@ -33,6 +33,7 @@
 #include <timemory/macros/os.hpp>
 #include <timemory/mpl/type_traits.hpp>
 #include <timemory/mpl/types.hpp>
+#include <timemory/utility/transient_function.hpp>
 
 namespace tim
 {
@@ -64,6 +65,10 @@ struct roctracer
 
     void start();
     void stop();
+
+    // this function protects roctracer_flush_activty from being called
+    // when omnitrace exits during a callback
+    [[nodiscard]] static scope::transient_destructor protect_flush_activity();
 };
 
 #if !defined(OMNITRACE_USE_ROCTRACER)

source/lib/omnitrace/library/components/roctracer_callbacks.cpp

@@ -285,16 +285,20 @@ hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg)
 
     OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
 
-    sampling::block_signals();
+    static thread_local std::once_flag _once{};
+    std::call_once(_once, []() {
+        sampling::block_signals();
+        threading::set_thread_name("omni.roctracer");
+    });
+
+    auto&& _protect = comp::roctracer::protect_flush_activity();
+    (void) _protect;
 
     static const char* copy_op_name     = "hsa_async_copy";
     static const char* dispatch_op_name = "hsa_dispatch";
     static const char* barrier_op_name  = "hsa_barrier";
     const char**       _name            = nullptr;
 
-    static thread_local auto _once = (threading::set_thread_name("omni.roctracer"), true);
-    (void) _once;
-
     switch(op)
     {
         case HSA_OP_ID_DISPATCH: _name = &dispatch_op_name; break;
@@ -629,10 +633,14 @@ hip_activity_callback(const char* begin, const char* end, void*)
 
     OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
 
-    sampling::block_signals();
+    static thread_local std::once_flag _once{};
+    std::call_once(_once, []() {
+        sampling::block_signals();
+        threading::set_thread_name("omni.roctracer");
+    });
 
-    static thread_local auto _once = (threading::set_thread_name("omni.roctracer"), true);
-    (void) _once;
+    auto&& _protect = comp::roctracer::protect_flush_activity();
+    (void) _protect;
 
     using Device = critical_trace::Device;
     using Phase  = critical_trace::Phase;