Integrated perfetto + roctracer (#5)

- hosttrace library automatically collects and merges timestamps for HIP API calls and kernels with the host-side instrumentation
  - mostly eliminates the need for using external rocprof
- added thread_instruction_count in perfetto output
- increased hosttrace min_loop_address_range to 512
- disabled instrumenting functions with dynamic callsites by default
- miscellaneous cmake updates

* roctracer support

- fully integrated perfetto + roctracer outputs
- thread_instruction_count in perfetto
- increased min_loop_address_range to 512
- disabled instrumenting functions with dynamic callsites by default
- updated timemory submodule

* hosttrace_launch_compiler

- support for using an alternative compiler as needed via launch compiler
- elfio added as submodule (not currently used)
- miscellaneous cmake updates

* README update + host/device categories + misc

- timemory fix for TIMEMORY_ROCTRACER_ENABLED
- transpose fix

* papi_tuple_t -> papi_tot_ins

- minor fix to Findroctracer.cmake

src/hosttrace-details.cpp

@@ -272,7 +272,7 @@ find_function(image_t* app_image, const std::string& _name, strset_t _extra)
     }
 
     if(!_func)
-        verbprintf(0, "timemory-run: Unable to find function %s\n", _name.c_str());
+        verbprintf(2, "hosttrace: Unable to find function %s\n", _name.c_str());
 
     return _func;
 }

src/hosttrace.cpp

@@ -30,10 +30,11 @@
 
 static bool     is_driver                                         = false;
 static bool     allow_overlapping                                 = false;
+static bool     instr_dynamic_callsites                           = false;
 static size_t   batch_size                                        = 50;
 static strset_t extra_libs                                        = {};
 static size_t   min_address_range                                 = (1 << 9);  // 512
-static size_t   min_loop_address_range                            = (1 << 6);  // 64
+static size_t   min_loop_address_range                            = (1 << 9);  // 512
 static std::vector<std::pair<uint64_t, string_t>> hash_ids        = {};
 static std::map<string_t, bool>                   use_stubs       = {};
 static std::map<string_t, procedure_t*>           beg_stubs       = {};
@@ -282,6 +283,15 @@ main(int argc, char** argv)
         .count(1)
         .dtype("size_t")
         .action([](parser_t& p) { batch_size = p.get<size_t>("batch-size"); });
+    parser
+        .add_argument({ "--dynamic-callsites" },
+                      "Force instrumentation if a function has dynamic callsites (e.g. "
+                      "function pointers)")
+        .max_count(1)
+        .dtype("boolean")
+        .action([](parser_t& p) {
+            instr_dynamic_callsites = p.get<bool>("dynamic-callsites");
+        });
     parser
         .add_argument({ "-r", "--min-address-range" },
                       "If the address range of a function is less than this value, "
@@ -1361,10 +1371,10 @@ main(int argc, char** argv)
             if(cfg)
                 cfg->getOuterLoops(basic_loop);
 
-            // if the function has dynamic callsites and we are in binary rewrite mode,
-            // force the instrumentation
+            // if the function has dynamic callsites and user specified instrumenting
+            // dynamic callsites, force the instrumentation
             bool _force_instr = false;
-            if(cfg && binary_rewrite)
+            if(cfg && instr_dynamic_callsites)
                 _force_instr = cfg->containsDynamicCallsites();
 
             auto _address_range = module_function{ mod, itr }.address_range;
@@ -1395,26 +1405,12 @@ main(int argc, char** argv)
                            (unsigned long) min_loop_address_range);
                 continue;
             }
-            else if(_address_range >= min_loop_address_range &&
-                    _address_range < min_address_range && _has_loop_entries)
-            {
-                verbprintf(
-                    1,
-                    "Enabling function [min-loop-address-range]: %s / %s despite not "
-                    "satisfy minimum loop address range (address range = %lu, minimum "
-                    "= %lu) because it has at least one loop (found: %lu)\n",
-                    name.m_name.c_str(), name.get().c_str(),
-                    (unsigned long) _address_range,
-                    (unsigned long) min_loop_address_range,
-                    (unsigned long) _num_loop_entries);
-            }
-            else if(_address_range < min_address_range && _force_instr)
+            else if(_force_instr)
             {
                 verbprintf(1,
-                           "Enabling function [min-address-range]: %s / %s despite not "
+                           "Enabling function [dynamic-callsite]: %s / %s despite not "
                            "satisfy minimum address range (address range = %lu, minimum "
-                           "= %lu) because contains dynamic callsites which may not be "
-                           "instrumented in binary rewrite mode\n",
+                           "= %lu) because contains dynamic callsites\n",
                            name.m_name.c_str(), name.get().c_str(),
                            (unsigned long) _address_range,
                            (unsigned long) min_address_range);

src/library.cpp

@@ -15,11 +15,7 @@ get_state()
     return _v;
 }
 
-//--------------------------------------------------------------------------------------//
-
-namespace
-{
-auto
+bool
 get_use_perfetto()
 {
     // if using timemory, default to perfetto being off
@@ -29,7 +25,7 @@ get_use_perfetto()
     return _v;
 }
 
-auto
+bool
 get_use_timemory()
 {
     // default to opposite of whether perfetto setting
@@ -39,11 +35,32 @@ get_use_timemory()
     return _v;
 }
 
+//--------------------------------------------------------------------------------------//
+
+namespace
+{
+size_t&
+get_sample_rate()
+{
+    static auto _v = tim::get_env<size_t>("HOSTTRACE_SAMPLE_RATE", 1);
+    return _v;
+}
+
+std::vector<bool>&
+get_sample_data()
+{
+    static thread_local auto _v = std::vector<bool>{};
+    return _v;
+}
+
 bool&
 get_use_mpi()
 {
-    // this does not enable anything particularly useful when not using timemory
-    static bool _v = tim::get_env("HOSTTRACE_USE_MPI", false, get_use_timemory());
+#if defined(TIMEMORY_USE_MPI)
+    static bool _v = tim::get_env("HOSTTRACE_USE_MPI", false);
+#else
+    static bool _v = false;
+#endif
     return _v;
 }
 
@@ -74,7 +91,7 @@ auto
 ensure_finalization()
 {
     HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);
-    return tim::scope::destructor{ []() { hosttrace_trace_finalize(); } };
+    return scope::destructor{ []() { hosttrace_trace_finalize(); } };
 }
 
 auto&
@@ -103,10 +120,10 @@ get_perfetto_output_filename()
         if(_pos != std::string::npos)
             _tmp.replace(_pos, _key.length(), std::to_string(_val()));
     };
-    _replace("%pid%", []() { return tim::process::get_id(); });
+    _replace("%pid%", []() { return process::get_id(); });
     _replace("%rank%", []() { return tim::mpi::rank(); });
     // backwards compatibility
-    _replace("%p", []() { return tim::process::get_id(); });
+    _replace("%p", []() { return process::get_id(); });
     return _tmp;
 }
 
@@ -179,7 +196,7 @@ hosttrace_init_tooling()
     if(get_use_perfetto() && get_use_mpi())
         tim::settings::time_output() = true;
 
-    auto _cmd = tim::read_command_line(tim::process::get_id());
+    auto _cmd = tim::read_command_line(process::get_id());
     auto _exe = (_cmd.empty()) ? "hosttrace" : _cmd.front();
     auto _pos = _exe.find_last_of('/');
     if(_pos < _exe.length() - 1)
@@ -187,6 +204,10 @@ hosttrace_init_tooling()
 
     tim::timemory_init({ _exe }, "hosttrace-");
 
+    if(get_sample_rate() < 1)
+        get_sample_rate() = 1;
+    get_sample_data().reserve(512);
+
     if(get_use_timemory())
     {
         comp::user_global_bundle::global_init();
@@ -215,9 +236,13 @@ hosttrace_init_tooling()
     }
 
     // always activate gotcha wrappers
-    auto& _fork_gotcha = get_main_bundle();
-    _fork_gotcha->start();
-    assert(_fork_gotcha->get<mpi_gotcha_t>()->get_is_running());
+    auto& _main_bundle = get_main_bundle();
+    _main_bundle->start();
+    assert(_main_bundle->get<mpi_gotcha_t>()->get_is_running());
+#if defined(HOSTTRACE_USE_ROCTRACER)
+    assert(_main_bundle->get<comp::roctracer>() != nullptr);
+    assert(_main_bundle->get<comp::roctracer>()->get_is_running());
+#endif
 
     perfetto::TracingInitArgs               args{};
     perfetto::TraceConfig                   cfg{};
@@ -253,8 +278,18 @@ hosttrace_init_tooling()
         (void) get_perfetto_output_filename();
     }
 
+    static auto _thread_init = [_exe]() {
+        hosttrace_thread_data<hosttrace_thread_bundle_t>::construct(
+            TIMEMORY_JOIN("", _exe, "/thread-", threading::get_id()),
+            quirk::config<quirk::auto_start>{});
+        static thread_local auto _dtor = scope::destructor{ []() {
+            hosttrace_thread_data<hosttrace_thread_bundle_t>::instance()->stop();
+        } };
+        (void) _dtor;
+    };
     // functors for starting and stopping timemory
     static auto _push_timemory = [](const char* name) {
+        _thread_init();
         auto& _data = get_timemory_data();
         // this generates a hash for the raw string array
         auto  _hash   = tim::add_hash_id(tim::string_view_t{ name });
@@ -264,6 +299,19 @@ hosttrace_init_tooling()
         _bundle->start();
     };
 
+    static auto _push_perfetto = [](const char* name) {
+        _thread_init();
+        TRACE_EVENT_BEGIN("host", perfetto::StaticString(name),
+                          [&](perfetto::EventContext ctx) {
+                              // compile-time check
+                              IF_CONSTEXPR(trait::is_available<papi_tot_ins>::value)
+                              {
+                                  ctx.event()->set_thread_instruction_count_absolute(
+                                      papi_tot_ins::record().at(0));
+                              }
+                          });
+    };
+
     static auto _pop_timemory = [](const char* name) {
         auto& _data = get_timemory_data();
         if(_data.bundles.empty())
@@ -278,25 +326,31 @@ hosttrace_init_tooling()
         _data.bundles.pop_back();
     };
 
+    static auto _pop_perfetto = [](const char*) {
+        TRACE_EVENT_END("host", [&](perfetto::EventContext ctx) {
+            IF_CONSTEXPR(trait::is_available<papi_tot_ins>::value)
+            {
+                ctx.event()->set_thread_instruction_count_absolute(
+                    papi_tot_ins::record().at(0));
+            }
+        });
+    };
+
     if(get_use_perfetto() && get_use_timemory())
     {
-        // if both are used, then use perfetto overload for calling lambda to launch
-        // timemory
         get_functors().first = [](const char* name) {
-            TRACE_EVENT_BEGIN("hosttrace", perfetto::StaticString(name),
-                              [&](perfetto::EventContext) { _push_timemory(name); });
+            _push_perfetto(name);
+            _push_timemory(name);
         };
         get_functors().second = [](const char* name) {
-            TRACE_EVENT_END("hosttrace",
-                            [&](perfetto::EventContext) { _pop_timemory(name); });
+            _pop_timemory(name);
+            _pop_perfetto(name);
         };
     }
     else if(get_use_perfetto())
     {
-        get_functors().first = [](const char* name) {
-            TRACE_EVENT_BEGIN("hosttrace", perfetto::StaticString(name));
-        };
-        get_functors().second = [](const char*) { TRACE_EVENT_END("hosttrace"); };
+        get_functors().first  = _push_perfetto;
+        get_functors().second = _pop_perfetto;
     }
     else if(get_use_timemory())
     {
@@ -304,7 +358,7 @@ hosttrace_init_tooling()
         get_functors().second = _pop_timemory;
     }
 
-    if(tim::dmp::rank() == 0)
+    if(dmp::rank() == 0)
     {
         tim::print_env(std::cerr,
                        [](const std::string& _v) { return _v.find("HOSTTRACE_") == 0; });
@@ -339,7 +393,7 @@ hosttrace_init_tooling()
     // ends the tracing session
     static auto _ensure_finalization = ensure_finalization();
 
-    if(tim::dmp::rank() == 0)
+    if(dmp::rank() == 0)
         puts("");
     return true;
 }
@@ -366,7 +420,12 @@ extern "C"
             HOSTTRACE_DEBUG("[%s] %s\n", __FUNCTION__, name);
         }
 
-        get_functors().first(name);
+        static auto                _sample_rate = std::max<size_t>(get_sample_rate(), 1);
+        static thread_local size_t _sample_idx  = 0;
+        auto                       _enabled     = (_sample_idx++ % _sample_rate == 0);
+        get_sample_data().emplace_back(_enabled);
+        if(_enabled)
+            get_functors().first(name);
     }
 
     void hosttrace_pop_trace(const char* name)
@@ -374,7 +433,13 @@ extern "C"
         if(get_state() == State::Active)
         {
             HOSTTRACE_DEBUG("[%s] %s\n", __FUNCTION__, name);
-            get_functors().second(name);
+            auto& _sample_data = get_sample_data();
+            if(!_sample_data.empty())
+            {
+                if(_sample_data.back())
+                    get_functors().second(name);
+                _sample_data.pop_back();
+            }
         }
         else
         {
@@ -396,22 +461,38 @@ extern "C"
 
         HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);
 
-        if(tim::dmp::rank() == 0)
+        if(dmp::rank() == 0)
             puts("");
 
         get_state() = State::Finalized;
 
+        // stop the main bundle and report the high-level metrics
         if(get_main_bundle())
         {
             get_main_bundle()->stop();
-            int64_t _id = (get_use_mpi()) ? tim::dmp::rank() : tim::process::get_id();
+            int64_t           _id = (get_use_mpi()) ? dmp::rank() : process::get_id();
             std::stringstream _ss{};
             _ss << "[" << __FUNCTION__ << "][" << _id << "] " << *get_main_bundle()
                 << "\n";
-            std::cout << _ss.str();
+            std::cerr << _ss.str();
             get_main_bundle().reset();
         }
 
+        // print out thread-data if they are not still running
+        // if they are still running (e.g. thread-pool still alive), the
+        // thread-specific data will be wrong if try to stop them from
+        // the main thread.
+        for(auto& itr : hosttrace_thread_data<hosttrace_thread_bundle_t>::instances())
+        {
+            if(itr && itr->get<comp::wall_clock>() &&
+               !itr->get<comp::wall_clock>()->get_is_running())
+            {
+                std::stringstream _ss{};
+                _ss << *itr << "\n";
+                std::cerr << _ss.str();
+            }
+        }
+
         // ensure that all the MT instances are flushed
         for(auto& itr : hosttrace_timemory_data::instances())
         {
@@ -443,9 +524,14 @@ extern "C"
                 return;
             }
             // Write the trace into a file.
-            fprintf(stderr, "[%s]> Outputting '%s'. Trace data: %lu bytes...\n",
+            fprintf(stderr,
+                    "[%s]> Outputting '%s'. Trace data: %lu B (%.2f KB / %.2f MB / %.2f "
+                    "GB)...\n",
                     __FUNCTION__, get_perfetto_output_filename().c_str(),
-                    (unsigned long) trace_data.size());
+                    (unsigned long) trace_data.size(),
+                    static_cast<double>(trace_data.size()) / units::KB,
+                    static_cast<double>(trace_data.size()) / units::MB,
+                    static_cast<double>(trace_data.size()) / units::GB);
             std::ofstream output{};
             output.open(get_perfetto_output_filename(), std::ios::out | std::ios::binary);
             if(!output)
@@ -472,8 +558,8 @@ extern "C"
                         (attached) ? "y" : "n");
         if(use && !attached)
         {
-            auto& _fork_gotcha = get_main_bundle();
-            _fork_gotcha->start();
+            auto& _main_bundle = get_main_bundle();
+            _main_bundle->start();
             tim::set_env("HOSTTRACE_USE_MPI", "ON", 1);
             get_use_mpi() = true;
             get_state()   = State::DelayedInit;

src/roctracer.cpp

@@ -0,0 +1,317 @@
+
+#include "roctracer.hpp"
+#include "library.hpp"
+
+#include <roctracer.h>
+#include <roctracer_hcc.h>
+#include <roctracer_hip.h>
+
+// Macro to check ROC-tracer calls status
+#define ROCTRACER_CALL(call)                                                             \
+    do                                                                                   \
+    {                                                                                    \
+        int err = call;                                                                  \
+        if(err != 0)                                                                     \
+        {                                                                                \
+            std::cerr << roctracer_error_string() << " in: " << #call << std::flush;     \
+        }                                                                                \
+    } while(0)
+
+using roctracer_bundle_t = tim::component_tuple<comp::roctracer_data>;
+
+namespace units = tim::units;
+
+namespace
+{
+static auto&
+get_roctracer_kernels()
+{
+    static auto _v = std::unordered_set<uint64_t>{};
+    return _v;
+}
+
+static auto&
+get_roctracer_data_map()
+{
+    static auto _v = std::unordered_map<uint64_t, roctracer_bundle_t>{};
+    return _v;
+}
+
+static auto&
+get_roctracer_key_map()
+{
+    static auto _v = std::unordered_map<uint64_t, const char*>{};
+    return _v;
+}
+
+using data_type_mutex_t = std::decay_t<decltype(get_roctracer_data_map())>;
+
+// HIP API callback function
+void
+hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg)
+{
+    if(!trait::runtime_enabled<comp::roctracer>::get())
+        return;
+    const hip_api_data_t* data = reinterpret_cast<const hip_api_data_t*>(callback_data);
+    HOSTTRACE_DEBUG("<%-30s id(%u)\tcorrelation_id(%lu) %s>\n",
+                    roctracer_op_string(ACTIVITY_DOMAIN_HIP_API, cid, 0), cid,
+                    data->correlation_id,
+                    (data->phase == ACTIVITY_API_PHASE_ENTER) ? "on-enter" : "on-exit");
+    if(data->phase == ACTIVITY_API_PHASE_ENTER)
+    {
+        switch(cid)
+        {
+            case HIP_API_ID___hipPushCallConfiguration:
+            case HIP_API_ID___hipPopCallConfiguration: break;
+            case HIP_API_ID_hipLaunchKernel: {
+                const char* _name =
+                    hipKernelNameRefByPtr(data->args.hipLaunchKernel.function_address,
+                                          data->args.hipLaunchKernel.stream);
+                tim::auto_lock_t _lk{ tim::type_mutex<data_type_mutex_t>() };
+                get_roctracer_kernels().emplace(data->correlation_id);
+                if(get_use_perfetto())
+                {
+                    get_roctracer_key_map().emplace(data->correlation_id, _name);
+                }
+                if(get_use_timemory())
+                {
+                    get_roctracer_data_map().emplace(
+                        data->correlation_id,
+                        roctracer_bundle_t{ tim::static_string{ _name }() });
+                }
+                break;
+            }
+            case HIP_API_ID_hipModuleLaunchKernel: {
+                const char* _name = hipKernelNameRef(data->args.hipModuleLaunchKernel.f);
+                tim::auto_lock_t _lk{ tim::type_mutex<data_type_mutex_t>() };
+                get_roctracer_kernels().emplace(data->correlation_id);
+                if(get_use_perfetto())
+                {
+                    get_roctracer_key_map().emplace(data->correlation_id, _name);
+                }
+                if(get_use_timemory())
+                {
+                    get_roctracer_data_map().emplace(
+                        data->correlation_id,
+                        roctracer_bundle_t{ tim::static_string{ _name }(),
+                                            tim::scope::get_default() });
+                }
+                break;
+            }
+            default: {
+                tim::auto_lock_t _lk{ tim::type_mutex<data_type_mutex_t>() };
+                const char* _name = roctracer_op_string(ACTIVITY_DOMAIN_HIP_API, cid, 0);
+                if(get_use_perfetto())
+                {
+                    get_roctracer_key_map().emplace(data->correlation_id, _name);
+                }
+                if(get_use_timemory())
+                {
+                    get_roctracer_data_map().emplace(
+                        data->correlation_id,
+                        roctracer_bundle_t{ tim::static_string{ _name }(),
+                                            tim::scope::get_default() });
+                }
+                break;
+            }
+        }
+    }
+    tim::consume_parameters(domain, arg);
+}
+
+// Activity tracing callback
+void
+activity_callback(const char* begin, const char* end, void*)
+{
+    if(!trait::runtime_enabled<comp::roctracer>::get())
+        return;
+    static auto _kernel_names        = std::unordered_map<const char*, std::string>{};
+    const roctracer_record_t* record = reinterpret_cast<const roctracer_record_t*>(begin);
+    const roctracer_record_t* end_record =
+        reinterpret_cast<const roctracer_record_t*>(end);
+    std::unordered_set<uint64_t> _indexes{};
+
+    tim::auto_lock_t _lk{ tim::type_mutex<data_type_mutex_t>() };
+    auto&            _data    = get_roctracer_data_map();
+    auto&            _keys    = get_roctracer_key_map();
+    auto&            _kernels = get_roctracer_kernels();
+
+    HOSTTRACE_DEBUG("Activity records:\n");
+    while(record < end_record)
+    {
+        HOSTTRACE_DEBUG("\t%-30s\tcorrelation_id(%lu) time_ns(%lu:%lu) device_id(%d) "
+                        "stream_id(%lu)\n",
+                        roctracer_op_string(record->domain, record->correlation_id, 0),
+                        record->correlation_id, record->begin_ns, record->end_ns,
+                        record->device_id, record->queue_id);
+
+        auto _is_kernel = _kernels.find(record->correlation_id) != _kernels.end();
+        if(_is_kernel && record->device_id != 0 && record->queue_id != 0)
+        {
+            // these are overheads associated with the kernel launch, not kernel runtime
+            ROCTRACER_CALL(roctracer_next_record(record, &record));
+            continue;
+        }
+
+        auto kitr =
+            (get_use_perfetto()) ? _keys.find(record->correlation_id) : _keys.end();
+        if(kitr != _keys.end())
+        {
+            if(_kernel_names.find(kitr->second) == _kernel_names.end())
+                _kernel_names.emplace(kitr->second, tim::demangle(kitr->second));
+            TRACE_EVENT_BEGIN(
+                "device",
+                perfetto::StaticString{ _kernel_names.at(kitr->second).c_str() },
+                record->begin_ns);
+            TRACE_EVENT_END("device", record->end_ns);
+            _indexes.emplace(kitr->first);
+        }
+
+        auto itr =
+            (get_use_timemory()) ? _data.find(record->correlation_id) : _data.end();
+        if(itr != _data.end())
+        {
+            itr->second.start()
+                .store(std::plus<double>{},
+                       static_cast<double>(record->end_ns - record->begin_ns))
+                .stop();
+            _indexes.emplace(kitr->first);
+        }
+        // code
+        ROCTRACER_CALL(roctracer_next_record(record, &record));
+    }
+
+    if(get_use_perfetto())
+    {
+        for(auto& itr : _indexes)
+            _keys.erase(itr);
+    }
+
+    if(get_use_timemory())
+    {
+        for(auto& itr : _indexes)
+            _data.erase(itr);
+    }
+
+    HOSTTRACE_DEBUG("[%s] recorded %lu phases\n", __FUNCTION__,
+                    (unsigned long) _indexes.size());
+}
+
+bool&
+roctracer_is_setup()
+{
+    static bool _v = false;
+    return _v;
+}
+}  // namespace
+
+#if !defined(HOSTTRACE_ROCTRACER_LIBKFDWRAPPER)
+#    define HOSTTRACE_ROCTRACER_LIBKFDWRAPPER "libkfdwrapper64.so"
+#endif
+
+namespace tim
+{
+namespace component
+{
+void
+roctracer::preinit()
+{
+    HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);
+    roctracer_data::label()       = "roctracer";
+    roctracer_data::description() = "ROCm tracer (activity API)";
+}
+
+void
+roctracer::setup()
+{
+    if(!get_use_timemory() && !get_use_perfetto())
+        return;
+
+    auto_lock_t _lk{ type_mutex<roctracer>() };
+    if(roctracer_is_setup())
+        return;
+    roctracer_is_setup() = true;
+    HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);
+
+    auto libpath   = tim::get_env<std::string>("HOSTTRACE_ROCTRACER_LIBKFDWRAPPER",
+                                             HOSTTRACE_ROCTRACER_LIBKFDWRAPPER, false);
+    auto libhandle = dlopen(libpath.c_str(), RTLD_NOW | RTLD_GLOBAL);
+    if(!libhandle)
+        fprintf(stderr, "%s\n", dlerror());
+    dlerror();  // Clear any existing error
+
+    // Allocating tracing pool
+    roctracer_properties_t properties{};
+    properties.buffer_size         = 0x1000;
+    properties.buffer_callback_fun = activity_callback;
+    ROCTRACER_CALL(roctracer_open_pool(&properties));
+    ROCTRACER_CALL(roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr));
+
+    // Enable API callbacks, all domains
+    ROCTRACER_CALL(roctracer_enable_callback(hip_api_callback, nullptr));
+    /*ROCTRACER_CALL(roctracer_enable_op_callback(ACTIVITY_DOMAIN_HIP_API,
+                                                HIP_API_ID_hipModuleLaunchKernel,
+                                                hip_api_callback, nullptr));
+    ROCTRACER_CALL(roctracer_enable_op_callback(
+        ACTIVITY_DOMAIN_HIP_API, HIP_API_ID_hipLaunchKernel, hip_api_callback, nullptr));
+    ROCTRACER_CALL(roctracer_enable_op_callback(
+        ACTIVITY_DOMAIN_HIP_API, HIP_API_ID_hipMalloc, hip_api_callback, nullptr));
+    ROCTRACER_CALL(roctracer_enable_op_callback(
+        ACTIVITY_DOMAIN_HIP_API, HIP_API_ID_hipMemcpy, hip_api_callback, nullptr));
+    ROCTRACER_CALL(roctracer_enable_op_callback(
+        ACTIVITY_DOMAIN_HIP_API, HIP_API_ID_hipFree, hip_api_callback, nullptr));*/
+    // Enable activity tracing, all domains
+    ROCTRACER_CALL(roctracer_enable_activity());
+
+    if(libhandle)
+        dlclose(libhandle);
+}
+
+void
+roctracer::tear_down()
+{
+    auto_lock_t _lk{ type_mutex<roctracer>() };
+    if(!roctracer_is_setup())
+        return;
+    roctracer_is_setup() = false;
+    HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);
+
+    // flush all the activity
+    ROCTRACER_CALL(roctracer_flush_activity());
+    // Disable tracing and closing the pool
+    ROCTRACER_CALL(roctracer_disable_callback());
+    /*ROCTRACER_CALL(roctracer_disable_op_callback(ACTIVITY_DOMAIN_HIP_API,
+                                                 HIP_API_ID_hipModuleLaunchKernel));
+    ROCTRACER_CALL(roctracer_disable_op_callback(ACTIVITY_DOMAIN_HIP_API,
+                                                 HIP_API_ID_hipLaunchKernel));
+    ROCTRACER_CALL(
+        roctracer_disable_op_callback(ACTIVITY_DOMAIN_HIP_API, HIP_API_ID_hipMemcpy));
+    ROCTRACER_CALL(
+        roctracer_disable_op_callback(ACTIVITY_DOMAIN_HIP_API, HIP_API_ID_hipMalloc));
+    ROCTRACER_CALL(
+        roctracer_disable_op_callback(ACTIVITY_DOMAIN_HIP_API, HIP_API_ID_hipFree));
+        */
+    ROCTRACER_CALL(roctracer_disable_activity());
+    ROCTRACER_CALL(roctracer_close_pool());
+}
+
+void
+roctracer::start()
+{
+    if(tracker_type::start() == 0)
+        setup();
+}
+
+void
+roctracer::stop()
+{
+    // flush all the activity
+    ROCTRACER_CALL(roctracer_flush_activity());
+    if(tracker_type::stop() == 0)
+        tear_down();
+}
+}  // namespace component
+}  // namespace tim
+
+TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(roctracer, false, void)
+TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(roctracer_data, true, double)