GitHub CI (#11)

* Continuous integration * linux-ci on * fix for parallel-overhead * Updated CMAKE_INSTALL_PREFIX * Updated installed boost libraries * Dyninst updates fixing TBB install * timemory + dyninst submodule updates - fixes some timemory package option handling - fixes dyninst libiberty handling * Update dyninst submodule with libiberty fix * dyninst submodule update with TBB internal build fixes * Updated linux-ci + tests + timemeory + dyninst - updated timemory submodule - update dyninst submodule - delay OnLoad implementation - DYNINST_RT_API handling improvement - CI for ubuntu-bionic in addition to focal - CTest in CI * Update TBB handling in CI * Update dyninst with symLite fix * Update dyninst submodule with ElfUtils-External fix * Dyninst::ElfUtils fix * Modified dyninst submodule TPL install * Update dyninst submodule with improved interface libraries * Fix to Dyninst::ElfUtils in dyninst submodule * Updated CI build matrix + test install * Updated CI * CI stage updates * Tweak * Dyninst updates + RPATH for hosttrace exe - hosttrace will rpath to dyninst - Dyninst will statically link to boost by default - Fixes to double init w/ MPI + runtime instrumentation - minor cleanup to hosttrace - hosttrace help exits with zero - CI updates * Dyninst + CI updates * Removed ELFUTILS_BUILD_STATIC option in Dyninst * Dyninst + visibility + roctracer + tests - Dyninst submodule updates with dynamic pcontrol lib - hosttrace visibility is hidden - improved handling of DYNINST_API_RT in hosttrace - roctracer::tear_down in finalize fixes issues with timemory - throw error if cannot create perfetto output file - roctracer_default_pool() safety guards - resume calling roctracer_close_pool() - fixed working dir of tests - fixed env of tests (cmake and CI) - simplified CI * Removed stray CI if condition * Disable hidden visibility for now * ifdef for roctracer::tear_down + reenable hidden * Better CI environment handling + dyninst packaging * Fix for dyninst-package CI * Fixes for cmake 3.15 issues with aliasing imported lib * Fix to ubuntu-focal-dyninst-package * Dyninst updates for packaging - fixes issues with hard-coded paths to libraries after relocation via package installer * Restrict CI to main and develop branches [ROCm/rocprofiler-systems commit: 35ab6b0110]
2021-09-15 16:45:49 -05:00
@@ -72,8 +72,31 @@ int
 main(int argc, char** argv)
 {
 #if defined(DYNINST_API_RT)
-    tim::set_env<string_t>("DYNINSTAPI_RT_LIB", DYNINST_API_RT, 0);
+    auto _dyn_api_rt_paths = tim::delimit(DYNINST_API_RT, ":");
+#else
+    auto _dyn_api_rt_paths = std::vector<std::string>{};
 #endif
+    auto _dyn_api_rt_abs = get_absolute_lib_filepath("libdyninstAPI_RT.so");
+    _dyn_api_rt_paths.insert(_dyn_api_rt_paths.begin(), _dyn_api_rt_abs);
+    for(auto&& itr : _dyn_api_rt_paths)
+    {
+        auto file_exists = [](const std::string& _fname) {
+            struct stat _buffer;
+            if(stat(_fname.c_str(), &_buffer) == 0)
+                return (S_ISREG(_buffer.st_mode) != 0 || S_ISLNK(_buffer.st_mode) != 0);
+            return false;
+        };
+        if(file_exists(itr))
+            tim::set_env<string_t>("DYNINSTAPI_RT_LIB", itr, 0);
+        else if(file_exists(TIMEMORY_JOIN('/', itr, "libdyninstAPI_RT.so")))
+            tim::set_env<string_t>("DYNINSTAPI_RT_LIB",
+                                   TIMEMORY_JOIN('/', itr, "libdyninstAPI_RT.so"), 0);
+        else if(file_exists(TIMEMORY_JOIN('/', itr, "libdyninstAPI_RT.a")))
+            tim::set_env<string_t>("DYNINSTAPI_RT_LIB",
+                                   TIMEMORY_JOIN('/', itr, "libdyninstAPI_RT.a"), 0);
+    }
+    verbprintf(0, "[hosttrace] DYNINST_API_RT: %s\n",
+               tim::get_env<string_t>("DYNINSTAPI_RT_LIB", "").c_str());

    argv0 = argv[0];

@@ -156,12 +179,12 @@ main(int argc, char** argv)

    if(verbose_level > 1)
    {
-        std::cout << "[original]: " << cmd_string(argc, argv) << std::endl;
-        std::cout << "[cfg-args]: " << cmd_string(_argc, _argv) << std::endl;
+        std::cout << "[hosttrace][original]: " << cmd_string(argc, argv) << std::endl;
+        std::cout << "[hosttrace][cfg-args]: " << cmd_string(_argc, _argv) << std::endl;
    }

    if(_cmdc > 0)
-        std::cout << "\n [command]: " << cmd_string(_cmdc, _cmdv) << "\n\n";
+        std::cout << "\n[hosttrace][command]: " << cmd_string(_cmdc, _cmdv) << "\n\n";

    if(_cmdc > 0)
        cmdv0 = _cmdv[0];
@@ -380,12 +403,6 @@ main(int argc, char** argv)

    string_t extra_help = "-- <CMD> <ARGS>";
    auto     err        = parser.parse(_argc, _argv);
-    if(err)
-    {
-        std::cerr << err << std::endl;
-        parser.print_help(extra_help);
-        return -1;
-    }

    if(parser.exists("h") || parser.exists("help"))
    {
@@ -393,6 +410,13 @@ main(int argc, char** argv)
        return 0;
    }

+    if(err)
+    {
+        std::cerr << err << std::endl;
+        parser.print_help(extra_help);
+        return -1;
+    }
+
    if(parser.exists("e"))
        werror = true;

@@ -892,7 +916,7 @@ main(int argc, char** argv)
        use_mpi = true;

    bool use_mpip = false;
-    if(use_mpi && binary_rewrite)
+    if(use_mpi)
        use_mpip = true;

    //----------------------------------------------------------------------------------//
@@ -1146,8 +1170,9 @@ main(int argc, char** argv)
    auto trace_call_args =
        hosttrace_call_expr("HOSTTRACE_COMPONENTS", default_components);
    auto use_mpi_call_args  = hosttrace_call_expr("HOSTTRACE_USE_MPI", "ON");
-    auto use_mpip_call_args = hosttrace_call_expr("HOSTTRACE_USE_MPIP", "ON");
-    auto none_call_args     = hosttrace_call_expr();
+    auto use_mpip_call_args = hosttrace_call_expr(
+        "HOSTTRACE_USE_MPIP", (binary_rewrite && use_mpi && use_mpip) ? "ON" : "OFF");
+    auto none_call_args = hosttrace_call_expr();

    verbprintf(2, "Done\n");
    verbprintf(2, "Getting call snippets... ");
@@ -1231,7 +1256,7 @@ main(int argc, char** argv)
    if(use_mpi && umpi_call)
        init_names.push_back(umpi_call.get());

-    if(init_call)
+    if(init_call && binary_rewrite)
        init_names.push_back(init_call.get());

    if(binary_rewrite)
@@ -1317,11 +1342,11 @@ main(int argc, char** argv)
            if(std::string{ modname }.find("libdyninst") != std::string::npos)
                continue;

-            if(module_constraint(modname) || !process_file_for_instrumentation(modname))
-            {
-                verbprintf(1, "Skipping constrained module: '%s'\n", modname);
+            if(module_constraint(modname))
+                continue;
+
+            if(!instrument_module(modname))
                continue;
-            }

            itr->getName(fname, FUNCNAMELEN);

@@ -1334,18 +1359,10 @@ main(int argc, char** argv)
            }

            if(routine_constraint(name.m_name.c_str()))
-            {
-                verbprintf(1, "Skipping function [constrained]: %s\n",
-                           name.m_name.c_str());
                continue;
-            }

            if(!instrument_entity(name.m_name))
-            {
-                verbprintf(1, "Skipping function [excluded]: %s / %s\n",
-                           name.m_name.c_str(), name.get().c_str());
                continue;
-            }

            if(is_static_exe && has_debug_info && string_t{ fname } == "_fini" &&
               string_t{ modname } == "DEFAULT_MODULE")
@@ -1816,8 +1833,19 @@ main(int argc, char** argv)
 //======================================================================================//

 bool
-process_file_for_instrumentation(const string_t& file_name)
+instrument_module(const string_t& file_name)
 {
+    auto _report = [&file_name](const string_t& _action, const string_t& _reason,
+                                int _lvl) {
+        static strset_t already_reported{};
+        if(already_reported.count(file_name) == 0)
+        {
+            verbprintf(_lvl, "%s module [%s] : '%s'...\n", _action.c_str(),
+                       _reason.c_str(), file_name.c_str());
+            already_reported.insert(file_name);
+        }
+    };
+
    auto is_include = [&](bool _if_empty) {
        if(file_include.empty())
            return _if_empty;
@@ -1835,28 +1863,22 @@ process_file_for_instrumentation(const string_t& file_name)
        for(auto& itr : file_exclude)
        {
            if(std::regex_search(file_name, itr))
-            {
-                verbprintf(2, "Excluding module [user-regex] : '%s'...\n",
-                           file_name.c_str());
                return true;
-            }
        }
        return false;
    };

-    auto _user_include = is_include(false) && !is_exclude();
+    auto _user_include = is_include(false);
+    auto _user_exclude = is_exclude();

-    if(_user_include)
-    {
-        verbprintf(2, "Including module [user-regex] : '%s'...\n", file_name.c_str());
-        return true;
-    }
+    if(_user_include && !_user_exclude)
+        return (_report("Including", "user-regex", 2), true);

    string_t          ext_str = "\\.(s|S)$";
    static std::regex ext_regex(ext_str, regex_opts);
    static std::regex sys_regex("^(s|k|e|w)_[A-Za-z_0-9\\-]+\\.(c|C)$", regex_opts);
    static std::regex userlib_regex(
-        "^lib(hosttrace|caliper|gotcha|papi|cupti|TAU|likwid|"
+        "^(lib|)(hosttrace|caliper|gotcha|papi|cupti|TAU|likwid|"
        "profiler|tcmalloc|dyninst|pfm|nvtx|upcxx|pthread|nvperf|hsa)",
        regex_opts);
    static std::regex corelib_regex("^lib(rt-|dl-|util-|python)", regex_opts);
@@ -1887,37 +1909,27 @@ process_file_for_instrumentation(const string_t& file_name)

    if(std::regex_search(file_name, ext_regex))
    {
-        verbprintf(3, "Excluding instrumentation [file extension] : '%s'...\n",
-                   file_name.c_str());
-        return false;
+        return (_report("Excluding", "file extension", 3), false);
    }

    if(std::regex_search(file_name, sys_regex))
    {
-        verbprintf(3, "Excluding instrumentation [system library] : '%s'...\n",
-                   file_name.c_str());
-        return false;
+        return (_report("Excluding", "system library", 3), false);
    }

    if(std::regex_search(file_name, corelib_regex))
    {
-        verbprintf(3, "Excluding instrumentation [core library] : '%s'...\n",
-                   file_name.c_str());
-        return false;
+        return (_report("Excluding", "core library", 3), false);
    }

    if(std::regex_search(file_name, userlib_regex))
    {
-        verbprintf(3, "Excluding instrumentation [instr library] : '%s'...\n",
-                   file_name.c_str());
-        return false;
+        return (_report("Excluding", "instrumentation", 3), false);
    }

    if(std::regex_search(file_name, prefix_regex))
    {
-        verbprintf(3, "Excluding instrumentation [prefix match] : '%s'...\n",
-                   file_name.c_str());
-        return false;
+        return (_report("Excluding", "prefix match", 3), false);
    }

    /*if(std::regex_search(file_name, suffix_regex))
@@ -1927,17 +1939,12 @@ process_file_for_instrumentation(const string_t& file_name)
        return false;
    }*/

-    bool use = is_include(true) && !is_exclude();
-    if(use)
-    {
-        static strset_t already_reported;
-        if(already_reported.count(file_name) == 0)
-        {
-            verbprintf(2, "%s |> [ %s ]\n", __FUNCTION__, file_name.c_str());
-            already_reported.insert(file_name);
-        }
-    }
-    return use;
+    if(_user_exclude)
+        return (_report("Excluding", "user-regex", 2), false);
+
+    _report("Including", "no constraint", 2);
+
+    return true;
 }

 //======================================================================================//
@@ -1990,7 +1997,7 @@ instrument_entity(const string_t& function_name)
        "delete|std::allocat|"
        "nvtx|gcov|main\\.cold|TAU|tau|Tau|dyn|RT|dl|sys|pthread|posix|clone|virtual "
        "thunk|non-virtual thunk|transaction "
-        "clone|RtsLayer|DYNINST|PthreadLayer|threaded_func|targ8)",
+        "clone|RtsLayer|DYNINST|PthreadLayer|threaded_func|targ8|PMPI)",
        regex_opts);
    static std::regex trailing("(\\.part\\.[0-9]+|\\.constprop\\.[0-9]+|\\.|\\.[0-9]+)$",
                               regex_opts);
@@ -2140,18 +2147,7 @@ module_constraint(char* fname)
    if(_fname == "DEFAULT_MODULE" || _fname == "LIBRARY_MODULE")
        return false;

-    // auto _valid_file_extension = std::regex_search(
-    //    _fname, std::regex{ "\\.(a|c|f|o|cc|so|cxx|cpp|C|F|CC|f90|F90|so\\.[0-9\\.]+)$",
-    //                        regex_opts });
-
-    auto _valid_file_regex = process_file_for_instrumentation(_fname);
-
-    // if module compiled from C, C++, or Fortran or a library
-    // if(_valid_file_extension && _valid_file_regex)
-    //    return false;
-
-    // apply regex expressions
-    if(_valid_file_regex)
+    if(instrument_module(_fname))
        return false;

    // do not instrument
@@ -2207,7 +2203,7 @@ get_absolute_exe_filepath(std::string exe_name)
            if(file_exists(TIMEMORY_JOIN('/', pitr, exe_name)))
            {
                exe_name = TIMEMORY_JOIN('/', pitr, exe_name);
-                verbprintf(0, "Resolved '%s' to '%s'...\n", _exe_orig.c_str(),
+                verbprintf(0, "[hosttrace] Resolved '%s' to '%s'...\n", _exe_orig.c_str(),
                           exe_name.c_str());
                break;
            }
@@ -2242,7 +2238,7 @@ get_absolute_lib_filepath(std::string lib_name)
            if(file_exists(TIMEMORY_JOIN('/', pitr, lib_name)))
            {
                lib_name = TIMEMORY_JOIN('/', pitr, lib_name);
-                verbprintf(0, "Resolved '%s' to '%s'...\n", _lib_orig.c_str(),
+                verbprintf(0, "[hosttrace] Resolved '%s' to '%s'...\n", _lib_orig.c_str(),
                           lib_name.c_str());
                break;
            }
@@ -167,8 +167,10 @@ get_functors()
 bool
 hosttrace_init_tooling()
 {
-    if(get_state() != State::PreInit)
+    static bool _once = false;
+    if(get_state() != State::PreInit || _once)
        return false;
+    _once = true;

    HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);

@@ -466,6 +468,11 @@ extern "C"

        get_state() = State::Finalized;

+#if defined(HOSTTRACE_USE_ROCTRACER)
+        // ensure that threads running roctracer callbacks shutdown
+        comp::roctracer::tear_down();
+#endif
+
        // stop the main bundle and report the high-level metrics
        if(get_main_bundle())
        {
@@ -506,6 +513,7 @@ extern "C"
            }
        }

+        bool _perfetto_output_error = false;
        if(get_use_perfetto() && !is_system_backend())
        {
            // Make sure the last event is closed for this example.
@@ -535,14 +543,20 @@ extern "C"
            std::ofstream output{};
            output.open(get_perfetto_output_filename(), std::ios::out | std::ios::binary);
            if(!output)
+            {
                fprintf(stderr, "[%s]> Error opening '%s'...\n", __FUNCTION__,
                        get_perfetto_output_filename().c_str());
+                _perfetto_output_error = true;
+            }
            else
                output.write(&trace_data[0], trace_data.size());
            output.close();
        }

        tim::timemory_finalize();
+
+        if(_perfetto_output_error)
+            throw std::runtime_error("Unable to create perfetto output file");
    }

    void hosttrace_trace_set_env(const char* env_name, const char* env_val)
@@ -145,6 +145,14 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
                    TRACE_EVENT_END("device", end_timestamp);
                }

+                /*if(get_use_timemory())
+                {
+                    static auto _scope = scope::flat() + scope::timeline();
+                    roctracer_bundle_t{ _name, _scope }
+                        .start()
+                        .store(end_timestamp - hsa_begin_timestamp)
+                        .stop();
+                }*/
                // timemory is disabled in this callback because collecting data in this
                // thread causes strange segmentation faults
            }
@@ -341,6 +349,22 @@ roctracer_is_setup()
    static bool _v = false;
    return _v;
 }
+
+using roctracer_functions_t = std::vector<std::pair<std::string, std::function<void()>>>;
+
+auto&
+roctracer_setup_routines()
+{
+    static auto _v = roctracer_functions_t{};
+    return _v;
+}
+
+auto&
+roctracer_tear_down_routines()
+{
+    static auto _v = roctracer_functions_t{};
+    return _v;
+}
 }  // namespace

 #if !defined(HOSTTRACE_ROCTRACER_LIBKFDWRAPPER)
@@ -391,6 +415,52 @@ roctracer::preinit()
    roctracer_data::description() = "ROCm tracer (activity API)";
 }

+bool
+roctracer::is_setup()
+{
+    return roctracer_is_setup();
+}
+
+void
+roctracer::add_setup(const std::string& _lbl, std::function<void()>&& _func)
+{
+    roctracer_setup_routines().emplace_back(_lbl, std::move(_func));
+}
+
+void
+roctracer::add_tear_down(const std::string& _lbl, std::function<void()>&& _func)
+{
+    roctracer_tear_down_routines().emplace_back(_lbl, std::move(_func));
+}
+
+void
+roctracer::remove_setup(const std::string& _lbl)
+{
+    auto& _data = roctracer_setup_routines();
+    for(auto itr = _data.begin(); itr != _data.end(); ++itr)
+    {
+        if(itr->first == _lbl)
+        {
+            _data.erase(itr);
+            break;
+        }
+    }
+}
+
+void
+roctracer::remove_tear_down(const std::string& _lbl)
+{
+    auto& _data = roctracer_setup_routines();
+    for(auto itr = _data.begin(); itr != _data.end(); ++itr)
+    {
+        if(itr->first == _lbl)
+        {
+            _data.erase(itr);
+            break;
+        }
+    }
+}
+
 void
 roctracer::setup()
 {
@@ -408,17 +478,25 @@ roctracer::setup()
    auto _kfdwrapper = dynamic_library{ "HOSTTRACE_ROCTRACER_LIBKFDWRAPPER",
                                        HOSTTRACE_ROCTRACER_LIBKFDWRAPPER };

-    // Allocating tracing pool
-    roctracer_properties_t properties{};
-    properties.buffer_size         = 0x1000;
-    properties.buffer_callback_fun = hip_activity_callback;
-    ROCTRACER_CALL(roctracer_open_pool(&properties));
    ROCTRACER_CALL(roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr));

+    if(roctracer_default_pool() == nullptr)
+    {
+        // Allocating tracing pool
+        roctracer_properties_t properties{};
+        properties.buffer_size         = 0x1000;
+        properties.buffer_callback_fun = hip_activity_callback;
+        ROCTRACER_CALL(roctracer_open_pool(&properties));
+    }
+
    // Enable API callbacks, all domains
    ROCTRACER_CALL(roctracer_enable_callback(hip_api_callback, nullptr));
    // Enable activity tracing, all domains
    ROCTRACER_CALL(roctracer_enable_activity());
+
+    // callback for HSA
+    for(auto& itr : roctracer_setup_routines())
+        itr.second();
 }

 void
@@ -431,22 +509,21 @@ roctracer::tear_down()
    HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__);

    // flush all the activity
-    ROCTRACER_CALL(roctracer_flush_activity());
-
+    if(roctracer_default_pool() != nullptr)
+    {
+        ROCTRACER_CALL(roctracer_flush_activity());
+    }
    // flush all buffers
    roctracer_flush_buf();

-    ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API));
-
-    ROCTRACER_CALL(
-        roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
+    // callback for hsa
+    for(auto& itr : roctracer_tear_down_routines())
+        itr.second();

    // Disable tracing and closing the pool
    ROCTRACER_CALL(roctracer_disable_callback());
    ROCTRACER_CALL(roctracer_disable_activity());
-
-    // closing the pool with HSA enabled causes segfaults
-    // ROCTRACER_CALL(roctracer_close_pool());
+    ROCTRACER_CALL(roctracer_close_pool());
 }

 void
@@ -459,8 +536,6 @@ roctracer::start()
 void
 roctracer::stop()
 {
-    // flush all the activity
-    ROCTRACER_CALL(roctracer_flush_activity());
    if(tracker_type::stop() == 0)
        tear_down();
 }
@@ -471,84 +546,112 @@ TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(roctracer, false, void)
 TIMEMORY_INSTANTIATE_EXTERN_COMPONENT(roctracer_data, true, double)

 // HSA-runtime tool on-load method
-extern "C" TIMEMORY_VISIBILITY("default") bool OnLoad(
-    HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,
-    const char* const* failed_tool_names)
+extern "C"
 {
-    puts(__FUNCTION__);
-    tim::consume_parameters(table, runtime_version, failed_tool_count, failed_tool_names);
+    TIMEMORY_VISIBILITY("default")
+    bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,
+                const char* const* failed_tool_names);
+    TIMEMORY_VISIBILITY("default") void OnUnload();

-    // ONLOAD_TRACE_BEG();
-    // on_exit(exit_handler, nullptr);
-
-    get_hsa_timer() = std::make_unique<hsa_timer_t>(table->core_->hsa_system_get_info_fn);
-
-    // const char* output_prefix = getenv("ROCP_OUTPUT_DIR");
-    const char* output_prefix = nullptr;
-
-    // App begin timestamp begin_ts_file.txt
-    // begin_ts_file_handle = open_output_file(output_prefix, "begin_ts_file.txt");
-    // const timestamp_t app_start_time = timer->timestamp_fn_ns();
-    // fprintf(begin_ts_file_handle, "%lu\n", app_start_time);
-
-    bool trace_hsa_api = tim::get_env("HOSTTRACE_ROCTRACER_HSA_API", true);
-    std::vector<std::string> hsa_api_vec =
-        tim::delimit(tim::get_env<std::string>("HOSTTRACE_ROCTRACER_HSA_API_TYPES", ""));
-
-    // Enable HSA API callbacks/activity
-    if(trace_hsa_api)
+    bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,
+                const char* const* failed_tool_names)
    {
-        // hsa_api_file_handle = open_output_file(output_prefix, "hsa_api_trace.txt");
+        puts(__FUNCTION__);
+        tim::consume_parameters(table, runtime_version, failed_tool_count,
+                                failed_tool_names);

-        // initialize HSA tracing
-        roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table);
+        // ONLOAD_TRACE_BEG();
+        // on_exit(exit_handler, nullptr);

-        fprintf(stdout, "    HSA-trace(");
-        fflush(stdout);
-        if(!hsa_api_vec.empty())
-        {
-            for(unsigned i = 0; i < hsa_api_vec.size(); ++i)
+        auto _setup = [=]() {
+            get_hsa_timer() =
+                std::make_unique<hsa_timer_t>(table->core_->hsa_system_get_info_fn);
+
+            // const char* output_prefix = getenv("ROCP_OUTPUT_DIR");
+            const char* output_prefix = nullptr;
+
+            // App begin timestamp begin_ts_file.txt
+            // begin_ts_file_handle = open_output_file(output_prefix,
+            // "begin_ts_file.txt"); const timestamp_t app_start_time =
+            // timer->timestamp_fn_ns(); fprintf(begin_ts_file_handle, "%lu\n",
+            // app_start_time);
+
+            bool trace_hsa_api = tim::get_env("HOSTTRACE_ROCTRACER_HSA_API", true);
+            std::vector<std::string> hsa_api_vec = tim::delimit(
+                tim::get_env<std::string>("HOSTTRACE_ROCTRACER_HSA_API_TYPES", ""));
+
+            // Enable HSA API callbacks/activity
+            if(trace_hsa_api)
            {
-                uint32_t    cid = HSA_API_ID_NUMBER;
-                const char* api = hsa_api_vec[i].c_str();
-                ROCTRACER_CALL(
-                    roctracer_op_code(ACTIVITY_DOMAIN_HSA_API, api, &cid, nullptr));
-                ROCTRACER_CALL(roctracer_enable_op_callback(ACTIVITY_DOMAIN_HSA_API, cid,
-                                                            hsa_api_callback, nullptr));
-                printf(" %s", api);
+                // hsa_api_file_handle = open_output_file(output_prefix,
+                // "hsa_api_trace.txt");
+
+                // initialize HSA tracing
+                roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table);
+
+                fprintf(stdout, "    HSA-trace(");
+                fflush(stdout);
+                if(!hsa_api_vec.empty())
+                {
+                    for(unsigned i = 0; i < hsa_api_vec.size(); ++i)
+                    {
+                        uint32_t    cid = HSA_API_ID_NUMBER;
+                        const char* api = hsa_api_vec[i].c_str();
+                        ROCTRACER_CALL(roctracer_op_code(ACTIVITY_DOMAIN_HSA_API, api,
+                                                         &cid, nullptr));
+                        ROCTRACER_CALL(roctracer_enable_op_callback(
+                            ACTIVITY_DOMAIN_HSA_API, cid, hsa_api_callback, nullptr));
+                        printf(" %s", api);
+                    }
+                }
+                else
+                {
+                    ROCTRACER_CALL(roctracer_enable_domain_callback(
+                        ACTIVITY_DOMAIN_HSA_API, hsa_api_callback, nullptr));
+                }
+                printf(")\n");
            }
-        }
-        else
-        {
-            ROCTRACER_CALL(roctracer_enable_domain_callback(ACTIVITY_DOMAIN_HSA_API,
-                                                            hsa_api_callback, nullptr));
-        }
-        printf(")\n");
-    }

-    bool trace_hsa_activity = tim::get_env("HOSTTRACE_ROCTRACER_HSA_ACTIVITY", true);
-    // Enable HSA GPU activity
-    if(trace_hsa_activity)
-    {
-        // initialize HSA tracing
-        roctracer::hsa_ops_properties_t ops_properties{
-            table, reinterpret_cast<activity_async_callback_t>(hsa_activity_callback),
-            nullptr, output_prefix
+            bool trace_hsa_activity =
+                tim::get_env("HOSTTRACE_ROCTRACER_HSA_ACTIVITY", true);
+            // Enable HSA GPU activity
+            if(trace_hsa_activity)
+            {
+                // initialize HSA tracing
+                roctracer::hsa_ops_properties_t ops_properties{
+                    table,
+                    reinterpret_cast<activity_async_callback_t>(hsa_activity_callback),
+                    nullptr, output_prefix
+                };
+                roctracer_set_properties(ACTIVITY_DOMAIN_HSA_OPS, &ops_properties);
+
+                fprintf(stdout, "    HSA-activity-trace()\n");
+                fflush(stdout);
+                ROCTRACER_CALL(roctracer_enable_op_activity(ACTIVITY_DOMAIN_HSA_OPS,
+                                                            HSA_OP_ID_COPY));
+            }
        };
-        roctracer_set_properties(ACTIVITY_DOMAIN_HSA_OPS, &ops_properties);

-        fprintf(stdout, "    HSA-activity-trace()\n");
-        fflush(stdout);
-        ROCTRACER_CALL(
-            roctracer_enable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
+        auto _tear_down = []() {
+            ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API));
+
+            ROCTRACER_CALL(
+                roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
+        };
+
+        if(comp::roctracer::is_setup())
+            _setup();
+
+        comp::roctracer::add_setup("hsa", std::move(_setup));
+        comp::roctracer::add_tear_down("hsa", std::move(_tear_down));
+
+        return true;
    }

-    return true;
-}
-
-// HSA-runtime on-unload method
-extern "C" TIMEMORY_VISIBILITY("default") void OnUnload()
-{
-    puts(__FUNCTION__);
-    // ONLOAD_TRACE("");
+    // HSA-runtime on-unload method
+    void OnUnload()
+    {
+        puts(__FUNCTION__);
+        // ONLOAD_TRACE("");
+    }
 }