ROCm environment fixes + workflow updates (#117)

* Improve dlopen of ROCm libraries + rocprofiler test - Use PROJECT_BINARY_DIR in tests - Added rocprofiler test * Revert OMNITRACE_FORCE_ROCPROFILER_INIT * omnitrace-avail --all test * Fix ROCP_METRICS for ROCm 5.2.0 * Fix ROCP_METRICS for ROCm 5.2.0 * Restrict containers workflow to AMDResearch/omnitrace * Bump version to 1.3.1 * Update cpack workflow - generate release draft - upload installers as release assets * Test rocprofiler w/o roctracer enabled * Fix formatting * verbose message
2022-07-27 06:36:52 -05:00
commit 7e31d9f450
@@ -201,6 +201,13 @@ omnitrace_add_bin_test(
        ".*\\\[omnitrace-avail\\\] Usage:.*\\\[CATEGORIES\\\].*\\\[VIEW OPTIONS\\\].*\\\[COLUMN OPTIONS\\\].*\\\[WIDTH OPTIONS\\\].*\\\[OUTPUT OPTIONS\\\].*"
    )

+omnitrace_add_bin_test(
+    NAME omnitrace-avail-all
+    TARGET omnitrace-avail
+    ARGS --all
+    LABELS "omnitrace-avail"
+    TIMEOUT 45)
+
 omnitrace_add_bin_test(
    NAME omnitrace-avail-all-csv
    TARGET omnitrace-avail
@@ -132,6 +132,11 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
 #endif

 #if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
+#    if OMNITRACE_HIP_VERSION >= 50200
+#        define ROCPROFILER_METRICS_DIR "lib/rocprofiler"
+#    else
+#        define ROCPROFILER_METRICS_DIR "rocprofiler/lib"
+#    endif
    setenv("HSA_TOOLS_LIB", _omnilib.c_str(), 0);
    setenv("ROCP_TOOL_LIB", _omnilib.c_str(), 0);
    setenv("ROCPROFILER_LOG", "1", 0);
@@ -142,7 +147,8 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
        if(getenv(itr))
        {
            setenv("ROCP_METRICS",
-                   common::join('/', getenv(itr), "rocprofiler/lib/metrics.xml").c_str(),
+                   common::join('/', getenv(itr), ROCPROFILER_METRICS_DIR, "metrics.xml")
+                       .c_str(),
                   0);
            setenv("OMNITRACE_ROCPROFILER_LIBRARY",
                   common::join('/', getenv(itr), "rocprofiler/lib/librocprofiler64.so")
@@ -153,7 +159,8 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
    }
    // default path
    setenv("ROCP_METRICS",
-           common::join('/', OMNITRACE_DEFAULT_ROCM_PATH, "rocprofiler/lib/metrics.xml")
+           common::join('/', OMNITRACE_DEFAULT_ROCM_PATH, ROCPROFILER_METRICS_DIR,
+                        "metrics.xml")
               .c_str(),
           0);
 #endif
@@ -123,11 +123,18 @@ roctracer::setup()

    OMNITRACE_VERBOSE_F(1, "setting up roctracer...\n");

-    dynamic_library _amdhip64{ "OMNITRACE_ROCTRACER_LIBAMDHIP64", "libamdhip64.so" };
+    dynamic_library _amdhip64{ "OMNITRACE_ROCTRACER_LIBAMDHIP64",
+                               find_library_path("libamdhip64.so",
+                                                 { "OMNITRACE_ROCM_PATH", "ROCM_PATH" },
+                                                 { OMNITRACE_DEFAULT_ROCM_PATH }) };

 #if OMNITRACE_HIP_VERSION_MAJOR == 4 && OMNITRACE_HIP_VERSION_MINOR < 4
-    dynamic_library _kfdwrapper{ "OMNITRACE_ROCTRACER_LIBKFDWRAPPER",
-                                 OMNITRACE_ROCTRACER_LIBKFDWRAPPER };
+    dynamic_library _kfdwrapper{
+        "OMNITRACE_ROCTRACER_LIBKFDWRAPPER",
+        find_library_path("libkfdwrapper64.so", { "OMNITRACE_ROCM_PATH", "ROCM_PATH" },
+                          { OMNITRACE_DEFAULT_ROCM_PATH },
+                          { "roctracer/lib", "roctracer/lib64", "lib", "lib64" })
+    };
 #endif

    ROCTRACER_CALL(roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr));
@@ -24,14 +24,6 @@

 #include "common/defines.h"

-// clang-format off
-#if defined(OMNITRACE_USE_ROCTRACER)
-#    define OMNITRACE_ROCTRACER_LIBKFDWRAPPER "@roctracer_kfdwrapper_LIBRARY@"
-#else
-#    define OMNITRACE_ROCTRACER_LIBKFDWRAPPER "/opt/rocm/roctracer/lib/libkfdwrapper64.so"
-#endif
-// clang-format on
-
 #define TIMEMORY_USER_COMPONENT_ENUM                                                     \
    OMNITRACE_COMPONENT_idx, OMNITRACE_USER_REGION_idx, OMNITRACE_ROCTRACER_idx,         \
        OMNITRACE_ROCPROFILER_idx, OMNITRACE_SAMPLING_WALL_CLOCK_idx,                    \
@@ -21,20 +21,90 @@
 // SOFTWARE.

 #include "library/dynamic_library.hpp"
+#include "common/defines.h"
+#include "library/common.hpp"
 #include "library/debug.hpp"
 #include "library/defines.hpp"

 #include <timemory/environment.hpp>
+#include <timemory/utility/delimit.hpp>
+#include <timemory/utility/filepath.hpp>
+
+#include <string>
+#include <utility>

 namespace omnitrace
 {
-dynamic_library::dynamic_library(const char* _env, const char* _fname, int _flags,
-                                 bool _store)
-: envname{ _env }
-, filename{ tim::get_env<std::string>(_env, _fname, _store) }
+std::string
+find_library_path(const std::string& _name, const std::vector<std::string>& _env_vars,
+                  const std::vector<std::string>& _hints,
+                  const std::vector<std::string>& _path_suffixes)
+{
+    if(_name.find('/') == 0) return _name;
+
+    auto _paths = std::vector<std::string>{};
+    for(const std::string& itr : _env_vars)
+    {
+        auto _env_val = get_env(itr, std::string{});
+        for(auto vitr : tim::delimit(_env_val, ":"))
+            if(!vitr.empty()) _paths.emplace_back(vitr);
+    }
+
+    for(const std::string& itr : _hints)
+    {
+        if(!itr.empty()) _paths.emplace_back(itr);
+    }
+
+    for(auto& itr : _paths)
+    {
+        auto _v = JOIN('/', itr, _name);
+        if(filepath::exists(_v)) return _v;
+        for(const auto& litr : _path_suffixes)
+        {
+            _v = JOIN('/', itr, litr, _name);
+            if(filepath::exists(_v)) return _v;
+        }
+    }
+
+    return _name;
+}
+
+dynamic_library::dynamic_library(std::string _env, std::string _fname, int _flags,
+                                 bool _open, bool _query_env, bool _store)
+: envname{ std::move(_env) }
+, filename{ std::move(_fname) }
 , flags{ _flags }
 {
-    open();
+    if(_query_env)
+    {
+        auto _env_val = get_env(envname, std::string{}, _store);
+        // if the environment variable is set to an absolute path that exists,
+        // override with value
+        if(!_env_val.empty())
+        {
+            if(_env_val.find('/') == 0 && filepath::exists(_env_val))
+            {
+                filename = _env_val;
+            }
+            else if(_env_val.find('/') == 0)
+            {
+                OMNITRACE_VERBOSE_F(1,
+                                    "Ignoring environment variable %s=\"%s\" because the "
+                                    "filepath does not exist. Using \"%s\" instead...\n",
+                                    envname.c_str(), _env_val.c_str(), filename.c_str())
+            }
+            else if(_env_val.find('/') != 0 && filename.find('/') == 0)
+            {
+                OMNITRACE_VERBOSE_F(
+                    1,
+                    "Ignoring environment variable %s=\"%s\" because the "
+                    "filepath is relative. Using absolute path \"%s\" instead...\n",
+                    envname.c_str(), _env_val.c_str(), filename.c_str())
+            }
+        }
+    }
+
+    if(_open) open();
 }

 dynamic_library::~dynamic_library() { close(); }
@@ -47,8 +117,8 @@ dynamic_library::open()
        handle = dlopen(filename.c_str(), flags);
        if(!handle)
        {
-            OMNITRACE_VERBOSE(2, "[dynamic_library][%s][%s] %s\n", envname.c_str(),
-                              filename.c_str(), dlerror());
+            OMNITRACE_VERBOSE(2, "[dynamic_library] Error opening %s=\"%s\" :: %s.\n",
+                              envname.c_str(), filename.c_str(), dlerror());
        }
        dlerror();  // Clear any existing error
    }
@@ -61,4 +131,10 @@ dynamic_library::close() const
    if(handle) return dlclose(handle);
    return -1;
 }
+
+bool
+dynamic_library::is_open() const
+{
+    return (handle != nullptr);
+}
 }  // namespace omnitrace
@@ -27,9 +27,15 @@
 #include <dlfcn.h>
 #include <string>
 #include <unistd.h>
+#include <vector>

 namespace omnitrace
 {
+std::string
+find_library_path(const std::string& _name, const std::vector<std::string>& _env_vars,
+                  const std::vector<std::string>& _hints,
+                  const std::vector<std::string>& _path_suffixes = { "lib", "lib64" });
+
 struct dynamic_library
 {
    dynamic_library()                           = delete;
@@ -38,37 +44,43 @@ struct dynamic_library
    dynamic_library& operator=(const dynamic_library&) = delete;
    dynamic_library& operator=(dynamic_library&&) noexcept = default;

-    dynamic_library(const char* _env, const char* _fname,
-                    int _flags = (RTLD_LAZY | RTLD_GLOBAL), bool _store = false);
+    dynamic_library(std::string _env, std::string _fname,
+                    int _flags = (RTLD_LAZY | RTLD_GLOBAL), bool _open = true,
+                    bool _query_env = true, bool _store = true);

    ~dynamic_library();

    bool open();
    int  close() const;
+    bool is_open() const;

    template <typename RetT, typename... Args>
-    RetT invoke(std::string_view _name, RetT (*&_func)(Args...), Args... _args)
-    {
-        if(!handle) open();
-        if(handle)
-        {
-            *(void**) (&_func) = dlsym(handle, _name.data());
-            if(_func)
-            {
-                return (*_func)(_args...);
-            }
-            else
-            {
-                fprintf(stderr, "[omnitrace][pid=%i]> %s :: %s\n", getpid(), _name.data(),
-                        dlerror());
-            }
-        }
-        return RetT{};
-    }
+    RetT invoke(std::string_view, RetT (*&_func)(Args...), Args...);

    std::string envname  = {};
    std::string filename = {};
    int         flags    = 0;
    void*       handle   = nullptr;
 };
+
+template <typename RetT, typename... Args>
+inline RetT
+dynamic_library::invoke(std::string_view _name, RetT (*&_func)(Args...), Args... _args)
+{
+    if(!handle) open();
+    if(handle)
+    {
+        *(void**) (&_func) = dlsym(handle, _name.data());
+        if(_func)
+        {
+            return (*_func)(_args...);
+        }
+        else
+        {
+            fprintf(stderr, "[omnitrace][pid=%i]> %s :: %s\n", getpid(), _name.data(),
+                    dlerror());
+        }
+    }
+    return RetT{};
+}
 }  // namespace omnitrace
@@ -28,6 +28,7 @@
 #include "library/config.hpp"
 #include "library/critical_trace.hpp"
 #include "library/debug.hpp"
+#include "library/gpu.hpp"
 #include "library/rocprofiler.hpp"
 #include "library/rocprofiler/hsa_rsrc_factory.hpp"
 #include "library/roctracer.hpp"
@@ -103,13 +104,20 @@ extern "C"
        rocm::lock_t _lk{ rocm::rocm_mutex, std::defer_lock };
        if(!_lk.owns_lock()) _lk.lock();

-        if(rocm::is_loaded) return;
+        if(rocm::is_loaded)
+        {
+            OMNITRACE_BASIC_VERBOSE_F(1, "rocprofiler is already loaded\n");
+            return;
+        }
        rocm::is_loaded = true;

        _lk.unlock();

        // Enable timestamping
-        settings->timestamp_on = 1u;
+        settings->timestamp_on     = 1;
+        settings->intercept_mode   = 1;
+        settings->hsa_intercepting = 1;
+        settings->k_concurrent     = 1;

        // Initialize profiling
        omnitrace::rocprofiler::rocm_initialize();
@@ -127,7 +135,7 @@ extern "C"

        roctracer_is_init() = true;
        pthread_gotcha::push_enable_sampling_on_child_threads(false);
-        OMNITRACE_BASIC_VERBOSE_F(1, "\n");
+        OMNITRACE_BASIC_VERBOSE_F(1, "Loading ROCm tooling...\n");

        tim::consume_parameters(table, runtime_version, failed_tool_count,
                                failed_tool_names);
@@ -156,7 +164,6 @@ extern "C"
                    // initialize HSA tracing
                    roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table);

-                    OMNITRACE_VERBOSE(1, "    HSA-trace(");
                    if(!hsa_api_vec.empty())
                    {
                        for(const auto& itr : hsa_api_vec)
@@ -168,15 +175,15 @@ extern "C"
                            ROCTRACER_CALL(roctracer_enable_op_callback(
                                ACTIVITY_DOMAIN_HSA_API, cid, hsa_api_callback, nullptr));

-                            OMNITRACE_VERBOSE(1, " %s", api);
+                            OMNITRACE_VERBOSE(1, "    HSA-trace(%s)", api);
                        }
                    }
                    else
                    {
+                        OMNITRACE_VERBOSE(1, "    HSA-trace()\n");
                        ROCTRACER_CALL(roctracer_enable_domain_callback(
                            ACTIVITY_DOMAIN_HSA_API, hsa_api_callback, nullptr));
                    }
-                    OMNITRACE_VERBOSE(1, " )\n");
                }

                bool trace_hsa_activity = get_trace_hsa_activity();
@@ -212,17 +219,21 @@ extern "C"
                roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
        };

+        OMNITRACE_VERBOSE_F(1, "Computing the roctracer clock skew...\n");
        (void) omnitrace::get_clock_skew();

        comp::roctracer::add_setup("hsa", _setup);
        comp::roctracer::add_shutdown("hsa", _shutdown);

+        OMNITRACE_VERBOSE_F(1, "Setting rocm_smi state to active...\n");
        rocm_smi::set_state(State::Active);
+
+        OMNITRACE_VERBOSE_F(1, "Requesting roctracer to setup...\n");
        comp::roctracer::setup();

 #if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
        bool _force_rocprofiler_init =
-            tim::get_env("OMNITRACE_FORCE_ROCPROFILE_INIT", false, false);
+            tim::get_env("OMNITRACE_FORCE_ROCPROFILER_INIT", false, false);
 #else
        bool _force_rocprofiler_init = false;
 #endif
@@ -233,8 +244,17 @@ extern "C"
        if(_force_rocprofiler_init || (get_use_rocprofiler() && !_is_empty))
        {
            auto _rocprof =
-                dynamic_library{ "OMNITRACE_ROCPROFILER_LIBRARY", "librocprofiler64.so",
-                                 (RTLD_LAZY | RTLD_GLOBAL), true };
+                dynamic_library{ "OMNITRACE_ROCPROFILER_LIBRARY",
+                                 find_library_path("librocprofiler64.so",
+                                                   { "OMNITRACE_ROCM_PATH", "ROCM_PATH" },
+                                                   { OMNITRACE_DEFAULT_ROCM_PATH },
+                                                   { "lib", "lib64", "rocprofiler/lib",
+                                                     "rocprofiler/lib64" }),
+                                 (RTLD_LAZY | RTLD_GLOBAL), false };
+
+            OMNITRACE_VERBOSE_F(1, "Loading rocprofiler library (%s=%s)...\n",
+                                _rocprof.envname.c_str(), _rocprof.filename.c_str());
+            _rocprof.open();

            on_load_t _rocprof_load = nullptr;
            _success = _rocprof.invoke("OnLoad", _rocprof_load, table, runtime_version,