rocprofiler and roctx updates (#261)

* Improve locating ROCP_METRICS - moved common::path to common/path.hpp - added more functionality to common::path - common::path::exists now returns true for directories - check for metrics.xml and gfx_metrics.xml before setting ROCP_METRICS - throw error if path for ROCP_METRICS is cannot be explicitly determined * Fix roctxRangePop handling - message is nullptr -> keep thread-local stack [ROCm/rocprofiler-systems commit: 9eafb23602]
2023-03-22 00:49:14 -05:00
@@ -22,6 +22,8 @@

 #pragma once

+#include "common/defines.h"
+
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
@@ -30,6 +32,47 @@
 #include <string>
 #include <string_view>
 #include <type_traits>
+#include <unistd.h>
+
+#if !defined(OMNITRACE_ENVIRON_LOG_NAME)
+#    if defined(OMNITRACE_COMMON_LIBRARY_NAME)
+#        define OMNITRACE_ENVIRON_LOG_NAME "[" OMNITRACE_COMMON_LIBRARY_NAME "]"
+#    else
+#        define OMNITRACE_ENVIRON_LOG_NAME
+#    endif
+#endif
+
+#if !defined(OMNITRACE_ENVIRON_LOG_START)
+#    if defined(OMNITRACE_COMMON_LIBRARY_LOG_START)
+#        define OMNITRACE_ENVIRON_LOG_START OMNITRACE_COMMON_LIBRARY_LOG_START
+#    elif defined(TIMEMORY_LOG_COLORS_AVAILABLE)
+#        define OMNITRACE_ENVIRON_LOG_START                                              \
+            fprintf(stderr, "%s", ::tim::log::color::info());
+#    else
+#        define OMNITRACE_ENVIRON_LOG_START
+#    endif
+#endif
+
+#if !defined(OMNITRACE_ENVIRON_LOG_END)
+#    if defined(OMNITRACE_COMMON_LIBRARY_LOG_END)
+#        define OMNITRACE_ENVIRON_LOG_END OMNITRACE_COMMON_LIBRARY_LOG_END
+#    elif defined(TIMEMORY_LOG_COLORS_AVAILABLE)
+#        define OMNITRACE_ENVIRON_LOG_END fprintf(stderr, "%s", ::tim::log::color::end());
+#    else
+#        define OMNITRACE_ENVIRON_LOG_END
+#    endif
+#endif
+
+#define OMNITRACE_ENVIRON_LOG(CONDITION, ...)                                            \
+    if(CONDITION)                                                                        \
+    {                                                                                    \
+        fflush(stderr);                                                                  \
+        OMNITRACE_ENVIRON_LOG_START                                                      \
+        fprintf(stderr, "[omnitrace]" OMNITRACE_ENVIRON_LOG_NAME "[%i] ", getpid());     \
+        fprintf(stderr, __VA_ARGS__);                                                    \
+        OMNITRACE_ENVIRON_LOG_END                                                        \
+        fflush(stderr);                                                                  \
+    }

 namespace omnitrace
 {
@@ -101,6 +144,7 @@ get_env_impl(std::string_view env_id, bool _default)
    }
    return _default;
 }
+}  // namespace

 template <typename Tp>
 inline auto
@@ -117,6 +161,20 @@ get_env(std::string_view env_id, Tp&& _default)
        return get_env_impl(env_id, std::forward<Tp>(_default));
    }
 }
-}  // namespace
+
+struct OMNITRACE_INTERNAL_API env_config
+{
+    std::string env_name  = {};
+    std::string env_value = {};
+    int         override  = 0;
+
+    auto operator()(bool _verbose = false) const
+    {
+        if(env_name.empty()) return -1;
+        OMNITRACE_ENVIRON_LOG(_verbose, "setenv(\"%s\", \"%s\", %i)\n", env_name.c_str(),
+                              env_value.c_str(), override);
+        return setenv(env_name.c_str(), env_value.c_str(), override);
+    }
+};
 }  // namespace common
 }  // namespace omnitrace
@@ -0,0 +1,414 @@
+// MIT License
+//
+// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "common/defines.h"
+#include "common/delimit.hpp"
+#include "common/environment.hpp"
+#include "common/join.hpp"
+
+#include <cstdlib>
+#include <cstring>
+#include <dlfcn.h>
+#include <fstream>
+#include <link.h>
+#include <linux/limits.h>
+#include <string>
+#include <string_view>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#if !defined(OMNITRACE_PATH_LOG_NAME)
+#    if defined(OMNITRACE_COMMON_LIBRARY_NAME)
+#        define OMNITRACE_PATH_LOG_NAME "[" OMNITRACE_COMMON_LIBRARY_NAME "]"
+#    else
+#        define OMNITRACE_PATH_LOG_NAME
+#    endif
+#endif
+
+#if !defined(OMNITRACE_PATH_LOG_START)
+#    if defined(OMNITRACE_COMMON_LIBRARY_LOG_START)
+#        define OMNITRACE_PATH_LOG_START OMNITRACE_COMMON_LIBRARY_LOG_START
+#    elif defined(TIMEMORY_LOG_COLORS_AVAILABLE)
+#        define OMNITRACE_PATH_LOG_START fprintf(stderr, "%s", ::tim::log::color::info());
+#    else
+#        define OMNITRACE_PATH_LOG_START
+#    endif
+#endif
+
+#if !defined(OMNITRACE_PATH_LOG_END)
+#    if defined(OMNITRACE_COMMON_LIBRARY_LOG_END)
+#        define OMNITRACE_PATH_LOG_END OMNITRACE_COMMON_LIBRARY_LOG_END
+#    elif defined(TIMEMORY_LOG_COLORS_AVAILABLE)
+#        define OMNITRACE_PATH_LOG_END fprintf(stderr, "%s", ::tim::log::color::end());
+#    else
+#        define OMNITRACE_PATH_LOG_END
+#    endif
+#endif
+
+#define OMNITRACE_PATH_LOG(CONDITION, ...)                                               \
+    if(CONDITION)                                                                        \
+    {                                                                                    \
+        fflush(stderr);                                                                  \
+        OMNITRACE_PATH_LOG_START                                                         \
+        fprintf(stderr, "[omnitrace]" OMNITRACE_PATH_LOG_NAME "[%i] ", getpid());        \
+        fprintf(stderr, __VA_ARGS__);                                                    \
+        OMNITRACE_PATH_LOG_END                                                           \
+        fflush(stderr);                                                                  \
+    }
+
+namespace omnitrace
+{
+inline namespace common
+{
+namespace path
+{
+inline std::vector<std::string>
+get_link_map(const char*, std::vector<int>&& = { (RTLD_LAZY | RTLD_NOLOAD) },
+             bool _include_self = false) OMNITRACE_INTERNAL_API;
+
+inline auto
+get_link_map(const char* _name, bool&& _include_self,
+             std::vector<int>&& _open_modes = {
+                 (RTLD_LAZY | RTLD_NOLOAD) }) OMNITRACE_INTERNAL_API;
+
+inline std::string
+get_origin(const std::string&,
+           std::vector<int>&& = { (RTLD_LAZY | RTLD_NOLOAD) }) OMNITRACE_INTERNAL_API;
+
+inline bool
+exists(const std::string& _fname) OMNITRACE_INTERNAL_API;
+
+template <typename RetT = std::string>
+inline RetT
+get_default_lib_search_paths() OMNITRACE_INTERNAL_API;
+
+inline std::string
+find_path(const std::string& _path, int _verbose,
+          const std::string& _search_paths = {}) OMNITRACE_INTERNAL_API;
+
+inline std::string
+dirname(const std::string& _fname) OMNITRACE_INTERNAL_API;
+
+inline std::string
+realpath(const std::string& _relpath,
+         std::string*       _resolved = nullptr) OMNITRACE_INTERNAL_API;
+
+inline bool
+is_text_file(const std::string& filename) OMNITRACE_INTERNAL_API;
+
+inline bool
+is_link(const std::string& _path) OMNITRACE_INTERNAL_API;
+
+inline std::string
+readlink(const std::string& _path) OMNITRACE_INTERNAL_API;
+
+struct OMNITRACE_INTERNAL_API path_type
+{
+    enum path_type_e
+    {
+        directory = 0,
+        regular,
+        link,
+        unknown
+    };
+
+    inline path_type(const std::string&);
+    ~path_type()                = default;
+    path_type(const path_type&) = default;
+    path_type(path_type&&)      = default;
+    path_type& operator=(const path_type&) = default;
+    path_type& operator=(path_type&&) = default;
+
+    bool     exists() const { return m_type < unknown; }
+    explicit operator bool() const { return exists(); }
+
+private:
+    path_type_e m_type = unknown;
+};
+
+//--------------------------------------------------------------------------------------//
+//
+//      Implementation
+//
+//--------------------------------------------------------------------------------------//
+
+path_type::path_type(const std::string& _fname)
+{
+    struct stat _buffer;
+    if(lstat(_fname.c_str(), &_buffer) == 0)
+    {
+        if(S_ISDIR(_buffer.st_mode) != 0)
+            m_type = directory;
+        else if(S_ISREG(_buffer.st_mode) != 0)
+            m_type = regular;
+        else if(S_ISLNK(_buffer.st_mode) != 0)
+            m_type = link;
+    }
+}
+
+bool
+exists(const std::string& _fname)
+{
+    struct stat _buffer;
+    if(lstat(_fname.c_str(), &_buffer) == 0)
+        return (S_ISDIR(_buffer.st_mode) != 0 || S_ISREG(_buffer.st_mode) != 0 ||
+                S_ISLNK(_buffer.st_mode) != 0);
+    return false;
+}
+
+template <typename RetT>
+RetT
+get_default_lib_search_paths()
+{
+    auto _paths = join(":", get_env("OMNITRACE_PATH", ""), get_env("LD_LIBRARY_PATH", ""),
+                       get_env("LIBRARY_PATH", ""), get_env("PWD", ""), ".");
+    if constexpr(std::is_same<RetT, std::string>::value)
+        return _paths;
+    else
+        return delimit(_paths, ":");
+}
+
+std::string
+find_path(const std::string& _path, int _verbose, const std::string& _search_paths)
+{
+    if(exists(_path) && !_path.empty() && _path.at(0) == '/') return _path;
+
+    auto _paths = delimit(_search_paths, ":");
+    if(_paths.empty())
+    {
+        _paths = get_default_lib_search_paths<std::vector<std::string>>();
+    }
+
+    constexpr int _verbose_lvl = 2;
+    for(const auto& itr : _paths)
+    {
+        auto _f = join('/', itr, _path);
+        OMNITRACE_PATH_LOG(_verbose >= _verbose_lvl + 1,
+                           "searching for '%s' in '%s' ...\n", _path.c_str(),
+                           itr.c_str());
+        if(exists(_f))
+        {
+            OMNITRACE_PATH_LOG(_verbose >= _verbose_lvl, "found '%s' in '%s' ...\n",
+                               _path.c_str(), itr.c_str());
+            return _f;
+        }
+    }
+
+    for(const auto& itr : _paths)
+    {
+        if(std::string_view{ ::basename(itr.c_str()) }.find("lib") ==
+               std::string_view::npos &&
+           !dirname(itr).empty())
+        {
+            for(const auto* sitr : { "lib", "lib64", "../lib", "../lib64" })
+            {
+                auto _f = join('/', dirname(itr), sitr, _path);
+                OMNITRACE_PATH_LOG(_verbose >= _verbose_lvl + 1,
+                                   "searching for '%s' in '%s' ...\n", _path.c_str(),
+                                   common::join('/', itr, sitr).c_str());
+                if(exists(_f))
+                {
+                    OMNITRACE_PATH_LOG(_verbose >= _verbose_lvl,
+                                       "found '%s' in '%s' ...\n", _path.c_str(),
+                                       itr.c_str());
+                    return _f;
+                }
+            }
+        }
+    }
+
+    return _path;
+}
+
+std::string
+dirname(const std::string& _fname)
+{
+    if(_fname.find('/') != std::string::npos)
+        return _fname.substr(0, _fname.find_last_of('/'));
+    return std::string{};
+}
+
+bool
+is_link(const std::string& _path)
+{
+    struct stat _buffer;
+    if(lstat(_path.c_str(), &_buffer) == 0) return (S_ISLNK(_buffer.st_mode) != 0);
+    return false;
+}
+
+std::string
+readlink(const std::string& _path)
+{
+    constexpr size_t MaxLen = PATH_MAX;
+    // if not a symbolic link, just return the path
+    if(!is_link(_path)) return _path;
+
+    char    _buffer[MaxLen];
+    ssize_t _buffer_len = MaxLen;
+    _buffer_len         = ::readlink(_path.c_str(), _buffer, _buffer_len);
+    if(_buffer_len < 0 || _buffer_len == (MaxLen))
+    {
+        auto* _path_rp = ::realpath(_path.c_str(), nullptr);
+        if(_path_rp)
+        {
+            auto _ret = std::string{ _path_rp };
+            free(_path_rp);
+            return _ret;
+        }
+    }
+    else
+    {
+        _buffer[_buffer_len] = '\0';
+        return _buffer;
+    }
+    return _path;
+}
+
+std::string
+realpath(const std::string& _relpath, std::string* _resolved)
+{
+    constexpr size_t MaxLen = PATH_MAX;
+    auto             _len   = std::min<size_t>(_relpath.length(), MaxLen);
+
+    char        _buffer[MaxLen] = { '\0' };
+    const char* _result         = _buffer;
+
+    if(::realpath(_relpath.c_str(), _buffer) == nullptr)
+    {
+        _result = _relpath.data();
+    }
+
+    if(_resolved)
+    {
+        _resolved->clear();
+        _len = strnlen(_result, MaxLen);
+        _resolved->resize(_len);
+        for(size_t i = 0; i < _len; ++i)
+            (*_resolved)[i] = _result[i];
+    }
+
+    return (_resolved) ? *_resolved : std::string{ _result };
+}
+
+bool
+is_text_file(const std::string& filename)
+{
+    std::ifstream _file{ filename, std::ios::in | std::ios::binary };
+    if(!_file.is_open())
+    {
+        OMNITRACE_PATH_LOG(0, "Error! '%s' could not be opened...\n", filename.c_str());
+        return false;
+    }
+
+    constexpr size_t buffer_size = 1024;
+    char             buffer[buffer_size];
+    while(_file.read(buffer, sizeof(buffer)))
+    {
+        for(char itr : buffer)
+        {
+            if(itr == '\0') return false;
+        }
+    }
+
+    if(_file.gcount() > 0)
+    {
+        for(std::streamsize i = 0; i < _file.gcount(); ++i)
+        {
+            if(buffer[i] == '\0') return false;
+        }
+    }
+
+    return true;
+}
+
+std::vector<std::string>
+get_link_map(const char* _name, std::vector<int>&& _open_modes, bool _include_self)
+{
+    void* _handle = nullptr;
+    bool  _noload = false;
+    for(auto _mode : _open_modes)
+    {
+        _handle = dlopen(_name, _mode);
+        _noload = (_mode & RTLD_NOLOAD) == RTLD_NOLOAD;
+        if(_handle) break;
+    }
+
+    auto _chain = std::vector<std::string>{};
+    if(_handle)
+    {
+        struct link_map* _link_map = nullptr;
+        dlinfo(_handle, RTLD_DI_LINKMAP, &_link_map);
+        // if include_self is false, start at next library
+        struct link_map* _next = (_include_self) ? _link_map : _link_map->l_next;
+        while(_next)
+        {
+            if(_next->l_name != nullptr && !std::string_view{ _next->l_name }.empty())
+            {
+                _chain.emplace_back(_next->l_name);
+            }
+            _next = _next->l_next;
+        }
+
+        if(_noload == false) dlclose(_handle);
+    }
+    return _chain;
+}
+
+auto
+get_link_map(const char* _name, bool&& _include_self, std::vector<int>&& _open_modes)
+{
+    return get_link_map(_name, std::move(_open_modes), _include_self);
+}
+
+std::string
+get_origin(const std::string& _filename, std::vector<int>&& _open_modes)
+{
+    void* _handle = nullptr;
+    bool  _noload = false;
+    for(auto _mode : _open_modes)
+    {
+        _handle = dlopen(_filename.c_str(), _mode);
+        _noload = (_mode & RTLD_NOLOAD) == RTLD_NOLOAD;
+        if(_handle) break;
+    }
+
+    auto _chain = std::vector<std::string>{};
+    if(_handle)
+    {
+        char _buffer[PATH_MAX];
+        memset(_buffer, '\0', PATH_MAX * sizeof(char));
+        if(dlinfo(_handle, RTLD_DI_ORIGIN, &_buffer) == 0)
+        {
+            auto _origin = std::string{ _buffer };
+            if(exists(_origin)) return _origin;
+        }
+
+        if(_noload == false) dlclose(_handle);
+    }
+
+    return std::string{};
+}
+}  // namespace path
+}  // namespace common
+}  // namespace omnitrace
@@ -26,9 +26,16 @@
 #include "common/delimit.hpp"
 #include "common/environment.hpp"
 #include "common/join.hpp"
+#include "common/path.hpp"

+#include <algorithm>
 #include <cstdlib>
 #include <cstring>
+#include <dlfcn.h>
+#include <fstream>
+#include <ios>
+#include <link.h>
+#include <linux/limits.h>
 #include <string>
 #include <string_view>
 #include <sys/stat.h>
@@ -78,81 +85,32 @@ namespace omnitrace
 {
 inline namespace common
 {
-namespace path
+inline std::vector<env_config>
+get_environ(int _verbose, std::string _search_paths = {},
+            std::string _omnilib    = "libomnitrace.so",
+            std::string _omnilib_dl = "libomnitrace-dl.so")
 {
-inline std::string
-find_path(const std::string& _path, int _verbose, std::string _search_paths = {})
-{
-    auto _exists = [](std::string_view _name) {
-        struct stat _buffer;
-        if(stat(_name.data(), &_buffer) == 0)
-            return (S_ISREG(_buffer.st_mode) != 0 || S_ISLNK(_buffer.st_mode) != 0);
-        return false;
-    };
+    auto _data            = std::vector<env_config>{};
+    auto _omnilib_path    = path::get_origin(_omnilib);
+    auto _omnilib_dl_path = path::get_origin(_omnilib_dl);

-    if(_exists(_path) && !_path.empty() && _path.at(0) == '/') return _path;
-
-    auto _default_search_paths =
-        join(":", get_env("OMNITRACE_PATH", ""), get_env("PWD", ""), ".",
-             get_env("LD_LIBRARY_PATH", ""), get_env("LIBRARY_PATH", ""));
-
-    if(_search_paths.empty()) _search_paths = _default_search_paths;
-
-    auto _paths = delimit(_search_paths, ":");
-
-    if(_paths.empty())
+    if(!_omnilib_path.empty())
    {
-        _search_paths = _default_search_paths;
-        _paths        = delimit(_search_paths, ":");
+        _omnilib      = join('/', _omnilib_path, ::basename(_omnilib.c_str()));
+        _search_paths = join(':', _omnilib_path, _search_paths);
    }

-    int _verbose_lvl = 2;
-    for(const auto& itr : _paths)
+    if(!_omnilib_dl_path.empty())
    {
-        auto _f = join('/', itr, _path);
-        OMNITRACE_SETUP_LOG(_verbose >= _verbose_lvl, "searching for '%s' in '%s' ...\n",
-                            _path.c_str(), itr.c_str());
-        if(_exists(_f)) return _f;
-        for(const auto* sitr : { "lib", "lib64", "../lib", "../lib64" })
-        {
-            _f = join('/', itr, sitr, _path);
-            OMNITRACE_SETUP_LOG(_verbose >= _verbose_lvl,
-                                "searching for '%s' in '%s' ...\n", _path.c_str(),
-                                common::join('/', itr, sitr).c_str());
-            if(_exists(_f)) return _f;
-        }
+        _omnilib_dl   = join('/', _omnilib_dl_path, ::basename(_omnilib_dl.c_str()));
+        _search_paths = join(':', _omnilib_dl_path, _search_paths);
    }
-    return _path;
-}

-inline std::string
-dirname(const std::string& _fname)
-{
-    if(_fname.find('/') != std::string::npos)
-        return _fname.substr(0, _fname.find_last_of('/'));
-    return std::string{};
-}
-
-inline bool
-exists(const std::string& _fname)
-{
-    struct stat _buffer;
-    if(stat(_fname.c_str(), &_buffer) == 0)
-        return (S_ISREG(_buffer.st_mode) != 0 || S_ISLNK(_buffer.st_mode) != 0);
-    return false;
-}
-}  // namespace path
-
-inline void
-setup_environ(int _verbose, const std::string& _search_paths = {},
-              std::string _omnilib    = "libomnitrace.so",
-              std::string _omnilib_dl = "libomnitrace-dl.so")
-{
    _omnilib    = common::path::find_path(_omnilib, _verbose, _search_paths);
    _omnilib_dl = common::path::find_path(_omnilib_dl, _verbose, _search_paths);

 #if defined(OMNITRACE_USE_ROCTRACER) && OMNITRACE_USE_ROCTRACER > 0
-    setenv("HSA_TOOLS_LIB", _omnilib.c_str(), 0);
+    _data.emplace_back(env_config{ "HSA_TOOLS_LIB", _omnilib.c_str(), 0 });
 #endif

 #if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
@@ -161,11 +119,11 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
 #    else
 #        define ROCPROFILER_METRICS_DIR "rocprofiler/lib"
 #    endif
-    setenv("HSA_TOOLS_LIB", _omnilib.c_str(), 0);
-    setenv("ROCP_TOOL_LIB", _omnilib.c_str(), 0);
-    setenv("ROCPROFILER_LOG", "1", 0);
-    setenv("ROCP_HSA_INTERCEPT", "1", 0);
-    setenv("HSA_TOOLS_REPORT_LOAD_FAILURE", "1", 0);
+    _data.emplace_back(env_config{ "HSA_TOOLS_LIB", _omnilib.c_str(), 0 });
+    _data.emplace_back(env_config{ "ROCP_TOOL_LIB", _omnilib.c_str(), 0 });
+    _data.emplace_back(env_config{ "ROCPROFILER_LOG", "1", 0 });
+    _data.emplace_back(env_config{ "ROCP_HSA_INTERCEPT", "1", 0 });
+    _data.emplace_back(env_config{ "HSA_TOOLS_REPORT_LOAD_FAILURE", "1", 0 });

    auto _possible_rocp_metrics = std::vector<std::string>{};
    auto _possible_rocprof_libs = std::vector<std::string>{};
@@ -174,11 +132,11 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
        if(getenv(itr))
        {
            _possible_rocp_metrics.emplace_back(
-                common::join('/', getenv(itr), "lib/rocprofiler", "metrics.xml"));
+                common::join('/', getenv(itr), "lib/rocprofiler"));
            _possible_rocprof_libs.emplace_back(
                common::join('/', getenv(itr), "lib/rocprofiler", "librocprofiler64.so"));
            _possible_rocp_metrics.emplace_back(
-                common::join('/', getenv(itr), "rocprofiler/lib", "metrics.xml"));
+                common::join('/', getenv(itr), "rocprofiler/lib"));
            _possible_rocprof_libs.emplace_back(
                common::join('/', getenv(itr), "rocprofiler/lib", "librocprofiler64.so"));
        }
@@ -186,50 +144,137 @@ setup_environ(int _verbose, const std::string& _search_paths = {},

    // default path
    _possible_rocp_metrics.emplace_back(
-        common::join('/', OMNITRACE_DEFAULT_ROCM_PATH, "lib/rocprofiler", "metrics.xml"));
+        common::join('/', OMNITRACE_DEFAULT_ROCM_PATH, "lib/rocprofiler"));
    _possible_rocp_metrics.emplace_back(
-        common::join('/', OMNITRACE_DEFAULT_ROCM_PATH, "rocprofiler/lib", "metrics.xml"));
+        common::join('/', OMNITRACE_DEFAULT_ROCM_PATH, "rocprofiler/lib"));
+
+    auto _realpath_and_unique = [](const auto& _inp_v) {
+        auto _out_v = decltype(_inp_v){};
+        for(auto& itr : _inp_v)
+        {
+            if(path::exists(itr)) _out_v.emplace_back(path::realpath(itr));
+        }
+
+        _out_v.erase(std::unique(_out_v.begin(), _out_v.end()), _out_v.end());
+        return _out_v;
+    };
+
+    _possible_rocprof_libs = _realpath_and_unique(_possible_rocprof_libs);

    for(const auto& itr : _possible_rocprof_libs)
    {
        if(path::exists(itr))
        {
-            setenv("OMNITRACE_ROCPROFILER_LIBRARY", itr.c_str(), 0);
+            _data.emplace_back(
+                env_config{ "OMNITRACE_ROCPROFILER_LIBRARY", itr.c_str(), 0 });
            _possible_rocp_metrics.emplace(
                _possible_rocp_metrics.begin(),
-                common::join('/', path::dirname(itr),
-                             "../../lib/rocprofiler/metrics.xml"));
-            _possible_rocp_metrics.emplace(
-                _possible_rocp_metrics.begin(),
-                common::join('/', path::dirname(itr), "metrics.xml"));
+                common::join('/', path::dirname(itr), "../../lib/rocprofiler"));
+            _possible_rocp_metrics.emplace(_possible_rocp_metrics.begin(),
+                                           common::join('/', path::dirname(itr)));
        }
    }

-    for(const auto& itr : _possible_rocp_metrics)
-        if(path::exists(itr)) setenv("ROCP_METRICS", itr.c_str(), 0);
+    _possible_rocp_metrics = _realpath_and_unique(_possible_rocp_metrics);

-    // default if none of above succeeded
-    setenv("ROCP_METRICS",
-           common::join('/', OMNITRACE_DEFAULT_ROCM_PATH, ROCPROFILER_METRICS_DIR,
-                        "metrics.xml")
-               .c_str(),
-           0);
+    auto _env_rocp_metrics = get_env("ROCP_METRICS", "");
+    if(!_env_rocp_metrics.empty())
+    {
+        if(!path::exists(_env_rocp_metrics))
+            throw std::runtime_error(join("", "Error! ROCP_METRICS file \"",
+                                          _env_rocp_metrics, "\" does not exist"));
+        _possible_rocp_metrics.clear();
+        _possible_rocp_metrics.emplace_back(
+            common::join('/', path::dirname(_env_rocp_metrics)));
+    }

+    auto _found_rocp_metrics = (!_env_rocp_metrics.empty())
+                                   ? get_env("OMNITRACE_ROCP_METRICS_FORCE_VALID", false)
+                                   : false;
+
+    if(!_found_rocp_metrics)
+    {
+        for(const auto& itr : _possible_rocp_metrics)
+        {
+            auto _metrics_path = join('/', itr, "metrics.xml");
+            if(path::exists(itr) && path::exists(_metrics_path) &&
+               path::exists(join('/', itr, "gfx_metrics.xml")))
+            {
+                _found_rocp_metrics = true;
+                _data.emplace_back(
+                    env_config{ "ROCP_METRICS", _metrics_path.c_str(), 0 });
+                break;
+            }
+        }
+    }
+
+    // handle error
+    if(!_found_rocp_metrics)
+    {
+        auto _msg = std::stringstream{};
+        _msg << std::boolalpha;
+        if(!_env_rocp_metrics.empty())
+        {
+            auto _env_rocp_metrics_dir = path::dirname(_env_rocp_metrics);
+            auto _rocp_metrics_xml     = join('/', _env_rocp_metrics_dir, "metrics.xml");
+            auto _rocp_gfx_metrics_xml =
+                join('/', _env_rocp_metrics_dir, "gfx_metrics.xml");
+            _msg << "Error! ROCP_METRICS=\"" << _env_rocp_metrics
+                 << "\" in the environment but the directory (" << _env_rocp_metrics_dir
+                 << ") does not contain "
+                    "metrics.xml (found: "
+                 << path::exists(_rocp_metrics_xml) << ") and/or gfx_metrics.xml (found: "
+                 << path::exists(_rocp_gfx_metrics_xml)
+                 << "). To ignore this error, set "
+                    "OMNITRACE_ROCP_METRICS_FORCE_VALID=true in the environment";
+        }
+        else
+        {
+            _msg << "Error! ROCP_METRICS not set in environment and OmniTrace could not "
+                    "find a suitable path. Please set ROCP_METRICS=/path/to/metrics.xml "
+                    "in the environment. This file is typically located in the same "
+                    "folder as the librocprofiler64.so library.\nAdditional note: "
+                    "metrics.xml typically contains:\n\t#include "
+                    "\"gfx_metrics.xml\"\nMake sure the provided path also contains this "
+                    "file.\nExample:\n\texport ROCP_METRICS="
+                 << OMNITRACE_DEFAULT_ROCM_PATH << "/" << ROCPROFILER_METRICS_DIR
+                 << "/metrics.xml\n";
+        }
+        throw std::runtime_error(_msg.str());
+    }
 #endif

 #if defined(OMNITRACE_USE_OMPT) && OMNITRACE_USE_OMPT > 0
-    std::string _omni_omp_libs = _omnilib_dl;
-    const char* _omp_libs      = getenv("OMP_TOOL_LIBRARIES");
-    if(_omp_libs != nullptr &&
-       std::string_view{ _omp_libs }.find(_omnilib_dl) == std::string::npos)
-        _omni_omp_libs = common::join(':', _omp_libs, _omnilib_dl);
-    OMNITRACE_SETUP_LOG(_verbose >= 2, "setting OMP_TOOL_LIBRARIES to '%s'\n",
-                        _omni_omp_libs.c_str());
-    setenv("OMP_TOOL_LIBRARIES", _omni_omp_libs.c_str(), 1);
+    if(get_env("OMNITRACE_USE_OMPT", true))
+    {
+        std::string _omni_omp_libs = _omnilib_dl;
+        const char* _omp_libs      = getenv("OMP_TOOL_LIBRARIES");
+        int         _override      = 0;
+        if(_omp_libs != nullptr &&
+           std::string_view{ _omp_libs }.find(_omnilib_dl) == std::string::npos)
+        {
+            _override      = 1;
+            _omni_omp_libs = common::join(':', _omp_libs, _omnilib_dl);
+        }
+        OMNITRACE_SETUP_LOG(_verbose >= 2, "setting OMP_TOOL_LIBRARIES to '%s'\n",
+                            _omni_omp_libs.c_str());
+        _data.emplace_back(
+            env_config{ "OMP_TOOL_LIBRARIES", _omni_omp_libs.c_str(), _override });
+    }
 #endif

-    (void) _omnilib;
-    (void) _omnilib_dl;
+    return _data;
+}
+
+inline void
+setup_environ(int _verbose, const std::string& _search_paths = {},
+              std::string _omnilib    = "libomnitrace.so",
+              std::string _omnilib_dl = "libomnitrace-dl.so")
+{
+    auto _data =
+        get_environ(_verbose, _search_paths, std::move(_omnilib), std::move(_omnilib_dl));
+    for(const auto& itr : _data)
+        itr(_verbose >= 3);
 }
 }  // namespace common
 }  // namespace omnitrace
@@ -448,20 +448,35 @@ roctx_api_callback(uint32_t domain, uint32_t cid, const void* callback_data,

    if(domain != ACTIVITY_DOMAIN_ROCTX) return;

-    static auto _range_map  = std::unordered_map<roctx_range_id_t, std::string_view>{};
+    static auto _range_map  = std::unordered_map<roctx_range_id_t, std::string>{};
    static auto _range_lock = std::mutex{};
    const auto* _data       = reinterpret_cast<const roctx_api_data_t*>(callback_data);
+    static thread_local auto _range_stack = std::vector<std::string>{};

    switch(cid)
    {
        case ROCTX_API_ID_roctxRangePushA:
        {
-            component::category_region<category::rocm_roctx>::start(_data->args.message);
+            if(_data->args.message)
+            {
+                auto& itr = _range_stack.emplace_back(std::string{ _data->args.message });
+                component::category_region<category::rocm_roctx>::start(itr.c_str());
+            }
            break;
        }
        case ROCTX_API_ID_roctxRangePop:
        {
-            component::category_region<category::rocm_roctx>::stop(_data->args.message);
+            if(!_range_stack.empty())
+            {
+                auto& itr = _range_stack.back();
+                component::category_region<category::rocm_roctx>::stop(itr.c_str());
+                _range_stack.pop_back();
+            }
+            else
+            {
+                OMNITRACE_THROW("Error! roctxRangePop stack is empty! Expected "
+                                "roctxRangePush/roctxRangePop on same thread\n");
+            }
            break;
        }
        case ROCTX_API_ID_roctxRangeStartA:
@@ -470,7 +485,7 @@ roctx_api_callback(uint32_t domain, uint32_t cid, const void* callback_data,
                std::unique_lock<std::mutex> _lk{ _range_lock, std::defer_lock };
                if(!_lk.owns_lock()) _lk.lock();
                _range_map.emplace(roctx_range_id_t{ _data->args.id },
-                                   std::string_view{ _data->args.message });
+                                   std::string{ _data->args.message });
            }

            component::category_region<category::rocm_roctx>::start(_data->args.message);
@@ -506,7 +521,14 @@ roctx_api_callback(uint32_t domain, uint32_t cid, const void* callback_data,
            break;
        }
        case ROCTX_API_ID_roctxMarkA:
-            // we do nothing with marker events...for now
+        {
+            if(_data->args.message)
+            {
+                component::category_region<category::rocm_roctx>::mark(
+                    _data->args.message);
+            }
+            break;
+        }
        default: break;
    }
 }