ROCm environment fixes + workflow updates (#117)

* Improve dlopen of ROCm libraries + rocprofiler test - Use PROJECT_BINARY_DIR in tests - Added rocprofiler test * Revert OMNITRACE_FORCE_ROCPROFILER_INIT * omnitrace-avail --all test * Fix ROCP_METRICS for ROCm 5.2.0 * Fix ROCP_METRICS for ROCm 5.2.0 * Restrict containers workflow to AMDResearch/omnitrace * Bump version to 1.3.1 * Update cpack workflow - generate release draft - upload installers as release assets * Test rocprofiler w/o roctracer enabled * Fix formatting * verbose message
2022-07-27 06:36:52 -05:00
commit 7e31d9f450
@@ -14,6 +14,8 @@ env:

 jobs:
  omnitrace-ci:
+    if: github.repository == 'AMDResearch/omnitrace'
+
    runs-on: ubuntu-20.04

    strategy:
@@ -57,6 +59,8 @@ jobs:
          popd

  omnitrace-release-ubuntu:
+    if: github.repository == 'AMDResearch/omnitrace'
+
    runs-on: ubuntu-20.04

    strategy:
@@ -98,6 +102,8 @@ jobs:
          popd

  omnitrace-release-opensuse:
+    if: github.repository == 'AMDResearch/omnitrace'
+    
    runs-on: ubuntu-20.04

    strategy:
@@ -13,6 +13,22 @@ env:
  GIT_DISCOVERY_ACROSS_FILESYSTEM: 1

 jobs:
+  release:
+    if: github.repository == 'AMDResearch/omnitrace'
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: write
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Generate Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          draft: True
+          generate_release_notes: True
+
  ubuntu:
    runs-on: ubuntu-20.04
    strategy:
@@ -77,7 +93,6 @@ jobs:
      # before testing remove any artifacts of the build
      - name: Remove Build
        timeout-minutes: 10
-        continue-on-error: true
        run: |
          mv build-release/stgz/*.sh ./
          mv build-release/deb/*.deb ./
@@ -86,7 +101,6 @@ jobs:

      - name: Test STGZ Install
        timeout-minutes: 20
-        continue-on-error: true
        run: |
          set -v
          for i in omnitrace-*.sh
@@ -96,7 +110,6 @@ jobs:

      - name: Test DEB Install
        timeout-minutes: 20
-        continue-on-error: true
        run: |
          set -v
          for i in omnitrace_*.deb
@@ -104,6 +117,14 @@ jobs:
            ./docker/test-docker-release.sh --distro ubuntu --versions ${{ matrix.os }} --rocm-versions ${CI_ROCM_VERSION} -- --deb ${i}
          done

+      - name: Upload Release Assets
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/') && github.repository == 'AMDResearch/omnitrace'
+        with:
+          files: |
+            build-release/stgz/*.sh
+            build-release/deb/*.deb
+
  opensuse:
    runs-on: ubuntu-20.04
    strategy:
@@ -173,7 +194,6 @@ jobs:
      # before testing remove any artifacts of the build
      - name: Remove Build
        timeout-minutes: 10
-        continue-on-error: true
        run: |
          mv build-release/stgz/*.sh ./
          mv build-release/rpm/*.rpm ./
@@ -182,7 +202,6 @@ jobs:

      - name: Test STGZ Install
        timeout-minutes: 20
-        continue-on-error: true
        run: |
          set -v
          for i in omnitrace-*.sh
@@ -192,10 +211,17 @@ jobs:

      - name: Test RPM Install
        timeout-minutes: 20
-        continue-on-error: true
        run: |
          set -v
          for i in omnitrace-*.rpm
          do
            ./docker/test-docker-release.sh --distro opensuse --versions ${{ matrix.os }} --rocm-versions ${CI_ROCM_VERSION} -- --rpm ${i}
          done
+
+      - name: Upload Release Assets
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/') && github.repository == 'AMDResearch/omnitrace'
+        with:
+          files: |
+            build-release/stgz/*.sh
+            build-release/rpm/*.rpm
@@ -1 +1 @@
-1.3.0
+1.3.1
@@ -124,7 +124,8 @@ function(OMNITRACE_STRIP_TARGET _TARGET)
                --keep-symbol="omnitrace_pop_region" --keep-symbol="omnitrace_set_env"
                --keep-symbol="omnitrace_set_mpi" --keep-symbol="omnitrace_user_*"
                --keep-symbol="ompt_start_tool" --keep-symbol="kokkosp_*"
-                --keep-symbol="OnLoad" --keep-symbol="OnUnload" ${ARGN}
+                --keep-symbol="OnLoad" --keep-symbol="OnUnload"
+                --keep-symbol="OnLoadToolProp" --keep-symbol="OnUnloadTool" ${ARGN}
                $<TARGET_FILE:${_TARGET}>
            WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
            COMMENT "Stripping ${_TARGET}...")
@@ -201,6 +201,13 @@ omnitrace_add_bin_test(
        ".*\\\[omnitrace-avail\\\] Usage:.*\\\[CATEGORIES\\\].*\\\[VIEW OPTIONS\\\].*\\\[COLUMN OPTIONS\\\].*\\\[WIDTH OPTIONS\\\].*\\\[OUTPUT OPTIONS\\\].*"
    )

+omnitrace_add_bin_test(
+    NAME omnitrace-avail-all
+    TARGET omnitrace-avail
+    ARGS --all
+    LABELS "omnitrace-avail"
+    TIMEOUT 45)
+
 omnitrace_add_bin_test(
    NAME omnitrace-avail-all-csv
    TARGET omnitrace-avail
@@ -132,6 +132,11 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
 #endif

 #if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
+#    if OMNITRACE_HIP_VERSION >= 50200
+#        define ROCPROFILER_METRICS_DIR "lib/rocprofiler"
+#    else
+#        define ROCPROFILER_METRICS_DIR "rocprofiler/lib"
+#    endif
    setenv("HSA_TOOLS_LIB", _omnilib.c_str(), 0);
    setenv("ROCP_TOOL_LIB", _omnilib.c_str(), 0);
    setenv("ROCPROFILER_LOG", "1", 0);
@@ -142,7 +147,8 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
        if(getenv(itr))
        {
            setenv("ROCP_METRICS",
-                   common::join('/', getenv(itr), "rocprofiler/lib/metrics.xml").c_str(),
+                   common::join('/', getenv(itr), ROCPROFILER_METRICS_DIR, "metrics.xml")
+                       .c_str(),
                   0);
            setenv("OMNITRACE_ROCPROFILER_LIBRARY",
                   common::join('/', getenv(itr), "rocprofiler/lib/librocprofiler64.so")
@@ -153,7 +159,8 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
    }
    // default path
    setenv("ROCP_METRICS",
-           common::join('/', OMNITRACE_DEFAULT_ROCM_PATH, "rocprofiler/lib/metrics.xml")
+           common::join('/', OMNITRACE_DEFAULT_ROCM_PATH, ROCPROFILER_METRICS_DIR,
+                        "metrics.xml")
               .c_str(),
           0);
 #endif
@@ -123,11 +123,18 @@ roctracer::setup()

    OMNITRACE_VERBOSE_F(1, "setting up roctracer...\n");

-    dynamic_library _amdhip64{ "OMNITRACE_ROCTRACER_LIBAMDHIP64", "libamdhip64.so" };
+    dynamic_library _amdhip64{ "OMNITRACE_ROCTRACER_LIBAMDHIP64",
+                               find_library_path("libamdhip64.so",
+                                                 { "OMNITRACE_ROCM_PATH", "ROCM_PATH" },
+                                                 { OMNITRACE_DEFAULT_ROCM_PATH }) };

 #if OMNITRACE_HIP_VERSION_MAJOR == 4 && OMNITRACE_HIP_VERSION_MINOR < 4
-    dynamic_library _kfdwrapper{ "OMNITRACE_ROCTRACER_LIBKFDWRAPPER",
-                                 OMNITRACE_ROCTRACER_LIBKFDWRAPPER };
+    dynamic_library _kfdwrapper{
+        "OMNITRACE_ROCTRACER_LIBKFDWRAPPER",
+        find_library_path("libkfdwrapper64.so", { "OMNITRACE_ROCM_PATH", "ROCM_PATH" },
+                          { OMNITRACE_DEFAULT_ROCM_PATH },
+                          { "roctracer/lib", "roctracer/lib64", "lib", "lib64" })
+    };
 #endif

    ROCTRACER_CALL(roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr));
@@ -24,14 +24,6 @@

 #include "common/defines.h"

-// clang-format off
-#if defined(OMNITRACE_USE_ROCTRACER)
-#    define OMNITRACE_ROCTRACER_LIBKFDWRAPPER "@roctracer_kfdwrapper_LIBRARY@"
-#else
-#    define OMNITRACE_ROCTRACER_LIBKFDWRAPPER "/opt/rocm/roctracer/lib/libkfdwrapper64.so"
-#endif
-// clang-format on
-
 #define TIMEMORY_USER_COMPONENT_ENUM                                                     \
    OMNITRACE_COMPONENT_idx, OMNITRACE_USER_REGION_idx, OMNITRACE_ROCTRACER_idx,         \
        OMNITRACE_ROCPROFILER_idx, OMNITRACE_SAMPLING_WALL_CLOCK_idx,                    \
@@ -21,20 +21,90 @@
 // SOFTWARE.

 #include "library/dynamic_library.hpp"
+#include "common/defines.h"
+#include "library/common.hpp"
 #include "library/debug.hpp"
 #include "library/defines.hpp"

 #include <timemory/environment.hpp>
+#include <timemory/utility/delimit.hpp>
+#include <timemory/utility/filepath.hpp>
+
+#include <string>
+#include <utility>

 namespace omnitrace
 {
-dynamic_library::dynamic_library(const char* _env, const char* _fname, int _flags,
-                                 bool _store)
-: envname{ _env }
-, filename{ tim::get_env<std::string>(_env, _fname, _store) }
+std::string
+find_library_path(const std::string& _name, const std::vector<std::string>& _env_vars,
+                  const std::vector<std::string>& _hints,
+                  const std::vector<std::string>& _path_suffixes)
+{
+    if(_name.find('/') == 0) return _name;
+
+    auto _paths = std::vector<std::string>{};
+    for(const std::string& itr : _env_vars)
+    {
+        auto _env_val = get_env(itr, std::string{});
+        for(auto vitr : tim::delimit(_env_val, ":"))
+            if(!vitr.empty()) _paths.emplace_back(vitr);
+    }
+
+    for(const std::string& itr : _hints)
+    {
+        if(!itr.empty()) _paths.emplace_back(itr);
+    }
+
+    for(auto& itr : _paths)
+    {
+        auto _v = JOIN('/', itr, _name);
+        if(filepath::exists(_v)) return _v;
+        for(const auto& litr : _path_suffixes)
+        {
+            _v = JOIN('/', itr, litr, _name);
+            if(filepath::exists(_v)) return _v;
+        }
+    }
+
+    return _name;
+}
+
+dynamic_library::dynamic_library(std::string _env, std::string _fname, int _flags,
+                                 bool _open, bool _query_env, bool _store)
+: envname{ std::move(_env) }
+, filename{ std::move(_fname) }
 , flags{ _flags }
 {
-    open();
+    if(_query_env)
+    {
+        auto _env_val = get_env(envname, std::string{}, _store);
+        // if the environment variable is set to an absolute path that exists,
+        // override with value
+        if(!_env_val.empty())
+        {
+            if(_env_val.find('/') == 0 && filepath::exists(_env_val))
+            {
+                filename = _env_val;
+            }
+            else if(_env_val.find('/') == 0)
+            {
+                OMNITRACE_VERBOSE_F(1,
+                                    "Ignoring environment variable %s=\"%s\" because the "
+                                    "filepath does not exist. Using \"%s\" instead...\n",
+                                    envname.c_str(), _env_val.c_str(), filename.c_str())
+            }
+            else if(_env_val.find('/') != 0 && filename.find('/') == 0)
+            {
+                OMNITRACE_VERBOSE_F(
+                    1,
+                    "Ignoring environment variable %s=\"%s\" because the "
+                    "filepath is relative. Using absolute path \"%s\" instead...\n",
+                    envname.c_str(), _env_val.c_str(), filename.c_str())
+            }
+        }
+    }
+
+    if(_open) open();
 }

 dynamic_library::~dynamic_library() { close(); }
@@ -47,8 +117,8 @@ dynamic_library::open()
        handle = dlopen(filename.c_str(), flags);
        if(!handle)
        {
-            OMNITRACE_VERBOSE(2, "[dynamic_library][%s][%s] %s\n", envname.c_str(),
-                              filename.c_str(), dlerror());
+            OMNITRACE_VERBOSE(2, "[dynamic_library] Error opening %s=\"%s\" :: %s.\n",
+                              envname.c_str(), filename.c_str(), dlerror());
        }
        dlerror();  // Clear any existing error
    }
@@ -61,4 +131,10 @@ dynamic_library::close() const
    if(handle) return dlclose(handle);
    return -1;
 }
+
+bool
+dynamic_library::is_open() const
+{
+    return (handle != nullptr);
+}
 }  // namespace omnitrace
@@ -27,9 +27,15 @@
 #include <dlfcn.h>
 #include <string>
 #include <unistd.h>
+#include <vector>

 namespace omnitrace
 {
+std::string
+find_library_path(const std::string& _name, const std::vector<std::string>& _env_vars,
+                  const std::vector<std::string>& _hints,
+                  const std::vector<std::string>& _path_suffixes = { "lib", "lib64" });
+
 struct dynamic_library
 {
    dynamic_library()                           = delete;
@@ -38,37 +44,43 @@ struct dynamic_library
    dynamic_library& operator=(const dynamic_library&) = delete;
    dynamic_library& operator=(dynamic_library&&) noexcept = default;

-    dynamic_library(const char* _env, const char* _fname,
-                    int _flags = (RTLD_LAZY | RTLD_GLOBAL), bool _store = false);
+    dynamic_library(std::string _env, std::string _fname,
+                    int _flags = (RTLD_LAZY | RTLD_GLOBAL), bool _open = true,
+                    bool _query_env = true, bool _store = true);

    ~dynamic_library();

    bool open();
    int  close() const;
+    bool is_open() const;

    template <typename RetT, typename... Args>
-    RetT invoke(std::string_view _name, RetT (*&_func)(Args...), Args... _args)
-    {
-        if(!handle) open();
-        if(handle)
-        {
-            *(void**) (&_func) = dlsym(handle, _name.data());
-            if(_func)
-            {
-                return (*_func)(_args...);
-            }
-            else
-            {
-                fprintf(stderr, "[omnitrace][pid=%i]> %s :: %s\n", getpid(), _name.data(),
-                        dlerror());
-            }
-        }
-        return RetT{};
-    }
+    RetT invoke(std::string_view, RetT (*&_func)(Args...), Args...);

    std::string envname  = {};
    std::string filename = {};
    int         flags    = 0;
    void*       handle   = nullptr;
 };
+
+template <typename RetT, typename... Args>
+inline RetT
+dynamic_library::invoke(std::string_view _name, RetT (*&_func)(Args...), Args... _args)
+{
+    if(!handle) open();
+    if(handle)
+    {
+        *(void**) (&_func) = dlsym(handle, _name.data());
+        if(_func)
+        {
+            return (*_func)(_args...);
+        }
+        else
+        {
+            fprintf(stderr, "[omnitrace][pid=%i]> %s :: %s\n", getpid(), _name.data(),
+                    dlerror());
+        }
+    }
+    return RetT{};
+}
 }  // namespace omnitrace
@@ -28,6 +28,7 @@
 #include "library/config.hpp"
 #include "library/critical_trace.hpp"
 #include "library/debug.hpp"
+#include "library/gpu.hpp"
 #include "library/rocprofiler.hpp"
 #include "library/rocprofiler/hsa_rsrc_factory.hpp"
 #include "library/roctracer.hpp"
@@ -103,13 +104,20 @@ extern "C"
        rocm::lock_t _lk{ rocm::rocm_mutex, std::defer_lock };
        if(!_lk.owns_lock()) _lk.lock();

-        if(rocm::is_loaded) return;
+        if(rocm::is_loaded)
+        {
+            OMNITRACE_BASIC_VERBOSE_F(1, "rocprofiler is already loaded\n");
+            return;
+        }
        rocm::is_loaded = true;

        _lk.unlock();

        // Enable timestamping
-        settings->timestamp_on = 1u;
+        settings->timestamp_on     = 1;
+        settings->intercept_mode   = 1;
+        settings->hsa_intercepting = 1;
+        settings->k_concurrent     = 1;

        // Initialize profiling
        omnitrace::rocprofiler::rocm_initialize();
@@ -127,7 +135,7 @@ extern "C"

        roctracer_is_init() = true;
        pthread_gotcha::push_enable_sampling_on_child_threads(false);
-        OMNITRACE_BASIC_VERBOSE_F(1, "\n");
+        OMNITRACE_BASIC_VERBOSE_F(1, "Loading ROCm tooling...\n");

        tim::consume_parameters(table, runtime_version, failed_tool_count,
                                failed_tool_names);
@@ -156,7 +164,6 @@ extern "C"
                    // initialize HSA tracing
                    roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table);

-                    OMNITRACE_VERBOSE(1, "    HSA-trace(");
                    if(!hsa_api_vec.empty())
                    {
                        for(const auto& itr : hsa_api_vec)
@@ -168,15 +175,15 @@ extern "C"
                            ROCTRACER_CALL(roctracer_enable_op_callback(
                                ACTIVITY_DOMAIN_HSA_API, cid, hsa_api_callback, nullptr));

-                            OMNITRACE_VERBOSE(1, " %s", api);
+                            OMNITRACE_VERBOSE(1, "    HSA-trace(%s)", api);
                        }
                    }
                    else
                    {
+                        OMNITRACE_VERBOSE(1, "    HSA-trace()\n");
                        ROCTRACER_CALL(roctracer_enable_domain_callback(
                            ACTIVITY_DOMAIN_HSA_API, hsa_api_callback, nullptr));
                    }
-                    OMNITRACE_VERBOSE(1, " )\n");
                }

                bool trace_hsa_activity = get_trace_hsa_activity();
@@ -212,17 +219,21 @@ extern "C"
                roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
        };

+        OMNITRACE_VERBOSE_F(1, "Computing the roctracer clock skew...\n");
        (void) omnitrace::get_clock_skew();

        comp::roctracer::add_setup("hsa", _setup);
        comp::roctracer::add_shutdown("hsa", _shutdown);

+        OMNITRACE_VERBOSE_F(1, "Setting rocm_smi state to active...\n");
        rocm_smi::set_state(State::Active);
+
+        OMNITRACE_VERBOSE_F(1, "Requesting roctracer to setup...\n");
        comp::roctracer::setup();

 #if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
        bool _force_rocprofiler_init =
-            tim::get_env("OMNITRACE_FORCE_ROCPROFILE_INIT", false, false);
+            tim::get_env("OMNITRACE_FORCE_ROCPROFILER_INIT", false, false);
 #else
        bool _force_rocprofiler_init = false;
 #endif
@@ -233,8 +244,17 @@ extern "C"
        if(_force_rocprofiler_init || (get_use_rocprofiler() && !_is_empty))
        {
            auto _rocprof =
-                dynamic_library{ "OMNITRACE_ROCPROFILER_LIBRARY", "librocprofiler64.so",
-                                 (RTLD_LAZY | RTLD_GLOBAL), true };
+                dynamic_library{ "OMNITRACE_ROCPROFILER_LIBRARY",
+                                 find_library_path("librocprofiler64.so",
+                                                   { "OMNITRACE_ROCM_PATH", "ROCM_PATH" },
+                                                   { OMNITRACE_DEFAULT_ROCM_PATH },
+                                                   { "lib", "lib64", "rocprofiler/lib",
+                                                     "rocprofiler/lib64" }),
+                                 (RTLD_LAZY | RTLD_GLOBAL), false };
+
+            OMNITRACE_VERBOSE_F(1, "Loading rocprofiler library (%s=%s)...\n",
+                                _rocprof.envname.c_str(), _rocprof.filename.c_str());
+            _rocprof.open();

            on_load_t _rocprof_load = nullptr;
            _success = _rocprof.invoke("OnLoad", _rocprof_load, table, runtime_version,
@@ -190,7 +190,7 @@ endif()
 # -------------------------------------------------------------------------------------- #

 function(OMNITRACE_WRITE_TEST_CONFIG _FILE _ENV)
-    set(_ENV_ONLY "OMNITRACE_(USE_MPIP|DEBUG_SETTINGS)=")
+    set(_ENV_ONLY "OMNITRACE_(USE_MPIP|DEBUG_SETTINGS|FORCE_ROCPROFILER_INIT)=")
    set(_FILE_CONTENTS)
    set(_ENV_CONTENTS)

@@ -467,34 +467,34 @@ function(OMNITRACE_ADD_PYTHON_TEST)

    if(NOT TEST_COMMAND)
        list(APPEND TEST_ENVIRONMENT "OMNITRACE_CI=ON"
-             "OMNITRACE_OUTPUT_PATH=omnitrace-tests-output"
+             "OMNITRACE_OUTPUT_PATH=${PROJECT_BINARY_DIR}/omnitrace-tests-output"
             "OMNITRACE_OUTPUT_PREFIX=${TEST_NAME}/${TEST_PYTHON_VERSION}/")
        get_filename_component(_TEST_FILE "${TEST_FILE}" NAME)
        set(_TEST_FILE
-            ${CMAKE_BINARY_DIR}/python/tests/${TEST_PYTHON_VERSION}/${_TEST_FILE})
+            ${PROJECT_BINARY_DIR}/python/tests/${TEST_PYTHON_VERSION}/${_TEST_FILE})
        configure_file(${TEST_FILE} ${_TEST_FILE} @ONLY)
        if(TEST_STANDALONE)
            add_test(
                NAME ${TEST_NAME}-${TEST_PYTHON_VERSION}
                COMMAND ${TEST_PYTHON_EXECUTABLE} ${_TEST_FILE} ${TEST_RUN_ARGS}
-                WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+                WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
        else()
            add_test(
                NAME ${TEST_NAME}-${TEST_PYTHON_VERSION}
                COMMAND ${TEST_PYTHON_EXECUTABLE} -m omnitrace ${TEST_PROFILE_ARGS} --
                        ${_TEST_FILE} ${TEST_RUN_ARGS}
-                WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+                WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
        endif()
    else()
        list(APPEND TEST_LABELS "python-check" "python-${TEST_PYTHON_VERSION}-check")
        add_test(
            NAME ${TEST_NAME}-${TEST_PYTHON_VERSION}
            COMMAND ${TEST_COMMAND} ${TEST_FILE}
-            WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+            WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
        add_test(
            NAME ${TEST_NAME}-${TEST_PYTHON_VERSION}-inverse
            COMMAND ${TEST_COMMAND} ${TEST_FILE}
-            WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+            WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
    endif()

    foreach(_TEST ${TEST_NAME}-${TEST_PYTHON_VERSION}
@@ -632,6 +632,40 @@ omnitrace_add_test(
    ENVIRONMENT "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF"
    REWRITE_FAIL_REGEX "0 instrumented loops in procedure transpose")

+set(OMNITRACE_ROCM_EVENTS_TEST
+    "GRBM_COUNT,GPUBusy,SQ_WAVES,SQ_INSTS_VALU,VALUInsts,TCC_HIT_sum,TA_TA_BUSY[0]:device=0,TA_TA_BUSY[11]:device=0"
+    )
+
+omnitrace_add_test(
+    SKIP_BASELINE SKIP_SAMPLING SKIP_RUNTIME
+    NAME transpose-rocprofiler
+    TARGET transpose
+    LABELS "rocprofiler"
+    MPI ${TRANSPOSE_USE_MPI}
+    GPU ON
+    NUM_PROCS ${NUM_PROCS}
+    REWRITE_ARGS -e -v 2 -E uniform_int_distribution
+    ENVIRONMENT
+        "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_COUT_OUTPUT=ON;OMNITRACE_ROCM_EVENTS=${OMNITRACE_ROCM_EVENTS_TEST}"
+    REWRITE_RUN_PASS_REGEX
+        "rocprof-device-0-GRBM_COUNT.txt(.*)rocprof-device-0-GPUBusy.txt(.*)rocprof-device-0-SQ_WAVES.txt(.*)rocprof-device-0-SQ_INSTS_VALU.txt(.*)rocprof-device-0-VALUInsts.txt(.*)rocprof-device-0-TCC_HIT_sum.txt(.*)rocprof-device-0-TA_TA_BUSY_0.txt(.*)rocprof-device-0-TA_TA_BUSY_11.txt"
+    )
+
+omnitrace_add_test(
+    SKIP_BASELINE SKIP_SAMPLING SKIP_RUNTIME
+    NAME transpose-rocprofiler-no-roctracer
+    TARGET transpose
+    LABELS "rocprofiler"
+    MPI ${TRANSPOSE_USE_MPI}
+    GPU ON
+    NUM_PROCS ${NUM_PROCS}
+    REWRITE_ARGS -e -v 2 -E uniform_int_distribution
+    ENVIRONMENT
+        "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_COUT_OUTPUT=ON;OMNITRACE_USE_ROCTRACER=OFF;OMNITRACE_ROCM_EVENTS=${OMNITRACE_ROCM_EVENTS_TEST}"
+    REWRITE_RUN_PASS_REGEX
+        "rocprof-device-0-GRBM_COUNT.txt(.*)rocprof-device-0-GPUBusy.txt(.*)rocprof-device-0-SQ_WAVES.txt(.*)rocprof-device-0-SQ_INSTS_VALU.txt(.*)rocprof-device-0-VALUInsts.txt(.*)rocprof-device-0-TCC_HIT_sum.txt(.*)rocprof-device-0-TA_TA_BUSY_0.txt(.*)rocprof-device-0-TA_TA_BUSY_11.txt"
+    REWRITE_RUN_FAIL_REGEX "roctracer.txt")
+
 omnitrace_add_test(
    NAME parallel-overhead
    TARGET parallel-overhead
@@ -1072,7 +1106,7 @@ if(TARGET parallel-overhead AND _VALID_PTRACE_SCOPE)
            ${CMAKE_CURRENT_LIST_DIR}/run-omnitrace-pid.sh $<TARGET_FILE:omnitrace-exe>
            -ME "\.c$" -e -v 1 --label return args file -l --
            $<TARGET_FILE:parallel-overhead> 30 8 1000
-        WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR})

    set(_parallel_overhead_attach_environ
        "${_attach_environment}" "OMNITRACE_OUTPUT_PATH=omnitrace-tests-output"
@@ -1 +1 @@
 .3.0
 .3.1