roctracer: device and stream tracks (#209)

* roctracer: use multiple tracks for HIP streams

Use different perfetto tracks for each stream, and set the name of
these tracks to the stream pointer values. Setting the name like this
matches the args in the API traces.
This fixes overlapping work on multiple streams appearing as a call
stack.

* Fix -pedantic

* Run clang-format

* Add option to disable per stream tracks in perfetto

* Updated scheme for roctracer activity + general roctracer fixes

- Per-device tracks
- Handle HSA OPS in ROCm 5.3
  - the changes in ROCm 5.3 were causing HSA ops to get discarded
- Default for OMNITRACE_ROCTRACER_DISCARD_INVALID is now zero
  - i.e. default behavior is to flip beg_ns and end_ns when beg_ns > end_ns

* Flush perfetto at end of hip_activity_callback

- fixes unterminated regions

* GitHub Actions and run-ci script updates

- improve reliability

* Set OMNITRACE_TMPDIR in testing

- files in /tmp get occasionally deleted during CI

Co-authored-by: Gergely Meszaros <gergely@streamhpc.com>

.github/workflows/cpack.yml

@@ -161,16 +161,11 @@ jobs:
           echo "CI_SCRIPT_ARGS=--rocm +python" >> $GITHUB_ENV
 
       - name: Build Base Container
-        timeout-minutes: 20
-        uses: nick-fields/retry@v2
-        with:
-          retry_wait_seconds: 120
-          timeout_minutes: 20
-          max_attempts: 5
-          command: |
-            pushd docker
-            ./build-docker.sh --distro ${{ matrix.os-distro }} --versions ${{ matrix.os-version }} --rocm-versions ${{ matrix.rocm-version }}
-            popd
+        timeout-minutes: 30
+        run: |
+          pushd docker
+          ./build-docker.sh --distro ${{ matrix.os-distro }} --versions ${{ matrix.os-version }} --rocm-versions ${{ matrix.rocm-version }}
+          popd
 
       - name: Build Release
         timeout-minutes: 150

.github/workflows/opensuse.yml

@@ -20,6 +20,7 @@ concurrency:
 env:
   OMNITRACE_VERBOSE: 1
   OMNITRACE_CI: ON
+  OMNITRACE_TMPDIR: "%env{PWD}%/testing-tmp"
 
 jobs:
   opensuse:
@@ -37,11 +38,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done

.github/workflows/ubuntu-bionic.yml

@@ -23,6 +23,7 @@ env:
   OMNITRACE_VERBOSE: 1
   OMNITRACE_CI: ON
   GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
+  OMNITRACE_TMPDIR: "%env{PWD}%/testing-tmp"
 
 jobs:
   ubuntu-bionic:
@@ -37,7 +38,7 @@ jobs:
 
     steps:
     - name: Patch Git
-      timeout-minutes: 10
+      timeout-minutes: 25
       run: |
         apt-get update
         apt-get install -y software-properties-common
@@ -51,11 +52,11 @@ jobs:
         submodules: recursive
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           apt-get update &&

.github/workflows/ubuntu-focal.yml

@@ -20,6 +20,7 @@ concurrency:
 env:
   OMNITRACE_VERBOSE: 1
   OMNITRACE_CI: ON
+  OMNITRACE_TMPDIR: "%env{PWD}%/testing-tmp"
 
 jobs:
   ubuntu-focal-external:
@@ -67,11 +68,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           apt-get update &&
@@ -244,11 +245,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           apt-get update &&
@@ -408,11 +409,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           sudo apt-get update &&
@@ -555,11 +556,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           apt-get update &&

.github/workflows/ubuntu-jammy.yml

@@ -20,6 +20,7 @@ concurrency:
 env:
   OMNITRACE_VERBOSE: 1
   OMNITRACE_CI: ON
+  OMNITRACE_TMPDIR: "%env{PWD}%/testing-tmp"
 
 jobs:
   ubuntu-jammy-external:
@@ -70,11 +71,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           apt-get update &&
@@ -87,12 +88,12 @@ jobs:
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
 
     - name: Install ROCm Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       if: ${{ matrix.rocm-version != '0.0' }}
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         shell: bash
         command: |

docker/build-docker.sh

@@ -7,6 +7,7 @@
 : ${PYTHON_VERSIONS:="6 7 8 9 10"}
 : ${BUILD_CI:=""}
 : ${PUSH:=0}
+: ${RETRY:=3}
 
 set -e
 
@@ -32,7 +33,9 @@ usage()
     print_default_option versions "[VERSION] [VERSION...]" "Ubuntu or OpenSUSE release" "${VERSIONS}"
     print_default_option rocm-versions "[VERSION] [VERSION...]" "ROCm versions" "${ROCM_VERSIONS}"
     print_default_option python-versions "[VERSION] [VERSION...]" "Python 3 minor releases" "${PYTHON_VERSIONS}"
-    print_default_option user "[USERNAME]" "DockerHub username" "${USER}"
+    print_default_option "user -u" "[USERNAME]" "DockerHub username" "${USER}"
+    print_default_option "retry -r" "[N]" "Number of attempts to build (to account for network errors" "${RETRY}"
+    print_default_option push "" "Push the image to Dockerhub" ""
     #print_default_option lto "[on|off]" "Enable LTO" "${LTO}"
 }
 
@@ -46,7 +49,31 @@ send-error()
 verbose-run()
 {
     echo -e "\n### Executing \"${@}\"... ###\n"
-    eval $@
+    eval "${@}"
+}
+
+verbose-build()
+{
+    echo -e "\n### Executing \"${@}\" a maximum of ${RETRY} times... ###\n"
+    for i in $(seq 1 1 ${RETRY})
+    do
+        set +e
+        eval "${@}"
+        local RETC=$?
+        set -e
+        if [ "${RETC}" -eq 0 ]; then
+            break
+        else
+            echo -en "\n### Command failed with error code ${RETC}... "
+            if [ "${i}" -ne "${RETRY}" ]; then
+                echo -e "Retrying... ###\n"
+                sleep 3
+            else
+                echo -e "Exiting... ###\n"
+                exit ${RETC}
+            fi
+        fi
+    done
 }
 
 reset-last()
@@ -92,6 +119,11 @@ do
         --push)
             PUSH=1
             ;;
+        --retry|-r)
+            shift
+            RETRY=${1}
+            reset-last
+            ;;
         "--*")
             send-error "Unsupported argument at position $((${n} + 1)) :: ${1}"
             ;;
@@ -149,7 +181,7 @@ do
                 *)
                     ;;
             esac
-            verbose-run docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg ROCM_REPO_VERSION=${ROCM_REPO_VERSION} --build-arg ROCM_REPO_DIST=${ROCM_REPO_DIST} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
+            verbose-build docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg ROCM_REPO_VERSION=${ROCM_REPO_VERSION} --build-arg ROCM_REPO_DIST=${ROCM_REPO_DIST} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
         elif [ "${DISTRO}" = "centos" ]; then
             case "${VERSION}" in
                 7)
@@ -192,7 +224,7 @@ do
                     send-error "Unsupported combination :: ${DISTRO}-${VERSION} + ROCm ${ROCM_VERSION}"
                     ;;
             esac
-            verbose-run docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg TOOLSET_VERSION=${TOOLSET_VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
+            verbose-build docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg TOOLSET_VERSION=${TOOLSET_VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
         elif [ "${DISTRO}" = "opensuse" ]; then
             case "${VERSION}" in
                 15.*)
@@ -225,7 +257,7 @@ do
                     send-error "Unsupported combination :: ${DISTRO}-${VERSION} + ROCm ${ROCM_VERSION}"
                 ;;
             esac
-            verbose-run docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO_IMAGE} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
+            verbose-build docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO_IMAGE} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\"
         fi
         if [ "${PUSH}" -ne 0 ]; then
             docker push ${CONTAINER}

scripts/run-ci.py

@@ -4,6 +4,7 @@
 import os
 import re
 import sys
+import glob
 import socket
 import shutil
 import argparse
@@ -240,15 +241,34 @@ if __name__ == "__main__":
     for itr in args.stages:
         dashboard_args.append(f"{args.mode}{itr}")
 
-    run(
-        [CTEST_CMD]
-        + dashboard_args
-        + [
-            "-S",
-            os.path.join(args.binary_dir, "dashboard.cmake"),
-            "--output-on-failure",
-            "-VV",
-        ]
-        + ctest_args,
-        check=True,
-    )
+    try:
+        run(
+            [CTEST_CMD]
+            + dashboard_args
+            + [
+                "-S",
+                os.path.join(args.binary_dir, "dashboard.cmake"),
+                "--output-on-failure",
+                "-V",
+            ]
+            + ctest_args,
+            check=True,
+        )
+    finally:
+        if "-VV" not in ctest_args:
+            for file in glob.glob(
+                os.path.join(args.binary_dir, "Testing/**"), recursive=True
+            ):
+                if not os.path.isfile(file):
+                    continue
+                print(f"\n\n\n###### Reading {file}... ######\n\n\n")
+                with open(file, "r") as inpf:
+                    fdata = inpf.read()
+                    if "LastTest" not in file and "Coverage" not in file:
+                        print(fdata)
+                    oname = os.path.basename(file)
+                    if oname.endswith(".log"):
+                        oname += ".log"
+                    with open(os.path.join(args.binary_dir, oname), "w") as outf:
+                        print(f"\n\n###### Writing {oname}... ######\n\n")
+                        outf.write(fdata)

source/lib/omnitrace/library/config.cpp

@@ -576,6 +576,12 @@ configure_settings(bool _init)
                              _config->get<bool>("collapse_processes"), "perfetto", "data",
                              "advanced");
 
+    OMNITRACE_CONFIG_SETTING(
+        bool, "OMNITRACE_PERFETTO_ROCTRACER_PER_STREAM",
+        "Separate roctracer GPU side traces (copies, kernels) into separate "
+        "tracks based on the stream they're enqueued into",
+        true, "perfetto", "roctracer", "rocm", "advanced");
+
     OMNITRACE_CONFIG_SETTING(
         std::string, "OMNITRACE_PERFETTO_FILL_POLICY",
         "Behavior when perfetto buffer is full. 'discard' will ignore new entries, "
@@ -1518,6 +1524,17 @@ get_use_roctracer()
 #endif
 }
 
+bool
+get_perfetto_roctracer_per_stream()
+{
+#if defined(OMNITRACE_USE_ROCTRACER) && OMNITRACE_USE_ROCTRACER > 0
+    static auto _v = get_config()->find("OMNITRACE_PERFETTO_ROCTRACER_PER_STREAM");
+    return static_cast<tim::tsettings<bool>&>(*_v->second).get();
+#else
+    return false;
+#endif
+}
+
 bool
 get_use_rocprofiler()
 {

source/lib/omnitrace/library/config.hpp

@@ -266,6 +266,9 @@ get_backend();
 std::string
 get_perfetto_output_filename();
 
+bool
+get_perfetto_roctracer_per_stream() OMNITRACE_HOT;
+
 int64_t
 get_critical_trace_count();
 

source/lib/omnitrace/library/roctracer.cpp

@@ -32,6 +32,7 @@
 
 #include <timemory/backends/cpu.hpp>
 #include <timemory/backends/threading.hpp>
+#include <timemory/hash/types.hpp>
 #include <timemory/utility/types.hpp>
 
 #include <atomic>
@@ -118,7 +119,7 @@ get_roctracer_tid_data()
     return _v;
 }
 
-using cid_tuple_t = std::tuple<uint64_t, uint64_t, uint32_t>;
+using cid_tuple_t = std::tuple<uint64_t, uint64_t, uint32_t, uintptr_t>;
 struct cid_data : cid_tuple_t
 {
     using cid_tuple_t::cid_tuple_t;
@@ -128,10 +129,12 @@ struct cid_data : cid_tuple_t
     auto& cid() { return std::get<0>(*this); }
     auto& pcid() { return std::get<1>(*this); }
     auto& depth() { return std::get<2>(*this); }
+    auto& queue() { return std::get<3>(*this); }
 
     auto cid() const { return std::get<0>(*this); }
     auto pcid() const { return std::get<1>(*this); }
     auto depth() const { return std::get<2>(*this); }
+    auto queue() const { return std::get<3>(*this); }
 };
 
 auto&
@@ -344,24 +347,13 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
 }
 
 void
-hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg)
+hsa_activity_callback(uint32_t op, const activity_record_t* record, void* arg)
 {
     if(get_state() != State::Active || !trait::runtime_enabled<comp::roctracer>::get())
         return;
 
     OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
 
-    static thread_local std::once_flag _once{};
-    std::call_once(_once, []() {
-        threading::offset_this_id(true);
-        if(threading::get_id() != 0)
-        {
-            sampling::block_signals();
-            threading::set_thread_name("roctracer.hsa");
-            sampling::shutdown();
-        }
-    });
-
     auto&& _protect = comp::roctracer::protect_flush_activity();
     (void) _protect;
 
@@ -760,8 +752,8 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
                 critical_trace::add_hash_id(op_name), _depth);
         }
 
-        get_roctracer_cid_data(_tid).emplace(_corr_id,
-                                             cid_data{ _cid, _parent_cid, _depth });
+        get_roctracer_cid_data(_tid).emplace(
+            _corr_id, cid_data{ _cid, _parent_cid, _depth, _queue });
 
         hip_exec_activity_callbacks(_tid);
     }
@@ -769,7 +761,8 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
     {
         hip_exec_activity_callbacks(_tid);
 
-        std::tie(_cid, _parent_cid, _depth) = get_roctracer_cid_data(_tid).at(_corr_id);
+        std::tie(_cid, _parent_cid, _depth, std::ignore) =
+            get_roctracer_cid_data(_tid).at(_corr_id);
 
         if(get_use_perfetto())
         {
@@ -814,24 +807,13 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void*
 
 // Activity tracing callback
 void
-hip_activity_callback(const char* begin, const char* end, void*)
+hip_activity_callback(const char* begin, const char* end, void* arg)
 {
     if(get_state() != State::Active || !trait::runtime_enabled<comp::roctracer>::get())
         return;
 
     OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
 
-    static thread_local std::once_flag _once{};
-    std::call_once(_once, []() {
-        threading::offset_this_id(true);
-        if(threading::get_id() != 0)
-        {
-            sampling::block_signals();
-            threading::set_thread_name("roctracer.hip");
-            sampling::shutdown();
-        }
-    });
-
     auto&& _protect = comp::roctracer::protect_flush_activity();
     (void) _protect;
 
@@ -859,8 +841,12 @@ hip_activity_callback(const char* begin, const char* end, void*)
         assert(HIP_OP_ID_DISPATCH == 0);
         assert(HIP_OP_ID_COPY == 1);
         assert(HIP_OP_ID_BARRIER == 2);
-        assert(record->domain == ACTIVITY_DOMAIN_HIP_OPS);
 
+        if(record->domain == ACTIVITY_DOMAIN_HSA_OPS)
+        {
+            hsa_activity_callback(record->op, record, arg);
+            continue;
+        }
         if(record->domain != ACTIVITY_DOMAIN_HIP_OPS) continue;
         if(record->op > HIP_OP_ID_BARRIER) continue;
 
@@ -880,6 +866,7 @@ hip_activity_callback(const char* begin, const char* end, void*)
         uint64_t    _pcid           = 0;                     // parent corr_id
         int32_t     _devid          = record->device_id;     // device id
         int64_t     _queid          = record->queue_id;      // queue id
+        uintptr_t   _queue          = 0;                     // Host queue (stream)
         auto        _laps           = _indexes[_corr_id]++;  // see note #1
         const char* _name           = nullptr;
         bool        _found          = false;
@@ -903,7 +890,7 @@ hip_activity_callback(const char* begin, const char* end, void*)
         {
             auto& _cids = get_roctracer_cid_data(_tid);
             if(_cids.find(_corr_id) != _cids.end())
-                std::tie(_cid, _pcid, _depth) = _cids.at(_corr_id);
+                std::tie(_cid, _pcid, _depth, _queue) = _cids.at(_corr_id);
             else
             {
                 OMNITRACE_VERBOSE_F(3,
@@ -922,7 +909,7 @@ hip_activity_callback(const char* begin, const char* end, void*)
             auto          _verbose = []() { return get_verbose() >= 0 || get_debug(); };
             static size_t _n       = 0;
             static size_t _nmax =
-                get_env<size_t>("OMNITRACE_ROCTRACER_DISCARD_INVALID", 10);
+                get_env<size_t>("OMNITRACE_ROCTRACER_DISCARD_INVALID", 0);
             if(_nmax == 0) std::swap(_end_ns, _beg_ns);
             OMNITRACE_WARNING_IF_F(
                 _n < _nmax && _verbose(),
@@ -952,36 +939,36 @@ hip_activity_callback(const char* begin, const char* end, void*)
             if(_kernel_names.find(_name) == _kernel_names.end())
                 _kernel_names.emplace(_name, tim::demangle(_name));
 
+            auto _track_desc = [](int32_t _device_id, int64_t _queue_id) {
+                if(config::get_perfetto_roctracer_per_stream())
+                    return JOIN("", "HIP Activity Device ", _device_id, ", Queue ",
+                                _queue_id);
+                return JOIN("", "HIP Activity Device ", _device_id);
+            };
+
+            const auto _track = tracing::get_perfetto_track(
+                category::device_hip{}, _track_desc, _devid,
+                (get_perfetto_roctracer_per_stream()) ? _queid : 0);
+
             assert(_end_ns >= _beg_ns);
-            tracing::push_perfetto_ts(
-                category::device_hip{}, _kernel_names.at(_name).c_str(), _beg_ns,
+            tracing::push_perfetto_track(
+                category::device_hip{}, _kernel_names.at(_name).c_str(), _track, _beg_ns,
                 perfetto::Flow::ProcessScoped(_cid), [&](perfetto::EventContext ctx) {
                     if(config::get_perfetto_annotations())
                     {
                         tracing::add_perfetto_annotation(ctx, "begin_ns", _beg_ns);
+                        tracing::add_perfetto_annotation(ctx, "end_ns", _end_ns);
                         tracing::add_perfetto_annotation(ctx, "corr_id", _corr_id);
                         tracing::add_perfetto_annotation(ctx, "device", _devid);
                         tracing::add_perfetto_annotation(ctx, "queue", _queid);
                         tracing::add_perfetto_annotation(ctx, "tid", _tid);
+                        tracing::add_perfetto_annotation(
+                            ctx, "stream", JOIN("", "0x", std::hex, _queue));
                         tracing::add_perfetto_annotation(ctx, "op",
                                                          _op_id_names.at(record->op));
                     }
                 });
-            tracing::pop_perfetto_ts(
-                category::device_hip{}, "", _end_ns, [&](perfetto::EventContext ctx) {
-                    if(config::get_perfetto_annotations())
-                    {
-                        tracing::add_perfetto_annotation(ctx, "end_ns", _end_ns);
-                    }
-                });
-            // for some reason, this is necessary to make sure very last one ends
-            tracing::pop_perfetto_ts(
-                category::device_hip{}, "", _end_ns, [&](perfetto::EventContext ctx) {
-                    if(config::get_perfetto_annotations())
-                    {
-                        tracing::add_perfetto_annotation(ctx, "end_ns", _end_ns);
-                    }
-                });
+            tracing::pop_perfetto_track(category::device_hip{}, "", _track, _end_ns);
         }
 
         if(_critical_trace)
@@ -1013,6 +1000,9 @@ hip_activity_callback(const char* begin, const char* end, void*)
             _async_ops->emplace_back(std::move(_func));
         }
     }
+
+    // ensures that all the updates are written
+    if(get_use_perfetto()) ::perfetto::TrackEvent::Flush();
 }
 
 bool&

source/lib/omnitrace/library/roctracer.hpp

@@ -57,7 +57,7 @@ void
 hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg);
 
 void
-hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg);
+hsa_activity_callback(uint32_t op, const activity_record_t* record, void* arg);
 
 void
 hip_exec_activity_callbacks(int64_t _tid);

source/lib/omnitrace/library/tracing.cpp

@@ -43,6 +43,13 @@ get_perfetto_session()
     return _v;
 }
 
+std::unordered_map<hash_value_t, std::string>&
+get_perfetto_track_uuids()
+{
+    static thread_local auto _v = std::unordered_map<hash_value_t, std::string>{};
+    return _v;
+}
+
 std::vector<std::function<void()>>&
 get_finalization_functions()
 {

source/lib/omnitrace/library/tracing.hpp

@@ -38,13 +38,22 @@
 #include <timemory/hash/types.hpp>
 #include <timemory/mpl/type_traits.hpp>
 
+#include <atomic>
+#include <functional>
+#include <memory>
+#include <ratio>
+#include <string>
 #include <type_traits>
+#include <unordered_map>
+#include <utility>
+#include <vector>
 
 namespace omnitrace
 {
 namespace tracing
 {
 using interval_data_instances = thread_data<std::vector<bool>>;
+using hash_value_t            = tim::hash_value_t;
 
 perfetto::TraceConfig&
 get_perfetto_config();
@@ -52,6 +61,54 @@ get_perfetto_config();
 std::unique_ptr<perfetto::TracingSession>&
 get_perfetto_session();
 
+template <typename CategoryT, typename... Args>
+auto
+get_perfetto_category_uuid(Args&&... _args)
+{
+    return tim::hash::get_combined_hash_id(
+        tim::hash::get_hash_id(JOIN('_', "omnitrace", trait::name<CategoryT>::value)),
+        std::forward<Args>(_args)...);
+}
+
+std::unordered_map<hash_value_t, std::string>&
+get_perfetto_track_uuids();
+
+template <typename CategoryT, typename TrackT = ::perfetto::Track, typename FuncT,
+          typename... Args>
+auto
+get_perfetto_track(CategoryT, FuncT&& _desc_generator, Args&&... _args)
+{
+    auto  _uuid = get_perfetto_category_uuid<CategoryT>(std::forward<Args>(_args)...);
+    auto& _track_uuids = get_perfetto_track_uuids();
+    if(_track_uuids.find(_uuid) == _track_uuids.end())
+    {
+        const auto _track = TrackT(_uuid);
+        auto       _desc  = _track.Serialize();
+
+        auto _name = std::forward<FuncT>(_desc_generator)(std::forward<Args>(_args)...);
+        _desc.set_name(_name);
+        ::perfetto::TrackEvent::SetTrackDescriptor(_track, _desc);
+
+        OMNITRACE_VERBOSE_F(4, "[%s] Created %s(%zu) with description: \"%s\"\n",
+                            trait::name<CategoryT>::value, demangle<TrackT>().c_str(),
+                            _uuid, _name.c_str());
+
+        _track_uuids.emplace(_uuid, _name);
+    }
+
+    // guard this with ppdefs in addition to runtime check to avoid
+    // overhead of generating string during releases
+#if defined(OMNITRACE_CI) && OMNITRACE_CI > 0
+    auto _name = std::forward<FuncT>(_desc_generator)(std::forward<Args>(_args)...);
+    OMNITRACE_CI_THROW(_track_uuids.at(_uuid) != _name,
+                       "Error! Multiple invocations of UUID %zu produced different "
+                       "descriptions: \"%s\" and \"%s\"\n",
+                       _uuid, _track_uuids.at(_uuid).c_str(), _name.c_str());
+#endif
+
+    return TrackT(_uuid);
+}
+
 std::vector<std::function<void()>>&
 get_finalization_functions();
 
@@ -292,5 +349,24 @@ pop_perfetto_ts(CategoryT, const char*, uint64_t _ts, Args&&... args)
 
     TRACE_EVENT_END(trait::name<CategoryT>::value, _ts, std::forward<Args>(args)...);
 }
+
+template <typename CategoryT, typename... Args>
+inline void
+push_perfetto_track(CategoryT, const char* name, perfetto::Track _track, uint64_t _ts,
+                    Args&&... args)
+{
+    TRACE_EVENT_BEGIN(trait::name<CategoryT>::value, perfetto::StaticString(name), _track,
+                      _ts, std::forward<Args>(args)...);
+}
+
+template <typename CategoryT, typename... Args>
+inline void
+pop_perfetto_track(CategoryT, const char*, perfetto::Track _track, uint64_t _ts,
+                   Args&&... args)
+{
+    TRACE_EVENT_END(trait::name<CategoryT>::value, _track, _ts,
+                    std::forward<Args>(args)...);
+}
+
 }  // namespace tracing
 }  // namespace omnitrace