Add 'projects/rocprofiler-sdk/' from commit 'bf0fad1d5406fbc51403ba1aa9621a9d4a9bce2b'

git-subtree-dir: projects/rocprofiler-sdk
git-subtree-mainline: 50a90550e9
git-subtree-split: bf0fad1d54

projects/rocprofiler-sdk/tests/CMakeLists.txt

@@ -0,0 +1,94 @@
+#
+# Integration tests
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(COMMAND rocprofiler_deactivate_clang_tidy)
+    rocprofiler_deactivate_clang_tidy()
+endif()
+
+project(rocprofiler-sdk-tests LANGUAGES C CXX)
+
+set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "tests")
+set(ROCPROFILER_SDK_TESTS_SOURCE_DIR "${PROJECT_SOURCE_DIR}")
+set(ROCPROFILER_SDK_TESTS_BINARY_DIR "${PROJECT_BINARY_DIR}")
+
+if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE
+        "Release"
+        CACHE STRING "" FORCE)
+endif()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+# this should be defaulted to OFF by ROCm 7.0.1 or 7.1 this should only used to disable
+# tests in extreme circumstances
+option(ROCPROFILER_DISABLE_UNSTABLE_CTESTS "Disable unstable tests" ON)
+
+enable_testing()
+include(CTest)
+
+include(GNUInstallDirs)
+# always use lib instead of lib64
+set(CMAKE_INSTALL_LIBDIR "lib")
+
+# define the library output directory
+if(PROJECT_IS_TOP_LEVEL)
+    set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
+    set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
+else()
+    set(CMAKE_MESSAGE_INDENT "[${PROJECT_NAME}] ")
+endif()
+
+# needed for validation
+find_package(Python3 REQUIRED)
+
+# generally needed
+find_package(rocprofiler-sdk REQUIRED)
+
+# get the gfx architectures that are present on the system
+rocprofiler_sdk_get_gfx_architectures(rocprofiler-sdk-tests-gfx-info ECHO)
+
+# configure python module <BINARY_DIR>/rocprofiler_sdk/pytest_utils
+add_subdirectory(pytest-packages)
+
+# common utilities
+add_subdirectory(common)
+
+# tool libraries used for data collection during integration tests
+add_subdirectory(tools)
+
+# libraries used by integration test applications
+add_subdirectory(lib)
+
+# applications used by integration tests
+add_subdirectory(bin)
+
+# validation tests
+add_subdirectory(kernel-tracing)
+add_subdirectory(async-copy-tracing)
+add_subdirectory(hsa-memory-allocation)
+add_subdirectory(scratch-memory-tracing)
+add_subdirectory(c-tool)
+add_subdirectory(thread-trace)
+add_subdirectory(pc_sampling)
+add_subdirectory(hip-graph-tracing)
+add_subdirectory(counter-collection)
+add_subdirectory(openmp-tools)
+add_subdirectory(rocdecode)
+add_subdirectory(rocjpeg)
+
+# rocpd validation tests
+add_subdirectory(rocpd)
+
+# rocprofv3 validation tests
+add_subdirectory(rocprofv3)
+
+# python bindings
+add_subdirectory(python-bindings)
+add_subdirectory(rocprofv3-avail)

projects/rocprofiler-sdk/tests/async-copy-tracing/CMakeLists.txt

@@ -0,0 +1,45 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(
+    rocprofiler-sdk-tests-async-copy-tracing
+    LANGUAGES CXX
+    VERSION 0.0.0)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+if(ROCPROFILER_MEMCHECK_PRELOAD_ENV)
+    set(PRELOAD_ENV
+        "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}:$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+else()
+    set(PRELOAD_ENV "LD_PRELOAD=$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+endif()
+
+add_test(NAME test-async-copy-tracing-execute COMMAND $<TARGET_FILE:transpose>)
+
+set(async-copy-tracing-env
+    "${PRELOAD_ENV}"
+    "ROCPROFILER_TOOL_OUTPUT_FILE=async-copy-tracing-test.json"
+    "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:rocprofiler-sdk::rocprofiler-sdk-shared-library>:$ENV{LD_LIBRARY_PATH}"
+    )
+
+set_tests_properties(
+    test-async-copy-tracing-execute
+    PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT
+               "${async-copy-tracing-env}" FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
+
+# copy to binary directory
+rocprofiler_configure_pytest_files(COPY validate.py conftest.py CONFIG pytest.ini)
+
+add_test(NAME test-async-copy-tracing-validate
+         COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --input
+                 ${CMAKE_CURRENT_BINARY_DIR}/async-copy-tracing-test.json)
+
+set_tests_properties(
+    test-async-copy-tracing-validate
+    PROPERTIES TIMEOUT 45 LABELS "integration-tests" DEPENDS
+               test-async-copy-tracing-execute FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}")

projects/rocprofiler-sdk/tests/async-copy-tracing/conftest.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import json
+import pytest
+
+from rocprofiler_sdk.pytest_utils.dotdict import dotdict
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--input",
+        action="store",
+        default="async-copy-tracing-test.json",
+        help="Input JSON",
+    )
+
+
+@pytest.fixture
+def input_data(request):
+    filename = request.config.getoption("--input")
+    with open(filename, "r") as inp:
+        return dotdict(json.load(inp))

projects/rocprofiler-sdk/tests/async-copy-tracing/pytest.ini

@@ -0,0 +1,5 @@
+
+[pytest]
+addopts = --durations=20 -rA -s -vv
+testpaths = validate.py
+pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages

projects/rocprofiler-sdk/tests/async-copy-tracing/validate.py

@@ -0,0 +1,526 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import pytest
+
+
+test_api_traces = [
+    "hsa_api_traces",
+    "marker_api_traces",
+    "hip_api_traces",
+    "rccl_api_traces",
+    "scratch_memory_traces",
+]
+
+
+# helper function
+def node_exists(name, data, min_len=1):
+    assert name in data
+    assert data[name] is not None
+    if isinstance(data[name], (list, tuple, dict, set)):
+        assert len(data[name]) >= min_len, f"{name}:\n{data}"
+
+
+def get_operation(record, kind_name, op_name=None):
+    for idx, itr in enumerate(record["names"]):
+        if kind_name == itr["kind"]:
+            if op_name is None:
+                return idx, itr["operations"]
+            else:
+                for oidx, oname in enumerate(itr["operations"]):
+                    if op_name == oname:
+                        return oidx
+
+    return None
+
+
+def get_operation_name(record, kind_idx, op_idx):
+    for idx, itr in enumerate(record["names"]):
+        if idx == kind_idx:
+            return itr["operations"][op_idx]
+
+    return None
+
+
+def groupby_corr_id(trace_item, op_id=None):
+    """
+    If op_id is not none, returns records only with that operation ID
+
+    {
+        corr_id-1: record with internal corr_id = corr_id-1
+        corr_id-2: record with internal corr_id = corr_id-2
+        ...
+    }
+    """
+    from rocprofiler_sdk.pytest_utils.dotdict import dotdict
+
+    ret = {}
+
+    for x in trace_item:
+        if op_id is not None and x.operation != op_id:
+            continue
+
+        corr_id = x.correlation_id["internal"]
+
+        if corr_id in ret.keys():
+            assert False, f"Duplicate internal corr_id {corr_id}"
+        else:
+            ret[corr_id] = x
+
+    return dotdict(ret)
+
+
+def test_data_structure(input_data):
+    """verify minimum amount of expected data is present"""
+    data = input_data
+
+    node_exists("rocprofiler-sdk-json-tool", data)
+
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    node_exists("metadata", sdk_data)
+    node_exists("pid", sdk_data["metadata"])
+    node_exists("main_tid", sdk_data["metadata"])
+    node_exists("init_time", sdk_data["metadata"])
+    node_exists("fini_time", sdk_data["metadata"])
+
+    node_exists("agents", sdk_data)
+    node_exists("call_stack", sdk_data)
+    node_exists("callback_records", sdk_data)
+    node_exists("buffer_records", sdk_data)
+
+    node_exists("names", sdk_data["callback_records"])
+    node_exists("code_objects", sdk_data["callback_records"])
+    node_exists("kernel_symbols", sdk_data["callback_records"])
+    node_exists("host_functions", sdk_data["callback_records"])
+    node_exists("hsa_api_traces", sdk_data["callback_records"])
+    node_exists("hip_api_traces", sdk_data["callback_records"], 0)
+    node_exists("marker_api_traces", sdk_data["callback_records"])
+    node_exists("kernel_dispatch", sdk_data["callback_records"])
+    node_exists("memory_copies", sdk_data["callback_records"], 24)
+
+    node_exists("names", sdk_data["buffer_records"])
+    node_exists("kernel_dispatch", sdk_data["buffer_records"])
+    node_exists("memory_copies", sdk_data["buffer_records"], 12)
+    node_exists("hsa_api_traces", sdk_data["buffer_records"])
+    node_exists("hip_api_traces", sdk_data["buffer_records"], 0)
+    node_exists("marker_api_traces", sdk_data["buffer_records"])
+    node_exists("retired_correlation_ids", sdk_data["buffer_records"])
+
+
+def test_size_entries(input_data):
+    # check that size fields are > 0 but account for function arguments
+    # which are named "size"
+    def check_size(data, bt):
+        if "size" in data.keys():
+            if isinstance(data["size"], str) and bt.endswith('["args"]'):
+                pass
+            else:
+                assert data["size"] > 0, f"origin: {bt}"
+
+    # recursively check the entire data structure
+    def iterate_data(data, bt):
+        if isinstance(data, (list, tuple)):
+            for i, itr in enumerate(data):
+                if isinstance(itr, dict):
+                    check_size(itr, f"{bt}[{i}]")
+                iterate_data(itr, f"{bt}[{i}]")
+        elif isinstance(data, dict):
+            check_size(data, f"{bt}")
+            for key, itr in data.items():
+                iterate_data(itr, f'{bt}["{key}"]')
+
+    # start recursive check over entire JSON dict
+    iterate_data(input_data, "input_data")
+
+
+def test_timestamps(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    cb_start = {}
+    cb_end = {}
+    for titr in test_api_traces:
+        for itr in sdk_data["callback_records"][titr]:
+            cid = itr["correlation_id"]["internal"]
+            phase = itr["phase"]
+            if phase == 1:
+                cb_start[cid] = itr["timestamp"]
+            elif phase == 2:
+                cb_end[cid] = itr["timestamp"]
+                assert cb_start[cid] <= itr["timestamp"]
+            else:
+                assert phase == 1 or phase == 2
+
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["start_timestamp"] <= itr["end_timestamp"]
+
+    for titr in ["kernel_dispatch", "memory_copies"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["start_timestamp"] < itr["end_timestamp"], f"[{titr}] {itr}"
+            assert itr["correlation_id"]["internal"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["init_time"] < itr["start_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["init_time"] < itr["end_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["fini_time"] > itr["start_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["fini_time"] > itr["end_timestamp"]
+            ), f"[{titr}] {itr}"
+
+            api_start = cb_start[itr["correlation_id"]["internal"]]
+            # api_end = cb_end[itr["correlation_id"]["internal"]]
+            assert api_start < itr["start_timestamp"], f"[{titr}] {itr}"
+            # assert api_end <= itr["end_timestamp"], f"[{titr}] {itr}"
+
+
+def test_internal_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    api_corr_ids = []
+    for titr in test_api_traces:
+        for itr in sdk_data["callback_records"][titr]:
+            api_corr_ids.append(itr["correlation_id"]["internal"])
+
+        for itr in sdk_data["buffer_records"][titr]:
+            api_corr_ids.append(itr["correlation_id"]["internal"])
+
+    api_corr_ids_sorted = sorted(api_corr_ids)
+    api_corr_ids_unique = list(set(api_corr_ids))
+
+    for itr in sdk_data["buffer_records"]["kernel_dispatch"]:
+        assert itr["correlation_id"]["internal"] in api_corr_ids_unique
+
+    for itr in sdk_data["buffer_records"]["memory_copies"]:
+        assert itr["correlation_id"]["internal"] in api_corr_ids_unique
+
+    len_corr_id_unq = len(api_corr_ids_unique)
+    assert len(api_corr_ids) != len_corr_id_unq
+    assert max(api_corr_ids_sorted) == len_corr_id_unq
+
+
+def test_external_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    extern_corr_ids = []
+    for titr in test_api_traces:
+        for itr in sdk_data["callback_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0
+            assert itr["thread_id"] == itr["correlation_id"]["external"]
+            extern_corr_ids.append(itr["correlation_id"]["external"])
+
+    extern_corr_ids = list(set(sorted(extern_corr_ids)))
+    for titr in test_api_traces:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert (
+                itr["thread_id"] == itr["correlation_id"]["external"]
+            ), f"[{titr}] {itr}"
+            assert itr["thread_id"] in extern_corr_ids, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
+
+    for titr in ["kernel_dispatch", "memory_copies"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
+
+        for itr in sdk_data["callback_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
+
+
+def test_kernel_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    symbol_info = {}
+    for itr in sdk_data["callback_records"]["kernel_symbols"]:
+        phase = itr["phase"]
+        payload = itr["payload"]
+        kern_id = payload["kernel_id"]
+
+        assert phase == 1 or phase == 2
+        assert kern_id > 0
+        if phase == 1:
+            assert len(payload["kernel_name"]) > 0
+            symbol_info[kern_id] = payload
+        elif phase == 2:
+            assert payload["kernel_id"] in symbol_info.keys()
+            assert payload["kernel_name"] == symbol_info[kern_id]["kernel_name"]
+
+    for itr in sdk_data["buffer_records"]["kernel_dispatch"]:
+        assert itr["dispatch_info"]["kernel_id"] in symbol_info.keys()
+
+    for itr in sdk_data["callback_records"]["kernel_dispatch"]:
+        assert itr["payload"]["dispatch_info"]["kernel_id"] in symbol_info.keys()
+
+
+def test_kernel_dispatch_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    num_dispatches = len(sdk_data["buffer_records"]["kernel_dispatch"])
+    num_cb_dispatches = len(sdk_data["callback_records"]["kernel_dispatch"])
+
+    assert num_cb_dispatches == (3 * num_dispatches)
+
+    bf_seq_ids = []
+    for itr in sdk_data["buffer_records"]["kernel_dispatch"]:
+        bf_seq_ids.append(itr["dispatch_info"]["dispatch_id"])
+
+    cb_seq_ids = []
+    for itr in sdk_data["callback_records"]["kernel_dispatch"]:
+        cb_seq_ids.append(itr["payload"]["dispatch_info"]["dispatch_id"])
+
+    bf_seq_ids = sorted(bf_seq_ids)
+    cb_seq_ids = sorted(cb_seq_ids)
+
+    assert (3 * len(bf_seq_ids)) == len(cb_seq_ids)
+
+    assert bf_seq_ids[0] == cb_seq_ids[0]
+    assert bf_seq_ids[-1] == cb_seq_ids[-1]
+
+    def get_uniq(data):
+        return list(set(data))
+
+    bf_seq_ids_uniq = get_uniq(bf_seq_ids)
+    cb_seq_ids_uniq = get_uniq(cb_seq_ids)
+
+    assert bf_seq_ids == bf_seq_ids_uniq
+    assert len(cb_seq_ids) == (3 * len(cb_seq_ids_uniq))
+    assert len(bf_seq_ids) == num_dispatches
+    assert len(bf_seq_ids_uniq) == num_dispatches
+    assert len(cb_seq_ids_uniq) == num_dispatches
+
+
+def test_async_copy_direction(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    # Direction values:
+    #   0 == ??? (unknown)
+    #   1 == H2H (host to host)
+    #   2 == H2D (host to device)
+    #   3 == D2H (device to host)
+    #   4 == D2D (device to device)
+    default_async_dir_cnt = dict([(idx, 0) for idx in range(0, 5)])
+    thread_async_dir_cnt = {}
+    for itr in sdk_data.buffer_records.memory_copies:
+        tid = itr.thread_id
+        if tid not in thread_async_dir_cnt.keys():
+            thread_async_dir_cnt[tid] = default_async_dir_cnt
+        op_id = itr.operation
+        assert op_id > 1, f"{itr}"
+        assert op_id < 4, f"{itr}"
+        thread_async_dir_cnt[tid][op_id] += 1
+
+    for itr in sdk_data.callback_records.memory_copies:
+        tid = itr.thread_id
+        if tid not in thread_async_dir_cnt.keys():
+            thread_async_dir_cnt[tid] = default_async_dir_cnt
+        op_id = itr.operation
+        assert op_id > 1, f"{itr}"
+        assert op_id < 4, f"{itr}"
+        thread_async_dir_cnt[tid][op_id] += 1
+
+        phase = itr.phase
+        pitr = itr.payload
+
+        assert phase is not None, f"{itr}"
+        assert pitr is not None, f"{itr}"
+
+        if phase == 1:
+            assert pitr.start_timestamp == 0, f"{itr}"
+            assert pitr.end_timestamp == 0, f"{itr}"
+        elif phase == 2:
+            assert pitr.start_timestamp > 0, f"{itr}"
+            assert pitr.end_timestamp > 0, f"{itr}"
+            assert pitr.end_timestamp >= pitr.start_timestamp, f"{itr}"
+        else:
+            assert phase == 1 or phase == 2, f"{itr}"
+
+    # in the transpose test which generates the input file,
+    # two threads each perform one H2D + one D2H memory copy.
+    # there are at least two callback records (phase start +
+    # phase end) and one buffer record for each memory copy,
+    # i.e., at least 3 records per memory copy
+    assert len(thread_async_dir_cnt) == 2, f"{thread_async_dir_cnt}"
+    for tid, async_dir_cnt in thread_async_dir_cnt.items():
+        min_copy_records = 3
+        assert async_dir_cnt[0] == 0
+        assert async_dir_cnt[1] == 0
+        assert async_dir_cnt[2] >= min_copy_records, f"TID={tid}:\n\t{async_dir_cnt}"
+        assert async_dir_cnt[3] >= min_copy_records, f"TID={tid}:\n\t{async_dir_cnt}"
+        assert async_dir_cnt[4] == 0
+        # HIP memory copies may be decomposed into more than one
+        # memory copy at the HSA level so require it to be a multiple
+        # of min_copy_records
+        assert (
+            async_dir_cnt[2] % min_copy_records
+        ) == 0, f"TID={tid}:\n\t{async_dir_cnt}"
+        assert (
+            async_dir_cnt[3] % min_copy_records
+        ) == 0, f"TID={tid}:\n\t{async_dir_cnt}"
+
+
+def test_ancestor_ids(input_data):
+    """
+    This test ensures that each memcpy can be traced back to
+    a hipMemcpyAsync through ancestor IDs
+    """
+    from rocprofiler_sdk.pytest_utils.dotdict import dotdict
+
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+    buffer_records = sdk_data.buffer_records
+    memcopies = buffer_records.memory_copies
+
+    _, hip_op_ids = get_operation(buffer_records, "HIP_RUNTIME_API")
+    hip_memcopy_id = get_operation(buffer_records, "HIP_RUNTIME_API", "hipMemcpyAsync")
+
+    # dict with { internal id : record }
+    hip_memcopies = groupby_corr_id(buffer_records.hip_api_traces, hip_memcopy_id)
+
+    hsa_records = groupby_corr_id(buffer_records.hsa_api_traces)
+    hip_records = groupby_corr_id(buffer_records.hip_api_traces)
+
+    accounted_for_hip_ids = []
+
+    for memcpy in memcopies:
+        parent_hsa_call = hsa_records[memcpy.correlation_id.internal]
+        parent_hip_call = hip_records[parent_hsa_call.correlation_id.ancestor]
+
+        assert (
+            parent_hip_call.thread_id == parent_hsa_call.thread_id
+        ), "Expected hsa and hip calls to be on the same thread"
+        assert hip_op_ids[parent_hip_call.operation] == "hipMemcpyAsync"
+        accounted_for_hip_ids.append(parent_hip_call.correlation_id.internal)
+
+    # Ensure we looked through all HIP entries
+    assert (
+        set(accounted_for_hip_ids) == set(hip_memcopies.keys()),
+        "Expected to account for all HIP memcpy calls through ancestor ID lookup",
+    )
+
+
+def test_retired_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+    buffer_records = sdk_data["buffer_records"]
+    api_name_info = {}
+
+    def _sort_dict(inp):
+        return dict(sorted(inp.items()))
+
+    api_corr_ids = {}
+    for titr in test_api_traces:
+        for itr in sdk_data["buffer_records"][titr]:
+            corr_id = itr["correlation_id"]["internal"]
+            name = get_operation_name(buffer_records, itr["kind"], itr["operation"])
+
+            assert corr_id not in api_corr_ids.keys()
+            assert name is not None, f"{itr}"
+
+            api_corr_ids[corr_id] = itr
+            api_name_info[corr_id] = name
+
+    async_corr_ids = {}
+    for titr in ["kernel_dispatch", "memory_copies"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            corr_id = itr["correlation_id"]["internal"]
+            assert corr_id not in async_corr_ids.keys()
+            async_corr_ids[corr_id] = itr
+
+    retired_corr_ids = {}
+    for itr in sdk_data["buffer_records"]["retired_correlation_ids"]:
+        corr_id = itr["internal_correlation_id"]
+        assert corr_id not in retired_corr_ids.keys()
+        retired_corr_ids[corr_id] = itr
+
+    api_corr_ids = _sort_dict(api_corr_ids)
+    async_corr_ids = _sort_dict(async_corr_ids)
+    retired_corr_ids = _sort_dict(retired_corr_ids)
+
+    #
+    # verify all the correlation ids were retired
+    #
+    num_api_corr_ids = len(api_corr_ids.keys())
+    num_retired_corr_ids = len(retired_corr_ids.keys())
+
+    missing_retired_corr_ids = [
+        itr for itr in api_corr_ids.keys() if itr not in retired_corr_ids.keys()
+    ]
+    # log in case of failure
+    sys.stderr.flush()
+    for itr in missing_retired_corr_ids:
+        name = api_name_info[itr]
+        info = api_corr_ids[itr]
+        sys.stderr.write(f"- unretired corr id: {itr} :: {name} :: {info}\n")
+    sys.stderr.flush()
+
+    assert (
+        num_api_corr_ids == num_retired_corr_ids
+    ), f"correlation ids not retired:\n\t{missing_retired_corr_ids}"
+
+    #
+    #   verify the retirement timestamp is >= the end timestamp of the records
+    #
+    for cid, itr in api_corr_ids.items():
+        assert cid in retired_corr_ids.keys()
+        retired_ts = retired_corr_ids[cid]["timestamp"]
+        end_ts = itr["end_timestamp"]
+        name = api_name_info[cid]
+        assert (
+            retired_ts - end_ts
+        ) >= 0, f"\n\tcorr: {cid}\n\tname: {name}\n\tdata: {itr}"
+
+    # allow the retired timestamp to be 10 usec earlier than async end timestamp
+    # since the async timestamps undergo conversion from the GPU clock domain to
+    # the CPU clock domain. 10 microseconds was arbitrarily chosen to be an
+    # acceptable amount of inaccuracy -- in an ideal world, retired_ts should
+    # always be >= end_ts
+    usec = 1000
+    supported_fuzzing = 10 * usec
+
+    for cid, itr in async_corr_ids.items():
+        assert cid in retired_corr_ids.keys()
+        retired_ts = retired_corr_ids[cid]["timestamp"]
+        end_ts = itr["end_timestamp"]
+        name = api_name_info[cid]
+        assert (
+            retired_ts - end_ts
+        ) >= -supported_fuzzing, f"\n\tcorr: {cid}\n\tname: {name}\n\tdata: {itr}"
+
+
+if __name__ == "__main__":
+    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
+    sys.exit(exit_code)

projects/rocprofiler-sdk/tests/bin/CMakeLists.txt

@@ -0,0 +1,41 @@
+#
+# Integration test applications
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(rocprofiler-sdk-tests-bin LANGUAGES C CXX)
+
+set(CMAKE_BUILD_RPATH
+    "\$ORIGIN:\$ORIGIN/../lib:$<TARGET_FILE_DIR:rocprofiler-sdk-roctx::rocprofiler-sdk-roctx-shared-library>"
+    )
+
+# Find rocDecode and rocJPEG packages for testing
+find_package(rocDecode)
+find_package(rocJPEG)
+
+# applications used by integration tests which DO link to rocprofiler-sdk-roctx
+add_subdirectory(reproducible-runtime)
+add_subdirectory(reproducible-dispatch-count)
+add_subdirectory(transpose)
+add_subdirectory(openmp)
+
+set(CMAKE_BUILD_RPATH "\$ORIGIN:\$ORIGIN/../lib")
+
+# applications used by integration tests which DO NOT link to rocprofiler-sdk-roctx
+add_subdirectory(simple-transpose)
+add_subdirectory(multistream)
+add_subdirectory(vector-operations)
+add_subdirectory(hip-in-libraries)
+add_subdirectory(scratch-memory)
+add_subdirectory(hsa-queue-dependency)
+add_subdirectory(hip-graph)
+add_subdirectory(hsa-memory-allocation)
+add_subdirectory(pc-sampling)
+if(rocDecode_FOUND AND rocDecode_VERSION VERSION_GREATER 0.8.0)
+    add_subdirectory(rocdecode)
+endif()
+if(rocJPEG_FOUND AND rocJPEG_VERSION VERSION_GREATER 0.6.0)
+    add_subdirectory(rocjpeg)
+endif()
+add_subdirectory(hsa-code-object)
+add_subdirectory(hip-streams)

projects/rocprofiler-sdk/tests/bin/hip-graph/CMakeLists.txt

@@ -0,0 +1,44 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-bin-hip-graph LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(hip-graph.cpp PROPERTIES LANGUAGE HIP)
+add_executable(hip-graph)
+target_sources(hip-graph PRIVATE hip-graph.cpp)
+target_compile_options(hip-graph PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow -Werror)
+
+find_package(Threads REQUIRED)
+target_link_libraries(hip-graph PRIVATE Threads::Threads)
+
+# find_package(rocprofiler-sdk-roctx REQUIRED) target_link_libraries(hip-graph PRIVATE
+# rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)

projects/rocprofiler-sdk/tests/bin/hip-graph/hip-graph.cpp

@@ -0,0 +1,220 @@
+/*
+Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include <libgen.h>
+#include <future>
+#include <iomanip>
+#include <iostream>
+#include <mutex>
+
+// hip header file
+#include <hip/hip_runtime.h>
+
+#include <stdio.h>
+#include <unistd.h>
+#include <regex>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace
+{
+using auto_lock_t    = std::unique_lock<std::mutex>;
+auto print_mutex     = std::mutex{};
+auto global_kern_num = std::atomic<uint64_t>{0};
+}  // namespace
+
+template <typename T>
+void
+check(T result, char const* const func, const char* const file, int const line)
+{
+    if(result)
+    {
+        fprintf(stderr,
+                "Hip error at %s:%d code=%d(%s) \"%s\" \n",
+                file,
+                line,
+                static_cast<unsigned int>(result),
+                hipGetErrorName(result),
+                func);
+        exit(EXIT_FAILURE);
+    }
+}
+#define checkHipErrors(val) check((val), #val, __FILE__, __LINE__)
+
+__global__ void
+kernel_foo(const int devid, const int kernid, const int kernid_global, const volatile int* streamid)
+{
+    printf("[hip-graph][device %2i][stream %2i] Kernel foo | %2i | %2i executing...\n",
+           devid,
+           *streamid,
+           kernid,
+           kernid_global);
+}
+
+__global__ void
+kernel_bar(const int devid, const int kernid, const int kernid_global, const volatile int* streamid)
+{
+    printf("[hip-graph][device %2i][stream %2i] Kernel bar | %2i | %2i executing...\n",
+           devid,
+           *streamid,
+           kernid,
+           kernid_global);
+}
+
+void
+run(uint64_t                        devid,
+    uint64_t                        nstream,
+    uint64_t                        nkernel_per_stream,
+    std::atomic<uint64_t>*          progress,
+    const std::shared_future<void>& future)
+{
+    auto prefix = [devid]() {
+        auto ss = std::stringstream{};
+        ss << "[hip-graph][device " << std::setw(2) << devid << "] ";
+        return ss.str();
+    }();
+
+    auto log_message = [&prefix](const auto& msg) {
+        auto _lk = auto_lock_t{print_mutex};
+        std::cout << prefix << msg << "..." << std::endl;
+    };
+
+    log_message("setting device");
+    checkHipErrors(hipSetDevice(devid));
+
+    auto streams    = std::vector<hipStream_t>(nstream, nullptr);
+    auto stream_num = std::vector<int*>(nstream, nullptr);
+
+    log_message("creating streams");
+    for(auto& itr : streams)
+        checkHipErrors(hipStreamCreate(&itr));
+
+    log_message("allocating data");
+    for(uint64_t i = 0; i < nstream; ++i)
+    {
+        auto& itr = stream_num.at(i);
+        auto* str = streams.at(i);
+        auto  val = i;
+        checkHipErrors(hipMallocAsync(&itr, sizeof(int), str));
+        checkHipErrors(hipMemcpyAsync(itr, &val, sizeof(int), hipMemcpyHostToDevice, str));
+    }
+
+    auto graphs = std::vector<hipGraph_t>(nstream);
+    auto execs  = std::vector<hipGraphExec_t>(nstream, nullptr);
+
+    uint64_t kern_num = 0;
+    for(uint64_t i = 0; i < nstream; ++i)
+    {
+        checkHipErrors(hipStreamBeginCapture(streams.at(i), hipStreamCaptureModeGlobal));
+
+        for(uint64_t j = 0; j < nkernel_per_stream; ++j)
+        {
+            auto kern_num_v      = kern_num++;
+            auto glob_kern_num_v = global_kern_num++;
+            auto kernel          = (j % 2 == 0) ? kernel_foo : kernel_bar;
+            hipLaunchKernelGGL(kernel,
+                               dim3(1),
+                               dim3(1),
+                               0,
+                               streams.at(i),
+                               devid,
+                               kern_num_v,
+                               glob_kern_num_v,
+                               stream_num.at(i));
+            checkHipErrors(hipGetLastError());
+        }
+
+        checkHipErrors(hipStreamEndCapture(streams.at(i), &graphs.at(i)));
+        checkHipErrors(hipGraphInstantiate(&execs.at(i), graphs.at(i), nullptr, nullptr, 0));
+    }
+
+    if(progress) progress->fetch_add(1);
+    future.wait();
+
+    log_message("launching graph");
+    for(uint64_t i = 0; i < nstream; ++i)
+        checkHipErrors(hipGraphLaunch(execs.at(i), streams.at(i)));
+
+    for(uint64_t i = 0; i < nstream; ++i)
+        checkHipErrors(hipStreamSynchronize(streams.at(i)));
+
+    log_message("destroying graph");
+    for(uint64_t i = 0; i < nstream; ++i)
+        checkHipErrors(hipGraphDestroy(graphs.at(i)));
+
+    log_message("freeing data");
+    for(auto& itr : stream_num)
+        checkHipErrors(hipFree(itr));
+
+    log_message("returning");
+}
+
+int
+main(int argc, char* argv[])
+{
+    std::cout << "[" << basename(argv[0]) << "] executing..." << std::endl;
+
+    int ndevice_real = 0;
+    checkHipErrors(hipGetDeviceCount(&ndevice_real));
+
+    uint64_t nstream            = 1;
+    uint64_t nkernel_per_stream = 12;
+    uint64_t ndevice            = ndevice_real;
+
+    if(argc > 1) nstream = std::stoul(argv[1]);
+    if(argc > 2) nkernel_per_stream = std::stoul(argv[2]);
+    if(argc > 3) ndevice = std::stoul(argv[3]);
+
+    ndevice = std::min<uint64_t>(ndevice, ndevice_real);
+
+    auto progress = std::atomic<uint64_t>{0};
+    auto promise  = std::promise<void>{};
+    auto future   = promise.get_future().share();
+    auto threads  = std::vector<std::thread>{};
+    threads.reserve(ndevice);
+
+    for(uint64_t i = 0; i < ndevice; ++i)
+        threads.emplace_back(run, i, nstream, nkernel_per_stream, &progress, future);
+
+    // wait for all threads to reach designated progress point
+    while(progress < ndevice)
+    {
+        std::this_thread::yield();
+        std::this_thread::sleep_for(std::chrono::milliseconds{1});
+    }
+
+    // release the threads
+    promise.set_value();
+
+    for(auto& itr : threads)
+        itr.join();
+
+    for(uint64_t i = 0; i < ndevice; ++i)
+    {
+        checkHipErrors(hipSetDevice(i));
+        checkHipErrors(hipDeviceSynchronize());
+    }
+
+    std::cout << "[" << basename(argv[0]) << "] complete" << std::endl;
+    return 0;
+}

projects/rocprofiler-sdk/tests/bin/hip-in-libraries/CMakeLists.txt

@@ -0,0 +1,29 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(rocprofiler-sdk-tests-bin-hip-in-libraries LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+add_executable(hip-in-libraries)
+target_sources(hip-in-libraries PRIVATE hip-in-libraries.cpp)
+target_compile_options(hip-in-libraries PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow
+                                                -Werror)
+target_link_libraries(hip-in-libraries PRIVATE transpose-shared-library
+                                               vector-ops-shared-library)
+
+find_package(hip REQUIRED)
+target_link_libraries(hip-in-libraries PRIVATE hip::host)
+
+find_package(Threads REQUIRED)
+target_link_libraries(hip-in-libraries PRIVATE Threads::Threads)
+
+if(TRANSPOSE_USE_MPI)
+    find_package(MPI REQUIRED)
+    target_compile_definitions(hip-in-libraries PRIVATE USE_MPI)
+    target_link_libraries(hip-in-libraries PRIVATE MPI::MPI_C)
+endif()

projects/rocprofiler-sdk/tests/bin/hip-in-libraries/hip-in-libraries.cpp

@@ -0,0 +1,152 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "transpose.hpp"
+#include "vector-ops.hpp"
+
+#include <hip/hip_runtime_api.h>
+
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <mutex>
+#include <stdexcept>
+#include <thread>
+
+#if defined(USE_MPI)
+#    include <mpi.h>
+#endif
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t = std::unique_lock<std::mutex>;
+auto   print_lock = std::mutex{};
+size_t nqueues    = 4;
+size_t nthreads   = 4;
+size_t nitr       = 500;
+size_t nsync      = 10;
+}  // namespace
+
+int
+main(int argc, char** argv)
+{
+    int rank = 0;
+    int size = 1;
+
+#if defined(USE_MPI)
+    MPI_Init(&argc, &argv);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+#else
+    (void) size;
+#endif
+
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{argv[i]};
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            if(rank == 0)
+            {
+                fprintf(stderr,
+                        "usage: hip-in-libraries [NUM_QUEUES (%zu)] [NUM_THREADS (%zu)] "
+                        "[NUM_ITERATION (%zu)] "
+                        "[SYNC_EVERY_N_ITERATIONS (%zu)]\n",
+                        nqueues,
+                        nthreads,
+                        nitr,
+                        nsync);
+            }
+            exit(EXIT_SUCCESS);
+        }
+    }
+
+    if(argc > 1) nqueues = atoll(argv[1]);
+    if(argc > 2) nthreads = atoll(argv[2]);
+    if(argc > 3) nitr = atoll(argv[3]);
+    if(argc > 4) nsync = atoll(argv[4]);
+
+    int ndevice = 0;
+    HIP_API_CALL(hipGetDeviceCount(&ndevice));
+
+    printf("[hip-in-libraries] Number of devices found: %i\n", ndevice);
+    printf("[hip-in-libraries] Number of queues: %zu\n", nqueues);
+    printf("[hip-in-libraries] Number of threads: %zu\n", nthreads);
+    printf("[hip-in-libraries] Number of iterations: %zu\n", nitr);
+    printf("[hip-in-libraries] Syncing every %zu iterations\n", nsync);
+
+    {
+        auto vector_ops_thread = std::thread{run_vector_ops, nthreads, nqueues};
+        std::this_thread::sleep_for(std::chrono::milliseconds{100});
+        auto transpose_thread = std::thread{run_transpose, nthreads, nitr, nsync};
+
+        vector_ops_thread.join();
+        transpose_thread.join();
+    }
+
+    // this is a temporary workaround in omnitrace when HIP + MPI is enabled
+
+#if defined(USE_MPI)
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    for(int i = 0; i < ndevice; ++i)
+    {
+        HIP_API_CALL(hipSetDevice(i));
+        HIP_API_CALL(hipDeviceSynchronize());
+    }
+
+#if defined(USE_MPI)
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    if(rank == 0)
+    {
+        for(int i = 0; i < ndevice; ++i)
+        {
+            HIP_API_CALL(hipSetDevice(i));
+            HIP_API_CALL(hipDeviceReset());
+        }
+    }
+
+#if defined(USE_MPI)
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    return 0;
+}

projects/rocprofiler-sdk/tests/bin/hip-streams/CMakeLists.txt

@@ -0,0 +1,41 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-bin-hip-streams LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(compute_comm_overlap.cpp PROPERTIES LANGUAGE HIP)
+add_executable(hip-streams)
+target_sources(hip-streams PRIVATE compute_comm_overlap.cpp)
+target_link_libraries(hip-streams PRIVATE rocprofiler-sdk::tests-build-flags)
+
+find_package(Threads REQUIRED)
+target_link_libraries(hip-streams PRIVATE Threads::Threads)

projects/rocprofiler-sdk/tests/bin/hip-streams/compute_comm_overlap.cpp

@@ -0,0 +1,160 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "hip/hip_runtime.h"
+
+#define BLOCKDIM 64
+
+/* Macro for checking GPU API return values */
+#define HIP_ASSERT(call)                                                                           \
+    do                                                                                             \
+    {                                                                                              \
+        hipError_t gpuErr = call;                                                                  \
+        if(hipSuccess != gpuErr)                                                                   \
+        {                                                                                          \
+            printf(                                                                                \
+                "GPU API Error - %s:%d: '%s'\n", __FILE__, __LINE__, hipGetErrorString(gpuErr));   \
+            exit(1);                                                                               \
+        }                                                                                          \
+    } while(0)
+
+// HIP kernel. Each thread takes care of one element of input
+__global__ void
+cube(double* input, double* output, size_t offset, size_t elements_per_stream)
+{
+    size_t tid     = blockIdx.x * blockDim.x + threadIdx.x;
+    size_t gstride = blockDim.x * gridDim.x;
+
+    // Span all elements assigned to this stream
+    for(size_t id = tid + offset; id < offset + elements_per_stream; id += gstride)
+        for(size_t i = 0; i < 1000; ++i)
+            output[id] = input[id] * input[id] * input[id];
+}
+
+int
+main()
+{
+    // number of streams
+    const int num_streams = 8;
+
+    // Number of threads in each thread block
+    const int blockSize = 512;
+
+    // Size of vectors
+    int n                   = 100000000;
+    int elements_per_stream = n / num_streams;
+    int bytes_per_stream    = elements_per_stream * sizeof(double);
+
+    // Host input vectors
+    double* h_input1{nullptr};
+    // Host output vector
+    double* h_output1{nullptr};
+
+    // Device input vectors
+    double* d_input1{nullptr};
+    // Device output vector
+    double* d_output1{nullptr};
+
+    // Creating events for timers
+    hipEvent_t start{}, stop{};
+    HIP_ASSERT(hipEventCreate(&start));
+    HIP_ASSERT(hipEventCreate(&stop));
+
+    // Creating streams
+    hipStream_t streams[num_streams];
+    for(int i = 0; i < num_streams; ++i)
+    {
+        HIP_ASSERT(hipStreamCreate(&streams[i]));
+    }
+
+    // Size, in bytes, of each vector
+    size_t bytes = n * sizeof(double);
+
+    // Allocate page locked memory for these vectors on host
+    HIP_ASSERT(hipHostMalloc(&h_input1, bytes));
+    HIP_ASSERT(hipHostMalloc(&h_output1, bytes));
+
+    // Allocate memory for each vector on GPU
+    HIP_ASSERT(hipMalloc(&d_input1, bytes));
+    HIP_ASSERT(hipMalloc(&d_output1, bytes));
+
+    // Initialize vectors on host
+    for(int i = 0; i < n; i++)
+    {
+        h_input1[i] = sin(i);
+    }
+
+    // Number of thread blocks in grid
+    const int gridSizePerStream = 104;  //(int)ceil((float)elements_per_stream/blockSize);
+
+    HIP_ASSERT(hipEventRecord(start));
+    // split H2D copies and kernel calls into separate loops
+    for(int i = 0; i < num_streams; i++)
+    {
+        int offset = i * elements_per_stream;
+        HIP_ASSERT(hipMemcpyAsync(&d_input1[offset],
+                                  &h_input1[offset],
+                                  bytes_per_stream,
+                                  hipMemcpyHostToDevice,
+                                  streams[i]));
+    }
+    for(int i = 0; i < num_streams; i++)
+    {
+        int offset = i * elements_per_stream;
+        cube<<<gridSizePerStream, blockSize, 0, streams[i]>>>(
+            d_input1, d_output1, offset, elements_per_stream);
+    }
+    for(int i = 0; i < num_streams; i++)
+    {
+        int offset = i * elements_per_stream;
+        HIP_ASSERT(hipMemcpyAsync(&h_output1[offset],
+                                  &d_output1[offset],
+                                  bytes_per_stream,
+                                  hipMemcpyDeviceToHost,
+                                  streams[i]));
+    }
+
+    HIP_ASSERT(hipEventRecord(stop));
+    HIP_ASSERT(hipEventSynchronize(stop));
+
+    float milliseconds = 0;
+    HIP_ASSERT(hipEventElapsedTime(&milliseconds, start, stop));
+
+    // Release device memory
+    HIP_ASSERT(hipFree(d_input1));
+    HIP_ASSERT(hipFree(d_output1));
+
+    // Release host memory
+    HIP_ASSERT(hipHostFree(h_input1));
+    HIP_ASSERT(hipHostFree(h_output1));
+
+    // Destroy streams
+    for(int i = 0; i < num_streams; ++i)
+    {
+        HIP_ASSERT(hipStreamDestroy(streams[i]));
+    }
+
+    return 0;
+}

projects/rocprofiler-sdk/tests/bin/hsa-code-object/CMakeLists.txt

@@ -0,0 +1,98 @@
+# MIT License
+#
+# Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+#
+# HSA multi-queue dependency test
+
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(rocprofiler-sdk-tests-bin-hsa-code-object LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+find_program(
+    amdclangpp_EXECUTABLE REQUIRED
+    NAMES amdclang++
+    HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+    PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+    PATH_SUFFIXES bin llvm/bin NO_CACHE)
+
+function(generate_hsaco TARGET_ID INPUT_FILE OUTPUT_FILE)
+    separate_arguments(
+        CLANG_ARG_LIST
+        UNIX_COMMAND
+        "-O2 -x cl -Xclang -finclude-default-header -cl-denorms-are-zero -cl-std=CL2.0 -Wl,--build-id=sha1
+    -target amdgcn-amd-amdhsa -mcpu=${TARGET_ID} -o ${OUTPUT_FILE} ${INPUT_FILE}")
+    add_custom_command(
+        OUTPUT ${PROJECT_BINARY_DIR}/${OUTPUT_FILE}
+        COMMAND ${amdclangpp_EXECUTABLE} ${CLANG_ARG_LIST}
+        COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_BINARY_DIR}/${OUTPUT_FILE}
+                ${CMAKE_BINARY_DIR}/tests/rocprofv3/advanced-thread-trace/${OUTPUT_FILE}
+        OUTPUT ${CMAKE_BINARY_DIR}/tests/rocprofv3/advanced-thread-trace/${OUTPUT_FILE}
+        COMMAND
+            ${CMAKE_COMMAND} -E copy
+            ${CMAKE_BINARY_DIR}/tests/rocprofv3/advanced-thread-trace/${OUTPUT_FILE}
+            ${CMAKE_BINARY_DIR}/rocprofv3/advanced-thread-trace/${OUTPUT_FILE}
+        COMMENT "Building ${OUTPUT_FILE}...")
+    set(HSACO_TARGET_LIST
+        ${HSACO_TARGET_LIST} ${PROJECT_BINARY_DIR}/${OUTPUT_FILE}
+        PARENT_SCOPE)
+endfunction(generate_hsaco)
+
+foreach(target_id ${GPU_TARGETS})
+    # generate kernel bitcodes
+    generate_hsaco(${target_id} ${CMAKE_CURRENT_SOURCE_DIR}/copy.cl
+                   ${target_id}_copy.hsaco)
+    generate_hsaco(${target_id} ${CMAKE_CURRENT_SOURCE_DIR}/copy_memory.cl
+                   ${target_id}_copy_memory.hsaco)
+endforeach()
+
+add_custom_target(generate_hsaco_targets_code_object DEPENDS ${HSACO_TARGET_LIST})
+
+add_executable(hsa_code_object_testapp)
+target_sources(hsa_code_object_testapp PRIVATE hsa_code_object_app.cpp)
+target_compile_options(hsa_code_object_testapp PRIVATE -W -Wall -Wextra -Wshadow -Werror)
+
+find_package(Threads REQUIRED)
+target_link_libraries(hsa_code_object_testapp PRIVATE stdc++fs Threads::Threads)
+
+find_package(rocprofiler-sdk REQUIRED)
+target_link_libraries(
+    hsa_code_object_testapp PRIVATE rocprofiler-sdk::rocprofiler-sdk
+                                    rocprofiler-sdk::tests-common-library)
+
+find_package(
+    hsa-runtime64
+    REQUIRED
+    CONFIG
+    HINTS
+    ${rocm_version_DIR}
+    ${ROCM_PATH}
+    PATHS
+    ${rocm_version_DIR}
+    ${ROCM_PATH})
+
+target_link_libraries(hsa_code_object_testapp PRIVATE hsa-runtime64::hsa-runtime64)
+
+add_dependencies(hsa_code_object_testapp generate_hsaco_targets_code_object)

projects/rocprofiler-sdk/tests/bin/hsa-code-object/copy.cl

@@ -0,0 +1,32 @@
+/* Copyright (c) 2022 Advanced Micro Devices, Inc.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE. */
+
+__kernel void copyA(__global unsigned int* a, __global unsigned int* b) {
+    uint tid = get_global_id(0);
+    a[tid] = b[tid];
+}
+__kernel void copyB(__global unsigned int* a, __global unsigned int* b) {
+    uint tid = get_global_id(0);
+    a[tid] = b[tid];
+}
+__kernel void copyC(__global unsigned int* a, __global unsigned int* b) {
+    uint tid = get_global_id(0);
+    a[tid] = b[tid];
+}

projects/rocprofiler-sdk/tests/bin/hsa-code-object/copy_memory.cl

@@ -0,0 +1,32 @@
+/* Copyright (c) 2022 Advanced Micro Devices, Inc.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE. */
+
+__kernel void copyD(__global unsigned int* a, __global unsigned int* b) {
+    uint tid = get_global_id(0);
+    a[tid] = b[tid];
+}
+__kernel void copyE(__global unsigned int* a, __global unsigned int* b) {
+    uint tid = get_global_id(0);
+    a[tid] = b[tid];
+}
+__kernel void copyF(__global unsigned int* a, __global unsigned int* b) {
+    uint tid = get_global_id(0);
+    a[tid] = b[tid];
+}

projects/rocprofiler-sdk/tests/bin/hsa-code-object/hsa_code_object_app.cpp

@@ -0,0 +1,492 @@
+// MIT License
+//
+// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+/** ROC Profiler Multi Queue Dependency Test
+ *
+ * The goal of this test is to ensure ROC profiler does not go to deadlock
+ * when multiple queue are created and they are dependent on each other
+ *
+ */
+
+#include "hsa_code_object_app.h"
+
+enum class storage_type
+{
+    CODE_OBJECT_STORAGE_FILE,
+    CODE_OBJECT_STORAGE_MEMORY
+};
+
+void
+code_object_load(MQDependencyTest&             obj,
+                 storage_type                  type,
+                 MQDependencyTest::CodeObject& code_object)
+{
+    hsa_status_t status;
+    obj.device_discovery();
+    char agent_name[64];
+    status = hsa_agent_get_info(obj.gpu[0].agent, HSA_AGENT_INFO_NAME, agent_name);
+    RET_IF_HSA_ERR(status)
+
+    if(type == storage_type::CODE_OBJECT_STORAGE_FILE)
+    {
+        std::string hasco_file_path = std::string(agent_name) + std::string("_copy.hsaco");
+        obj.search_hasco(fs::current_path(), hasco_file_path);
+        if(!obj.load_code_object(hasco_file_path, obj.gpu[0].agent, code_object))
+        {
+            printf("Kernel file not found or not usable with given agent.\n");
+            abort();
+        }
+    }
+    else
+    {
+        std::string hasco_file_path = std::string(agent_name) + std::string("_copy_memory.hsaco");
+        obj.search_hasco(fs::current_path(), hasco_file_path);
+        if(!obj.load_code_object_memory(hasco_file_path, obj.gpu[0].agent, code_object))
+        {
+            abort();
+        }
+    }
+}
+
+MQDependencyTest::Kernel
+get_kernel(MQDependencyTest::CodeObject& code_object,
+           std::string                   kernel_name,
+           MQDependencyTest&             obj)
+{
+    MQDependencyTest::Kernel copy;
+    if(!obj.get_kernel(code_object, kernel_name, obj.gpu[0].agent, copy))
+    {
+        printf("Test %s not found.\n", kernel_name.c_str());
+        abort();
+    }
+    return copy;
+}
+
+int
+main()
+{
+    hsa_status_t                 status;
+    MQDependencyTest             obj;
+    MQDependencyTest             obj_memory  = {};
+    MQDependencyTest::CodeObject code_object = {}, code_object_memory = {};
+
+    code_object_load(obj, storage_type::CODE_OBJECT_STORAGE_FILE, code_object);
+    code_object_load(obj_memory, storage_type::CODE_OBJECT_STORAGE_MEMORY, code_object_memory);
+
+    MQDependencyTest::Kernel copyA = get_kernel(code_object, "copyA", obj);
+    MQDependencyTest::Kernel copyB = get_kernel(code_object, "copyB", obj);
+    MQDependencyTest::Kernel copyC = get_kernel(code_object, "copyC", obj);
+
+    MQDependencyTest::Kernel copyD = get_kernel(code_object_memory, "copyD", obj_memory);
+    MQDependencyTest::Kernel copyE = get_kernel(code_object_memory, "copyE", obj_memory);
+    MQDependencyTest::Kernel copyF = get_kernel(code_object_memory, "copyF", obj_memory);
+
+    struct args_t
+    {
+        uint32_t*                       a      = nullptr;
+        uint32_t*                       b      = nullptr;
+        MQDependencyTest::OCLHiddenArgs hidden = {};
+    };
+
+    args_t* args = static_cast<args_t*>(obj.hsa_malloc(sizeof(args_t), obj.kernarg));
+    *args        = {};
+
+    uint32_t* a = static_cast<uint32_t*>(obj.hsa_malloc(64 * sizeof(uint32_t), obj.kernarg));
+    uint32_t* b = static_cast<uint32_t*>(obj.hsa_malloc(64 * sizeof(uint32_t), obj.kernarg));
+
+    memset(a, 0, 64 * sizeof(uint32_t));
+    memset(b, 1, 64 * sizeof(uint32_t));
+
+    args_t* args_memory =
+        static_cast<args_t*>(obj_memory.hsa_malloc(sizeof(args_t), obj_memory.kernarg));
+    *args_memory = {};
+
+    uint32_t* c =
+        static_cast<uint32_t*>(obj_memory.hsa_malloc(64 * sizeof(uint32_t), obj_memory.kernarg));
+    uint32_t* d =
+        static_cast<uint32_t*>(obj_memory.hsa_malloc(64 * sizeof(uint32_t), obj_memory.kernarg));
+
+    memset(c, 0, 64 * sizeof(uint32_t));
+    memset(d, 1, 64 * sizeof(uint32_t));
+
+    // Create queue in gpu agent and prepare a kernel dispatch packet
+    hsa_queue_t* queue1 = nullptr;
+    status              = hsa_queue_create(obj.gpu[0].agent,
+                              1024,
+                              HSA_QUEUE_TYPE_SINGLE,
+                              nullptr,
+                              nullptr,
+                              UINT32_MAX,
+                              UINT32_MAX,
+                              &queue1);
+    RET_IF_HSA_ERR(status)
+
+    // Create a signal with a value of 1 and attach it to the first kernel
+    // dispatch packet
+    hsa_signal_t completion_signal_1 = {};
+    status                           = hsa_signal_create(1, 0, nullptr, &completion_signal_1);
+    RET_IF_HSA_ERR(status)
+
+    // First dispath packet on queue 1, Kernel A
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_KERNEL_DISPATCH;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.dispatch.setup            = 1;
+        packet.dispatch.workgroup_size_x = 64;
+        packet.dispatch.workgroup_size_y = 1;
+        packet.dispatch.workgroup_size_z = 1;
+        packet.dispatch.grid_size_x      = 64;
+        packet.dispatch.grid_size_y      = 1;
+        packet.dispatch.grid_size_z      = 1;
+
+        packet.dispatch.group_segment_size   = copyA.group;
+        packet.dispatch.private_segment_size = copyA.scratch;
+        packet.dispatch.kernel_object        = copyA.handle;
+
+        packet.dispatch.kernarg_address   = args;
+        packet.dispatch.completion_signal = completion_signal_1;
+
+        args->a = a;
+        args->b = b;
+        // Tell packet processor of A to launch the first kernel dispatch packet
+        obj.submit_packet(queue1, packet);
+    }
+
+    // Create a signal with a value of 1 and attach it to the second kernel
+    // dispatch packet
+    hsa_signal_t completion_signal_2 = {};
+    status                           = hsa_signal_create(1, 0, nullptr, &completion_signal_2);
+    RET_IF_HSA_ERR(status)
+
+    hsa_signal_t completion_signal_3 = {};
+    status                           = hsa_signal_create(1, 0, nullptr, &completion_signal_3);
+    RET_IF_HSA_ERR(status)
+
+    // Create barrier-AND packet that is enqueued in queue 1
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_BARRIER_AND;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.barrier_and.dep_signal[0] = completion_signal_2;
+        obj.submit_packet(queue1, packet);
+    }
+
+    // Second dispath packet on queue 1, Kernel C
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_KERNEL_DISPATCH;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.dispatch.setup            = 1;
+        packet.dispatch.workgroup_size_x = 64;
+        packet.dispatch.workgroup_size_y = 1;
+        packet.dispatch.workgroup_size_z = 1;
+        packet.dispatch.grid_size_x      = 64;
+        packet.dispatch.grid_size_y      = 1;
+        packet.dispatch.grid_size_z      = 1;
+
+        packet.dispatch.group_segment_size   = copyC.group;
+        packet.dispatch.private_segment_size = copyC.scratch;
+        packet.dispatch.kernel_object        = copyC.handle;
+        packet.dispatch.completion_signal    = completion_signal_3;
+        packet.dispatch.kernarg_address      = args;
+
+        args->a = a;
+        args->b = b;
+        // Tell packet processor to launch the second kernel dispatch packet
+        obj.submit_packet(queue1, packet);
+    }
+
+    // Create queue 2
+    hsa_queue_t* queue2 = nullptr;
+    status              = hsa_queue_create(obj.gpu[0].agent,
+                              1024,
+                              HSA_QUEUE_TYPE_SINGLE,
+                              nullptr,
+                              nullptr,
+                              UINT32_MAX,
+                              UINT32_MAX,
+                              &queue2);
+    RET_IF_HSA_ERR(status)
+
+    // Create barrier-AND packet that is enqueued in queue 2
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_BARRIER_AND;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.barrier_and.dep_signal[0] = completion_signal_1;
+        obj.submit_packet(queue2, packet);
+    }
+
+    // Third dispath packet on queue 2, Kernel B
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_KERNEL_DISPATCH;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.dispatch.setup            = 1;
+        packet.dispatch.workgroup_size_x = 64;
+        packet.dispatch.workgroup_size_y = 1;
+        packet.dispatch.workgroup_size_z = 1;
+        packet.dispatch.grid_size_x      = 64;
+        packet.dispatch.grid_size_y      = 1;
+        packet.dispatch.grid_size_z      = 1;
+
+        packet.dispatch.group_segment_size   = copyB.group;
+        packet.dispatch.private_segment_size = copyB.scratch;
+        packet.dispatch.kernel_object        = copyB.handle;
+
+        packet.dispatch.kernarg_address   = args;
+        packet.dispatch.completion_signal = completion_signal_2;
+
+        args->a = a;
+        args->b = b;
+        // Tell packet processor to launch the third kernel dispatch packet
+        obj.submit_packet(queue2, packet);
+    }
+    // Create a signal with a value of 1 and attach it to the first kernel
+    // dispatch packet
+    hsa_signal_t completion_signal_4 = {};
+    status                           = hsa_signal_create(1, 0, nullptr, &completion_signal_4);
+    RET_IF_HSA_ERR(status)
+    // First dispath packet on queue 1, Kernel D
+    {
+        [[maybe_unused]] MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_KERNEL_DISPATCH;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.dispatch.setup            = 1;
+        packet.dispatch.workgroup_size_x = 64;
+        packet.dispatch.workgroup_size_y = 1;
+        packet.dispatch.workgroup_size_z = 1;
+        packet.dispatch.grid_size_x      = 64;
+        packet.dispatch.grid_size_y      = 1;
+        packet.dispatch.grid_size_z      = 1;
+
+        packet.dispatch.group_segment_size   = copyD.group;
+        packet.dispatch.private_segment_size = copyD.scratch;
+        packet.dispatch.kernel_object        = copyD.handle;
+
+        packet.dispatch.kernarg_address   = args_memory;
+        packet.dispatch.completion_signal = completion_signal_4;
+
+        args_memory->a = c;
+        args_memory->b = d;
+        // Tell packet processor of A to launch the first kernel dispatch packet
+        obj_memory.submit_packet(queue1, packet);
+    }
+
+    // Create a signal with a value of 1 and attach it to the second kernel
+    // dispatch packet
+    hsa_signal_t completion_signal_5 = {};
+    status                           = hsa_signal_create(1, 0, nullptr, &completion_signal_5);
+    RET_IF_HSA_ERR(status)
+    hsa_signal_t completion_signal_6 = {};
+    status                           = hsa_signal_create(1, 0, nullptr, &completion_signal_6);
+    RET_IF_HSA_ERR(status)
+
+    // Create barrier-AND packet that is enqueued in queue 1
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_BARRIER_AND;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.barrier_and.dep_signal[0] = completion_signal_5;
+        obj_memory.submit_packet(queue1, packet);
+    }
+    // Second dispath packet on queue 1, Kernel F
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_KERNEL_DISPATCH;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.dispatch.setup            = 1;
+        packet.dispatch.workgroup_size_x = 64;
+        packet.dispatch.workgroup_size_y = 1;
+        packet.dispatch.workgroup_size_z = 1;
+        packet.dispatch.grid_size_x      = 64;
+        packet.dispatch.grid_size_y      = 1;
+        packet.dispatch.grid_size_z      = 1;
+
+        packet.dispatch.group_segment_size   = copyF.group;
+        packet.dispatch.private_segment_size = copyF.scratch;
+        packet.dispatch.kernel_object        = copyF.handle;
+        packet.dispatch.completion_signal    = completion_signal_6;
+        packet.dispatch.kernarg_address      = args_memory;
+
+        args_memory->a = c;
+        args_memory->b = d;
+        // Tell packet processor to launch the second kernel dispatch packet
+        obj_memory.submit_packet(queue1, packet);
+    }
+    // Create barrier-AND packet that is enqueued in queue 2
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_BARRIER_AND;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.barrier_and.dep_signal[0] = completion_signal_4;
+        obj_memory.submit_packet(queue2, packet);
+    }
+    // Third dispath packet on queue 2, Kernel
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_KERNEL_DISPATCH;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.dispatch.setup            = 1;
+        packet.dispatch.workgroup_size_x = 64;
+        packet.dispatch.workgroup_size_y = 1;
+        packet.dispatch.workgroup_size_z = 1;
+        packet.dispatch.grid_size_x      = 64;
+        packet.dispatch.grid_size_y      = 1;
+        packet.dispatch.grid_size_z      = 1;
+
+        packet.dispatch.group_segment_size   = copyE.group;
+        packet.dispatch.private_segment_size = copyE.scratch;
+        packet.dispatch.kernel_object        = copyE.handle;
+
+        packet.dispatch.kernarg_address   = args_memory;
+        packet.dispatch.completion_signal = completion_signal_5;
+
+        args_memory->a = c;
+        args_memory->b = d;
+        // Tell packet processor to launch the third kernel dispatch packet
+        obj_memory.submit_packet(queue2, packet);
+    }
+    // Wait on the completion signal
+    hsa_signal_wait_relaxed(
+        completion_signal_1, HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, HSA_WAIT_STATE_BLOCKED);
+
+    // Wait on the completion signal
+    hsa_signal_wait_relaxed(
+        completion_signal_2, HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, HSA_WAIT_STATE_BLOCKED);
+
+    // Wait on the completion signal
+    hsa_signal_wait_relaxed(
+        completion_signal_3, HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, HSA_WAIT_STATE_BLOCKED);
+
+    // Wait on the completion signal
+    hsa_signal_wait_relaxed(
+        completion_signal_4, HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, HSA_WAIT_STATE_BLOCKED);
+
+    // Wait on the completion signal
+    hsa_signal_wait_relaxed(
+        completion_signal_5, HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, HSA_WAIT_STATE_BLOCKED);
+
+    // Wait on the completion signal
+    hsa_signal_wait_relaxed(
+        completion_signal_6, HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, HSA_WAIT_STATE_BLOCKED);
+
+    for(int i = 0; i < 64; i++)
+    {
+        if(a[i] != b[i])
+        {
+            printf("error at %d: expected %d, got %d\n", i, b[i], a[i]);
+            abort();
+        }
+    }
+
+    // Clearing data structures and memory
+    status = hsa_signal_destroy(completion_signal_1);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_signal_destroy(completion_signal_2);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_signal_destroy(completion_signal_3);
+    RET_IF_HSA_ERR(status)
+
+    // Clearing data structures and memory
+    status = hsa_signal_destroy(completion_signal_4);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_signal_destroy(completion_signal_5);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_signal_destroy(completion_signal_6);
+    RET_IF_HSA_ERR(status)
+
+    if(queue1 != nullptr)
+    {
+        status = hsa_queue_destroy(queue1);
+        RET_IF_HSA_ERR(status)
+    }
+
+    if(queue2 != nullptr)
+    {
+        status = hsa_queue_destroy(queue2);
+        RET_IF_HSA_ERR(status)
+    }
+
+    status = hsa_memory_free(a);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_memory_free(b);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_memory_free(c);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_memory_free(d);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_executable_destroy(code_object.executable);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_code_object_reader_destroy(code_object.code_obj_rdr);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_executable_destroy(code_object_memory.executable);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_code_object_reader_destroy(code_object_memory.code_obj_rdr);
+    RET_IF_HSA_ERR(status)
+
+    close(code_object.file);
+
+    close(code_object_memory.file);
+}

projects/rocprofiler-sdk/tests/bin/hsa-code-object/hsa_code_object_app.h

@@ -0,0 +1,415 @@
+// MIT License
+//
+// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#pragma once
+
+#include "common/filesystem.hpp"
+
+#include <hsa/hsa.h>
+#include <hsa/hsa_ext_amd.h>
+
+#include <dlfcn.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+namespace fs = common::fs;
+
+#define RET_IF_HSA_ERR(err)                                                                        \
+    {                                                                                              \
+        if((err) != HSA_STATUS_SUCCESS)                                                            \
+        {                                                                                          \
+            char  err_val[12];                                                                     \
+            char* err_str = nullptr;                                                               \
+            if(hsa_status_string(err, (const char**) &err_str) != HSA_STATUS_SUCCESS)              \
+            {                                                                                      \
+                sprintf(&(err_val[0]), "%#x", (uint32_t) err);                                     \
+                err_str = &(err_val[0]);                                                           \
+            }                                                                                      \
+            printf("hsa api call failure at: %s:%d\n", __FILE__, __LINE__);                        \
+            printf("Call returned %s\n", err_str);                                                 \
+            abort();                                                                               \
+        }                                                                                          \
+    }
+
+struct Device
+{
+    struct Memory
+    {
+        hsa_amd_memory_pool_t pool;
+        bool                  fine;
+        bool                  kernarg;
+        size_t                size;
+        size_t                granule;
+    };
+
+    hsa_agent_t                     agent;
+    char                            name[64];
+    std::vector<Memory>             pools;
+    uint32_t                        fine;
+    uint32_t                        coarse;
+    static std::vector<hsa_agent_t> all_devices;
+};
+
+class MQDependencyTest
+{
+public:
+    MQDependencyTest() { hsa_init(); }
+    ~MQDependencyTest() { hsa_shut_down(); }
+
+    std::vector<Device>      cpu;
+    std::vector<Device>      gpu;
+    Device::Memory           kernarg;
+    std::vector<hsa_agent_t> all_devices = {};
+
+    struct CodeObject
+    {
+        hsa_file_t               file         = 0;
+        hsa_code_object_reader_t code_obj_rdr = {};
+        hsa_executable_t         executable   = {};
+    };
+
+    struct Kernel
+    {
+        uint64_t handle        = 0;
+        uint32_t scratch       = 0;
+        uint32_t group         = 0;
+        uint32_t kernarg_size  = 0;
+        uint32_t kernarg_align = 0;
+    };
+
+    union AqlHeader
+    {
+        struct
+        {
+            uint16_t type     : 8;
+            uint16_t barrier  : 1;
+            uint16_t acquire  : 2;
+            uint16_t release  : 2;
+            uint16_t reserved : 3;
+        };
+        uint16_t raw = 0;
+    };
+
+    struct BarrierValue
+    {
+        AqlHeader          header            = {};
+        uint8_t            AmdFormat         = 0;
+        uint8_t            reserved          = 0;
+        uint32_t           reserved1         = 0;
+        hsa_signal_t       signal            = {};
+        hsa_signal_value_t value             = 0;
+        hsa_signal_value_t mask              = 0;
+        uint32_t           cond              = 0;
+        uint32_t           reserved2         = 0;
+        uint64_t           reserved3         = 0;
+        uint64_t           reserved4         = 0;
+        hsa_signal_t       completion_signal = {};
+    };
+
+    union Aql
+    {
+        AqlHeader                    header;
+        hsa_kernel_dispatch_packet_t dispatch;
+        hsa_barrier_and_packet_t     barrier_and;
+        hsa_barrier_or_packet_t      barrier_or;
+        BarrierValue                 barrier_value = {};
+    };
+
+    struct OCLHiddenArgs
+    {
+        uint64_t offset_x      = 0;
+        uint64_t offset_y      = 0;
+        uint64_t offset_z      = 0;
+        void*    printf_buffer = nullptr;
+        void*    enqueue       = nullptr;
+        void*    enqueue2      = nullptr;
+        void*    multi_grid    = nullptr;
+    };
+
+    bool load_code_object(const std::string& filename, hsa_agent_t agent, CodeObject& code_object)
+    {
+        hsa_status_t err;
+        code_object.file = open(filename.c_str(), O_RDONLY);
+        if(code_object.file == -1)
+        {
+            fprintf(stderr, "%s:%s\n", "Could not load code object", filename.c_str());
+            abort();
+            return false;
+        }
+
+        err = hsa_code_object_reader_create_from_file(code_object.file, &code_object.code_obj_rdr);
+        RET_IF_HSA_ERR(err);
+
+        err = hsa_executable_create_alt(HSA_PROFILE_FULL,
+                                        HSA_DEFAULT_FLOAT_ROUNDING_MODE_DEFAULT,
+                                        nullptr,
+                                        &code_object.executable);
+        RET_IF_HSA_ERR(err);
+
+        err = hsa_executable_load_agent_code_object(
+            code_object.executable, agent, code_object.code_obj_rdr, nullptr, nullptr);
+        if(err != HSA_STATUS_SUCCESS) return false;
+
+        err = hsa_executable_freeze(code_object.executable, nullptr);
+        RET_IF_HSA_ERR(err);
+
+        return true;
+    }
+
+    bool load_code_object_memory(const std::string& filename,
+                                 hsa_agent_t        agent,
+                                 CodeObject&        code_object)
+    {
+        hsa_status_t  err;
+        size_t        buffer_size = 0;
+        std::ifstream code_object_file(filename.c_str(), std::ios::binary | std::ios::ate);
+        if(!code_object_file.good()) return false;
+        buffer_size = code_object_file.tellg();
+        code_object_file.seekg(0, std::ios::beg);
+        uint8_t* binary = new(std::nothrow) uint8_t[buffer_size];
+        if(binary && !code_object_file.read(reinterpret_cast<char*>(binary), buffer_size))
+        {
+            delete[] binary;
+            binary = nullptr;
+            return false;
+        }
+
+        err = hsa_code_object_reader_create_from_memory(
+            binary, buffer_size, &code_object.code_obj_rdr);
+        RET_IF_HSA_ERR(err);
+
+        err = hsa_executable_create_alt(HSA_PROFILE_FULL,
+                                        HSA_DEFAULT_FLOAT_ROUNDING_MODE_DEFAULT,
+                                        nullptr,
+                                        &code_object.executable);
+        RET_IF_HSA_ERR(err);
+
+        err = hsa_executable_load_agent_code_object(
+            code_object.executable, agent, code_object.code_obj_rdr, nullptr, nullptr);
+        if(err != HSA_STATUS_SUCCESS) return false;
+
+        err = hsa_executable_freeze(code_object.executable, nullptr);
+        RET_IF_HSA_ERR(err);
+        delete[] binary;
+        binary = nullptr;
+        return true;
+    }
+
+    bool get_kernel(const CodeObject&  code_object,
+                    const std::string& kernel,
+                    hsa_agent_t        agent,
+                    Kernel&            kern)
+    {
+        hsa_executable_symbol_t symbol;
+        hsa_status_t            err = hsa_executable_get_symbol_by_name(
+            code_object.executable, kernel.c_str(), &agent, &symbol);
+        if(err != HSA_STATUS_SUCCESS)
+        {
+            err = hsa_executable_get_symbol_by_name(
+                code_object.executable, (kernel + ".kd").c_str(), &agent, &symbol);
+            if(err != HSA_STATUS_SUCCESS)
+            {
+                return false;
+            }
+        }
+        printf("\nkernel-name: %s\n", kernel.c_str());
+        err = hsa_executable_symbol_get_info(
+            symbol, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_OBJECT, &kern.handle);
+        RET_IF_HSA_ERR(err);
+
+        return true;
+    }
+
+    // Not for parallel insertion.
+    bool submit_packet(hsa_queue_t* queue, Aql& pkt)
+    {
+        size_t mask = queue->size - 1;
+        Aql*   ring = static_cast<Aql*>(queue->base_address);
+
+        uint64_t write = hsa_queue_load_write_index_relaxed(queue);
+        uint64_t read  = hsa_queue_load_read_index_relaxed(queue);
+        if(write - read + 1 > queue->size) return false;
+
+        Aql& dst = ring[write & mask];
+
+        uint16_t header = pkt.header.raw;
+        pkt.header.raw  = dst.header.raw;
+        dst             = pkt;
+        __atomic_store_n(&dst.header.raw, header, __ATOMIC_RELEASE);
+        pkt.header.raw = header;
+
+        hsa_queue_store_write_index_release(queue, write + 1);
+        hsa_signal_store_screlease(queue->doorbell_signal, write);
+
+        return true;
+    }
+
+    void* hsa_malloc(size_t size, const Device::Memory& mem)
+    {
+        void*        ret;
+        hsa_status_t err = hsa_amd_memory_pool_allocate(mem.pool, size, 0, &ret);
+        RET_IF_HSA_ERR(err);
+
+        err = hsa_amd_agents_allow_access(all_devices.size(), all_devices.data(), nullptr, ret);
+        RET_IF_HSA_ERR(err);
+        return ret;
+    }
+
+    void* hsa_malloc(size_t size, const Device& dev, bool fine)
+    {
+        uint32_t index = fine ? dev.fine : dev.coarse;
+        assert(index != -1u && "Memory type unavailable.");
+        return hsa_malloc(size, dev.pools[index]);
+    }
+
+    bool device_discovery()
+    {
+        hsa_status_t err;
+
+        err = hsa_iterate_agents(
+            [](hsa_agent_t agent, void* obj) {
+                hsa_status_t error;
+
+                Device dev;
+                dev.agent = agent;
+
+                dev.fine               = -1u;
+                dev.coarse             = -1u;
+                MQDependencyTest* _obj = reinterpret_cast<MQDependencyTest*>(obj);
+                error                  = hsa_agent_get_info(agent, HSA_AGENT_INFO_NAME, dev.name);
+                RET_IF_HSA_ERR(error)
+
+                hsa_device_type_t type;
+                error = hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &type);
+                RET_IF_HSA_ERR(error)
+
+                error = hsa_amd_agent_iterate_memory_pools(
+                    agent,
+                    [](hsa_amd_memory_pool_t pool, void* data) {
+                        auto&        pools = *reinterpret_cast<std::vector<Device::Memory>*>(data);
+                        hsa_status_t status;
+
+                        bool allowed = false;
+                        status       = hsa_amd_memory_pool_get_info(
+                            pool, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_ALLOWED, &allowed);
+
+                        if(!allowed) return HSA_STATUS_SUCCESS;
+
+                        hsa_amd_segment_t segment;
+                        status = hsa_amd_memory_pool_get_info(
+                            pool, HSA_AMD_MEMORY_POOL_INFO_SEGMENT, &segment);
+                        RET_IF_HSA_ERR(status)
+
+                        if(segment != HSA_AMD_SEGMENT_GLOBAL) return HSA_STATUS_SUCCESS;
+
+                        uint32_t flags;
+                        status = hsa_amd_memory_pool_get_info(
+                            pool, HSA_AMD_MEMORY_POOL_INFO_GLOBAL_FLAGS, &flags);
+                        RET_IF_HSA_ERR(status)
+
+                        Device::Memory mem;
+                        mem.pool = pool;
+                        mem.fine = ((flags & HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_FINE_GRAINED) != 0u);
+                        mem.kernarg =
+                            ((flags & HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_KERNARG_INIT) != 0u);
+
+                        status = hsa_amd_memory_pool_get_info(
+                            pool, HSA_AMD_MEMORY_POOL_INFO_SIZE, &mem.size);
+                        RET_IF_HSA_ERR(status)
+
+                        status = hsa_amd_memory_pool_get_info(
+                            pool, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE, &mem.granule);
+                        RET_IF_HSA_ERR(status)
+
+                        pools.push_back(mem);
+                        return HSA_STATUS_SUCCESS;
+                    },
+                    static_cast<void*>(&dev.pools));
+
+                if(!dev.pools.empty())
+                {
+                    for(size_t i = 0; i < dev.pools.size(); i++)
+                    {
+                        if(dev.pools[i].fine && dev.pools[i].kernarg && dev.fine == -1u)
+                            dev.fine = i;
+                        if(dev.pools[i].fine && !dev.pools[i].kernarg) dev.fine = i;
+                        if(!dev.pools[i].fine) dev.coarse = i;
+                    }
+
+                    if(type == HSA_DEVICE_TYPE_CPU)
+                        _obj->cpu.push_back(dev);
+                    else
+                        _obj->gpu.push_back(dev);
+
+                    _obj->all_devices.push_back(dev.agent);
+                }
+
+                return HSA_STATUS_SUCCESS;
+            },
+            this);
+
+        bool is_break = false;
+        for(auto& dev : cpu)
+        {
+            for(auto& mem : dev.pools)
+            {
+                if(mem.fine && mem.kernarg)
+                {
+                    kernarg  = mem;
+                    is_break = true;
+                    break;
+                }
+            }
+            if(is_break) break;
+        }
+
+        RET_IF_HSA_ERR(err);
+
+        if(cpu.empty() || gpu.empty() || kernarg.pool.handle == 0) return false;
+        return true;
+    }
+
+    void search_hasco(const fs::path& directory, std::string& filename)
+    {
+        for(const auto& entry : fs::directory_iterator(directory))
+        {
+            if(fs::is_regular_file(entry))
+            {
+                if(entry.path().filename() == filename)
+                {
+                    filename = entry.path();
+                }
+            }
+            else if(fs::is_directory(entry))
+            {
+                search_hasco(entry, filename);  // Recursive call for subdirectories
+            }
+        }
+    }
+};

projects/rocprofiler-sdk/tests/bin/hsa-memory-allocation/CMakeLists.txt

@@ -0,0 +1,43 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-tool-test-app-hsa-memory-allocation LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(hsa-memory-allocation.cpp PROPERTIES LANGUAGE HIP)
+add_executable(hsa-memory-allocation)
+target_sources(hsa-memory-allocation PRIVATE hsa-memory-allocation.cpp)
+target_compile_options(hsa-memory-allocation PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow
+                                                     -Werror)
+
+find_package(Threads REQUIRED)
+target_link_libraries(hsa-memory-allocation PRIVATE Threads::Threads hsa-runtime64
+                                                    rocprofiler-sdk::tests-common-library)

projects/rocprofiler-sdk/tests/bin/hsa-memory-allocation/hsa-memory-allocation.cpp

@@ -0,0 +1,307 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include <hip/hip_runtime.h>
+#include <hsa/hsa.h>
+#include <hsa/hsa_ext_amd.h>
+#include <cstdio>
+#include <fstream>
+#include <iostream>
+#include <vector>
+
+#define RET_IF_HSA_ERR(err)                                                                        \
+    {                                                                                              \
+        if((err) != HSA_STATUS_SUCCESS)                                                            \
+        {                                                                                          \
+            char  err_val[12];                                                                     \
+            char* err_str = nullptr;                                                               \
+            if(hsa_status_string(err, (const char**) &err_str) != HSA_STATUS_SUCCESS)              \
+            {                                                                                      \
+                sprintf(&(err_val[0]), "%#x", (uint32_t) err);                                     \
+                err_str = &(err_val[0]);                                                           \
+            }                                                                                      \
+            printf("hsa api call failure at: %s:%d\n", __FILE__, __LINE__);                        \
+            printf("Call returned %s\n", err_str);                                                 \
+            abort();                                                                               \
+        }                                                                                          \
+    }
+
+// Callback function to get the list of agents
+hsa_status_t
+get_agents(hsa_agent_t agent, void* data)
+{
+    hsa_agent_t** agent_list = (hsa_agent_t**) data;
+    **agent_list             = agent;
+    ++(*agent_list);
+
+    return HSA_STATUS_SUCCESS;
+}
+
+// Callback function to get the number of agents
+hsa_status_t
+get_num_agents(hsa_agent_t agent, void* data)
+{
+    (void) agent;
+    int* num_agents = (int*) data;
+    ++(*num_agents);
+
+    return HSA_STATUS_SUCCESS;
+}
+
+// Callback function to get the number of regions of an agent
+hsa_status_t
+callback_get_num_regions(hsa_region_t region, void* data)
+{
+    (void) region;
+    int* num_regions = (int*) data;
+    ++(*num_regions);
+    return HSA_STATUS_SUCCESS;
+}
+
+// Callback function to get the number of memory pools of an agent
+hsa_status_t
+callback_get_num_pools(hsa_amd_memory_pool_t memory_pool, void* data)
+{
+    (void) memory_pool;
+    int* num_pools = (int*) data;
+    ++(*num_pools);
+    return HSA_STATUS_SUCCESS;
+}
+
+// Callback function to get the list of regions of an agent
+hsa_status_t
+callback_get_regions(hsa_region_t region, void* data)
+{
+    hsa_region_t** region_list = (hsa_region_t**) data;
+    **region_list              = region;
+    ++(*region_list);
+    return HSA_STATUS_SUCCESS;
+}
+
+// Callback function to get the list of memory pools of an agent
+hsa_status_t
+callback_get_memory_pools(hsa_amd_memory_pool_t memory_pool, void* data)
+{
+    hsa_amd_memory_pool_t** pool_list = (hsa_amd_memory_pool_t**) data;
+    **pool_list                       = memory_pool;
+    ++(*pool_list);
+    return HSA_STATUS_SUCCESS;
+}
+
+std::vector<hsa_agent_t>
+get_agent_list()
+{
+    size_t       num_agents = 0;
+    hsa_status_t status;
+    // Get number of agents
+    status = hsa_iterate_agents(get_num_agents, &num_agents);
+    RET_IF_HSA_ERR(status)
+    if(num_agents < 2)
+    {
+        printf("Not enough HSA agents available\n");
+        abort();
+    }
+
+    // Create a array of size num_agents to store the agent list
+    std::vector<hsa_agent_t> agents(num_agents);
+
+    // Get the agent list
+    hsa_agent_t* agent_iter = agents.data();
+    status                  = hsa_iterate_agents(get_agents, &agent_iter);
+    RET_IF_HSA_ERR(status)
+
+    return agents;
+}
+
+hsa_agent_t
+get_cpu_agent(std::vector<hsa_agent_t>& agents)
+{
+    for(hsa_agent_t agent : agents)
+    {
+        hsa_device_type_t ag_type;
+        hsa_status_t      status = hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &ag_type);
+        RET_IF_HSA_ERR(status)
+
+        if(ag_type == HSA_DEVICE_TYPE_CPU)
+        {
+            return agent;
+        }
+    }
+    std::cerr << "No CPU agents available" << std::endl;
+    abort();
+}
+
+hsa_agent_t
+get_gpu_agent(std::vector<hsa_agent_t>& agents)
+{
+    for(hsa_agent_t agent : agents)
+    {
+        hsa_device_type_t ag_type;
+        hsa_status_t      status = hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &ag_type);
+        RET_IF_HSA_ERR(status)
+
+        if(ag_type == HSA_DEVICE_TYPE_GPU)
+        {
+            return agent;
+        }
+    }
+    std::cerr << "No GPU agents available" << std::endl;
+    abort();
+}
+
+void
+call_hsa_memory_allocate(const size_t i, const size_t base_size, hsa_agent_t agent)
+{
+    // Getting total number of regions for the agent
+    int          num_regions = 0;
+    hsa_status_t status = hsa_agent_iterate_regions(agent, callback_get_num_regions, &num_regions);
+    RET_IF_HSA_ERR(status)
+    if(num_regions < 1)
+    {
+        printf("No HSA regions available\n");
+        abort();
+    }
+    // Allocate memory to hold region list of an agent
+    std::vector<hsa_region_t> region_list(num_regions);
+    hsa_region_t*             ptr_reg = region_list.data();
+    status = hsa_agent_iterate_regions(agent, callback_get_regions, &ptr_reg);
+    RET_IF_HSA_ERR(status)
+    auto address_vec = std::vector<void*>{};
+    address_vec.reserve(i);
+
+    for(size_t j = 0; j < i; ++j)
+    {
+        void* addr = nullptr;
+
+        status = hsa_memory_allocate(region_list[0], base_size, &addr);
+        RET_IF_HSA_ERR(status)
+        address_vec.emplace_back(addr);
+    }
+    for(void* addr : address_vec)
+    {
+        status = hsa_memory_free(addr);
+        RET_IF_HSA_ERR(status)
+    }
+}
+
+void
+call_hsa_memory_pool_allocate(const size_t i, const size_t base_size, hsa_agent_t agent)
+{
+    // Getting total number of regions for the agent
+    int          num_pools = 0;
+    hsa_status_t status =
+        hsa_amd_agent_iterate_memory_pools(agent, callback_get_num_pools, &num_pools);
+    RET_IF_HSA_ERR(status)
+    if(num_pools < 1)
+    {
+        printf("No memory pools available\n");
+        abort();
+    }
+    // Allocate memory to hold region list of an agent
+    std::vector<hsa_amd_memory_pool_t> memory_pool_list(num_pools);
+    hsa_amd_memory_pool_t*             ptr_memory_pool = memory_pool_list.data();
+    status = hsa_amd_agent_iterate_memory_pools(agent, callback_get_memory_pools, &ptr_memory_pool);
+    RET_IF_HSA_ERR(status)
+    auto address_vec = std::vector<void*>{};
+    address_vec.reserve(i);
+
+    for(size_t j = 0; j < i; ++j)
+    {
+        void*    addr  = nullptr;
+        uint32_t flags = 0;
+
+        status = hsa_amd_memory_pool_allocate(memory_pool_list[0], base_size, flags, &addr);
+        RET_IF_HSA_ERR(status)
+        address_vec.emplace_back(addr);
+    }
+    for(void* addr : address_vec)
+    {
+        status = hsa_amd_memory_pool_free(addr);
+        RET_IF_HSA_ERR(status)
+    }
+}
+
+void
+call_hsa_vmem_allocate(const size_t i, hsa_agent_t agent)
+{
+    // Getting total number of regions for the agent
+    int          num_pools = 0;
+    hsa_status_t status =
+        hsa_amd_agent_iterate_memory_pools(agent, callback_get_num_pools, &num_pools);
+    RET_IF_HSA_ERR(status)
+    if(num_pools < 1)
+    {
+        printf("No memory pools available\n");
+        abort();
+    }
+    // Allocate memory to hold region list of an agent
+    std::vector<hsa_amd_memory_pool_t> memory_pool_list(num_pools);
+    hsa_amd_memory_pool_t*             ptr_memory_pool = memory_pool_list.data();
+    status = hsa_amd_agent_iterate_memory_pools(agent, callback_get_memory_pools, &ptr_memory_pool);
+    RET_IF_HSA_ERR(status)
+
+    // Ensure Virtual Memory API is supported
+    bool supp = false;
+    status    = hsa_system_get_info(HSA_AMD_SYSTEM_INFO_VIRTUAL_MEM_API_SUPPORTED, (void*) &supp);
+    RET_IF_HSA_ERR(status)
+    if(!supp)
+    {
+        std::cerr << "Virtual Memory API not supported" << std::endl;
+        abort();
+    }
+
+    // Get runtime allocation granule size. Required for vmem_handle_create
+    int size;
+    status = hsa_amd_memory_pool_get_info(
+        memory_pool_list[0], HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE, (void*) &size);
+    RET_IF_HSA_ERR(status)
+    for(size_t j = 0; j < i; ++j)
+    {
+        hsa_amd_vmem_alloc_handle_t memory_handle{};
+
+        status = hsa_amd_vmem_handle_create(
+            memory_pool_list[0], size, MEMORY_TYPE_NONE, 0, &memory_handle);
+        RET_IF_HSA_ERR(status)
+        status = hsa_amd_vmem_handle_release(memory_handle);
+        RET_IF_HSA_ERR(status)
+    }
+}
+
+int
+main()
+{
+    hsa_status_t status;
+    status = hsa_init();
+    RET_IF_HSA_ERR(status)
+
+    std::vector<hsa_agent_t> agents    = get_agent_list();
+    hsa_agent_t              cpu_agent = get_cpu_agent(agents);
+    hsa_agent_t              gpu_agent = get_gpu_agent(agents);
+    call_hsa_memory_allocate(6, 1024, cpu_agent);
+    call_hsa_memory_pool_allocate(9, 2048, gpu_agent);
+    // Virtual memory API not supported in CI. Will add back if this changes
+    // call_hsa_vmem_allocate(3, gpu_agent);
+
+    status = hsa_shut_down();
+    RET_IF_HSA_ERR(status)
+    return 0;
+}

projects/rocprofiler-sdk/tests/bin/hsa-queue-dependency/CMakeLists.txt

@@ -0,0 +1,74 @@
+#
+#
+# HSA multi-queue dependency test
+
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(rocprofiler-sdk-tests-bin-hsa-multiqueue-dependency LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+find_program(
+    amdclangpp_EXECUTABLE REQUIRED
+    NAMES amdclang++
+    HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+    PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+    PATH_SUFFIXES bin llvm/bin NO_CACHE)
+
+function(generate_hsaco TARGET_ID INPUT_FILE OUTPUT_FILE)
+    separate_arguments(
+        CLANG_ARG_LIST
+        UNIX_COMMAND
+        "-O2 -x cl -Xclang -finclude-default-header -cl-denorms-are-zero -cl-std=CL2.0 -Wl,--build-id=sha1
+    -target amdgcn-amd-amdhsa -mcpu=${TARGET_ID} -o ${OUTPUT_FILE} ${INPUT_FILE}")
+    add_custom_command(
+        OUTPUT ${PROJECT_BINARY_DIR}/${OUTPUT_FILE}
+        COMMAND ${amdclangpp_EXECUTABLE} ${CLANG_ARG_LIST}
+        COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_BINARY_DIR}/${OUTPUT_FILE}
+                ${CMAKE_BINARY_DIR}/tests/rocprofv3/hsa-queue-dependency/${OUTPUT_FILE}
+        OUTPUT ${CMAKE_BINARY_DIR}/tests/rocprofv3/hsa-queue-dependency/${OUTPUT_FILE}
+        COMMAND
+            ${CMAKE_COMMAND} -E copy
+            ${CMAKE_BINARY_DIR}/tests/rocprofv3/hsa-queue-dependency/${OUTPUT_FILE}
+            ${CMAKE_BINARY_DIR}/rocprofv3/hsa-queue-dependency/${OUTPUT_FILE}
+        COMMENT "Building ${OUTPUT_FILE}...")
+    set(HSACO_TARGET_LIST
+        ${HSACO_TARGET_LIST} ${PROJECT_BINARY_DIR}/${OUTPUT_FILE}
+        PARENT_SCOPE)
+endfunction(generate_hsaco)
+
+foreach(target_id ${GPU_TARGETS})
+    # generate kernel bitcodes
+    generate_hsaco(${target_id} ${CMAKE_CURRENT_SOURCE_DIR}/copy.cl
+                   ${target_id}_copy.hsaco)
+endforeach()
+
+add_custom_target(generate_hsaco_targets DEPENDS ${HSACO_TARGET_LIST})
+
+add_executable(multiqueue_testapp)
+target_sources(multiqueue_testapp PRIVATE multiqueue_app.cpp)
+target_compile_options(multiqueue_testapp PRIVATE -W -Wall -Wextra -Wshadow -Werror)
+
+find_package(Threads REQUIRED)
+target_link_libraries(multiqueue_testapp PRIVATE stdc++fs Threads::Threads)
+
+find_package(rocprofiler-sdk REQUIRED)
+target_link_libraries(multiqueue_testapp PRIVATE rocprofiler-sdk::rocprofiler-sdk
+                                                 rocprofiler-sdk::tests-common-library)
+
+find_package(
+    hsa-runtime64
+    REQUIRED
+    CONFIG
+    HINTS
+    ${rocm_version_DIR}
+    ${ROCM_PATH}
+    PATHS
+    ${rocm_version_DIR}
+    ${ROCM_PATH})
+
+target_link_libraries(multiqueue_testapp PRIVATE hsa-runtime64::hsa-runtime64)
+
+add_dependencies(multiqueue_testapp generate_hsaco_targets)

projects/rocprofiler-sdk/tests/bin/hsa-queue-dependency/copy.cl

@@ -0,0 +1,32 @@
+/* Copyright (c) 2022 Advanced Micro Devices, Inc.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE. */
+
+__kernel void copyA(__global unsigned int* a, __global unsigned int* b) {
+    uint tid = get_global_id(0);
+    a[tid] = b[tid];
+}
+__kernel void copyB(__global unsigned int* a, __global unsigned int* b) {
+    uint tid = get_global_id(0);
+    a[tid] = b[tid];
+}
+__kernel void copyC(__global unsigned int* a, __global unsigned int* b) {
+    uint tid = get_global_id(0);
+    a[tid] = b[tid];
+}

projects/rocprofiler-sdk/tests/bin/hsa-queue-dependency/multiqueue_app.cpp

@@ -0,0 +1,304 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+/** ROC Profiler Multi Queue Dependency Test
+ *
+ * The goal of this test is to ensure ROC profiler does not go to deadlock
+ * when multiple queue are created and they are dependent on each other
+ *
+ */
+
+#include "multiqueue_app.h"
+
+std::vector<hsa_agent_t> Device::all_devices;
+std::vector<Device>      MQDependencyTest::cpu;
+std::vector<Device>      MQDependencyTest::gpu;
+Device::Memory           MQDependencyTest::kernarg;
+
+int
+main()
+{
+    hsa_status_t     status;
+    MQDependencyTest obj;
+
+    // Get Agent info
+    obj.device_discovery();
+
+    char agent_name[64];
+    status = hsa_agent_get_info(obj.gpu[0].agent, HSA_AGENT_INFO_NAME, agent_name);
+    RET_IF_HSA_ERR(status)
+
+    // Getting hasco Path
+    std::string hasco_file_path = std::string(agent_name) + std::string("_copy.hsaco");
+    obj.search_hasco(fs::current_path(), hasco_file_path);
+
+    MQDependencyTest::CodeObject code_object;
+    if(!obj.load_code_object(hasco_file_path, obj.gpu[0].agent, code_object))
+    {
+        printf("Kernel file not found or not usable with given agent.\n");
+        abort();
+    }
+
+    MQDependencyTest::Kernel copyA;
+    if(!obj.get_kernel(code_object, "copyA", obj.gpu[0].agent, copyA))
+    {
+        printf("Test kernel A not found.\n");
+        abort();
+    }
+
+    MQDependencyTest::Kernel copyB;
+    if(!obj.get_kernel(code_object, "copyB", obj.gpu[0].agent, copyB))
+    {
+        printf("Test kernel B not found.\n");
+        abort();
+    }
+
+    MQDependencyTest::Kernel copyC;
+    if(!obj.get_kernel(code_object, "copyC", obj.gpu[0].agent, copyC))
+    {
+        printf("Test kernel C not found.\n");
+        abort();
+    }
+
+    struct args_t
+    {
+        uint32_t*                       a      = nullptr;
+        uint32_t*                       b      = nullptr;
+        MQDependencyTest::OCLHiddenArgs hidden = {};
+    };
+
+    args_t* args = static_cast<args_t*>(obj.hsa_malloc(sizeof(args_t), obj.kernarg));
+    *args        = {};
+
+    uint32_t* a = static_cast<uint32_t*>(obj.hsa_malloc(64 * sizeof(uint32_t), obj.kernarg));
+    uint32_t* b = static_cast<uint32_t*>(obj.hsa_malloc(64 * sizeof(uint32_t), obj.kernarg));
+
+    memset(a, 0, 64 * sizeof(uint32_t));
+    memset(b, 1, 64 * sizeof(uint32_t));
+
+    // Create queue in gpu agent and prepare a kernel dispatch packet
+    hsa_queue_t* queue1 = nullptr;
+    status              = hsa_queue_create(obj.gpu[0].agent,
+                              1024,
+                              HSA_QUEUE_TYPE_SINGLE,
+                              nullptr,
+                              nullptr,
+                              UINT32_MAX,
+                              UINT32_MAX,
+                              &queue1);
+    RET_IF_HSA_ERR(status)
+
+    // Create a signal with a value of 1 and attach it to the first kernel
+    // dispatch packet
+    hsa_signal_t completion_signal_1 = {};
+    status                           = hsa_signal_create(1, 0, nullptr, &completion_signal_1);
+    RET_IF_HSA_ERR(status)
+
+    // First dispath packet on queue 1, Kernel A
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_KERNEL_DISPATCH;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.dispatch.setup            = 1;
+        packet.dispatch.workgroup_size_x = 64;
+        packet.dispatch.workgroup_size_y = 1;
+        packet.dispatch.workgroup_size_z = 1;
+        packet.dispatch.grid_size_x      = 64;
+        packet.dispatch.grid_size_y      = 1;
+        packet.dispatch.grid_size_z      = 1;
+
+        packet.dispatch.group_segment_size   = copyA.group;
+        packet.dispatch.private_segment_size = copyA.scratch;
+        packet.dispatch.kernel_object        = copyA.handle;
+
+        packet.dispatch.kernarg_address   = args;
+        packet.dispatch.completion_signal = completion_signal_1;
+
+        args->a = a;
+        args->b = b;
+        // Tell packet processor of A to launch the first kernel dispatch packet
+        obj.submit_packet(queue1, packet);
+    }
+
+    // Create a signal with a value of 1 and attach it to the second kernel
+    // dispatch packet
+    hsa_signal_t completion_signal_2 = {};
+    status                           = hsa_signal_create(1, 0, nullptr, &completion_signal_2);
+    RET_IF_HSA_ERR(status)
+
+    hsa_signal_t completion_signal_3 = {};
+    status                           = hsa_signal_create(1, 0, nullptr, &completion_signal_3);
+    RET_IF_HSA_ERR(status)
+
+    // Create barrier-AND packet that is enqueued in queue 1
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_BARRIER_AND;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.barrier_and.dep_signal[0] = completion_signal_2;
+        obj.submit_packet(queue1, packet);
+    }
+
+    // Second dispath packet on queue 1, Kernel C
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_KERNEL_DISPATCH;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.dispatch.setup            = 1;
+        packet.dispatch.workgroup_size_x = 64;
+        packet.dispatch.workgroup_size_y = 1;
+        packet.dispatch.workgroup_size_z = 1;
+        packet.dispatch.grid_size_x      = 64;
+        packet.dispatch.grid_size_y      = 1;
+        packet.dispatch.grid_size_z      = 1;
+
+        packet.dispatch.group_segment_size   = copyC.group;
+        packet.dispatch.private_segment_size = copyC.scratch;
+        packet.dispatch.kernel_object        = copyC.handle;
+        packet.dispatch.completion_signal    = completion_signal_3;
+        packet.dispatch.kernarg_address      = args;
+
+        args->a = a;
+        args->b = b;
+        // Tell packet processor to launch the second kernel dispatch packet
+        obj.submit_packet(queue1, packet);
+    }
+
+    // Create queue 2
+    hsa_queue_t* queue2 = nullptr;
+    status              = hsa_queue_create(obj.gpu[0].agent,
+                              1024,
+                              HSA_QUEUE_TYPE_SINGLE,
+                              nullptr,
+                              nullptr,
+                              UINT32_MAX,
+                              UINT32_MAX,
+                              &queue2);
+    RET_IF_HSA_ERR(status)
+
+    // Create barrier-AND packet that is enqueued in queue 2
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_BARRIER_AND;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.barrier_and.dep_signal[0] = completion_signal_1;
+        obj.submit_packet(queue2, packet);
+    }
+
+    // Third dispath packet on queue 2, Kernel B
+    {
+        MQDependencyTest::Aql packet{};
+        packet.header.type    = HSA_PACKET_TYPE_KERNEL_DISPATCH;
+        packet.header.barrier = 1;
+        packet.header.acquire = HSA_FENCE_SCOPE_SYSTEM;
+        packet.header.release = HSA_FENCE_SCOPE_SYSTEM;
+
+        packet.dispatch.setup            = 1;
+        packet.dispatch.workgroup_size_x = 64;
+        packet.dispatch.workgroup_size_y = 1;
+        packet.dispatch.workgroup_size_z = 1;
+        packet.dispatch.grid_size_x      = 64;
+        packet.dispatch.grid_size_y      = 1;
+        packet.dispatch.grid_size_z      = 1;
+
+        packet.dispatch.group_segment_size   = copyB.group;
+        packet.dispatch.private_segment_size = copyB.scratch;
+        packet.dispatch.kernel_object        = copyB.handle;
+
+        packet.dispatch.kernarg_address   = args;
+        packet.dispatch.completion_signal = completion_signal_2;
+
+        args->a = a;
+        args->b = b;
+        // Tell packet processor to launch the third kernel dispatch packet
+        obj.submit_packet(queue2, packet);
+    }
+
+    // Wait on the completion signal
+    hsa_signal_wait_relaxed(
+        completion_signal_1, HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, HSA_WAIT_STATE_BLOCKED);
+
+    // Wait on the completion signal
+    hsa_signal_wait_relaxed(
+        completion_signal_2, HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, HSA_WAIT_STATE_BLOCKED);
+
+    // Wait on the completion signal
+    hsa_signal_wait_relaxed(
+        completion_signal_3, HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, HSA_WAIT_STATE_BLOCKED);
+
+    for(int i = 0; i < 64; i++)
+    {
+        if(a[i] != b[i])
+        {
+            printf("error at %d: expected %d, got %d\n", i, b[i], a[i]);
+            abort();
+        }
+    }
+
+    // Clearing data structures and memory
+    status = hsa_signal_destroy(completion_signal_1);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_signal_destroy(completion_signal_2);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_signal_destroy(completion_signal_3);
+    RET_IF_HSA_ERR(status)
+
+    if(queue1 != nullptr)
+    {
+        status = hsa_queue_destroy(queue1);
+        RET_IF_HSA_ERR(status)
+    }
+
+    if(queue2 != nullptr)
+    {
+        status = hsa_queue_destroy(queue2);
+        RET_IF_HSA_ERR(status)
+    }
+
+    status = hsa_memory_free(a);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_memory_free(b);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_executable_destroy(code_object.executable);
+    RET_IF_HSA_ERR(status)
+
+    status = hsa_code_object_reader_destroy(code_object.code_obj_rdr);
+    RET_IF_HSA_ERR(status)
+
+    close(code_object.file);
+}

projects/rocprofiler-sdk/tests/bin/hsa-queue-dependency/multiqueue_app.h

@@ -0,0 +1,388 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#pragma once
+
+#include "common/filesystem.hpp"
+
+#include <hsa/hsa.h>
+#include <hsa/hsa_ext_amd.h>
+
+#include <dlfcn.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace fs = common::fs;
+
+#define RET_IF_HSA_ERR(err)                                                                        \
+    {                                                                                              \
+        if((err) != HSA_STATUS_SUCCESS)                                                            \
+        {                                                                                          \
+            char  err_val[12];                                                                     \
+            char* err_str = nullptr;                                                               \
+            if(hsa_status_string(err, (const char**) &err_str) != HSA_STATUS_SUCCESS)              \
+            {                                                                                      \
+                sprintf(&(err_val[0]), "%#x", (uint32_t) err);                                     \
+                err_str = &(err_val[0]);                                                           \
+            }                                                                                      \
+            printf("hsa api call failure at: %s:%d\n", __FILE__, __LINE__);                        \
+            printf("Call returned %s\n", err_str);                                                 \
+            abort();                                                                               \
+        }                                                                                          \
+    }
+
+struct Device
+{
+    struct Memory
+    {
+        hsa_amd_memory_pool_t pool;
+        bool                  fine;
+        bool                  kernarg;
+        size_t                size;
+        size_t                granule;
+    };
+
+    hsa_agent_t                     agent;
+    char                            name[64];
+    std::vector<Memory>             pools;
+    uint32_t                        fine;
+    uint32_t                        coarse;
+    static std::vector<hsa_agent_t> all_devices;
+};
+
+class MQDependencyTest
+{
+public:
+    MQDependencyTest() { hsa_init(); }
+    ~MQDependencyTest() { hsa_shut_down(); }
+
+    static std::vector<Device> cpu;
+    static std::vector<Device> gpu;
+    static Device::Memory      kernarg;
+
+    struct CodeObject
+    {
+        hsa_file_t               file         = 0;
+        hsa_code_object_reader_t code_obj_rdr = {};
+        hsa_executable_t         executable   = {};
+    };
+
+    struct Kernel
+    {
+        uint64_t handle        = 0;
+        uint32_t scratch       = 0;
+        uint32_t group         = 0;
+        uint32_t kernarg_size  = 0;
+        uint32_t kernarg_align = 0;
+    };
+
+    union AqlHeader
+    {
+        struct
+        {
+            uint16_t type     : 8;
+            uint16_t barrier  : 1;
+            uint16_t acquire  : 2;
+            uint16_t release  : 2;
+            uint16_t reserved : 3;
+        };
+        uint16_t raw = 0;
+    };
+
+    struct BarrierValue
+    {
+        AqlHeader          header            = {};
+        uint8_t            AmdFormat         = 0;
+        uint8_t            reserved          = 0;
+        uint32_t           reserved1         = 0;
+        hsa_signal_t       signal            = {};
+        hsa_signal_value_t value             = 0;
+        hsa_signal_value_t mask              = 0;
+        uint32_t           cond              = 0;
+        uint32_t           reserved2         = 0;
+        uint64_t           reserved3         = 0;
+        uint64_t           reserved4         = 0;
+        hsa_signal_t       completion_signal = {};
+    };
+
+    union Aql
+    {
+        AqlHeader                    header;
+        hsa_kernel_dispatch_packet_t dispatch;
+        hsa_barrier_and_packet_t     barrier_and;
+        hsa_barrier_or_packet_t      barrier_or;
+        BarrierValue                 barrier_value = {};
+    };
+
+    struct OCLHiddenArgs
+    {
+        uint64_t offset_x      = 0;
+        uint64_t offset_y      = 0;
+        uint64_t offset_z      = 0;
+        void*    printf_buffer = nullptr;
+        void*    enqueue       = nullptr;
+        void*    enqueue2      = nullptr;
+        void*    multi_grid    = nullptr;
+    };
+
+    static bool load_code_object(const std::string& filename,
+                                 hsa_agent_t        agent,
+                                 CodeObject&        code_object)
+    {
+        hsa_status_t err;
+        code_object.file = open(filename.c_str(), O_RDONLY);
+        if(code_object.file == -1)
+        {
+            fprintf(stderr, "%s:%s\n", "Could not load code object", filename.c_str());
+            abort();
+            return false;
+        }
+
+        err = hsa_code_object_reader_create_from_file(code_object.file, &code_object.code_obj_rdr);
+        RET_IF_HSA_ERR(err);
+
+        err = hsa_executable_create_alt(HSA_PROFILE_FULL,
+                                        HSA_DEFAULT_FLOAT_ROUNDING_MODE_DEFAULT,
+                                        nullptr,
+                                        &code_object.executable);
+        RET_IF_HSA_ERR(err);
+
+        err = hsa_executable_load_agent_code_object(
+            code_object.executable, agent, code_object.code_obj_rdr, nullptr, nullptr);
+        if(err != HSA_STATUS_SUCCESS) return false;
+
+        err = hsa_executable_freeze(code_object.executable, nullptr);
+        RET_IF_HSA_ERR(err);
+
+        return true;
+    }
+
+    static bool get_kernel(const CodeObject&  code_object,
+                           const std::string& kernel,
+                           hsa_agent_t        agent,
+                           Kernel&            kern)
+    {
+        hsa_executable_symbol_t symbol;
+        hsa_status_t            err = hsa_executable_get_symbol_by_name(
+            code_object.executable, kernel.c_str(), &agent, &symbol);
+        if(err != HSA_STATUS_SUCCESS)
+        {
+            err = hsa_executable_get_symbol_by_name(
+                code_object.executable, (kernel + ".kd").c_str(), &agent, &symbol);
+            if(err != HSA_STATUS_SUCCESS)
+            {
+                return false;
+            }
+        }
+        printf("\nkernel-name: %s\n", kernel.c_str());
+        err = hsa_executable_symbol_get_info(
+            symbol, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_OBJECT, &kern.handle);
+        RET_IF_HSA_ERR(err);
+
+        return true;
+    }
+
+    // Not for parallel insertion.
+    static bool submit_packet(hsa_queue_t* queue, Aql& pkt)
+    {
+        size_t mask = queue->size - 1;
+        Aql*   ring = static_cast<Aql*>(queue->base_address);
+
+        uint64_t write = hsa_queue_load_write_index_relaxed(queue);
+        uint64_t read  = hsa_queue_load_read_index_relaxed(queue);
+        if(write - read + 1 > queue->size) return false;
+
+        Aql& dst = ring[write & mask];
+
+        uint16_t header = pkt.header.raw;
+        pkt.header.raw  = dst.header.raw;
+        dst             = pkt;
+        __atomic_store_n(&dst.header.raw, header, __ATOMIC_RELEASE);
+        pkt.header.raw = header;
+
+        hsa_queue_store_write_index_release(queue, write + 1);
+        hsa_signal_store_screlease(queue->doorbell_signal, write);
+
+        return true;
+    }
+
+    static void* hsa_malloc(size_t size, const Device::Memory& mem)
+    {
+#define LOCAL_HSA_AMD_INTERFACE_VERSION                                                            \
+    (10000 * HSA_AMD_INTERFACE_VERSION_MAJOR) + (100 * HSA_AMD_INTERFACE_VERSION_MINOR)
+
+#if LOCAL_HSA_AMD_INTERFACE_VERSION >= 10700
+        constexpr auto hsa_amd_memory_pool_executable_flag = HSA_AMD_MEMORY_POOL_EXECUTABLE_FLAG;
+#elif LOCAL_HSA_AMD_INTERFACE_VERSION == 10600
+        constexpr auto hsa_amd_memory_pool_executable_flag = (1 << 2);
+#else
+        constexpr auto hsa_amd_memory_pool_executable_flag = 0;
+#endif
+
+        void*        ret;
+        hsa_status_t err =
+            hsa_amd_memory_pool_allocate(mem.pool, size, hsa_amd_memory_pool_executable_flag, &ret);
+        RET_IF_HSA_ERR(err);
+
+        err = hsa_amd_agents_allow_access(
+            Device::all_devices.size(), Device::all_devices.data(), nullptr, ret);
+        RET_IF_HSA_ERR(err);
+        return ret;
+    }
+
+    static void* hsa_malloc(size_t size, const Device& dev, bool fine)
+    {
+        uint32_t index = fine ? dev.fine : dev.coarse;
+        assert(index != -1u && "Memory type unavailable.");
+        return hsa_malloc(size, dev.pools[index]);
+    }
+
+    static bool device_discovery()
+    {
+        hsa_status_t err;
+
+        err = hsa_iterate_agents(
+            [](hsa_agent_t agent, void*) {
+                hsa_status_t error;
+
+                Device dev;
+                dev.agent = agent;
+
+                dev.fine   = -1u;
+                dev.coarse = -1u;
+
+                error = hsa_agent_get_info(agent, HSA_AGENT_INFO_NAME, dev.name);
+                RET_IF_HSA_ERR(error)
+
+                hsa_device_type_t type;
+                error = hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &type);
+                RET_IF_HSA_ERR(error)
+
+                error = hsa_amd_agent_iterate_memory_pools(
+                    agent,
+                    [](hsa_amd_memory_pool_t pool, void* data) {
+                        auto&        pools = *reinterpret_cast<std::vector<Device::Memory>*>(data);
+                        hsa_status_t status;
+
+                        bool allowed = false;
+                        status       = hsa_amd_memory_pool_get_info(
+                            pool, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_ALLOWED, &allowed);
+
+                        if(!allowed) return HSA_STATUS_SUCCESS;
+
+                        hsa_amd_segment_t segment;
+                        status = hsa_amd_memory_pool_get_info(
+                            pool, HSA_AMD_MEMORY_POOL_INFO_SEGMENT, &segment);
+                        RET_IF_HSA_ERR(status)
+
+                        if(segment != HSA_AMD_SEGMENT_GLOBAL) return HSA_STATUS_SUCCESS;
+
+                        uint32_t flags;
+                        status = hsa_amd_memory_pool_get_info(
+                            pool, HSA_AMD_MEMORY_POOL_INFO_GLOBAL_FLAGS, &flags);
+                        RET_IF_HSA_ERR(status)
+
+                        Device::Memory mem;
+                        mem.pool = pool;
+                        mem.fine = ((flags & HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_FINE_GRAINED) != 0u);
+                        mem.kernarg =
+                            ((flags & HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_KERNARG_INIT) != 0u);
+
+                        status = hsa_amd_memory_pool_get_info(
+                            pool, HSA_AMD_MEMORY_POOL_INFO_SIZE, &mem.size);
+                        RET_IF_HSA_ERR(status)
+
+                        status = hsa_amd_memory_pool_get_info(
+                            pool, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE, &mem.granule);
+                        RET_IF_HSA_ERR(status)
+
+                        pools.push_back(mem);
+                        return HSA_STATUS_SUCCESS;
+                    },
+                    static_cast<void*>(&dev.pools));
+
+                if(!dev.pools.empty())
+                {
+                    for(size_t i = 0; i < dev.pools.size(); i++)
+                    {
+                        if(dev.pools[i].fine && dev.pools[i].kernarg && dev.fine == -1u)
+                            dev.fine = i;
+                        if(dev.pools[i].fine && !dev.pools[i].kernarg) dev.fine = i;
+                        if(!dev.pools[i].fine) dev.coarse = i;
+                    }
+
+                    if(type == HSA_DEVICE_TYPE_CPU)
+                        cpu.push_back(dev);
+                    else
+                        gpu.push_back(dev);
+
+                    Device::all_devices.push_back(dev.agent);
+                }
+
+                return HSA_STATUS_SUCCESS;
+            },
+            nullptr);
+
+        []() {
+            for(auto& dev : cpu)
+            {
+                for(auto& mem : dev.pools)
+                {
+                    if(mem.fine && mem.kernarg)
+                    {
+                        kernarg = mem;
+                        return;
+                    }
+                }
+            }
+        }();
+        RET_IF_HSA_ERR(err);
+
+        if(cpu.empty() || gpu.empty() || kernarg.pool.handle == 0) return false;
+        return true;
+    }
+
+    void search_hasco(const fs::path& directory, std::string& filename)
+    {
+        for(const auto& entry : fs::directory_iterator(directory))
+        {
+            if(fs::is_regular_file(entry))
+            {
+                if(entry.path().filename() == filename)
+                {
+                    filename = entry.path();
+                }
+            }
+            else if(fs::is_directory(entry))
+            {
+                search_hasco(entry, filename);  // Recursive call for subdirectories
+            }
+        }
+    }
+};

projects/rocprofiler-sdk/tests/bin/multistream/CMakeLists.txt

@@ -0,0 +1,41 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-bin-multistream LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(multistream_app.cpp PROPERTIES LANGUAGE HIP)
+add_executable(multistream)
+target_sources(multistream PRIVATE multistream_app.cpp)
+target_compile_options(multistream PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow -Werror)
+
+find_package(Threads REQUIRED)
+target_link_libraries(multistream PRIVATE Threads::Threads)

projects/rocprofiler-sdk/tests/bin/multistream/multistream_app.cpp

@@ -0,0 +1,116 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include <hip/hip_runtime.h>
+#include <vector>
+#define HIP_ASSERT(call)                                                                           \
+    do                                                                                             \
+    {                                                                                              \
+        hipError_t err = call;                                                                     \
+        if(err != hipSuccess)                                                                      \
+        {                                                                                          \
+            fprintf(stderr, "%s\n", hipGetErrorString(err));                                       \
+            abort();                                                                               \
+        }                                                                                          \
+    } while(0)
+
+__device__ int counter = 0;
+__global__ void
+add(int n, float* x, float* y)
+{
+    if(__hip_atomic_load(&counter, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_AGENT) != 0)
+    {
+        abort();
+    }
+    __hip_atomic_fetch_add(&counter, 1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM);
+
+    int index  = blockIdx.x * blockDim.x + threadIdx.x;
+    int stride = blockDim.x * gridDim.x;
+    for(int i = index; i < n; i += stride)
+        y[i] = x[i] + y[i];
+    __hip_atomic_fetch_add(&counter, -1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+void
+LaunchMultiStreamKernels()
+{
+    int    N = 1 << 4;
+    float* x = new float[N];
+    float* y = new float[N];
+    float* d_x;
+    float* d_y;
+    //   Allocate Unified Memory -- accessible from CPU or GPU
+    HIP_ASSERT(hipMallocManaged(&d_x, N * sizeof(float)));
+    HIP_ASSERT(hipMallocManaged(&d_y, N * sizeof(float)));
+
+    //   initialize x and y arrays on the host
+    for(int i = 0; i < N; i++)
+    {
+        x[i] = 1.0f;
+        y[i] = 2.0f;
+    }
+    std::vector<hipStream_t> hip_streams;
+    for(int i = 0; i < 100; i++)
+    {
+        hipStream_t stream;
+        HIP_ASSERT(hipStreamCreate(&stream));
+        hip_streams.push_back(stream);
+    }
+    HIP_ASSERT(hipMemcpy(d_x, x, N * sizeof(float), hipMemcpyHostToDevice));
+    HIP_ASSERT(hipMemcpy(d_y, y, N * sizeof(float), hipMemcpyHostToDevice));
+
+    // Launch kernel on one or two warps on the GPU
+    int blockSize = 64;
+    // This Kernel will always be launched with one wave
+    int numBlocks = 1;
+    for(int i = 0; i < 100; i++)
+    {
+        for(auto& hip_stream : hip_streams)
+        {
+            hipLaunchKernelGGL(add, numBlocks, blockSize, 0, hip_stream, N, d_x, d_y);
+        }
+    }
+
+    // Wait for GPU to finish before accessing on host
+    HIP_ASSERT(hipDeviceSynchronize());
+
+    HIP_ASSERT(hipMemcpy(x, d_x, N * sizeof(float), hipMemcpyDeviceToHost));
+    HIP_ASSERT(hipMemcpy(y, d_y, N * sizeof(float), hipMemcpyDeviceToHost));
+
+    //   Free memory
+    HIP_ASSERT(hipFree(d_x));
+    HIP_ASSERT(hipFree(d_y));
+
+    delete[] x;
+    delete[] y;
+
+    for(auto& hip_stream : hip_streams)
+    {
+        HIP_ASSERT(hipStreamDestroy(hip_stream));
+    }
+}
+
+int
+main()
+{
+    LaunchMultiStreamKernels();
+}

projects/rocprofiler-sdk/tests/bin/openmp/CMakeLists.txt

@@ -0,0 +1,33 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT OMP_TARGET_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(OMP_TARGET_COMPILER
+            "${amdclangpp_EXECUTABLE}"
+            CACHE FILEPATH "")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-bin-openmp LANGUAGES CXX)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+set(DEFAULT_GPU_TARGETS "gfx906" "gfx908" "gfx90a" "gfx942" "gfx950" "gfx1100" "gfx1101"
+                        "gfx1102")
+
+set(OPENMP_GPU_TARGETS
+    "${DEFAULT_GPU_TARGETS}"
+    CACHE STRING "GPU targets to compile for")
+
+add_subdirectory(target)

projects/rocprofiler-sdk/tests/bin/openmp/target/CMakeLists.txt

@@ -0,0 +1,23 @@
+#
+#
+#
+
+set(CMAKE_BUILD_TYPE "RelWithDebInfo")
+
+find_package(Threads REQUIRED)
+find_package(rocprofiler-sdk-roctx REQUIRED)
+
+add_executable(openmp-target)
+target_sources(openmp-target PRIVATE openmp-target.cpp)
+target_link_libraries(openmp-target PRIVATE Threads::Threads
+                                            rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
+target_compile_options(openmp-target PRIVATE -fopenmp)
+target_link_options(openmp-target PRIVATE -fopenmp)
+
+foreach(_TARGET ${OPENMP_GPU_TARGETS})
+    target_compile_options(openmp-target PRIVATE --offload-arch=${_TARGET})
+    target_link_options(openmp-target PRIVATE --offload-arch=${_TARGET})
+endforeach()
+
+include(rocprofiler-sdk-custom-compilation)
+rocprofiler_sdk_custom_compilation(TARGET openmp-target COMPILER ${OMP_TARGET_COMPILER})

projects/rocprofiler-sdk/tests/bin/openmp/target/openmp-target.cpp

@@ -0,0 +1,155 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include <rocprofiler-sdk-roctx/roctx.h>
+
+#include <math.h>
+#include <stdio.h>
+
+constexpr float  EPS_FLOAT  = 1.0e-7f;
+constexpr double EPS_DOUBLE = 1.0e-15;
+
+#pragma omp declare target
+template <typename T>
+T
+mul(T a, T b)
+{
+    volatile T c = a * b;
+    return c;
+}
+#pragma omp end declare target
+
+template <typename T>
+void
+vmul(T* a, T* b, T* c, int N)
+{
+#pragma omp target           map(to : a [0:N], b [0:N]) map(from : c [0:N])
+#pragma omp teams distribute parallel for
+    for(int i = 0; i < N; i++)
+    {
+        for(int j = 0; j < 100000; ++j)
+            c[i] = mul(a[i], b[i]);
+    }
+}
+
+int
+main()
+{
+    auto range_id = roctxRangeStart("main");
+
+    constexpr int N = 100000;
+    int           a_i[N], b_i[N], c_i[N], validate_i[N];
+    float         a_f[N], b_f[N], c_f[N], validate_f[N];
+    double        a_d[N], b_d[N], c_d[N], validate_d[N];
+    int           N_errors = 0;
+    bool          flag     = false;
+
+    roctxMark("initialization");
+
+#pragma omp parallel for
+    for(int i = 0; i < N; ++i)
+    {
+        a_f[i] = a_i[i] = i + 1;
+        b_f[i] = b_i[i] = i + 2;
+        a_d[i]          = a_i[i];
+        b_d[i]          = b_i[i];
+        validate_i[i]   = a_i[i] * b_i[i];
+        validate_f[i]   = a_f[i] * b_f[i];
+        validate_d[i]   = a_d[i] * b_d[i];
+    }
+
+    vmul(a_i, b_i, c_i, N);
+    vmul(a_f, b_f, c_f, N);
+
+    auto tid = roctx_thread_id_t{};
+    // get the thread id recognized by rocprofiler-sdk from roctx
+    roctxGetThreadId(&tid);
+    // pause API tracing
+    roctxProfilerPause(tid);
+
+    // we don't expect to see the third vmul
+    vmul(a_d, b_d, c_d, N);
+
+    // resume API tracing
+    roctxProfilerResume(tid);
+
+    for(int i = 0; i < N; i++)
+    {
+        if(c_i[i] != validate_i[i])
+        {
+            ++N_errors;
+            //       print 1st bad index
+            if(!flag)
+            {
+                printf(
+                    "First fail: c_i[%d](%d) != validate_i[%d](%d)\n", i, c_i[i], i, validate_i[i]);
+                flag = true;
+            }
+        }
+    }
+    flag = false;
+    for(int i = 0; i < N; i++)
+    {
+        if(fabs(c_f[i] - validate_f[i]) > EPS_FLOAT)
+        {
+            ++N_errors;
+            //      print 1st bad index
+            if(!flag)
+            {
+                printf("First fail: c_f[%d](%f) != validate_f[%d](%f)\n",
+                       i,
+                       static_cast<double>(c_f[i]),
+                       i,
+                       static_cast<double>(validate_f[i]));
+                flag = true;
+            }
+        }
+    }
+    flag = false;
+    for(int i = 0; i < N; i++)
+    {
+        if(fabs(c_d[i] - validate_d[i]) > EPS_DOUBLE)
+        {
+            ++N_errors;
+            //      print 1st bad index
+            if(!flag)
+            {
+                printf(
+                    "First fail: c_d[%d](%f) != validate_d[%d](%f)\n", i, c_d[i], i, validate_d[i]);
+                flag = true;
+            }
+        }
+    }
+    if(N_errors == 0)
+    {
+        printf("Success\n");
+        return 0;
+    }
+    else
+    {
+        printf("Total %d failures\n", N_errors);
+        printf("Fail\n");
+        return 1;
+    }
+
+    roctxRangeStop(range_id);
+}

projects/rocprofiler-sdk/tests/bin/pc-sampling/CMakeLists.txt

@@ -0,0 +1,11 @@
+#
+# Integration test applications
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(rocprofiler-sdk-tests-bin-pc-sampling LANGUAGES C CXX)
+
+set(CMAKE_BUILD_RPATH "\$ORIGIN:\$ORIGIN/../lib")
+
+# applications used by integration tests which DO NOT link to rocprofiler-sdk-roctx
+add_subdirectory(exec-mask-manipulation)

projects/rocprofiler-sdk/tests/bin/pc-sampling/exec-mask-manipulation/CMakeLists.txt

@@ -0,0 +1,43 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-bin-pc-sampling-exec-mask-manipulation LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(exec_mask_manipulation.cpp PROPERTIES LANGUAGE HIP)
+add_executable(exec-mask-manipulation)
+target_sources(exec-mask-manipulation PRIVATE exec_mask_manipulation.cpp)
+# debug symbols required for PC sampling decoding validation
+target_compile_options(exec-mask-manipulation PRIVATE -W -Wall -Wextra -Wpedantic
+                                                      -Wshadow -Werror -g)
+
+find_package(Threads REQUIRED)
+target_link_libraries(exec-mask-manipulation PRIVATE Threads::Threads)

projects/rocprofiler-sdk/tests/bin/pc-sampling/exec-mask-manipulation/exec_mask_manipulation.cpp

@@ -0,0 +1,537 @@
+/*
+Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include <iostream>
+#include <mutex>
+
+#include <hip/hip_runtime.h>
+
+#define ITER_NUM   16 * 1024
+#define BLOCK_SIZE 1024
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t = std::unique_lock<std::mutex>;
+auto print_lock   = std::mutex{};
+
+void
+check_hip_error(void);
+}  // namespace
+
+// ======================================================
+__global__ void
+kernel1(const int c)
+{
+    int a = 0;
+#pragma nounroll
+    for(int i = 0; i < ITER_NUM; i++)
+    {
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+        asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c));
+    }
+}
+
+__global__ void
+kernel2(const int c)
+{
+    int a = 0;
+#pragma nounroll
+    for(int i = 0; i < ITER_NUM; i++)
+    {
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+        asm volatile("s_mov_b32 %0 %1\n" : "=s"(a) : "s"(c));
+    }
+}
+
+__global__ void
+kernel3(const float c)
+{
+    double a        = threadIdx.x;
+    float  i        = 0;
+    float  d        = threadIdx.x;
+    float  e        = 0;
+    int    tid_even = threadIdx.x % 2;
+    for(int j = 0; j < ITER_NUM; j++)
+    {
+        if(tid_even == 0)
+        {
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+            asm volatile("v_rcp_f64 %0, %0\n" : "+v"(a), "=s"(i) : "s"(c));
+        }
+        else
+        {
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+            asm volatile("v_rcp_f32 %0, %0\n" : "+v"(d), "=s"(e) : "s"(c));
+        }
+    }
+}
+
+// ======================================================
+
+void
+run_kernel()
+{
+    for(int i = 1; i <= 64; i++)
+    {
+        if(i % 2 == 1)
+            kernel1<<<BLOCK_SIZE, i>>>(i);
+        else
+            kernel2<<<BLOCK_SIZE, i>>>(i);
+
+        check_hip_error();
+        HIP_API_CALL(hipDeviceSynchronize());
+    }
+
+    float arg = 0;
+    kernel3<<<BLOCK_SIZE, 4 * 64>>>(arg);
+    check_hip_error();
+    HIP_API_CALL(hipDeviceSynchronize());
+}
+
+int
+main()
+{
+    run_kernel();
+    return 0;
+}
+
+namespace
+{
+void
+check_hip_error(void)
+{
+    hipError_t err = hipGetLastError();
+    if(err != hipSuccess)
+    {
+        auto_lock_t _lk{print_lock};
+        std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
+        throw std::runtime_error("hip_api_call");
+    }
+}
+}  // namespace

projects/rocprofiler-sdk/tests/bin/reproducible-dispatch-count/CMakeLists.txt

@@ -0,0 +1,59 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-bin-reproducible-dispatch-count LANGUAGES CXX HIP)
+
+if(NOT CMAKE_BUILD_TYPE MATCHES "(Release|RelWithDebInfo)")
+    set(CMAKE_BUILD_TYPE "RelWithDebInfo")
+endif()
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+option(REPRODUCIBLE_DISPATCH_COUNT_USE_MPI
+       "Enable MPI support in reproducible-dispatch-count exe" OFF)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(reproducible-dispatch-count.cpp PROPERTIES LANGUAGE HIP)
+add_executable(reproducible-dispatch-count)
+target_sources(reproducible-dispatch-count PRIVATE reproducible-dispatch-count.cpp)
+target_compile_options(reproducible-dispatch-count PRIVATE -W -Wall -Wextra -Wpedantic
+                                                           -Wshadow -Werror)
+
+find_package(Threads REQUIRED)
+target_link_libraries(reproducible-dispatch-count PRIVATE Threads::Threads)
+
+find_package(rocprofiler-sdk-roctx REQUIRED)
+target_link_libraries(reproducible-dispatch-count
+                      PRIVATE rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
+
+if(REPRODUCIBLE_DISPATCH_COUNT_USE_MPI)
+    find_package(MPI REQUIRED)
+    target_compile_definitions(reproducible-dispatch-count PRIVATE USE_MPI)
+    target_link_libraries(reproducible-dispatch-count PRIVATE MPI::MPI_C)
+endif()

projects/rocprofiler-sdk/tests/bin/reproducible-dispatch-count/reproducible-dispatch-count.cpp

@@ -0,0 +1,255 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "hip/hip_runtime.h"
+#include "rocprofiler-sdk-roctx/roctx.h"
+
+#include <unistd.h>
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <iomanip>
+#include <iostream>
+#include <mutex>
+#include <random>
+#include <stdexcept>
+#include <thread>
+
+#if defined(USE_MPI)
+#    include <mpi.h>
+#endif
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error %i : %s\n",                                                \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    static_cast<int>(error_),                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t    = std::unique_lock<std::mutex>;
+auto     print_lock  = std::mutex{};
+size_t   niterations = 1000;
+uint32_t nspin       = 4 * 10000;
+size_t   nsync       = 1;
+size_t   nthreads    = 2;
+
+void
+check_hip_error(void);
+}  // namespace
+
+__global__ void
+reproducible_dispatch_count(uint32_t nspin);
+
+void
+run(int tid, int devid);
+
+void
+run_nsync(int tid, int devid);
+
+int
+main(int argc, char** argv)
+{
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{argv[i]};
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            fprintf(stderr,
+                    "usage: reproducible-dispatch-count [KERNEL ITERATIONS PER THREAD (default: "
+                    "%zu msec)] [NUM_THREADS (default: %zu)] [SPIN CYCLES PER KERNEL LAUNCH "
+                    "(default: %u)] [ITERATION PER SYNC (default: %zu)\n",
+                    niterations,
+                    nthreads,
+                    nspin,
+                    nsync);
+            exit(EXIT_SUCCESS);
+        }
+    }
+
+    if(argc > 1) niterations = std::stoll(argv[1]);
+    if(argc > 2) nthreads = std::stoll(argv[2]);
+    if(argc > 3) nspin = std::stoll(argv[3]);
+    if(argc > 4) nsync = std::stoll(argv[4]);
+
+    printf("[reproducible-dispatch-count] Kernel dispatches per thread: %zu\n", niterations);
+    printf("[reproducible-dispatch-count] Spin time per kernel: %u cycles\n", nspin);
+    printf("[reproducible-dispatch-count] Number of threads: %zu\n", nthreads);
+    printf("[reproducible-dispatch-count] Iterations per sync: %zu\n", nsync);
+
+    // this is a temporary workaround in omnitrace when HIP + MPI is enabled
+    int ndevice = 0;
+    HIP_API_CALL(hipGetDeviceCount(&ndevice));
+    printf("[reproducible-dispatch-count] Number of devices found: %i\n", ndevice);
+    auto _threads = std::vector<std::thread>{};
+    for(size_t i = 0; i < nthreads; ++i)
+    {
+        if(nsync <= 1)
+            _threads.emplace_back(run, i, i % ndevice);
+        else
+            _threads.emplace_back(run_nsync, i, i % ndevice);
+    }
+    for(auto& itr : _threads)
+        itr.join();
+    HIP_API_CALL(hipDeviceSynchronize());
+    HIP_API_CALL(hipDeviceReset());
+
+    return 0;
+}
+
+__global__ void
+reproducible_dispatch_count(uint32_t nspin_v)
+{
+    for(uint32_t i = 0; i < nspin_v / 64; i++)
+        asm volatile("s_sleep 1");
+    if(nspin_v > 64)
+        for(uint32_t i = 0; i < nspin_v % 64; i++)
+            asm volatile("s_sleep 1");
+}
+
+void
+run(int tid, int devid)
+{
+    auto roctx_range_id = roctxRangeStart("run");
+
+    constexpr int min_avail_simd = 128;
+    dim3          grid(min_avail_simd);
+    dim3          block(32);
+    double        time    = 0.0;
+    hipStream_t   stream  = {};
+    hipEvent_t    start   = {};
+    hipEvent_t    stop    = {};
+    uint64_t      nlaunch = 0;
+
+    HIP_API_CALL(hipSetDevice(devid));
+    HIP_API_CALL(hipStreamCreate(&stream));
+    HIP_API_CALL(hipEventCreate(&start));
+    HIP_API_CALL(hipEventCreate(&stop));
+
+    for(size_t i = 0; i < niterations; ++i)
+    {
+        roctxMark("iteration");
+        HIP_API_CALL(hipEventRecord(start, stream));
+        reproducible_dispatch_count<<<grid, block, 0, stream>>>(nspin);
+        HIP_API_CALL(hipEventRecord(stop, stream));
+        check_hip_error();
+        HIP_API_CALL(hipEventSynchronize(stop));
+        float elapsed = 0.0f;
+        HIP_API_CALL(hipEventElapsedTime(&elapsed, start, stop));
+        time += static_cast<double>(elapsed);
+        ++nlaunch;
+    }
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipEventDestroy(start));
+    HIP_API_CALL(hipEventDestroy(stop));
+
+    {
+        auto _msg = std::stringstream{};
+        _msg << '[' << getpid() << "][" << tid << "] Runtime of reproducible-dispatch-count is "
+             << std::setprecision(2) << std::fixed << time << " ms (" << std::setprecision(3)
+             << (time / 1000.0f) << " sec). Kernels dispatched: " << nlaunch << "\n";
+        auto_lock_t _lk{print_lock};
+        std::cout << _msg.str() << std::flush;
+    }
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipStreamDestroy(stream));
+
+    roctxRangeStop(roctx_range_id);
+}
+
+void
+run_nsync(int tid, int devid)
+{
+    auto roctx_range_id = roctxRangeStart("run");
+
+    constexpr int min_avail_simd = 128;
+    dim3          grid(min_avail_simd);
+    dim3          block(32);
+    hipStream_t   stream  = {};
+    uint64_t      nlaunch = 0;
+
+    HIP_API_CALL(hipSetDevice(devid));
+    HIP_API_CALL(hipStreamCreate(&stream));
+
+    auto _elapsed = std::chrono::steady_clock::duration{};
+    auto _beg     = std::chrono::steady_clock::now();
+    for(size_t i = 0; i < niterations; ++i)
+    {
+        roctxMark("iteration");
+        reproducible_dispatch_count<<<grid, block, 0, stream>>>(nspin);
+        if((i % nsync) == (nsync - 1))
+        {
+            HIP_API_CALL(hipStreamSynchronize(stream));
+            auto _end = std::chrono::steady_clock::now();
+            _elapsed += (_end - _beg);
+            _beg = std::chrono::steady_clock::now();
+        }
+        ++nlaunch;
+    }
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    auto _end = std::chrono::steady_clock::now();
+    _elapsed += (_end - _beg);
+
+    {
+        auto _time =
+            std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(_elapsed).count();
+        auto _msg = std::stringstream{};
+        _msg << '[' << getpid() << "][" << tid << "] Runtime of reproducible-dispatch-count is "
+             << std::setprecision(2) << std::fixed << _time << " ms (" << std::setprecision(3)
+             << (_time / 1000.0f) << " sec). Kernels dispatched: " << nlaunch << "\n";
+        auto_lock_t _lk{print_lock};
+        std::cout << _msg.str() << std::flush;
+    }
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipStreamDestroy(stream));
+
+    roctxRangeStop(roctx_range_id);
+}
+
+namespace
+{
+void
+check_hip_error(void)
+{
+    hipError_t err = hipGetLastError();
+    if(err != hipSuccess)
+    {
+        auto_lock_t _lk{print_lock};
+        std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
+        throw std::runtime_error("hip_api_call");
+    }
+}
+}  // namespace

projects/rocprofiler-sdk/tests/bin/reproducible-runtime/CMakeLists.txt

@@ -0,0 +1,58 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-bin-reproducible-runtime LANGUAGES CXX HIP)
+
+if(NOT CMAKE_BUILD_TYPE MATCHES "(Release|RelWithDebInfo)")
+    set(CMAKE_BUILD_TYPE "RelWithDebInfo")
+endif()
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+option(REPRODUCIBLE_RUNTIME_USE_MPI "Enable MPI support in reproducible-runtime exe" OFF)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(reproducible-runtime.cpp PROPERTIES LANGUAGE HIP)
+add_executable(reproducible-runtime)
+target_sources(reproducible-runtime PRIVATE reproducible-runtime.cpp)
+target_compile_options(reproducible-runtime PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow
+                                                    -Werror)
+
+find_package(Threads REQUIRED)
+target_link_libraries(reproducible-runtime PRIVATE Threads::Threads)
+
+find_package(rocprofiler-sdk-roctx REQUIRED)
+target_link_libraries(reproducible-runtime
+                      PRIVATE rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
+
+if(REPRODUCIBLE_RUNTIME_USE_MPI)
+    find_package(MPI REQUIRED)
+    target_compile_definitions(reproducible-runtime PRIVATE USE_MPI)
+    target_link_libraries(reproducible-runtime PRIVATE MPI::MPI_C)
+endif()

projects/rocprofiler-sdk/tests/bin/reproducible-runtime/reproducible-runtime.cpp

@@ -0,0 +1,201 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "hip/hip_runtime.h"
+#include "rocprofiler-sdk-roctx/roctx.h"
+
+#include <unistd.h>
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <iomanip>
+#include <iostream>
+#include <mutex>
+#include <random>
+#include <stdexcept>
+#include <thread>
+
+#if defined(USE_MPI)
+#    include <mpi.h>
+#endif
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error %i : %s\n",                                                \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    static_cast<int>(error_),                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t   = std::unique_lock<std::mutex>;
+auto     print_lock = std::mutex{};
+double   nruntime   = 500.0;  // ms
+uint32_t nspin      = 128 * 5000;
+size_t   nthreads   = 2;
+
+void
+check_hip_error(void);
+}  // namespace
+
+__global__ void
+reproducible_runtime(uint32_t nspin);
+
+void
+run(int tid, int devid);
+
+int
+main(int argc, char** argv)
+{
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{argv[i]};
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            fprintf(stderr,
+                    "usage: reproducible-runtime [KERNEL RUNTIME PER THREAD (default: %f msec)] "
+                    "[NUM_THREADS (default: %zu)] [SPIN CYCLES PER KERNEL LAUNCH (default: %u)]\n",
+                    nruntime,
+                    nthreads,
+                    nspin);
+            exit(EXIT_SUCCESS);
+        }
+    }
+
+    if(argc > 1) nruntime = std::stod(argv[1]);
+    if(argc > 2) nthreads = std::stoll(argv[2]);
+    if(argc > 3) nspin = std::stoll(argv[3]);
+
+    printf("[reproducible-runtime] Kernel runtime per thread: %.3f msec\n", nruntime);
+    printf("[reproducible-runtime] Spin time per kernel: %u cycles\n", nspin);
+    printf("[reproducible-runtime] Number of threads: %zu\n", nthreads);
+
+    // this is a temporary workaround in omnitrace when HIP + MPI is enabled
+    int ndevice = 0;
+    HIP_API_CALL(hipGetDeviceCount(&ndevice));
+    printf("[reproducible-runtime] Number of devices found: %i\n", ndevice);
+    auto _threads = std::vector<std::thread>{};
+    for(size_t i = 0; i < nthreads; ++i)
+        _threads.emplace_back(run, i, i % ndevice);
+    for(auto& itr : _threads)
+        itr.join();
+    HIP_API_CALL(hipDeviceSynchronize());
+    HIP_API_CALL(hipDeviceReset());
+
+    return 0;
+}
+
+__global__ void
+reproducible_runtime(uint32_t nspin_v)
+{
+    for(uint32_t i = 0; i < nspin_v / 64; i++)
+        asm volatile("s_sleep 1");
+    if(nspin_v > 64)
+        for(uint32_t i = 0; i < nspin_v % 64; i++)
+            asm volatile("s_sleep 1");
+}
+
+void
+run(int tid, int devid)
+{
+    auto roctx_range_id = roctxRangeStart("run");
+
+    constexpr int min_avail_simd = 128;
+    dim3          grid(min_avail_simd);
+    dim3          block(32);
+    double        time    = 0.0;
+    hipStream_t   stream  = {};
+    hipEvent_t    start   = {};
+    hipEvent_t    stop    = {};
+    uint64_t      nlaunch = 0;
+
+    HIP_API_CALL(hipSetDevice(devid));
+    HIP_API_CALL(hipStreamCreate(&stream));
+    HIP_API_CALL(hipEventCreate(&start));
+    HIP_API_CALL(hipEventCreate(&stop));
+
+    do
+    {
+        roctxMark("iteration");
+        uint32_t cyclesleft = 1000 * 1000 * (nruntime - static_cast<double>(time));
+        HIP_API_CALL(hipEventRecord(start, stream));
+        reproducible_runtime<<<grid, block, 0, stream>>>(std::min<uint32_t>(nspin, cyclesleft));
+        HIP_API_CALL(hipEventRecord(stop, stream));
+        check_hip_error();
+        HIP_API_CALL(hipEventSynchronize(stop));
+        float elapsed = 0.0f;
+        HIP_API_CALL(hipEventElapsedTime(&elapsed, start, stop));
+        time += static_cast<double>(elapsed);
+        ++nlaunch;
+    } while(time < nruntime);
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipEventDestroy(start));
+    HIP_API_CALL(hipEventDestroy(stop));
+
+    {
+        auto _msg = std::stringstream{};
+        _msg << '[' << getpid() << "][" << tid << "] Runtime of reproducible-runtime is "
+             << std::setprecision(2) << std::fixed << time << " ms (" << std::setprecision(3)
+             << (time / 1000.0f) << " sec). Kernels dispatched: " << nlaunch << "\n";
+        auto_lock_t _lk{print_lock};
+        std::cout << _msg.str() << std::flush;
+    }
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipStreamDestroy(stream));
+
+    roctxRangeStop(roctx_range_id);
+
+    constexpr auto scale = 1.1;
+    if(time > scale * nruntime)
+    {
+        auto _msg = std::stringstream{};
+        _msg << "total kernel runtime exceeded (" << scale << " * " << nruntime << " = "
+             << (scale * nruntime) << ") :: " << time << " ms";
+        throw std::runtime_error{_msg.str()};
+    }
+}
+
+namespace
+{
+void
+check_hip_error(void)
+{
+    hipError_t err = hipGetLastError();
+    if(err != hipSuccess)
+    {
+        auto_lock_t _lk{print_lock};
+        std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
+        throw std::runtime_error("hip_api_call");
+    }
+}
+}  // namespace

projects/rocprofiler-sdk/tests/bin/rocdecode/CMakeLists.txt

@@ -0,0 +1,44 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-tool-test-app-rocdecode LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(rocdecode.cpp PROPERTIES LANGUAGE HIP)
+add_executable(rocdecode-demo)
+target_sources(rocdecode-demo PRIVATE rocdecode.cpp)
+
+find_package(Threads REQUIRED)
+find_package(rocDecode REQUIRED)
+target_link_libraries(
+    rocdecode-demo
+    PRIVATE rocprofiler-sdk::tests-build-flags Threads::Threads hsa-runtime64
+            rocprofiler-sdk::tests-common-library rocDecode::rocDecode)

projects/rocprofiler-sdk/tests/bin/rocdecode/rocdecode.cpp

@@ -0,0 +1,134 @@
+/*
+Copyright (c) 2024 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include <rocdecode/roc_bitstream_reader.h>
+#include <rocdecode/rocdecode.h>
+#include <rocdecode/rocparser.h>
+#include <cstring>
+#include <iostream>
+
+int
+main(int argc, char** argv)
+{
+    // Get input file
+    std::string input_file_path{};
+    for(int i = 1; i < argc; i++)
+    {
+        if(!strcmp(argv[i], "-i"))
+        {
+            if(++i == argc)
+            {
+                std::cerr << "Provide path to input file" << std::endl;
+            }
+            input_file_path = argv[i];
+            continue;
+        }
+    }
+    // Set up bitstreamreader
+    RocdecBitstreamReader bs_reader = nullptr;
+    rocDecVideoCodec      rocdec_codec_id{};
+    int                   bit_depth{};
+    if(rocDecCreateBitstreamReader(&bs_reader, input_file_path.c_str()) != ROCDEC_SUCCESS)
+    {
+        std::cerr << "Failed to create the bitstream reader." << std::endl;
+        return 1;
+    }
+    if(rocDecGetBitstreamCodecType(bs_reader, &rocdec_codec_id) != ROCDEC_SUCCESS)
+    {
+        std::cerr << "Failed to get stream codec type." << std::endl;
+        return 1;
+    }
+    if(rocdec_codec_id >= rocDecVideoCodec_NumCodecs)
+    {
+        std::cerr << "Unsupported stream file type or codec type by the bitstream reader. Exiting."
+                  << std::endl;
+        return 1;
+    }
+    if(rocDecGetBitstreamBitDepth(bs_reader, &bit_depth) != ROCDEC_SUCCESS)
+    {
+        std::cerr << "Failed to get stream bit depth." << std::endl;
+        return 1;
+    }
+
+    uint8_t* pvideo        = nullptr;
+    int      n_video_bytes = 0;
+    int64_t  pts           = 0;
+    if(rocDecGetBitstreamPicData(bs_reader, &pvideo, &n_video_bytes, &pts) != ROCDEC_SUCCESS)
+    {
+        std::cerr << "Failed to get picture data." << std::endl;
+        return 1;
+    }
+
+    // The following commands were originally used via the RocVideoDecoder from the
+    // roc_video_dec.cpp file from the rocDecode repository. However, this file kept causing
+    // compiler issues, so the file was removed. The following commands will return errors, but they
+    // should should be tracked for testing purposes
+
+    // rocDecCreateVideoParser
+    RocdecVideoParser  rocdec_parser_     = nullptr;
+    RocdecParserParams parser_params      = {};
+    parser_params.codec_type              = rocdec_codec_id;
+    parser_params.max_num_decode_surfaces = 1;
+    parser_params.clock_rate              = 0;
+    parser_params.max_display_delay       = 1;
+    parser_params.pfn_sequence_callback   = nullptr;
+    parser_params.pfn_decode_picture      = nullptr;
+    rocDecCreateVideoParser(&rocdec_parser_, &parser_params);
+
+    // rocDecGetDecoderCaps
+    RocdecDecodeCaps decode_caps{};
+    decode_caps.codec_type        = rocdec_codec_id;
+    decode_caps.chroma_format     = rocDecVideoChromaFormat_420;
+    decode_caps.bit_depth_minus_8 = 0;
+    rocDecGetDecoderCaps(&decode_caps);
+
+    // rocDecCreateDecoder
+    rocDecDecoderHandle  roc_decoder_          = nullptr;
+    RocDecoderCreateInfo videoDecodeCreateInfo = {};
+    videoDecodeCreateInfo.device_id            = 0;
+    videoDecodeCreateInfo.codec_type           = rocdec_codec_id;
+    videoDecodeCreateInfo.chroma_format        = rocDecVideoChromaFormat_420;
+    videoDecodeCreateInfo.bit_depth_minus_8    = 0;
+    rocDecCreateDecoder(&roc_decoder_, &videoDecodeCreateInfo);
+
+    // rocDecDecodeFrame
+    RocdecPicParams* pPicParams = nullptr;
+    rocDecDecodeFrame(roc_decoder_, pPicParams);
+
+    // rocDecParseVideoData
+    RocdecSourceDataPacket packet = {};
+    rocDecParseVideoData(rocdec_parser_, &packet);
+
+    // rocDecGetVideoFrame
+    void*            src_dev_ptr[3]    = {0};
+    uint32_t         src_pitch[3]      = {0};
+    RocdecProcParams video_proc_params = {};
+    rocDecGetVideoFrame(roc_decoder_, 0, src_dev_ptr, src_pitch, &video_proc_params);
+
+    // rocDecGetDecodeStatus
+    RocdecDecodeStatus dec_status{};
+    rocDecGetDecodeStatus(roc_decoder_, 0, &dec_status);
+    if(bs_reader)
+    {
+        rocDecDestroyBitstreamReader(bs_reader);
+    }
+}

projects/rocprofiler-sdk/tests/bin/rocjpeg/CMakeLists.txt

@@ -0,0 +1,69 @@
+# MIT License
+#
+# Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-tool-test-app-rocjpeg LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+find_path(
+    ROCJPEG_SHARE_DIR
+    NAMES images
+    PATHS ${ROCM_PATH}/share/rocjpeg/)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(rocjpeg.cpp PROPERTIES LANGUAGE HIP)
+add_executable(rocjpeg-demo)
+target_sources(rocjpeg-demo PRIVATE rocjpeg.cpp)
+
+find_package(Threads REQUIRED)
+find_package(rocJPEG REQUIRED)
+target_link_libraries(
+    rocjpeg-demo
+    PRIVATE Threads::Threads hsa-runtime64 rocprofiler-sdk::tests-common-library
+            rocprofiler-sdk::tests-build-flags rocJPEG::rocJPEG)

projects/rocprofiler-sdk/tests/bin/rocjpeg/rocjpeg.cpp

@@ -0,0 +1,279 @@
+/*
+Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include <rocjpeg/rocjpeg.h>
+#include "rocjpeg_samples_utils.h"
+
+int
+main(int argc, char** argv)
+{
+    int                      device_id   = 0;
+    bool                     save_images = false;
+    uint8_t                  num_components;
+    uint32_t                 widths[ROCJPEG_MAX_COMPONENT]              = {};
+    uint32_t                 heights[ROCJPEG_MAX_COMPONENT]             = {};
+    uint32_t                 channel_sizes[ROCJPEG_MAX_COMPONENT]       = {};
+    uint32_t                 prior_channel_sizes[ROCJPEG_MAX_COMPONENT] = {};
+    uint32_t                 num_channels                               = 0;
+    int                      total_images                               = 0;
+    double                   time_per_image_all                         = 0;
+    std::string              chroma_sub_sampling                        = "";
+    std::string              input_path, output_file_path;
+    std::vector<std::string> file_paths = {};
+    bool                     is_dir     = false;
+    bool                     is_file    = false;
+    RocJpegChromaSubsampling subsampling;
+    RocJpegBackend           rocjpeg_backend       = ROCJPEG_BACKEND_HARDWARE;
+    RocJpegHandle            rocjpeg_handle        = nullptr;
+    RocJpegStreamHandle      rocjpeg_stream_handle = nullptr;
+    RocJpegImage             output_image          = {};
+    RocJpegDecodeParams      decode_params         = {};
+    RocJpegUtils             rocjpeg_utils;
+    uint64_t                 num_bad_jpegs                         = 0;
+    uint64_t                 num_jpegs_with_411_subsampling        = 0;
+    uint64_t                 num_jpegs_with_unknown_subsampling    = 0;
+    uint64_t                 num_jpegs_with_unsupported_resolution = 0;
+
+    RocJpegUtils::ParseCommandLine(input_path,
+                                   output_file_path,
+                                   save_images,
+                                   device_id,
+                                   rocjpeg_backend,
+                                   decode_params,
+                                   nullptr,
+                                   nullptr,
+                                   argc,
+                                   argv);
+
+    bool     is_roi_valid = false;
+    uint32_t roi_width;
+    uint32_t roi_height;
+    roi_width  = decode_params.crop_rectangle.right - decode_params.crop_rectangle.left;
+    roi_height = decode_params.crop_rectangle.bottom - decode_params.crop_rectangle.top;
+
+    if(!RocJpegUtils::GetFilePaths(input_path, file_paths, is_dir, is_file))
+    {
+        std::cerr << "ERROR: Failed to get input file paths!" << std::endl;
+        return EXIT_FAILURE;
+    }
+    if(!RocJpegUtils::InitHipDevice(device_id))
+    {
+        std::cerr << "ERROR: Failed to initialize HIP!" << std::endl;
+        return EXIT_FAILURE;
+    }
+
+    // CHECK_ROCJPEG(rocJpegCreate(rocjpeg_backend, device_id, &rocjpeg_handle));
+    if(rocJpegCreate(rocjpeg_backend, device_id, &rocjpeg_handle) != ROCJPEG_STATUS_SUCCESS)
+    {
+        std::cerr << "rocJPEG tests not supported" << std::endl;
+        return 0;
+    }
+    CHECK_ROCJPEG(rocJpegStreamCreate(&rocjpeg_stream_handle));
+    std::vector<char> file_data;
+    for(auto file_path : file_paths)
+    {
+        std::string base_file_name = file_path.substr(file_path.find_last_of("/\\") + 1);
+        int         image_count    = 0;
+
+        // Read an image from disk.
+        std::ifstream input(file_path.c_str(), std::ios::in | std::ios::binary | std::ios::ate);
+        if(!(input.is_open()))
+        {
+            std::cerr << "ERROR: Cannot open image: " << file_path << std::endl;
+            return EXIT_FAILURE;
+        }
+        // Get the size
+        std::streamsize file_size = input.tellg();
+        input.seekg(0, std::ios::beg);
+        // resize if buffer is too small
+        if(file_data.size() < static_cast<size_t>(file_size))
+        {
+            file_data.resize(file_size);
+        }
+        if(!input.read(file_data.data(), file_size))
+        {
+            std::cerr << "ERROR: Cannot read from file: " << file_path << std::endl;
+            return EXIT_FAILURE;
+        }
+
+        RocJpegStatus rocjpeg_status = rocJpegStreamParse(
+            reinterpret_cast<uint8_t*>(file_data.data()), file_size, rocjpeg_stream_handle);
+        if(rocjpeg_status != ROCJPEG_STATUS_SUCCESS)
+        {
+            if(is_dir)
+            {
+                num_bad_jpegs++;
+                continue;
+            }
+            else
+            {
+                std::cerr << "ERROR: Failed to parse the input jpeg stream with "
+                          << rocJpegGetErrorName(rocjpeg_status) << std::endl;
+                return EXIT_FAILURE;
+            }
+        }
+
+        CHECK_ROCJPEG(rocJpegGetImageInfo(
+            rocjpeg_handle, rocjpeg_stream_handle, &num_components, &subsampling, widths, heights));
+
+        if(roi_width > 0 && roi_height > 0 && roi_width <= widths[0] && roi_height <= heights[0])
+        {
+            is_roi_valid = true;
+        }
+        rocjpeg_utils.GetChromaSubsamplingStr(subsampling, chroma_sub_sampling);
+        if(widths[0] < 64 || heights[0] < 64)
+        {
+            std::cerr << "The image resolution is not supported by VCN Hardware" << std::endl;
+            if(is_dir)
+            {
+                num_jpegs_with_unsupported_resolution++;
+                continue;
+            }
+            else
+                return EXIT_FAILURE;
+        }
+        if(subsampling == ROCJPEG_CSS_411 || subsampling == ROCJPEG_CSS_UNKNOWN)
+        {
+            std::cerr << "The chroma sub-sampling is not supported by VCN Hardware" << std::endl;
+            if(is_dir)
+            {
+                if(subsampling == ROCJPEG_CSS_411) num_jpegs_with_411_subsampling++;
+                if(subsampling == ROCJPEG_CSS_UNKNOWN) num_jpegs_with_unknown_subsampling++;
+                continue;
+            }
+            else
+                return EXIT_FAILURE;
+        }
+
+        if(rocjpeg_utils.GetChannelPitchAndSizes(decode_params,
+                                                 subsampling,
+                                                 widths,
+                                                 heights,
+                                                 num_channels,
+                                                 output_image,
+                                                 channel_sizes))
+        {
+            std::cerr << "ERROR: Failed to get the channel pitch and sizes" << std::endl;
+            return EXIT_FAILURE;
+        }
+
+        // allocate memory for each channel and reuse them if the sizes remain unchanged for a new
+        // image.
+        for(uint32_t i = 0; i < num_channels; i++)
+        {
+            if(prior_channel_sizes[i] != channel_sizes[i])
+            {
+                if(output_image.channel[i] != nullptr)
+                {
+                    CHECK_HIP(hipFree((void*) output_image.channel[i]));
+                    output_image.channel[i] = nullptr;
+                }
+                CHECK_HIP(hipMalloc(&output_image.channel[i], channel_sizes[i]));
+            }
+        }
+
+        if(is_roi_valid)
+        {}
+        auto start_time = std::chrono::high_resolution_clock::now();
+        CHECK_ROCJPEG(
+            rocJpegDecode(rocjpeg_handle, rocjpeg_stream_handle, &decode_params, &output_image));
+        auto   end_time = std::chrono::high_resolution_clock::now();
+        double time_per_image_in_milli_sec =
+            std::chrono::duration<double, std::milli>(end_time - start_time).count();
+        image_count++;
+
+        if(save_images)
+        {
+            std::string image_save_path = output_file_path;
+            // if ROI is present, need to pass roi_width and roi_height
+            uint32_t width  = is_roi_valid ? roi_width : widths[0];
+            uint32_t height = is_roi_valid ? roi_height : heights[0];
+            if(is_dir)
+            {
+                rocjpeg_utils.GetOutputFileExt(decode_params.output_format,
+                                               base_file_name,
+                                               width,
+                                               height,
+                                               subsampling,
+                                               image_save_path);
+            }
+            rocjpeg_utils.SaveImage(image_save_path,
+                                    &output_image,
+                                    width,
+                                    height,
+                                    subsampling,
+                                    decode_params.output_format);
+        }
+
+        if(is_dir)
+        {
+            total_images += image_count;
+            time_per_image_all += time_per_image_in_milli_sec;
+        }
+        for(int i = 0; i < ROCJPEG_MAX_COMPONENT; i++)
+        {
+            prior_channel_sizes[i] = channel_sizes[i];
+        }
+    }
+
+    for(uint32_t i = 0; i < num_channels; i++)
+    {
+        if(output_image.channel[i] != nullptr)
+        {
+            CHECK_HIP(hipFree((void*) output_image.channel[i]));
+            output_image.channel[i] = nullptr;
+        }
+    }
+
+    if(is_dir)
+    {
+        time_per_image_all = time_per_image_all / total_images;
+        if(num_bad_jpegs || num_jpegs_with_411_subsampling || num_jpegs_with_unknown_subsampling ||
+           num_jpegs_with_unsupported_resolution)
+        {
+            if(num_bad_jpegs)
+            {
+                std::cout << " ,total images that cannot be parsed: " << num_bad_jpegs;
+            }
+            if(num_jpegs_with_411_subsampling)
+            {
+                std::cout << " ,total images with YUV 4:1:1 chroam subsampling: "
+                          << num_jpegs_with_411_subsampling;
+            }
+            if(num_jpegs_with_unknown_subsampling)
+            {
+                std::cout << " ,total images with unknwon chroam subsampling: "
+                          << num_jpegs_with_unknown_subsampling;
+            }
+            if(num_jpegs_with_unsupported_resolution)
+            {
+                std::cout << " ,total images with unsupported_resolution: "
+                          << num_jpegs_with_unsupported_resolution;
+            }
+            std::cout << std::endl;
+        }
+    }
+
+    CHECK_ROCJPEG(rocJpegDestroy(rocjpeg_handle));
+    CHECK_ROCJPEG(rocJpegStreamDestroy(rocjpeg_stream_handle));
+    return EXIT_SUCCESS;
+}

projects/rocprofiler-sdk/tests/bin/rocjpeg/rocjpeg_samples_utils.h

@@ -0,0 +1,873 @@
+/*
+Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef ROC_JPEG_SAMPLES_COMMON
+#define ROC_JPEG_SAMPLES_COMMON
+#pragma once
+
+#include "common/filesystem.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <condition_variable>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <mutex>
+#include <queue>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include <rocjpeg/rocjpeg.h>
+
+namespace fs = common::fs;
+
+#define CHECK_ROCJPEG(call)                                                                        \
+    {                                                                                              \
+        RocJpegStatus _rocjpeg_status = (call);                                                    \
+        if(_rocjpeg_status != ROCJPEG_STATUS_SUCCESS)                                              \
+        {                                                                                          \
+            std::cerr << #call << " returned " << rocJpegGetErrorName(_rocjpeg_status) << " at "   \
+                      << __FILE__ << ":" << __LINE__ << std::endl;                                 \
+            exit(1);                                                                               \
+        }                                                                                          \
+    }
+
+#define CHECK_HIP(call)                                                                            \
+    {                                                                                              \
+        hipError_t _hip_status = (call);                                                           \
+        if(_hip_status != hipSuccess)                                                              \
+        {                                                                                          \
+            std::cout << "rocJPEG failure: '#" << _hip_status << "' at " << __FILE__ << ":"        \
+                      << __LINE__ << std::endl;                                                    \
+            exit(1);                                                                               \
+        }                                                                                          \
+    }
+
+/**
+ * @class RocJpegUtils
+ * @brief Utility class for rocJPEG samples.
+ *
+ * This class provides utility functions for rocJPEG samples, such as parsing command line
+ * arguments, getting file paths, initializing HIP device, getting chroma subsampling string,
+ * getting channel pitch and sizes, getting output file extension, and saving images.
+ */
+class RocJpegUtils
+{
+public:
+    /**
+     * @brief Parses the command line arguments.
+     *
+     * This function parses the command line arguments and sets the corresponding variables.
+     *
+     * @param input_path The input path.
+     * @param output_file_path The output file path.
+     * @param save_images Flag indicating whether to save images.
+     * @param device_id The device ID.
+     * @param rocjpeg_backend The rocJPEG backend.
+     * @param decode_params The rocJPEG decode parameters.
+     * @param num_threads The number of threads.
+     * @param crop The crop rectangle.
+     * @param argc The number of command line arguments.
+     * @param argv The command line arguments.
+     */
+    static void ParseCommandLine(std::string&         input_path,
+                                 std::string&         output_file_path,
+                                 bool&                save_images,
+                                 int&                 device_id,
+                                 RocJpegBackend&      rocjpeg_backend,
+                                 RocJpegDecodeParams& decode_params,
+                                 int*                 num_threads,
+                                 int*                 batch_size,
+                                 int                  argc,
+                                 char*                argv[])
+    {
+        if(argc <= 1)
+        {
+            ShowHelpAndExit("", num_threads != nullptr, batch_size != nullptr);
+        }
+        for(int i = 1; i < argc; i++)
+        {
+            if(!strcmp(argv[i], "-h"))
+            {
+                ShowHelpAndExit("", num_threads != nullptr, batch_size != nullptr);
+            }
+            if(!strcmp(argv[i], "-i"))
+            {
+                if(++i == argc)
+                {
+                    ShowHelpAndExit("-i", num_threads != nullptr, batch_size != nullptr);
+                }
+                input_path = argv[i];
+                continue;
+            }
+            if(!strcmp(argv[i], "-o"))
+            {
+                if(++i == argc)
+                {
+                    ShowHelpAndExit("-o", num_threads != nullptr, batch_size != nullptr);
+                }
+                output_file_path = argv[i];
+                save_images      = true;
+                continue;
+            }
+            if(!strcmp(argv[i], "-d"))
+            {
+                if(++i == argc)
+                {
+                    ShowHelpAndExit("-d", num_threads != nullptr, batch_size != nullptr);
+                }
+                device_id = atoi(argv[i]);
+                continue;
+            }
+            if(!strcmp(argv[i], "-be"))
+            {
+                if(++i == argc)
+                {
+                    ShowHelpAndExit("-be", num_threads != nullptr, batch_size != nullptr);
+                }
+                rocjpeg_backend = static_cast<RocJpegBackend>(atoi(argv[i]));
+                continue;
+            }
+            if(!strcmp(argv[i], "-fmt"))
+            {
+                if(++i == argc)
+                {
+                    ShowHelpAndExit("-fmt", num_threads != nullptr, batch_size != nullptr);
+                }
+                std::string selected_output_format = argv[i];
+                if(selected_output_format == "native")
+                {
+                    decode_params.output_format = ROCJPEG_OUTPUT_NATIVE;
+                }
+                else if(selected_output_format == "yuv_planar")
+                {
+                    decode_params.output_format = ROCJPEG_OUTPUT_YUV_PLANAR;
+                }
+                else if(selected_output_format == "y")
+                {
+                    decode_params.output_format = ROCJPEG_OUTPUT_Y;
+                }
+                else if(selected_output_format == "rgb")
+                {
+                    decode_params.output_format = ROCJPEG_OUTPUT_RGB;
+                }
+                else if(selected_output_format == "rgb_planar")
+                {
+                    decode_params.output_format = ROCJPEG_OUTPUT_RGB_PLANAR;
+                }
+                else
+                {
+                    ShowHelpAndExit(argv[i], num_threads != nullptr);
+                }
+                continue;
+            }
+            if(!strcmp(argv[i], "-t"))
+            {
+                if(++i == argc)
+                {
+                    ShowHelpAndExit("-t", num_threads != nullptr, batch_size != nullptr);
+                }
+                if(num_threads != nullptr)
+                {
+                    *num_threads = atoi(argv[i]);
+                    if(*num_threads <= 0 || *num_threads > 32)
+                    {
+                        ShowHelpAndExit(argv[i], num_threads != nullptr, batch_size != nullptr);
+                    }
+                }
+                continue;
+            }
+            if(!strcmp(argv[i], "-b"))
+            {
+                if(++i == argc)
+                {
+                    ShowHelpAndExit("-b", num_threads != nullptr, batch_size != nullptr);
+                }
+                if(batch_size != nullptr) *batch_size = atoi(argv[i]);
+                continue;
+            }
+            if(!strcmp(argv[i], "-crop"))
+            {
+                if(++i == argc || 4 != sscanf(argv[i],
+                                              "%hd,%hd,%hd,%hd",
+                                              &decode_params.crop_rectangle.left,
+                                              &decode_params.crop_rectangle.top,
+                                              &decode_params.crop_rectangle.right,
+                                              &decode_params.crop_rectangle.bottom))
+                {
+                    ShowHelpAndExit("-crop");
+                }
+                if((&decode_params.crop_rectangle.right - &decode_params.crop_rectangle.left) % 2 ==
+                       1 ||
+                   (&decode_params.crop_rectangle.bottom - &decode_params.crop_rectangle.top) % 2 ==
+                       1)
+                {
+                    std::cout << "output crop rectangle must have width and height of even numbers"
+                              << std::endl;
+                    exit(1);
+                }
+                continue;
+            }
+            ShowHelpAndExit(argv[i], num_threads != nullptr, batch_size != nullptr);
+        }
+    }
+
+    /**
+     * Checks if a file is a JPEG file.
+     *
+     * @param filePath The path to the file to be checked.
+     * @return True if the file is a JPEG file, false otherwise.
+     */
+    static bool IsJPEG(const std::string& filePath)
+    {
+        std::ifstream file(filePath, std::ios::binary);
+        if(!file.is_open())
+        {
+            std::cerr << "Failed to open file: " << filePath << std::endl;
+            return false;
+        }
+
+        unsigned char buffer[2];
+        file.read(reinterpret_cast<char*>(buffer), 2);
+        file.close();
+
+        // The first two bytes of every JPEG stream are always 0xFFD8, which represents the Start of
+        // Image (SOI) marker.
+        return buffer[0] == 0xFF && buffer[1] == 0xD8;
+    }
+
+    /**
+     * @brief Gets the file paths.
+     *
+     * This function gets the file paths based on the input path and sets the corresponding
+     * variables.
+     *
+     * @param input_path The input path.
+     * @param file_paths The vector to store the file paths.
+     * @param is_dir Flag indicating whether the input path is a directory.
+     * @param is_file Flag indicating whether the input path is a file.
+     * @return True if successful, false otherwise.
+     */
+    static bool GetFilePaths(std::string&              input_path,
+                             std::vector<std::string>& file_paths,
+                             bool&                     is_dir,
+                             bool&                     is_file)
+    {
+        if(!fs::exists(input_path))
+        {
+            std::cerr << "ERROR: the input path does not exist!" << std::endl;
+            return false;
+        }
+        is_dir  = fs::is_directory(input_path);
+        is_file = fs::is_regular_file(input_path);
+        if(is_dir)
+        {
+            for(const auto& entry : fs::recursive_directory_iterator(input_path))
+            {
+                if(fs::is_regular_file(entry) && IsJPEG(entry.path().string()))
+                {
+                    file_paths.push_back(entry.path().string());
+                }
+            }
+        }
+        else if(is_file && IsJPEG(input_path))
+        {
+            file_paths.push_back(input_path);
+        }
+        else
+        {
+            std::cerr << "ERROR: the input path does not contain JPEG files!" << std::endl;
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * @brief Initializes the HIP device.
+     *
+     * This function initializes the HIP device with the specified device ID.
+     *
+     * @param device_id The device ID.
+     * @return True if successful, false otherwise.
+     */
+    static bool InitHipDevice(int device_id)
+    {
+        int             num_devices;
+        hipDeviceProp_t hip_dev_prop;
+        CHECK_HIP(hipGetDeviceCount(&num_devices));
+        if(num_devices < 1)
+        {
+            std::cerr << "ERROR: didn't find any GPU!" << std::endl;
+            return false;
+        }
+        if(device_id >= num_devices)
+        {
+            std::cerr << "ERROR: the requested device_id is not found!" << std::endl;
+            return false;
+        }
+        CHECK_HIP(hipSetDevice(device_id));
+        CHECK_HIP(hipGetDeviceProperties(&hip_dev_prop, device_id));
+
+        return true;
+    }
+
+    /**
+     * @brief Gets the chroma subsampling string.
+     *
+     * This function gets the chroma subsampling string based on the specified subsampling value.
+     *
+     * @param subsampling The chroma subsampling value.
+     * @param chroma_sub_sampling The string to store the chroma subsampling.
+     */
+    void GetChromaSubsamplingStr(RocJpegChromaSubsampling subsampling,
+                                 std::string&             chroma_sub_sampling)
+    {
+        switch(subsampling)
+        {
+            case ROCJPEG_CSS_444: chroma_sub_sampling = "YUV 4:4:4"; break;
+            case ROCJPEG_CSS_440: chroma_sub_sampling = "YUV 4:4:0"; break;
+            case ROCJPEG_CSS_422: chroma_sub_sampling = "YUV 4:2:2"; break;
+            case ROCJPEG_CSS_420: chroma_sub_sampling = "YUV 4:2:0"; break;
+            case ROCJPEG_CSS_411: chroma_sub_sampling = "YUV 4:1:1"; break;
+            case ROCJPEG_CSS_400: chroma_sub_sampling = "YUV 4:0:0"; break;
+            case ROCJPEG_CSS_UNKNOWN: chroma_sub_sampling = "UNKNOWN"; break;
+            default: chroma_sub_sampling = ""; break;
+        }
+    }
+
+    /**
+     * @brief Gets the channel pitch and sizes.
+     *
+     * This function gets the channel pitch and sizes based on the specified output format, chroma
+     * subsampling, output image, and channel sizes.
+     *
+     * @param decode_params The decode parameters that specify the output format and crop rectangle.
+     * @param subsampling The chroma subsampling.
+     * @param widths The array to store the channel widths.
+     * @param heights The array to store the channel heights.
+     * @param num_channels The number of channels.
+     * @param output_image The output image.
+     * @param channel_sizes The array to store the channel sizes.
+     * @return The channel pitch.
+     */
+    int GetChannelPitchAndSizes(RocJpegDecodeParams      decode_params,
+                                RocJpegChromaSubsampling subsampling,
+                                uint32_t*                widths,
+                                uint32_t*                heights,
+                                uint32_t&                num_channels,
+                                RocJpegImage&            output_image,
+                                uint32_t*                channel_sizes)
+    {
+        bool     is_roi_valid = false;
+        uint32_t roi_width;
+        uint32_t roi_height;
+        roi_width  = decode_params.crop_rectangle.right - decode_params.crop_rectangle.left;
+        roi_height = decode_params.crop_rectangle.bottom - decode_params.crop_rectangle.top;
+        if(roi_width > 0 && roi_height > 0 && roi_width <= widths[0] && roi_height <= heights[0])
+        {
+            is_roi_valid = true;
+        }
+        switch(decode_params.output_format)
+        {
+            case ROCJPEG_OUTPUT_NATIVE:
+                switch(subsampling)
+                {
+                    case ROCJPEG_CSS_444:
+                        num_channels          = 3;
+                        output_image.pitch[2] = output_image.pitch[1] = output_image.pitch[0] =
+                            is_roi_valid ? roi_width : widths[0];
+                        channel_sizes[2] = channel_sizes[1] = channel_sizes[0] =
+                            align(output_image.pitch[0] * (is_roi_valid ? roi_height : heights[0]),
+                                  mem_alignment);
+                        break;
+                    case ROCJPEG_CSS_440:
+                        num_channels          = 3;
+                        output_image.pitch[2] = output_image.pitch[1] = output_image.pitch[0] =
+                            is_roi_valid ? roi_width : widths[0];
+                        channel_sizes[0] =
+                            align(output_image.pitch[0] * (is_roi_valid ? roi_height : heights[0]),
+                                  mem_alignment);
+                        channel_sizes[2] = channel_sizes[1] = align(
+                            output_image.pitch[0] * ((is_roi_valid ? roi_height : heights[0]) >> 1),
+                            mem_alignment);
+                        break;
+                    case ROCJPEG_CSS_422:
+                        num_channels          = 1;
+                        output_image.pitch[0] = (is_roi_valid ? roi_width : widths[0]) * 2;
+                        channel_sizes[0] =
+                            align(output_image.pitch[0] * (is_roi_valid ? roi_height : heights[0]),
+                                  mem_alignment);
+                        break;
+                    case ROCJPEG_CSS_420:
+                        num_channels          = 2;
+                        output_image.pitch[1] = output_image.pitch[0] =
+                            is_roi_valid ? roi_width : widths[0];
+                        channel_sizes[0] =
+                            align(output_image.pitch[0] * (is_roi_valid ? roi_height : heights[0]),
+                                  mem_alignment);
+                        channel_sizes[1] = align(
+                            output_image.pitch[1] * ((is_roi_valid ? roi_height : heights[0]) >> 1),
+                            mem_alignment);
+                        break;
+                    case ROCJPEG_CSS_400:
+                        num_channels          = 1;
+                        output_image.pitch[0] = is_roi_valid ? roi_width : widths[0];
+                        channel_sizes[0] =
+                            align(output_image.pitch[0] * (is_roi_valid ? roi_height : heights[0]),
+                                  mem_alignment);
+                        break;
+                    default:
+                        std::cout << "Unknown chroma subsampling!" << std::endl;
+                        return EXIT_FAILURE;
+                }
+                break;
+            case ROCJPEG_OUTPUT_YUV_PLANAR:
+                if(subsampling == ROCJPEG_CSS_400)
+                {
+                    num_channels          = 1;
+                    output_image.pitch[0] = is_roi_valid ? roi_width : widths[0];
+                    channel_sizes[0] =
+                        align(output_image.pitch[0] * (is_roi_valid ? roi_height : heights[0]),
+                              mem_alignment);
+                }
+                else
+                {
+                    num_channels          = 3;
+                    output_image.pitch[0] = is_roi_valid ? roi_width : widths[0];
+                    output_image.pitch[1] = is_roi_valid ? roi_width : widths[1];
+                    output_image.pitch[2] = is_roi_valid ? roi_width : widths[2];
+                    channel_sizes[0] =
+                        align(output_image.pitch[0] * (is_roi_valid ? roi_height : heights[0]),
+                              mem_alignment);
+                    channel_sizes[1] =
+                        align(output_image.pitch[1] * (is_roi_valid ? roi_height : heights[1]),
+                              mem_alignment);
+                    channel_sizes[2] =
+                        align(output_image.pitch[2] * (is_roi_valid ? roi_height : heights[2]),
+                              mem_alignment);
+                }
+                break;
+            case ROCJPEG_OUTPUT_Y:
+                num_channels          = 1;
+                output_image.pitch[0] = is_roi_valid ? roi_width : widths[0];
+                channel_sizes[0] =
+                    align(output_image.pitch[0] * (is_roi_valid ? roi_height : heights[0]),
+                          mem_alignment);
+                break;
+            case ROCJPEG_OUTPUT_RGB:
+                num_channels          = 1;
+                output_image.pitch[0] = (is_roi_valid ? roi_width : widths[0]) * 3;
+                channel_sizes[0] =
+                    align(output_image.pitch[0] * (is_roi_valid ? roi_height : heights[0]),
+                          mem_alignment);
+                break;
+            case ROCJPEG_OUTPUT_RGB_PLANAR:
+                num_channels          = 3;
+                output_image.pitch[2] = output_image.pitch[1] = output_image.pitch[0] =
+                    is_roi_valid ? roi_width : widths[0];
+                channel_sizes[2] = channel_sizes[1] = channel_sizes[0] =
+                    align(output_image.pitch[0] * (is_roi_valid ? roi_height : heights[0]),
+                          mem_alignment);
+                break;
+            default: std::cout << "Unknown output format!" << std::endl; return EXIT_FAILURE;
+        }
+        return EXIT_SUCCESS;
+    }
+
+    /**
+     * @brief Gets the output file extension.
+     *
+     * This function gets the output file extension based on the specified output format, base file
+     * name, image width, image height, and file name for saving.
+     *
+     * @param output_format The output format.
+     * @param base_file_name The base file name.
+     * @param image_width The image width.
+     * @param image_height The image height.
+     * @param file_name_for_saving The string to store the file name for saving.
+     */
+    void GetOutputFileExt(RocJpegOutputFormat      output_format,
+                          std::string&             base_file_name,
+                          uint32_t                 image_width,
+                          uint32_t                 image_height,
+                          RocJpegChromaSubsampling subsampling,
+                          std::string&             file_name_for_saving)
+    {
+        std::string                  file_extension;
+        std::string::size_type const p(base_file_name.find_last_of('.'));
+        std::string                  file_name_no_ext   = base_file_name.substr(0, p);
+        std::string                  format_description = "";
+        switch(output_format)
+        {
+            case ROCJPEG_OUTPUT_NATIVE:
+                file_extension = "yuv";
+                switch(subsampling)
+                {
+                    case ROCJPEG_CSS_444: format_description = "444"; break;
+                    case ROCJPEG_CSS_440: format_description = "440"; break;
+                    case ROCJPEG_CSS_422: format_description = "422_yuyv"; break;
+                    case ROCJPEG_CSS_420: format_description = "nv12"; break;
+                    case ROCJPEG_CSS_400: format_description = "400"; break;
+                    default: std::cout << "Unknown chroma subsampling!" << std::endl; return;
+                }
+                break;
+            case ROCJPEG_OUTPUT_YUV_PLANAR:
+                file_extension     = "yuv";
+                format_description = "planar";
+                break;
+            case ROCJPEG_OUTPUT_Y:
+                file_extension     = "yuv";
+                format_description = "400";
+                break;
+            case ROCJPEG_OUTPUT_RGB:
+                file_extension     = "rgb";
+                format_description = "packed";
+                break;
+            case ROCJPEG_OUTPUT_RGB_PLANAR:
+                file_extension     = "rgb";
+                format_description = "planar";
+                break;
+            default: file_extension = ""; break;
+        }
+        file_name_for_saving += "//" + file_name_no_ext + "_" + std::to_string(image_width) + "x" +
+                                std::to_string(image_height) + "_" + format_description + "." +
+                                file_extension;
+    }
+
+    /**
+     * @brief Saves the image.
+     *
+     * This function saves the image to the specified output file name based on the output image,
+     * image width, image height, chroma subsampling, and output format.
+     *
+     * @param output_file_name The output file name.
+     * @param output_image The output image.
+     * @param img_width The image width.
+     * @param img_height The image height.
+     * @param subsampling The chroma subsampling.
+     * @param output_format The output format.
+     */
+    void SaveImage(std::string              output_file_name,
+                   RocJpegImage*            output_image,
+                   uint32_t                 img_width,
+                   uint32_t                 img_height,
+                   RocJpegChromaSubsampling subsampling,
+                   RocJpegOutputFormat      output_format)
+    {
+        uint8_t* hst_ptr = nullptr;
+        FILE*    fp;
+
+        if(output_image == nullptr || output_image->channel[0] == nullptr ||
+           output_image->pitch[0] == 0)
+        {
+            return;
+        }
+
+        uint32_t widths[ROCJPEG_MAX_COMPONENT]  = {};
+        uint32_t heights[ROCJPEG_MAX_COMPONENT] = {};
+
+        switch(output_format)
+        {
+            case ROCJPEG_OUTPUT_NATIVE:
+                switch(subsampling)
+                {
+                    case ROCJPEG_CSS_444:
+                        widths[2] = widths[1] = widths[0] = img_width;
+                        heights[2] = heights[1] = heights[0] = img_height;
+                        break;
+                    case ROCJPEG_CSS_440:
+                        widths[2] = widths[1] = widths[0] = img_width;
+                        heights[0]                        = img_height;
+                        heights[2] = heights[1] = img_height >> 1;
+                        break;
+                    case ROCJPEG_CSS_422:
+                        widths[0]  = img_width * 2;
+                        heights[0] = img_height;
+                        break;
+                    case ROCJPEG_CSS_420:
+                        widths[1] = widths[0] = img_width;
+                        heights[0]            = img_height;
+                        heights[1]            = img_height >> 1;
+                        break;
+                    case ROCJPEG_CSS_400:
+                        widths[0]  = img_width;
+                        heights[0] = img_height;
+                        break;
+                    default: std::cout << "Unknown chroma subsampling!" << std::endl; return;
+                }
+                break;
+            case ROCJPEG_OUTPUT_YUV_PLANAR:
+                switch(subsampling)
+                {
+                    case ROCJPEG_CSS_444:
+                        widths[2] = widths[1] = widths[0] = img_width;
+                        heights[2] = heights[1] = heights[0] = img_height;
+                        break;
+                    case ROCJPEG_CSS_440:
+                        widths[2] = widths[1] = widths[0] = img_width;
+                        heights[0]                        = img_height;
+                        heights[2] = heights[1] = img_height >> 1;
+                        break;
+                    case ROCJPEG_CSS_422:
+                        widths[0] = img_width;
+                        widths[2] = widths[1] = widths[0] >> 1;
+                        heights[2] = heights[1] = heights[0] = img_height;
+                        break;
+                    case ROCJPEG_CSS_420:
+                        widths[0] = img_width;
+                        widths[2] = widths[1] = widths[0] >> 1;
+                        heights[0]            = img_height;
+                        heights[2] = heights[1] = img_height >> 1;
+                        break;
+                    case ROCJPEG_CSS_400:
+                        widths[0]  = img_width;
+                        heights[0] = img_height;
+                        break;
+                    default: std::cout << "Unknown chroma subsampling!" << std::endl; return;
+                }
+                break;
+            case ROCJPEG_OUTPUT_Y:
+                widths[0]  = img_width;
+                heights[0] = img_height;
+                break;
+            case ROCJPEG_OUTPUT_RGB:
+                widths[0]  = img_width * 3;
+                heights[0] = img_height;
+                break;
+            case ROCJPEG_OUTPUT_RGB_PLANAR:
+                widths[2] = widths[1] = widths[0] = img_width;
+                heights[2] = heights[1] = heights[0] = img_height;
+                break;
+            default: std::cout << "Unknown output format!" << std::endl; return;
+        }
+
+        uint32_t channel0_size = output_image->pitch[0] * heights[0];
+        uint32_t channel1_size = output_image->pitch[1] * heights[1];
+        uint32_t channel2_size = output_image->pitch[2] * heights[2];
+
+        uint32_t output_image_size = channel0_size + channel1_size + channel2_size;
+
+        if(hst_ptr == nullptr)
+        {
+            hst_ptr = new uint8_t[output_image_size];
+        }
+
+        CHECK_HIP(hipMemcpyDtoH((void*) hst_ptr, output_image->channel[0], channel0_size));
+
+        uint8_t* tmp_hst_ptr = hst_ptr;
+        fp                   = fopen(output_file_name.c_str(), "wb");
+        if(fp)
+        {
+            // write channel0
+            if(widths[0] == output_image->pitch[0])
+            {
+                fwrite(hst_ptr, 1, channel0_size, fp);
+            }
+            else
+            {
+                for(uint32_t i = 0; i < heights[0]; i++)
+                {
+                    fwrite(tmp_hst_ptr, 1, widths[0], fp);
+                    tmp_hst_ptr += output_image->pitch[0];
+                }
+            }
+            // write channel1
+            if(channel1_size != 0 && output_image->channel[1] != nullptr)
+            {
+                uint8_t* channel1_hst_ptr = hst_ptr + channel0_size;
+                CHECK_HIP(hipMemcpyDtoH(
+                    (void*) channel1_hst_ptr, output_image->channel[1], channel1_size));
+                if(widths[1] == output_image->pitch[1])
+                {
+                    fwrite(channel1_hst_ptr, 1, channel1_size, fp);
+                }
+                else
+                {
+                    for(uint32_t i = 0; i < heights[1]; i++)
+                    {
+                        fwrite(channel1_hst_ptr, 1, widths[1], fp);
+                        channel1_hst_ptr += output_image->pitch[1];
+                    }
+                }
+            }
+            // write channel2
+            if(channel2_size != 0 && output_image->channel[2] != nullptr)
+            {
+                uint8_t* channel2_hst_ptr = hst_ptr + channel0_size + channel1_size;
+                CHECK_HIP(hipMemcpyDtoH(
+                    (void*) channel2_hst_ptr, output_image->channel[2], channel2_size));
+                if(widths[2] == output_image->pitch[2])
+                {
+                    fwrite(channel2_hst_ptr, 1, channel2_size, fp);
+                }
+                else
+                {
+                    for(uint32_t i = 0; i < heights[2]; i++)
+                    {
+                        fwrite(channel2_hst_ptr, 1, widths[2], fp);
+                        channel2_hst_ptr += output_image->pitch[2];
+                    }
+                }
+            }
+            fclose(fp);
+        }
+
+        if(hst_ptr != nullptr)
+        {
+            delete[] hst_ptr;
+            hst_ptr     = nullptr;
+            tmp_hst_ptr = nullptr;
+        }
+    }
+
+private:
+    static const int mem_alignment = 4 * 1024 * 1024;
+    /**
+     * @brief Shows the help message and exits.
+     *
+     * This function shows the help message and exits the program.
+     *
+     * @param option The option to display in the help message (optional).
+     * @param show_threads Flag indicating whether to show the number of threads in the help
+     * message.
+     */
+    static void ShowHelpAndExit(const char* option          = nullptr,
+                                bool        show_threads    = false,
+                                bool        show_batch_size = false)
+    {
+        (void) option;
+        std::cout
+            << "Options:\n"
+               "-i     [input path] - input path to a single JPEG image or a directory containing "
+               "JPEG images - [required]\n"
+               "-be    [backend] - select rocJPEG backend (0 for hardware-accelerated JPEG "
+               "decoding using VCN,\n"
+               "                                           1 for hybrid JPEG decoding using CPU "
+               "and GPU HIP kernels (currently not supported)) [optional - default: 0]\n"
+               "-fmt   [output format] - select rocJPEG output format for decoding, one of the "
+               "[native, yuv_planar, y, rgb, rgb_planar] - [optional - default: native]\n"
+               "-o     [output path] - path to an output file or a path to an existing directory - "
+               "write decoded images to a file or an existing directory based on selected output "
+               "format - [optional]\n"
+               "-crop  [crop rectangle] - crop rectangle for output in a comma-separated format: "
+               "left,top,right,bottom - [optional]\n"
+               "-d     [device id] - specify the GPU device id for the desired device (use 0 for "
+               "the first device, 1 for the second device, and so on) [optional - default: 0]\n";
+        if(show_threads)
+        {
+            std::cout << "-t     [threads] - number of threads (<= 32) for parallel JPEG decoding "
+                         "- [optional - default: 1]\n";
+        }
+        if(show_batch_size)
+        {
+            std::cout << "-b     [batch_size] - decode images from input by batches of a specified "
+                         "size - [optional - default: 1]\n";
+        }
+        exit(0);
+    }
+    /**
+     * @brief Aligns a value to a specified alignment.
+     *
+     * This function takes a value and aligns it to the specified alignment. It returns the aligned
+     * value.
+     *
+     * @param value The value to be aligned.
+     * @param alignment The alignment value.
+     * @return The aligned value.
+     */
+    static inline int align(int value, int alignment)
+    {
+        return (value + alignment - 1) & ~(alignment - 1);
+    }
+};
+
+class ThreadPool
+{
+public:
+    ThreadPool(int nthreads)
+    : shutdown_(false)
+    {
+        // Create the specified number of threads
+        threads_.reserve(nthreads);
+        for(int i = 0; i < nthreads; ++i)
+            threads_.emplace_back(std::bind(&ThreadPool::ThreadEntry, this, i));
+    }
+
+    ~ThreadPool() {}
+
+    void JoinThreads()
+    {
+        {
+            // Unblock any threads and tell them to stop
+            std::unique_lock<std::mutex> lock(mutex_);
+            shutdown_ = true;
+            cond_var_.notify_all();
+        }
+
+        // Wait for all threads to stop
+        for(auto& thread : threads_)
+            thread.join();
+    }
+
+    void ExecuteJob(std::function<void()> func)
+    {
+        // Place a job on the queue and unblock a thread
+        std::unique_lock<std::mutex> lock(mutex_);
+        decode_jobs_queue_.emplace(std::move(func));
+        cond_var_.notify_one();
+    }
+
+protected:
+    void ThreadEntry(int i)
+    {
+        (void) i;
+        std::function<void()> execute_decode_job;
+
+        while(true)
+        {
+            {
+                std::unique_lock<std::mutex> lock(mutex_);
+                cond_var_.wait(lock, [&] { return shutdown_ || !decode_jobs_queue_.empty(); });
+                if(decode_jobs_queue_.empty())
+                {
+                    // No jobs to do; shutting down
+                    return;
+                }
+
+                execute_decode_job = std::move(decode_jobs_queue_.front());
+                decode_jobs_queue_.pop();
+            }
+
+            // Execute the decode job without holding any locks
+            execute_decode_job();
+        }
+    }
+
+    std::mutex                        mutex_;
+    std::condition_variable           cond_var_;
+    bool                              shutdown_;
+    std::queue<std::function<void()>> decode_jobs_queue_;
+    std::vector<std::thread>          threads_;
+};
+#endif  // ROC_JPEG_SAMPLES_COMMON

projects/rocprofiler-sdk/tests/bin/scratch-memory/CMakeLists.txt

@@ -0,0 +1,43 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-tool-test-app-scratch-memory LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(scratch-memory.cpp PROPERTIES LANGUAGE HIP)
+add_executable(scratch-memory)
+target_sources(scratch-memory PRIVATE scratch-memory.cpp)
+target_compile_options(scratch-memory PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow
+                                              -Werror)
+
+find_package(Threads REQUIRED)
+target_link_libraries(scratch-memory PRIVATE Threads::Threads hsa-runtime64
+                                             rocprofiler-sdk::tests-common-library)

projects/rocprofiler-sdk/tests/bin/scratch-memory/scratch-memory.cpp

@@ -0,0 +1,246 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include <hip/hip_runtime.h>
+#include <hsa/hsa.h>
+#include <hsa/hsa_ext_amd.h>
+#include <cstdio>
+#include <iomanip>
+#include <iostream>
+#include <numeric>
+#include <vector>
+
+#include "common/defines.hpp"
+
+#define hipCheckErr(errval)                                                                        \
+    do                                                                                             \
+    {                                                                                              \
+        hipCheckAndFail((errval), __FILE__, __LINE__);                                             \
+    } while(0)
+
+#define hipCheckLastError()                                                                        \
+    do                                                                                             \
+    {                                                                                              \
+        hipCheckErr(hipGetLastError());                                                            \
+    } while(0)
+
+#define HSA_CALL2(cmd)                                                                             \
+    do                                                                                             \
+    {                                                                                              \
+        hsa_status_t error = (cmd);                                                                \
+        if(error != HSA_STATUS_SUCCESS)                                                            \
+        {                                                                                          \
+            const char* errorStr;                                                                  \
+            hsa_status_string(error, &errorStr);                                                   \
+            std::cout << "Encountered HSA error (" << errorStr << ") at line " << __LINE__         \
+                      << " in file " << __FILE__ << "\n";                                          \
+            exit(-1);                                                                              \
+        }                                                                                          \
+    } while(0)
+
+namespace
+{
+inline void
+hipCheckAndFail(hipError_t errval, const char* file, int line)
+{
+    if(errval != hipSuccess)
+    {
+        std::cerr << "hip error: " << hipGetErrorString(errval) << std::endl;
+        std::cerr << "    Location: " << file << ":" << line << std::endl;
+        exit(errval);
+    }
+}
+
+hsa_status_t
+find_gpu_agents(hsa_agent_t agent, void* data)
+{
+    hsa_status_t      status;
+    hsa_device_type_t device_type;
+    status = hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &device_type);
+    if(status == HSA_STATUS_SUCCESS && device_type == HSA_DEVICE_TYPE_GPU)
+    {
+        std::vector<hsa_agent_t>* agents = reinterpret_cast<std::vector<hsa_agent_t>*>(data);
+        agents->push_back(agent);
+    }
+    return HSA_STATUS_SUCCESS;
+}
+}  // namespace
+
+__global__ void
+test_kern_large(uint64_t* output)
+{
+    uint64_t result = 0;
+    int      test[4000];
+    memset(test, 5, 4000);
+    for(int& i : test)
+    {
+        i = i + 7;
+        *output += i;
+        result += i;
+    }
+    *output ^= result;
+    *output ^= result;
+}
+
+__global__ void
+test_kern_medium(uint64_t* output)
+{
+    uint64_t result = 0;
+    int      test[175];
+    memset(test, 5, 175);
+    for(int& i : test)
+    {
+        i = i + 7;
+        *output += i;
+        result += i;
+    }
+    *output ^= result;
+    *output ^= result;
+}
+
+__global__ void
+test_kern_small(uint64_t* output)
+{
+    uint64_t result = 0;
+    int      test[2];
+    for(int& i : test)
+    {
+        i = i + 7;
+        *output += i;
+        result += i;
+    }
+    *output ^= result;
+    *output ^= result;
+}
+
+// Checks whether we get a request-more-scratch when grid-x is incremented
+int
+test_gridx(uint64_t* data_ptr)
+{
+    *data_ptr = 0;
+    printf("Running Medium\n");
+    test_kern_medium<<<1000, 1>>>(data_ptr);
+    hipCheckErr(hipDeviceSynchronize());
+    printf("Running Medium - done\n");
+
+    printf("Running Medium-2 - should trigger more-scratch requests\n");
+    test_kern_medium<<<1500, 1>>>(data_ptr);
+    hipCheckErr(hipDeviceSynchronize());
+
+    printf("Running Medium-2 - done\n");
+    return 0;
+}
+
+// 1st allocation should go to primary, then large should still trigger a USO
+int
+test_primary_then_uso(uint64_t* data_ptr)
+{
+    printf("Running Medium - all slots\n");
+    test_kern_medium<<<10000, 1>>>(data_ptr);
+    hipCheckErr(hipDeviceSynchronize());
+    printf("Running Medium - done\n");
+
+    printf("Running Large - should trigger USO\n");
+    test_kern_large<<<1100, 1>>>(data_ptr);
+    hipCheckErr(hipDeviceSynchronize());
+    printf("Running Large - done\n");
+    return 0;
+}
+
+int
+test_scratch()
+{
+    uint64_t* data_ptr = nullptr;
+    hipCheckErr(HIP_HOST_ALLOC_FUNC(&data_ptr, sizeof(uint64_t), 0));
+
+    auto   host_floats = std::vector<float>(1024, 0.0f);
+    float* dev         = nullptr;
+
+    std::iota(host_floats.begin(), host_floats.end(), 1.0f);
+
+    hipCheckErr(hipMalloc((void**) &dev, host_floats.size() * sizeof(float)));
+    hipCheckErr(hipMemcpy(
+        dev, host_floats.data(), host_floats.size() * sizeof(float), hipMemcpyHostToDevice));
+
+    *data_ptr = 0;
+
+    printf("Running test_primary_then_uso========================\n");
+    test_primary_then_uso(data_ptr);
+    printf("=====================================================\n");
+
+    printf("Running test_gridx===================================\n");
+    test_gridx(data_ptr);
+    printf("=====================================================\n");
+
+    printf("Running Small\n");
+    test_kern_small<<<1000, 1>>>(data_ptr);
+    hipCheckErr(hipDeviceSynchronize());
+    printf("Running Small - done\n");
+
+    printf("Running Medium\n");
+    test_kern_medium<<<1000, 1>>>(data_ptr);
+    hipCheckErr(hipDeviceSynchronize());
+    printf("Running Medium - done\n");
+
+    printf("Running Small\n");
+    test_kern_small<<<1000, 1>>>(data_ptr);
+    hipCheckErr(hipDeviceSynchronize());
+    printf("Running Small - done\n");
+
+    printf("Running Large\n");
+    test_kern_large<<<1100, 1>>>(data_ptr);
+    hipCheckErr(hipDeviceSynchronize());
+    printf("Running Large - done\n");
+
+    printf("Running Large\n");
+    test_kern_large<<<1000, 1>>>(data_ptr);
+    hipCheckErr(hipDeviceSynchronize());
+    printf("Running Large - done\n");
+
+    printf("Running Large\n");
+    test_kern_large<<<1000, 1>>>(data_ptr);
+    hipCheckErr(hipFree(dev));
+    hipCheckErr(hipDeviceSynchronize());
+    printf("Running Large - done\n");
+
+    return 0;
+}
+
+int
+main()
+{
+    hipCheckErr(hipInit(0));
+
+    std::vector<hsa_agent_t> agents;
+    HSA_CALL2(hsa_iterate_agents(find_gpu_agents, &agents));
+    size_t numAgents = agents.size();
+    printf("Detected %ld agents\n", numAgents);
+
+    for(size_t i = 0; i < agents.size(); ++i)
+    {
+        printf("Testing scratch on device %zu\n", i);
+        hipCheckErr(hipSetDevice(i));
+        test_scratch();
+    }
+
+    return 0;
+}

projects/rocprofiler-sdk/tests/bin/simple-transpose/CMakeLists.txt

@@ -0,0 +1,46 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-bin-simple-transpose LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(simple-transpose.cpp PROPERTIES LANGUAGE HIP)
+add_executable(simple-transpose)
+target_sources(simple-transpose PRIVATE simple-transpose.cpp)
+target_compile_options(simple-transpose PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow
+                                                -Werror)
+
+find_package(Threads REQUIRED)
+target_link_libraries(simple-transpose PRIVATE Threads::Threads)
+
+find_package(rocprofiler-sdk-roctx REQUIRED)
+target_link_libraries(simple-transpose
+                      PRIVATE rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)

projects/rocprofiler-sdk/tests/bin/simple-transpose/simple-transpose.cpp

@@ -0,0 +1,179 @@
+/*
+Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include <iostream>
+#include <mutex>
+
+// hip header file
+#include <hip/hip_runtime.h>
+
+// ROCTx header file
+#include <rocprofiler-sdk-roctx/roctx.h>
+
+#define WIDTH 1024
+
+#define NUM (WIDTH * WIDTH)
+
+#define THREADS_PER_BLOCK_X 4
+#define THREADS_PER_BLOCK_Y 4
+#define THREADS_PER_BLOCK_Z 1
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t = std::unique_lock<std::mutex>;
+auto print_lock   = std::mutex{};
+}  // namespace
+
+// Device (Kernel) function, it must be void
+__global__ void
+matrixTranspose(float* out, float* in, const int width)
+{
+    int x = blockDim.x * blockIdx.x + threadIdx.x;
+    int y = blockDim.y * blockIdx.y + threadIdx.y;
+
+    out[y * width + x] = in[x * width + y];
+}
+
+// CPU implementation of matrix transpose
+void
+matrixTransposeCPUReference(float* output, float* input, const unsigned int width)
+{
+    for(unsigned int j = 0; j < width; j++)
+    {
+        for(unsigned int i = 0; i < width; i++)
+        {
+            output[i * width + j] = input[j * width + i];
+        }
+    }
+}
+
+int
+main()
+{
+    roctxRangePush("main");
+
+    float* Matrix;
+    float* TransposeMatrix;
+    float* cpuTransposeMatrix;
+
+    float* gpuMatrix;
+    float* gpuTransposeMatrix;
+
+    hipDeviceProp_t devProp;
+    HIP_API_CALL(hipGetDeviceProperties(&devProp, 0));
+
+    std::cout << "Device name " << devProp.name << std::endl;
+
+    int i;
+    int errors;
+
+    Matrix             = (float*) malloc(NUM * sizeof(float));
+    TransposeMatrix    = (float*) malloc(NUM * sizeof(float));
+    cpuTransposeMatrix = (float*) malloc(NUM * sizeof(float));
+
+    // initialize the input data
+    for(i = 0; i < NUM; i++)
+    {
+        Matrix[i] = (float) i * 10.0f;
+    }
+
+    // allocate the memory on the device side
+    HIP_API_CALL(hipMalloc((void**) &gpuMatrix, NUM * sizeof(float)));
+    HIP_API_CALL(hipMalloc((void**) &gpuTransposeMatrix, NUM * sizeof(float)));
+
+    // Memory transfer from host to device
+    HIP_API_CALL(hipMemcpy(gpuMatrix, Matrix, NUM * sizeof(float), hipMemcpyHostToDevice));
+
+    auto tid = roctx_thread_id_t{};
+    roctxGetThreadId(&tid);
+    roctxProfilerPause(tid);
+    // Memory transfer that should be hidden by profiling tool
+    HIP_API_CALL(
+        hipMemcpy(gpuTransposeMatrix, gpuMatrix, NUM * sizeof(float), hipMemcpyDeviceToDevice));
+    roctxProfilerResume(tid);
+
+    roctxMark("pre-kernel-launch");
+    // Lauching kernel from host
+    hipLaunchKernelGGL(matrixTranspose,
+                       dim3(WIDTH / THREADS_PER_BLOCK_X, WIDTH / THREADS_PER_BLOCK_Y),
+                       dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y),
+                       0,
+                       0,
+                       gpuTransposeMatrix,
+                       gpuMatrix,
+                       WIDTH);
+    roctxMark("post-kernel-launch");
+
+    // Memory transfer from device to host
+    HIP_API_CALL(
+        hipMemcpy(TransposeMatrix, gpuTransposeMatrix, NUM * sizeof(float), hipMemcpyDeviceToHost));
+
+    // CPU MatrixTranspose computation
+    matrixTransposeCPUReference(cpuTransposeMatrix, Matrix, WIDTH);
+
+    // verify the results
+    errors     = 0;
+    double eps = 1.0E-6;
+    for(i = 0; i < NUM; i++)
+    {
+        if(std::abs(TransposeMatrix[i] - cpuTransposeMatrix[i]) > eps)
+        {
+            errors++;
+        }
+    }
+    if(errors != 0)
+    {
+        printf("FAILED: %d errors\n", errors);
+    }
+    else
+    {
+        printf("PASSED!\n");
+    }
+
+    // free the resources on device side
+    HIP_API_CALL(hipFree(gpuMatrix));
+    HIP_API_CALL(hipFree(gpuTransposeMatrix));
+
+    // free the resources on host side
+    free(Matrix);
+    free(TransposeMatrix);
+    free(cpuTransposeMatrix);
+
+    roctxRangePop();
+
+    return errors;
+}

projects/rocprofiler-sdk/tests/bin/transpose/CMakeLists.txt

@@ -0,0 +1,100 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-bin-transpose LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+option(TRANSPOSE_USE_MPI "Enable MPI support in transpose exe" OFF)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(transpose.cpp PROPERTIES LANGUAGE HIP)
+
+function(transpose_build_target _NAME _COMPILE_DEFS _LINK_TARGETS)
+    add_executable(${_NAME})
+    target_sources(${_NAME} PRIVATE transpose.cpp)
+    target_compile_definitions(${_NAME} PRIVATE ${_COMPILE_DEFS})
+    target_compile_options(${_NAME} PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow -Werror)
+    target_link_libraries(
+        ${_NAME} PRIVATE Threads::Threads rocprofiler-sdk::tests-build-flags
+                         ${_LINK_TARGETS})
+
+    if(TRANSPOSE_USE_MPI)
+        find_package(MPI REQUIRED)
+        target_compile_definitions(${_NAME} PRIVATE USE_MPI=1)
+        target_link_libraries(${_NAME} PRIVATE MPI::MPI_C)
+    endif()
+
+endfunction()
+
+find_package(Threads REQUIRED)
+find_package(rocprofiler-sdk-roctx REQUIRED)
+
+transpose_build_target(transpose "" rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
+
+#
+# using old roctracer roctx
+#
+find_path(
+    roctracer_roctx_ROOT_DIR
+    NAMES include/roctracer/roctx.h
+          lib/${CMAKE_SHARED_LIBRARY_PREFIX}roctx64${CMAKE_SHARED_LIBRARY_SUFFIX}
+    HINTS ${hip_DIR} ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+    PATHS ${hip_DIR} ${ROCM_PATH} ENV ROCM_PATH /opt/rocm)
+
+find_path(
+    roctracer_roctx_INCLUDE_DIR
+    NAMES roctracer/roctx.h
+    HINTS ${roctracer_roctx_ROOT_DIR}
+    PATHS ${roctracer_roctx_ROOT_DIR}
+    PATH_SUFFIXES include)
+
+find_library(
+    roctracer_roctx_LIBRARY
+    NAMES roctx64
+    HINTS ${roctracer_roctx_ROOT_DIR}
+    PATHS ${roctracer_roctx_ROOT_DIR}
+    PATH_SUFFIXES lib lib64)
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(roctracer-roctx DEFAULT_MSG roctracer_roctx_ROOT_DIR
+                                  roctracer_roctx_INCLUDE_DIR roctracer_roctx_LIBRARY)
+
+if(roctracer-roctx_FOUND)
+    add_library(roctracer-roctx INTERFACE IMPORTED)
+    add_library(roctracer-roctx::roctracer-roctx ALIAS roctracer-roctx)
+    target_include_directories(roctracer-roctx INTERFACE ${roctracer_roctx_INCLUDE_DIR})
+    target_link_libraries(roctracer-roctx INTERFACE ${roctracer_roctx_LIBRARY})
+endif()
+
+if(TARGET roctracer-roctx::roctracer-roctx)
+    transpose_build_target(transpose-roctracer-roctx "USE_ROCTRACER_ROCTX=1"
+                           roctracer-roctx::roctracer-roctx)
+endif()

projects/rocprofiler-sdk/tests/bin/transpose/transpose.cpp

@@ -0,0 +1,339 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#if defined(USE_ROCTRACER_ROCTX)
+#    include <roctracer/roctx.h>
+#else
+#    include <rocprofiler-sdk-roctx/roctx.h>
+#endif
+
+#include <hip/hip_runtime.h>
+
+#if defined(USE_MPI)
+#    include <mpi.h>
+#endif
+
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <iomanip>
+#include <iostream>
+#include <mutex>
+#include <random>
+#include <sstream>
+#include <stdexcept>
+#include <thread>
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t                      = std::unique_lock<std::mutex>;
+auto               print_lock          = std::mutex{};
+size_t             nthreads            = 2;
+size_t             nitr                = 500;
+size_t             nsync               = 10;
+constexpr unsigned shared_mem_tile_dim = 32;
+
+void
+check_hip_error(void);
+
+void
+verify(int* in, int* out, int M, int N);
+}  // namespace
+
+__global__ void
+transpose(const int* in, int* out, int M, int N);
+
+void
+run(int rank, int tid, int ndevice, int argc, char** argv);
+
+int
+main(int argc, char** argv)
+{
+    int rank = 0;
+    int size = 1;
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{argv[i]};
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            fprintf(stderr,
+                    "usage: transpose [NUM_THREADS (%zu)] [NUM_ITERATION (%zu)] "
+                    "[SYNC_EVERY_N_ITERATIONS (%zu)]\n",
+                    nthreads,
+                    nitr,
+                    nsync);
+            exit(EXIT_SUCCESS);
+        }
+    }
+    if(argc > 1) nthreads = atoll(argv[1]);
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    printf("[transpose] Number of threads: %zu\n", nthreads);
+    printf("[transpose] Number of iterations: %zu\n", nitr);
+    printf("[transpose] Syncing every %zu iterations\n", nsync);
+
+#if defined(USE_ROCTRACER_ROCTX)
+    {
+        auto _roctracer_roctx_ss = std::stringstream{};
+        _roctracer_roctx_ss << "roctracer/roctx v" << roctx_version_major() << "."
+                            << roctx_version_minor();
+        roctxMark(_roctracer_roctx_ss.str().c_str());
+    }
+#endif
+
+#if defined(USE_MPI)
+    MPI_Init(&argc, &argv);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+#else
+    (void) size;
+#endif
+    // this is a temporary workaround in omnitrace when HIP + MPI is enabled
+    int ndevice = 0;
+    HIP_API_CALL(hipGetDeviceCount(&ndevice));
+    printf("[transpose] Number of devices found: %i\n", ndevice);
+    auto devids = std::vector<int>{};
+    devids.resize(size * nthreads, 0);
+    int devid = 0;
+    for(size_t i = 0; i < nthreads; ++i)
+    {
+        for(int j = 0; j < size; ++j)
+        {
+            auto idx       = (j * nthreads) + i;
+            devids.at(idx) = devid++ % ndevice;
+        }
+    }
+    auto devid_offset = (rank * nthreads);
+    auto _threads     = std::vector<std::thread>{};
+    for(size_t i = 1; i < nthreads; ++i)
+        _threads.emplace_back(run, rank, i, devids.at(devid_offset + i), argc, argv);
+    run(rank, 0, devids.at(devid_offset + 0), argc, argv);
+    for(auto& itr : _threads)
+        itr.join();
+
+#if defined(USE_MPI)
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    for(int i = 0; i < ndevice; ++i)
+    {
+        HIP_API_CALL(hipSetDevice(i));
+        HIP_API_CALL(hipDeviceSynchronize());
+    }
+
+#if defined(USE_MPI)
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    if(rank == 0)
+    {
+        for(int i = 0; i < ndevice; ++i)
+        {
+            HIP_API_CALL(hipSetDevice(i));
+            HIP_API_CALL(hipDeviceReset());
+        }
+    }
+
+#if defined(USE_MPI)
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    return 0;
+}
+
+__global__ void
+transpose(const int* in, int* out, int M, int N)
+{
+    __shared__ int tile[shared_mem_tile_dim][shared_mem_tile_dim];
+
+    int idx = (blockIdx.y * blockDim.y + threadIdx.y) * M + blockIdx.x * blockDim.x + threadIdx.x;
+    tile[threadIdx.y][threadIdx.x] = in[idx];
+    __syncthreads();
+    idx      = (blockIdx.x * blockDim.x + threadIdx.y) * N + blockIdx.y * blockDim.y + threadIdx.x;
+    out[idx] = tile[threadIdx.x][threadIdx.y];
+}
+
+void
+run(int rank, int tid, int devid, int argc, char** argv)
+{
+    auto roctx_run_id = roctxRangeStart("run");
+
+    const auto mark = [rank, tid, devid](std::string_view suffix) {
+        auto _ss = std::stringstream{};
+        _ss << "run/rank-" << rank << "/thread-" << tid << "/device-" << devid << "/" << suffix;
+        roctxMark(_ss.str().c_str());
+    };
+
+    mark("begin");
+
+    constexpr unsigned int M = 4960 * 2;
+    constexpr unsigned int N = 4960 * 2;
+
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    hipStream_t stream = {};
+
+    printf("[transpose] Rank %i, thread %i assigned to device %i\n", rank, tid, devid);
+    HIP_API_CALL(hipSetDevice(devid));
+    HIP_API_CALL(hipStreamCreate(&stream));
+
+    auto_lock_t _lk{print_lock};
+    std::cout << "[transpose][" << rank << "][" << tid << "] M: " << M << " N: " << N << std::endl;
+    _lk.unlock();
+
+    std::default_random_engine         _engine{std::random_device{}() * (rank + 1) * (tid + 1)};
+    std::uniform_int_distribution<int> _dist{0, 1000};
+
+    size_t size       = sizeof(int) * M * N;
+    int*   inp_matrix = new int[size];
+    int*   out_matrix = new int[size];
+    for(size_t i = 0; i < M * N; i++)
+    {
+        inp_matrix[i] = _dist(_engine);
+        out_matrix[i] = 0;
+    }
+    int* in  = nullptr;
+    int* out = nullptr;
+
+    // lock during malloc to get more accurate memory info
+    {
+        _lk.lock();
+        constexpr auto MiB           = (1024UL * 1024UL);
+        size_t         free_gpu_mem  = 0;
+        size_t         total_gpu_mem = 0;
+
+        HIP_API_CALL(hipMemGetInfo(&free_gpu_mem, &total_gpu_mem));
+        free_gpu_mem /= MiB;
+        total_gpu_mem /= MiB;
+
+        std::cout << "[transpose][" << rank << "][" << tid
+                  << "] Available GPU memory (MiB): " << std::setw(6) << free_gpu_mem << " / "
+                  << std::setw(6) << total_gpu_mem << std::endl;
+
+        HIP_API_CALL(hipMallocAsync(&in, size, stream));
+        HIP_API_CALL(hipMallocAsync(&out, size, stream));
+
+        _lk.unlock();
+    }
+
+    HIP_API_CALL(hipMemsetAsync(in, 0, size, stream));
+    HIP_API_CALL(hipMemsetAsync(out, 0, size, stream));
+    HIP_API_CALL(hipMemcpyAsync(in, inp_matrix, size, hipMemcpyHostToDevice, stream));
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    dim3 grid(M / 32, N / 32, 1);
+    dim3 block(32, 32, 1);  // transpose
+
+    auto t1 = std::chrono::high_resolution_clock::now();
+    for(size_t i = 0; i < nitr; ++i)
+    {
+        roctxRangePush("run/iteration");
+        transpose<<<grid, block, 0, stream>>>(in, out, M, N);
+        check_hip_error();
+        if(i % nsync == (nsync - 1))
+        {
+            roctxRangePush("run/iteration/sync");
+            HIP_API_CALL(hipStreamSynchronize(stream));
+            roctxRangePop();
+        }
+        roctxRangePop();
+    }
+    auto t2 = std::chrono::high_resolution_clock::now();
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipMemcpyAsync(out_matrix, out, size, hipMemcpyDeviceToHost, stream));
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+    float  GB   = (float) size * nitr * 2 / (1 << 30);
+
+    print_lock.lock();
+    std::cout << "[transpose][" << rank << "][" << tid << "] Runtime of transpose is " << time
+              << " sec\n";
+    std::cout << "[transpose][" << rank << "][" << tid
+              << "] The average performance of transpose is " << GB / time << " GBytes/sec"
+              << std::endl;
+    print_lock.unlock();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    // cpu_transpose(matrix, out_matrix, M, N);
+    verify(inp_matrix, out_matrix, M, N);
+
+    HIP_API_CALL(hipFreeAsync(in, stream));
+    HIP_API_CALL(hipFreeAsync(out, stream));
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipStreamDestroy(stream));
+
+    delete[] inp_matrix;
+    delete[] out_matrix;
+
+    mark("end");
+
+    roctxRangeStop(roctx_run_id);
+}
+
+namespace
+{
+void
+check_hip_error(void)
+{
+    hipError_t err = hipGetLastError();
+    if(err != hipSuccess)
+    {
+        auto_lock_t _lk{print_lock};
+        std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
+        throw std::runtime_error("hip_api_call");
+    }
+}
+
+void
+verify(int* in, int* out, int M, int N)
+{
+    for(int i = 0; i < 10; i++)
+    {
+        int row = rand() % M;
+        int col = rand() % N;
+        if(in[row * N + col] != out[col * M + row])
+        {
+            auto_lock_t _lk{print_lock};
+            std::cout << "mismatch: " << row << ", " << col << " : " << in[row * N + col] << " | "
+                      << out[col * M + row] << "\n";
+        }
+    }
+}
+}  // namespace

projects/rocprofiler-sdk/tests/bin/vector-operations/CMakeLists.txt

@@ -0,0 +1,42 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-bin-vector-operations LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(vector-ops.cpp PROPERTIES LANGUAGE HIP)
+add_executable(vector-ops)
+target_sources(vector-ops PRIVATE vector-ops.cpp)
+target_compile_options(vector-ops PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow -Werror)
+
+find_package(Threads REQUIRED)
+target_link_libraries(vector-ops PRIVATE Threads::Threads
+                                         rocprofiler-sdk::tests-common-library)

projects/rocprofiler-sdk/tests/bin/vector-operations/vector-ops.cpp

@@ -0,0 +1,293 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include <assert.h>
+#include <hip/hip_runtime.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <algorithm>
+#include <csignal>
+#include <iostream>
+#include <mutex>
+#include <vector>
+
+#include "common/defines.hpp"
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t = std::unique_lock<std::mutex>;
+auto print_lock   = std::mutex{};
+}  // namespace
+
+#define WIDTH  (1024)
+#define HEIGHT (1024)
+
+#define NUM (WIDTH * HEIGHT)
+
+#define THREADS_PER_BLOCK_X 64
+#define THREADS_PER_BLOCK_Y 1
+#define THREADS_PER_BLOCK_Z 1
+
+// Computes vectorAdd with matrix-multiply
+template <typename T>
+__global__ void
+addition_kernel(T* __restrict__ a,
+                const float* __restrict__ b,
+                const float* __restrict__ c,
+                int                  width,
+                [[maybe_unused]] int height)
+{
+    // printf("addition kernel\n");
+    int x = blockDim.x * blockIdx.x + threadIdx.x;
+    int y = blockDim.y * blockIdx.y + threadIdx.y;
+
+    if(x >= WIDTH || y >= HEIGHT) return;
+    int index = y * width + x;
+
+    a[index] = b[index] + c[index];
+}
+
+__global__ void
+subtract_kernel(float* __restrict__ a,
+                const float* __restrict__ b,
+                const float* __restrict__ c,
+                int                  width,
+                [[maybe_unused]] int height)
+{
+    // printf("subtract kernel\n");
+    int x = blockDim.x * blockIdx.x + threadIdx.x;
+    int y = blockDim.y * blockIdx.y + threadIdx.y;
+
+    if(x >= WIDTH || y >= HEIGHT) return;
+    int index = y * width + x;
+
+    a[index] = abs(b[index] - c[index]);
+}
+
+__global__ void
+multiply_kernel(float* __restrict__ a,
+                const float* __restrict__ b,
+                const float* __restrict__ c,
+                int                  width,
+                [[maybe_unused]] int height)
+{
+    // printf("multiply kernel\n");
+    int x = blockDim.x * blockIdx.x + threadIdx.x;
+    int y = blockDim.y * blockIdx.y + threadIdx.y;
+
+    if(x >= WIDTH || y >= HEIGHT) return;
+    int index = y * width + x;
+
+    a[index] = (b[index] - 1) * (c[index] - 1) + 1;
+}
+
+__global__ void
+divide_kernel(float* __restrict__ a,
+              const float* __restrict__ b,
+              const float* __restrict__ c,
+              int                  width,
+              [[maybe_unused]] int height)
+{
+    // printf("divide kernel\n");
+    int x = blockDim.x * blockIdx.x + threadIdx.x;
+    int y = blockDim.y * blockIdx.y + threadIdx.y;
+
+    if(x >= WIDTH || y >= HEIGHT) return;
+    int index = y * width + x;
+
+    a[index] = (b[index] - c[index]) / abs(c[index] + b[index]) + 1;
+}
+
+using namespace std;
+
+void
+run(int NUM_QUEUE, int DEVICE_ID)
+{
+    HIP_API_CALL(hipSetDevice(DEVICE_ID));
+
+    HIP_API_CALL(hipDeviceSynchronize());
+
+    std::vector<float*> hostA(NUM_QUEUE);
+    std::vector<float*> hostB(NUM_QUEUE);
+    std::vector<float*> hostC(NUM_QUEUE);
+
+    std::vector<float*> deviceA(NUM_QUEUE);
+    std::vector<float*> deviceB(NUM_QUEUE);
+    std::vector<float*> deviceC(NUM_QUEUE);
+
+    std::vector<hipStream_t> streams(NUM_QUEUE);
+
+    auto sync_stream = [NUM_QUEUE, &streams](int q) {
+        if(q < 0 || q >= NUM_QUEUE)
+            throw std::runtime_error{std::string{"invalid stream id: "} + std::to_string(q)};
+
+        HIP_API_CALL(hipStreamSynchronize(streams.at(q)));
+    };
+
+    auto sync_streams = [NUM_QUEUE, sync_stream]() {
+        for(int i = 0; i < NUM_QUEUE; ++i)
+            sync_stream(i);
+        HIP_API_CALL(hipDeviceSynchronize());
+    };
+
+    for(int q = 0; q < NUM_QUEUE; q++)
+    {
+        HIP_API_CALL(hipStreamCreateWithFlags(&streams[q], hipStreamNonBlocking));
+
+        HIP_API_CALL(HIP_HOST_ALLOC_FUNC(&hostA[q], NUM * sizeof(float), 0));
+        HIP_API_CALL(HIP_HOST_ALLOC_FUNC(&hostB[q], NUM * sizeof(float), 0));
+        HIP_API_CALL(HIP_HOST_ALLOC_FUNC(&hostC[q], NUM * sizeof(float), 0));
+
+        // initialize the input data
+        for(int i = 0; i < NUM; i++)
+        {
+            hostB[q][i] = static_cast<float>(i);
+            hostC[q][i] = static_cast<float>(i * 100.0f);
+        }
+
+        HIP_API_CALL(hipMallocAsync(&deviceA[q], NUM * sizeof(float), streams[q]));
+        HIP_API_CALL(hipMallocAsync(&deviceB[q], NUM * sizeof(float), streams[q]));
+        HIP_API_CALL(hipMallocAsync(&deviceC[q], NUM * sizeof(float), streams[q]));
+
+        HIP_API_CALL(hipMemcpyAsync(
+            deviceB[q], hostB[q], NUM * sizeof(float), hipMemcpyHostToDevice, streams[q]));
+        HIP_API_CALL(hipMemcpyAsync(
+            deviceC[q], hostC[q], NUM * sizeof(float), hipMemcpyHostToDevice, streams[q]));
+    }
+
+    sync_streams();
+
+    for(int q = 0; q < NUM_QUEUE; q++)
+    {
+        hipLaunchKernelGGL(addition_kernel,
+                           dim3(WIDTH / THREADS_PER_BLOCK_X, HEIGHT / THREADS_PER_BLOCK_Y),
+                           dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y),
+                           0,
+                           streams[q],
+                           deviceA[q],
+                           deviceB[q],
+                           deviceC[q],
+                           WIDTH,
+                           HEIGHT);
+
+        HIP_API_CALL(hipGetLastError());
+
+        hipLaunchKernelGGL(subtract_kernel,
+                           dim3(WIDTH / THREADS_PER_BLOCK_X, HEIGHT / THREADS_PER_BLOCK_Y),
+                           dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y),
+                           0,
+                           streams[q],
+                           deviceA[q],
+                           deviceB[q],
+                           deviceC[q],
+                           WIDTH,
+                           HEIGHT);
+
+        HIP_API_CALL(hipGetLastError());
+
+        if(getenv("ROCPROF_TESTING_RAISE_SIGNAL") != nullptr &&
+           std::stoi(getenv("ROCPROF_TESTING_RAISE_SIGNAL")) > 0)
+        {
+            ::raise(SIGINT);
+        }
+
+        hipLaunchKernelGGL(multiply_kernel,
+                           dim3(WIDTH / THREADS_PER_BLOCK_X, HEIGHT / THREADS_PER_BLOCK_Y),
+                           dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y),
+                           0,
+                           streams[q],
+                           deviceA[q],
+                           deviceB[q],
+                           deviceC[q],
+                           WIDTH,
+                           HEIGHT);
+
+        HIP_API_CALL(hipGetLastError());
+
+        hipLaunchKernelGGL(divide_kernel,
+                           dim3(WIDTH / THREADS_PER_BLOCK_X, HEIGHT / THREADS_PER_BLOCK_Y),
+                           dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y),
+                           0,
+                           streams[q],
+                           deviceB[q],
+                           deviceA[q],
+                           deviceC[q],
+                           WIDTH,
+                           HEIGHT);
+
+        HIP_API_CALL(hipGetLastError());
+    }
+
+    sync_streams();
+
+    for(int q = 0; q < NUM_QUEUE; q++)
+    {
+        HIP_API_CALL(hipMemcpyAsync(
+            hostA[q], deviceA[q], NUM * sizeof(float), hipMemcpyDeviceToHost, streams[q]));
+
+        sync_stream(q);
+
+        HIP_API_CALL(hipFree(deviceA[q]));
+        HIP_API_CALL(hipFree(deviceB[q]));
+        HIP_API_CALL(hipFree(deviceC[q]));
+
+        HIP_API_CALL(HIP_HOST_FREE_FUNC(hostA[q]));
+        HIP_API_CALL(HIP_HOST_FREE_FUNC(hostB[q]));
+        HIP_API_CALL(HIP_HOST_FREE_FUNC(hostC[q]));
+
+        HIP_API_CALL(hipStreamDestroy(streams[q]));
+    }
+
+    HIP_API_CALL(hipDeviceSynchronize());
+}
+
+int
+main(int argc, char** argv)
+{
+    int stream_count = 8;
+    int device_count = 0;
+    HIP_API_CALL(hipGetDeviceCount(&device_count));
+
+    if(argc > 1) stream_count = std::stoi(argv[1]);
+    if(argc > 2) device_count = std::stoi(argv[2]);
+
+    for(int i = 0; i < device_count; ++i)
+        run(stream_count, i);
+
+    return 0;
+}

projects/rocprofiler-sdk/tests/c-tool/CMakeLists.txt

@@ -0,0 +1,64 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(
+    rocprofiler-sdk-tests-c-tool
+    LANGUAGES CXX
+    VERSION 0.0.0)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+if(ROCPROFILER_MEMCHECK_PRELOAD_ENV)
+    set(PRELOAD_ENV
+        "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}:$<TARGET_FILE:rocprofiler-sdk-c-tool>")
+else()
+    set(PRELOAD_ENV "LD_PRELOAD=$<TARGET_FILE:rocprofiler-sdk-c-tool>")
+endif()
+
+add_test(NAME test-c-tool-execute COMMAND $<TARGET_FILE:transpose> 1)
+
+set(c-tool-env
+    "${PRELOAD_ENV}"
+    "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:rocprofiler-sdk::rocprofiler-sdk-shared-library>:$ENV{LD_LIBRARY_PATH}"
+    )
+
+set_tests_properties(
+    test-c-tool-execute
+    PROPERTIES
+        TIMEOUT
+        45
+        LABELS
+        "integration-tests"
+        ENVIRONMENT
+        "${c-tool-env}"
+        PASS_REGULAR_EXPRESSION
+        "Test C tool \\(priority=0\\) is using rocprofiler-sdk v([0-9]+\\.[0-9]+\\.[0-9]+)"
+        FAIL_REGULAR_EXPRESSION
+        "${ROCPROFILER_DEFAULT_FAIL_REGEX}|Internal thread for rocprofiler-sdk should not be created"
+    )
+
+# this test uses ROCP_TOOL_LIBRARIES instead of LD_PRELOAD
+add_test(NAME test-c-tool-rocp-tool-lib-execute COMMAND $<TARGET_FILE:transpose> 1)
+
+set(c-tool-rocp-tool-lib-env
+    "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}"
+    "ROCP_TOOL_LIBRARIES=$<TARGET_FILE:rocprofiler-sdk-c-tool>"
+    "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:rocprofiler-sdk::rocprofiler-sdk-shared-library>:$ENV{LD_LIBRARY_PATH}"
+    )
+
+set_tests_properties(
+    test-c-tool-rocp-tool-lib-execute
+    PROPERTIES
+        TIMEOUT
+        45
+        LABELS
+        "integration-tests"
+        ENVIRONMENT
+        "${c-tool-rocp-tool-lib-env}"
+        PASS_REGULAR_EXPRESSION
+        "Test C tool \\(priority=0\\) is using rocprofiler-sdk v([0-9]+\\.[0-9]+\\.[0-9]+)"
+        FAIL_REGULAR_EXPRESSION
+        "${ROCPROFILER_DEFAULT_FAIL_REGEX}|Internal thread for rocprofiler-sdk should not be created"
+    )

projects/rocprofiler-sdk/tests/common/CMakeLists.txt

@@ -0,0 +1,182 @@
+#
+# common utilities for tests
+#
+
+include(FetchContent)
+include(CMakeParseArguments)
+
+set(FETCHCONTENT_BASE_DIR ${PROJECT_BINARY_DIR}/external)
+
+# default FAIL_REGULAR_EXPRESSION for tests
+set(ROCPROFILER_DEFAULT_FAIL_REGEX
+    "threw an exception|Permission denied|Could not create logging file|failed with error code|Subprocess aborted"
+    CACHE INTERNAL "Default FAIL_REGULAR_EXPRESSION for tests")
+
+set(DEFAULT_GPU_TARGETS
+    "gfx900"
+    "gfx906"
+    "gfx908"
+    "gfx90a"
+    "gfx942"
+    "gfx950"
+    "gfx1030"
+    "gfx1010"
+    "gfx1100"
+    "gfx1101"
+    "gfx1102")
+
+set(GPU_TARGETS
+    "${DEFAULT_GPU_TARGETS}"
+    CACHE STRING "GPU targets to compile for")
+
+set(AMDGPU_TARGETS
+    "${GPU_TARGETS}"
+    CACHE STRING
+          "GPU targets to compile for AMDGPUs (update GPU_TARGETS, not this variable)"
+          FORCE)
+
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.24)
+    cmake_policy(SET CMP0135 NEW)
+endif()
+
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.30)
+    cmake_policy(SET CMP0167 NEW)
+    cmake_policy(SET CMP0169 OLD)
+endif()
+
+find_package(rocprofiler-sdk REQUIRED)
+
+# rocprofiler-sdk provides a Findlibdw.cmake
+find_package(libdw REQUIRED)
+
+# build flags
+add_library(rocprofiler-sdk-tests-build-flags INTERFACE)
+add_library(rocprofiler-sdk::tests-build-flags ALIAS rocprofiler-sdk-tests-build-flags)
+target_compile_options(rocprofiler-sdk-tests-build-flags INTERFACE -W -Wall -Wextra
+                                                                   -Wshadow)
+target_compile_features(rocprofiler-sdk-tests-build-flags INTERFACE cxx_std_17)
+
+if(ROCPROFILER_BUILD_CI OR ROCPROFILER_BUILD_WERROR)
+    target_compile_options(rocprofiler-sdk-tests-build-flags INTERFACE -Werror)
+endif()
+
+# serialization library
+if(NOT TARGET rocprofiler-sdk::rocprofiler-sdk-cereal)
+    get_filename_component(ROCPROFILER_SOURCE_DIR "${PROJECT_SOURCE_DIR}/.." REALPATH)
+
+    add_library(rocprofiler-sdk-cereal INTERFACE)
+    add_library(rocprofiler-sdk::rocprofiler-sdk-cereal ALIAS rocprofiler-sdk-cereal)
+    target_compile_definitions(rocprofiler-sdk-cereal
+                               INTERFACE $<BUILD_INTERFACE:CEREAL_THREAD_SAFE=1>)
+
+    if(EXISTS ${ROCPROFILER_SOURCE_DIR}/external AND COMMAND
+                                                     rocprofiler_checkout_git_submodule)
+        rocprofiler_checkout_git_submodule(
+            RECURSIVE
+            RELATIVE_PATH external/cereal
+            WORKING_DIRECTORY ${ROCPROFILER_SOURCE_DIR}
+            REPO_URL https://github.com/jrmadsen/cereal.git
+            REPO_BRANCH "rocprofiler")
+
+        target_include_directories(
+            rocprofiler-sdk-cereal SYSTEM
+            INTERFACE $<BUILD_INTERFACE:${ROCPROFILER_SOURCE_DIR}/external/cereal/include>
+            )
+    else()
+        fetchcontent_declare(
+            cereal
+            GIT_REPOSITORY https://github.com/jrmadsen/cereal.git
+            GIT_TAG rocprofiler
+            SOURCE_DIR ${PROJECT_BINARY_DIR}/external/cereal BINARY_DIR
+            ${PROJECT_BINARY_DIR}/external/build/cereal-build SUBBUILD_DIR
+            ${PROJECT_BINARY_DIR}/external/build/cereal-subdir)
+
+        fetchcontent_getproperties(cereal)
+
+        if(NOT cereal_POPULATED)
+            fetchcontent_populate(cereal)
+        endif()
+
+        target_include_directories(
+            rocprofiler-sdk-cereal SYSTEM
+            INTERFACE $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/external/cereal/include>)
+    endif()
+endif()
+
+if(NOT TARGET rocprofiler-sdk::rocprofiler-sdk-perfetto)
+    # perfetto
+    fetchcontent_declare(
+        perfetto
+        GIT_REPOSITORY https://github.com/google/perfetto
+        GIT_TAG v44.0
+        SOURCE_DIR ${PROJECT_BINARY_DIR}/external/perfetto BINARY_DIR
+        ${PROJECT_BINARY_DIR}/external/build/perfetto-build SUBBUILD_DIR
+        ${PROJECT_BINARY_DIR}/external/build/perfetto-subdir)
+
+    fetchcontent_getproperties(perfetto)
+
+    if(NOT perfetto_POPULATED)
+        fetchcontent_populate(perfetto)
+    endif()
+
+    add_library(rocprofiler-sdk-tests-perfetto STATIC)
+    add_library(rocprofiler-sdk::tests-perfetto ALIAS rocprofiler-sdk-tests-perfetto)
+    target_sources(
+        rocprofiler-sdk-tests-perfetto
+        PRIVATE ${PROJECT_BINARY_DIR}/external/perfetto/sdk/perfetto.h
+                ${PROJECT_BINARY_DIR}/external/perfetto/sdk/perfetto.cc)
+    target_include_directories(
+        rocprofiler-sdk-tests-perfetto SYSTEM
+        INTERFACE $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/external/perfetto/sdk>)
+    set_target_properties(rocprofiler-sdk-tests-perfetto
+                          PROPERTIES POSITION_INDEPENDENT_CODE ON)
+else()
+    add_library(rocprofiler-sdk-tests-perfetto INTERFACE)
+    add_library(rocprofiler-sdk::tests-perfetto ALIAS rocprofiler-sdk-tests-perfetto)
+    target_link_libraries(rocprofiler-sdk-tests-perfetto
+                          INTERFACE rocprofiler-sdk::rocprofiler-sdk-perfetto)
+endif()
+
+# common utilities
+cmake_path(GET CMAKE_CURRENT_SOURCE_DIR PARENT_PATH COMMON_LIBRARY_INCLUDE_DIR)
+
+add_library(rocprofiler-sdk-tests-common-library INTERFACE)
+add_library(rocprofiler-sdk::tests-common-library ALIAS
+            rocprofiler-sdk-tests-common-library)
+target_link_libraries(
+    rocprofiler-sdk-tests-common-library
+    INTERFACE rocprofiler-sdk::tests-build-flags rocprofiler-sdk::rocprofiler-sdk-cereal
+              libdw::libdw)
+target_compile_features(rocprofiler-sdk-tests-common-library INTERFACE cxx_std_17)
+target_include_directories(rocprofiler-sdk-tests-common-library
+                           INTERFACE ${COMMON_LIBRARY_INCLUDE_DIR})
+
+set(EXTERNAL_SUBMODULE_DIR "${PROJECT_SOURCE_DIR}/../external")
+cmake_path(ABSOLUTE_PATH EXTERNAL_SUBMODULE_DIR NORMALIZE)
+
+if(EXISTS ${EXTERNAL_SUBMODULE_DIR}/filesystem/include/ghc/filesystem.hpp)
+    target_compile_definitions(
+        rocprofiler-sdk-tests-common-library
+        INTERFACE $<BUILD_INTERFACE:ROCPROFILER_SAMPLES_HAS_GHC_LIB_FILESYSTEM=1>)
+    target_include_directories(
+        rocprofiler-sdk-tests-common-library SYSTEM
+        INTERFACE $<BUILD_INTERFACE:${EXTERNAL_SUBMODULE_DIR}/filesystem/include>)
+endif()
+
+function(rocprofiler_configure_pytest_files)
+    cmake_parse_arguments(RCPF "" "OUTPUT_DIRECTORY" "COPY;CONFIG" ${ARGN})
+
+    if(NOT RCPF_OUTPUT_DIRECTORY)
+        set(RCPF_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+    endif()
+
+    foreach(FILENAME ${RCPF_COPY})
+        configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${FILENAME}
+                       ${RCPF_OUTPUT_DIRECTORY}/${FILENAME} COPYONLY)
+    endforeach()
+
+    foreach(FILENAME ${RCPF_CONFIG})
+        configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${FILENAME}
+                       ${RCPF_OUTPUT_DIRECTORY}/${FILENAME} @ONLY)
+    endforeach()
+endfunction()

projects/rocprofiler-sdk/tests/common/defines.hpp

@@ -0,0 +1,48 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#define ROCPROFILER_CALL(result, msg)                                                              \
+    {                                                                                              \
+        rocprofiler_status_t CHECKSTATUS = result;                                                 \
+        if(CHECKSTATUS != ROCPROFILER_STATUS_SUCCESS)                                              \
+        {                                                                                          \
+            std::string status_name = rocprofiler_get_status_name(CHECKSTATUS);                    \
+            std::string status_msg  = rocprofiler_get_status_string(CHECKSTATUS);                  \
+            std::cerr << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg            \
+                      << " failed with error code " << status_name << " (" << CHECKSTATUS          \
+                      << "): " << status_msg << std::endl;                                         \
+            std::stringstream errmsg{};                                                            \
+            errmsg << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg               \
+                   << " failure (" << status_name << ": " << status_msg << ")";                    \
+            throw std::runtime_error(errmsg.str());                                                \
+        }                                                                                          \
+    }
+
+#if HIP_VERSION >= 60300000
+#    define HIP_HOST_ALLOC_FUNC hipHostMalloc
+#    define HIP_HOST_FREE_FUNC  hipHostFree
+#else
+#    define HIP_HOST_ALLOC_FUNC hipHostMalloc
+#    define HIP_HOST_FREE_FUNC  hipHostFree
+#endif

projects/rocprofiler-sdk/tests/common/filesystem.hpp

@@ -0,0 +1,77 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#if !defined(ROCPROFILER_TESTS_HAS_GHC_LIB_FILESYSTEM)
+#    if defined __has_include
+#        if __has_include(<ghc/filesystem.hpp>)
+#            define ROCPROFILER_TESTS_HAS_GHC_LIB_FILESYSTEM 1
+#        else
+#            define ROCPROFILER_TESTS_HAS_GHC_LIB_FILESYSTEM 0
+#        endif
+#    else
+#        define ROCPROFILER_TESTS_HAS_GHC_LIB_FILESYSTEM 0
+#    endif
+#endif
+
+#if ROCPROFILER_TESTS_HAS_GHC_LIB_FILESYSTEM == 0
+#    if defined __has_include
+#        if __has_include(<version>)
+#            include <version>
+#        endif
+#    endif
+
+#    if defined(__cpp_lib_filesystem)
+#        define ROCPROFILER_TESTS_HAS_CPP_LIB_FILESYSTEM 1
+#    else
+#        if defined __has_include
+#            if __has_include(<filesystem>)
+#                define ROCPROFILER_TESTS_HAS_CPP_LIB_FILESYSTEM 1
+#            endif
+#        endif
+#    endif
+#endif
+
+// include the correct filesystem header
+#if defined(ROCPROFILER_TESTS_HAS_GHC_LIB_FILESYSTEM) &&                                           \
+    ROCPROFILER_TESTS_HAS_GHC_LIB_FILESYSTEM > 0
+#    include <ghc/filesystem.hpp>
+#elif defined(ROCPROFILER_TESTS_HAS_CPP_LIB_FILESYSTEM) &&                                         \
+    ROCPROFILER_TESTS_HAS_CPP_LIB_FILESYSTEM > 0
+#    include <filesystem>
+#else
+#    include <experimental/filesystem>
+#endif
+
+namespace common
+{
+#if defined(ROCPROFILER_TESTS_HAS_GHC_LIB_FILESYSTEM) &&                                           \
+    ROCPROFILER_TESTS_HAS_GHC_LIB_FILESYSTEM > 0
+namespace fs = ::ghc::filesystem;  // NOLINT(misc-unused-alias-decls)
+#elif defined(ROCPROFILER_TESTS_HAS_CPP_LIB_FILESYSTEM) &&                                         \
+    ROCPROFILER_TESTS_HAS_CPP_LIB_FILESYSTEM > 0
+namespace fs = ::std::filesystem;  // NOLINT(misc-unused-alias-decls)
+#else
+namespace fs = ::std::experimental::filesystem;  // NOLINT(misc-unused-alias-decls)
+#endif
+}  // namespace common

projects/rocprofiler-sdk/tests/common/hash.hpp

@@ -0,0 +1,28 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <rocprofiler-sdk/agent.h>
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/cxx/hash.hpp>
+#include <rocprofiler-sdk/cxx/operators.hpp>

projects/rocprofiler-sdk/tests/common/name_info.hpp

@@ -0,0 +1,27 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+
+#pragma once
+
+#include <rocprofiler-sdk/cxx/name_info.hpp>
+#include <rocprofiler-sdk/cxx/serialization.hpp>

projects/rocprofiler-sdk/tests/common/perfetto.hpp

@@ -0,0 +1,25 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <rocprofiler-sdk/cxx/perfetto.hpp>

projects/rocprofiler-sdk/tests/common/serialization.hpp

@@ -0,0 +1,27 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+
+#pragma once
+
+// provided by the library
+#include <rocprofiler-sdk/cxx/serialization.hpp>

projects/rocprofiler-sdk/tests/counter-collection/CMakeLists.txt

@@ -0,0 +1,61 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(
+    rocprofiler-sdk-tests-counter-collection
+    LANGUAGES CXX
+    VERSION 0.0.0)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+if(ROCPROFILER_MEMCHECK_PRELOAD_ENV)
+    set(PRELOAD_ENV
+        "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}:$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+else()
+    set(PRELOAD_ENV "LD_PRELOAD=$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+endif()
+
+set(counter-collection-env
+    "${PRELOAD_ENV}" "ROCPROFILER_TOOL_OUTPUT_FILE=counter-collection-test.json"
+    "ROCPROFILER_TOOL_CONTEXTS=COUNTER_COLLECTION" "ROCPROF_COUNTERS=SQ_WAVES_sum")
+
+add_test(NAME test-counter-collection-execute COMMAND $<TARGET_FILE:multistream>)
+
+set_tests_properties(
+    test-counter-collection-execute
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "integration-tests"
+               ENVIRONMENT
+               "${counter-collection-env}"
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               DISABLED
+               "${ROCPROFILER_DISABLE_UNSTABLE_CTESTS}"
+               FIXTURES_SETUP
+               test-counter-collection)
+
+# copy to binary directory
+rocprofiler_configure_pytest_files(COPY validate.py conftest.py CONFIG pytest.ini)
+
+add_test(NAME test-counter-collection-validate
+         COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --input
+                 ${CMAKE_CURRENT_BINARY_DIR}/counter-collection-test.json)
+
+set_tests_properties(
+    test-counter-collection-validate
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "integration-tests"
+               DEPENDS
+               test-counter-collection-execute
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               DISABLED
+               "${ROCPROFILER_DISABLE_UNSTABLE_CTESTS}"
+               FIXTURES_REQUIRED
+               test-counter-collection)

projects/rocprofiler-sdk/tests/counter-collection/conftest.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import json
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--input",
+        action="store",
+        default="counter-collection-test.json",
+        help="Input JSON",
+    )
+
+
+@pytest.fixture
+def input_data(request):
+    filename = request.config.getoption("--input")
+    with open(filename, "r") as inp:
+        return json.load(inp)

projects/rocprofiler-sdk/tests/counter-collection/pytest.ini

@@ -0,0 +1,5 @@
+
+[pytest]
+addopts = --durations=20 -ras -vv
+testpaths = validate.py
+pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages

projects/rocprofiler-sdk/tests/counter-collection/validate.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import pytest
+
+
+# helper function
+def node_exists(name, data, min_len=1):
+    assert name in data
+    assert data[name] is not None
+    assert len(data[name]) >= min_len
+
+
+def test_data_structure(input_data):
+    """verify minimum amount of expected data is present"""
+    node_exists("rocprofiler-sdk-json-tool", input_data)
+    rocp_data = input_data
+    node_exists("names", rocp_data["rocprofiler-sdk-json-tool"]["buffer_records"])
+    node_exists(
+        "counter_collection", rocp_data["rocprofiler-sdk-json-tool"]["buffer_records"]
+    )
+
+
+def test_counter_values(input_data):
+    data = input_data["rocprofiler-sdk-json-tool"]
+    agent_data = data["agents"]
+    counter_info = data["counter_info"]
+    counter_data = data["buffer_records"]["counter_collection"]
+
+    for itr in counter_info:
+        if itr["is_constant"] == 1:
+            continue
+        assert itr["id"]["handle"] > 0, f"{itr}"
+        assert itr["is_constant"] in (0, 1), f"{itr}"
+        assert itr["is_derived"] in (0, 1), f"{itr}"
+        assert len(itr["name"]) >= 4, f"{itr}"
+        assert len(itr["description"]) >= 4, f"{itr}"
+        if itr["is_constant"] == 0:
+            if itr["is_derived"] == 0:
+                assert len(itr["block"]) > 0, f"{itr}"
+            if itr["is_derived"] == 1:
+                assert len(itr["expression"]) > 0, f"{itr}"
+
+    def get_agent(agent_id):
+        for itr in agent_data:
+            if itr["id"]["handle"] == agent_id["handle"]:
+                return itr
+        return None
+
+    def get_scaling_factor(agent_id):
+        agent = get_agent(agent_id)
+        assert agent is not None, f"id={agent_id}"
+        if agent["type"] == 2 and agent["wave_front_size"] > 0:
+            return 64 / agent["wave_front_size"]
+        return 0
+
+    for itr in counter_data:
+        assert itr["num_records"] == len(itr["records"]), f"itr={itr}"
+        agent_id = itr["dispatch_info"]["agent_id"]
+        agent = get_agent(agent_id)
+        scaling_factor = get_scaling_factor(agent_id)
+        assert agent is not None, f"itr={itr}\nagent={agent}"
+        assert itr["start_timestamp"] < itr["end_timestamp"]
+        for ritr in itr["records"]:
+            value = ritr["counter_value"]
+            if int(round(value, 0)) > 0:
+                assert int(round(value, 0)) == int(
+                    round(1 * scaling_factor, 0)
+                ), f"itr={itr}\nagent={agent}"
+
+
+if __name__ == "__main__":
+    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
+    sys.exit(exit_code)

projects/rocprofiler-sdk/tests/hip-graph-tracing/CMakeLists.txt

@@ -0,0 +1,51 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(
+    rocprofiler-sdk-tests-hip-graph-tracing
+    LANGUAGES CXX
+    VERSION 0.0.0)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+set(PYTEST_ARGS)
+if(ROCPROFILER_MEMCHECK MATCHES "(Address|Thread)Sanitizer" OR ROCPROFILER_BUILD_CODECOV)
+    set(PYTEST_ARGS -k "not test_total_runtime")
+endif()
+
+if(ROCPROFILER_MEMCHECK_PRELOAD_ENV)
+    set(PRELOAD_ENV
+        "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}:$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+else()
+    set(PRELOAD_ENV "LD_PRELOAD=$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+endif()
+
+add_test(NAME test-hip-graph-tracing-execute COMMAND $<TARGET_FILE:hip-graph>)
+
+set(hip-graph-tracing-env
+    "${PRELOAD_ENV}"
+    "ROCPROFILER_TOOL_OUTPUT_FILE=hip-graph-tracing-test.json"
+    "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:rocprofiler-sdk::rocprofiler-sdk-shared-library>:$ENV{LD_LIBRARY_PATH}"
+    "ROCPROFILER_TOOL_CONTEXTS=HIP_API_CALLBACK,HIP_API_BUFFERED,KERNEL_DISPATCH_CALLBACK,KERNEL_DISPATCH_BUFFERED,CODE_OBJECT"
+    )
+
+set_tests_properties(
+    test-hip-graph-tracing-execute
+    PROPERTIES TIMEOUT 100 LABELS "integration-tests" ENVIRONMENT
+               "${hip-graph-tracing-env}" FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
+
+rocprofiler_configure_pytest_files(COPY validate.py conftest.py CONFIG pytest.ini)
+
+add_test(
+    NAME test-hip-graph-tracing-validate
+    COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py ${PYTEST_ARGS}
+            --input ${CMAKE_CURRENT_BINARY_DIR}/hip-graph-tracing-test.json)
+
+set_tests_properties(
+    test-hip-graph-tracing-validate
+    PROPERTIES TIMEOUT 45 LABELS "integration-tests" DEPENDS
+               test-hip-graph-tracing-execute FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}")

projects/rocprofiler-sdk/tests/hip-graph-tracing/conftest.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import json
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--input",
+        action="store",
+        default="hip-graph-tracing-test.json",
+        help="Input JSON",
+    )
+
+
+@pytest.fixture
+def input_data(request):
+    filename = request.config.getoption("--input")
+    with open(filename, "r") as inp:
+        return json.load(inp)

projects/rocprofiler-sdk/tests/hip-graph-tracing/pytest.ini

@@ -0,0 +1,5 @@
+
+[pytest]
+addopts = --durations=20 -rA -s -vv
+testpaths = validate.py
+pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages

projects/rocprofiler-sdk/tests/hip-graph-tracing/validate.py

@@ -0,0 +1,258 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import pytest
+
+
+# helper function
+def node_exists(name, data, min_len=1):
+    assert name in data
+    assert data[name] is not None
+    if isinstance(data[name], (list, tuple, dict, set)):
+        assert len(data[name]) >= min_len
+
+
+def test_data_structure(input_data):
+    """verify minimum amount of expected data is present"""
+    data = input_data
+
+    node_exists("rocprofiler-sdk-json-tool", data)
+
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    node_exists("metadata", sdk_data)
+    node_exists("pid", sdk_data["metadata"])
+    node_exists("main_tid", sdk_data["metadata"])
+    node_exists("init_time", sdk_data["metadata"])
+    node_exists("fini_time", sdk_data["metadata"])
+
+    node_exists("agents", sdk_data)
+    node_exists("call_stack", sdk_data)
+    node_exists("callback_records", sdk_data)
+    node_exists("buffer_records", sdk_data)
+
+    node_exists("names", sdk_data["callback_records"])
+    node_exists("code_objects", sdk_data["callback_records"])
+    node_exists("kernel_symbols", sdk_data["callback_records"])
+    node_exists("host_functions", sdk_data["callback_records"])
+    node_exists("hip_api_traces", sdk_data["callback_records"])
+    node_exists("kernel_dispatch", sdk_data["callback_records"])
+
+    node_exists("names", sdk_data["buffer_records"])
+    node_exists("kernel_dispatch", sdk_data["buffer_records"])
+    node_exists("hip_api_traces", sdk_data["buffer_records"], 0)
+
+
+def test_size_entries(input_data):
+    # check that size fields are > 0 but account for function arguments
+    # which are named "size"
+    def check_size(data, bt):
+        if "size" in data.keys():
+            if isinstance(data["size"], str) and bt.endswith('["args"]'):
+                pass
+            else:
+                assert data["size"] > 0, f"origin: {bt}"
+
+    # recursively check the entire data structure
+    def iterate_data(data, bt):
+        if isinstance(data, (list, tuple)):
+            for i, itr in enumerate(data):
+                if isinstance(itr, dict):
+                    check_size(itr, f"{bt}[{i}]")
+                iterate_data(itr, f"{bt}[{i}]")
+        elif isinstance(data, dict):
+            check_size(data, f"{bt}")
+            for key, itr in data.items():
+                iterate_data(itr, f'{bt}["{key}"]')
+
+    # start recursive check over entire JSON dict
+    iterate_data(input_data, "input_data")
+
+
+def test_timestamps(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    cb_start = {}
+    cb_end = {}
+    for titr in ["hsa_api_traces", "marker_api_traces", "hip_api_traces"]:
+        for itr in sdk_data["callback_records"][titr]:
+            cid = itr["correlation_id"]["internal"]
+            phase = itr["phase"]
+            if phase == 1:
+                cb_start[cid] = itr["timestamp"]
+            elif phase == 2:
+                cb_end[cid] = itr["timestamp"]
+                assert cb_start[cid] <= itr["timestamp"]
+            else:
+                assert phase == 1 or phase == 2
+
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["start_timestamp"] <= itr["end_timestamp"]
+
+    for titr in ["kernel_dispatch", "memory_copies"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["start_timestamp"] < itr["end_timestamp"], f"[{titr}] {itr}"
+            assert itr["correlation_id"]["internal"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["init_time"] < itr["start_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["init_time"] < itr["end_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["fini_time"] > itr["start_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["fini_time"] > itr["end_timestamp"]
+            ), f"[{titr}] {itr}"
+
+            api_start = cb_start[itr["correlation_id"]["internal"]]
+            # api_end = cb_end[itr["correlation_id"]["internal"]]
+            assert api_start < itr["start_timestamp"], f"[{titr}] {itr}"
+            # assert api_end <= itr["end_timestamp"], f"[{titr}] {itr}"
+
+
+def test_internal_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    api_corr_ids = []
+    for titr in ["hsa_api_traces", "marker_api_traces", "hip_api_traces"]:
+        for itr in sdk_data["callback_records"][titr]:
+            api_corr_ids.append(itr["correlation_id"]["internal"])
+
+        for itr in sdk_data["buffer_records"][titr]:
+            api_corr_ids.append(itr["correlation_id"]["internal"])
+
+    api_corr_ids_sorted = sorted(api_corr_ids)
+    api_corr_ids_unique = list(set(api_corr_ids))
+
+    for itr in sdk_data["buffer_records"]["kernel_dispatch"]:
+        assert itr["correlation_id"]["internal"] in api_corr_ids_unique
+
+    for itr in sdk_data["buffer_records"]["memory_copies"]:
+        assert itr["correlation_id"]["internal"] in api_corr_ids_unique
+
+    len_corr_id_unq = len(api_corr_ids_unique)
+    assert len(api_corr_ids) != len_corr_id_unq
+    assert max(api_corr_ids_sorted) == len_corr_id_unq
+
+
+def test_external_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    extern_corr_ids = []
+    for titr in ["hsa_api_traces", "marker_api_traces", "hip_api_traces"]:
+        for itr in sdk_data["callback_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0
+            assert itr["thread_id"] == itr["correlation_id"]["external"]
+            extern_corr_ids.append(itr["correlation_id"]["external"])
+
+    extern_corr_ids = list(set(sorted(extern_corr_ids)))
+    for titr in ["hsa_api_traces", "marker_api_traces", "hip_api_traces"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert (
+                itr["thread_id"] == itr["correlation_id"]["external"]
+            ), f"[{titr}] {itr}"
+            assert itr["thread_id"] in extern_corr_ids, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
+
+    for titr in ["kernel_dispatch", "memory_copies"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
+
+
+def test_kernel_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    symbol_info = {}
+    for itr in sdk_data["callback_records"]["kernel_symbols"]:
+        phase = itr["phase"]
+        payload = itr["payload"]
+        kern_id = payload["kernel_id"]
+
+        assert phase == 1 or phase == 2
+        assert kern_id > 0
+        if phase == 1:
+            assert len(payload["kernel_name"]) > 0
+            symbol_info[kern_id] = payload
+        elif phase == 2:
+            assert payload["kernel_id"] in symbol_info.keys()
+            assert payload["kernel_name"] == symbol_info[kern_id]["kernel_name"]
+
+    for itr in sdk_data["buffer_records"]["kernel_dispatch"]:
+        assert itr["dispatch_info"]["kernel_id"] in symbol_info.keys()
+
+    for itr in sdk_data["callback_records"]["kernel_dispatch"]:
+        assert itr["payload"]["dispatch_info"]["kernel_id"] in symbol_info.keys()
+
+
+def test_kernel_dispatch_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    num_dispatches = len(sdk_data["buffer_records"]["kernel_dispatch"])
+    num_cb_dispatches = len(sdk_data["callback_records"]["kernel_dispatch"])
+
+    assert num_cb_dispatches == (3 * num_dispatches)
+
+    bf_seq_ids = []
+    for itr in sdk_data["buffer_records"]["kernel_dispatch"]:
+        bf_seq_ids.append(itr["dispatch_info"]["dispatch_id"])
+
+    cb_seq_ids = []
+    for itr in sdk_data["callback_records"]["kernel_dispatch"]:
+        cb_seq_ids.append(itr["payload"]["dispatch_info"]["dispatch_id"])
+
+    bf_seq_ids = sorted(bf_seq_ids)
+    cb_seq_ids = sorted(cb_seq_ids)
+
+    assert (3 * len(bf_seq_ids)) == len(cb_seq_ids)
+
+    assert bf_seq_ids[0] == cb_seq_ids[0]
+    assert bf_seq_ids[-1] == cb_seq_ids[-1]
+
+    def get_uniq(data):
+        return list(set(data))
+
+    bf_seq_ids_uniq = get_uniq(bf_seq_ids)
+    cb_seq_ids_uniq = get_uniq(cb_seq_ids)
+
+    assert bf_seq_ids == bf_seq_ids_uniq
+    assert len(cb_seq_ids) == (3 * len(cb_seq_ids_uniq))
+    assert len(bf_seq_ids) == num_dispatches
+    assert len(bf_seq_ids_uniq) == num_dispatches
+    assert len(cb_seq_ids_uniq) == num_dispatches
+
+
+if __name__ == "__main__":
+    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
+    sys.exit(exit_code)

projects/rocprofiler-sdk/tests/hsa-memory-allocation/CMakeLists.txt

@@ -0,0 +1,46 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(
+    rocprofiler-sdk-tests-memory-allocation-tracing
+    LANGUAGES CXX
+    VERSION 0.0.0)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+if(ROCPROFILER_MEMCHECK_PRELOAD_ENV)
+    set(PRELOAD_ENV
+        "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}:$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+else()
+    set(PRELOAD_ENV "LD_PRELOAD=$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+endif()
+
+add_test(NAME test-memory-allocation-tracing-execute
+         COMMAND $<TARGET_FILE:hsa-memory-allocation>)
+
+set(memory-allocation-tracing-env
+    "${PRELOAD_ENV}"
+    "ROCPROFILER_TOOL_OUTPUT_FILE=memory-allocation-tracing-test.json"
+    "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:rocprofiler-sdk::rocprofiler-sdk-shared-library>:$ENV{LD_LIBRARY_PATH}"
+    )
+
+set_tests_properties(
+    test-memory-allocation-tracing-execute
+    PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT
+               "${memory-allocation-tracing-env}" FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
+
+# copy to binary directory
+rocprofiler_configure_pytest_files(COPY validate.py conftest.py CONFIG pytest.ini)
+
+add_test(NAME test-memory-allocation-tracing-validate
+         COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --input
+                 ${CMAKE_CURRENT_BINARY_DIR}/memory-allocation-tracing-test.json)
+
+set_tests_properties(
+    test-memory-allocation-tracing-validate
+    PROPERTIES TIMEOUT 45 LABELS "integration-tests" DEPENDS
+               test-memory-allocation-tracing-execute FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}")

projects/rocprofiler-sdk/tests/hsa-memory-allocation/conftest.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import json
+import pytest
+
+from rocprofiler_sdk.pytest_utils.dotdict import dotdict
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--input",
+        action="store",
+        default="memory-allocation-tracing-test.json",
+        help="Input JSON",
+    )
+
+
+@pytest.fixture
+def input_data(request):
+    filename = request.config.getoption("--input")
+    with open(filename, "r") as inp:
+        return dotdict(json.load(inp))

projects/rocprofiler-sdk/tests/hsa-memory-allocation/pytest.ini

@@ -0,0 +1,5 @@
+
+[pytest]
+addopts = --durations=20 -rA -s -vv
+testpaths = validate.py
+pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages

projects/rocprofiler-sdk/tests/hsa-memory-allocation/validate.py

@@ -0,0 +1,310 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import pytest
+
+
+# helper function
+def node_exists(name, data, min_len=1):
+    assert name in data
+    assert data[name] is not None
+    if isinstance(data[name], (list, tuple, dict, set)):
+        assert len(data[name]) >= min_len, f"{name}:\n{data}"
+
+
+def test_data_structure(input_data):
+    """verify minimum amount of expected data is present"""
+    data = input_data
+
+    node_exists("rocprofiler-sdk-json-tool", data)
+
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    node_exists("metadata", sdk_data)
+    node_exists("pid", sdk_data["metadata"])
+    node_exists("main_tid", sdk_data["metadata"])
+    node_exists("init_time", sdk_data["metadata"])
+    node_exists("fini_time", sdk_data["metadata"])
+
+    node_exists("agents", sdk_data)
+    node_exists("call_stack", sdk_data)
+    node_exists("callback_records", sdk_data)
+    node_exists("buffer_records", sdk_data)
+
+    node_exists("names", sdk_data["callback_records"])
+    node_exists("hsa_api_traces", sdk_data["callback_records"])
+    node_exists("memory_allocations", sdk_data["callback_records"])
+
+    node_exists("names", sdk_data["buffer_records"])
+    node_exists("hsa_api_traces", sdk_data["callback_records"])
+    node_exists("memory_allocations", sdk_data["buffer_records"])
+
+
+def test_size_entries(input_data):
+    # check that size fields are > 0 but account for function arguments
+    # which are named "size"
+    def check_size(data, bt):
+        if "size" in data.keys():
+            if isinstance(data["size"], str) and bt.endswith('["args"]'):
+                pass
+            else:
+                assert data["size"] > 0, f"origin: {bt}"
+
+    # recursively check the entire data structure
+    def iterate_data(data, bt):
+        if isinstance(data, (list, tuple)):
+            for i, itr in enumerate(data):
+                if isinstance(itr, dict):
+                    check_size(itr, f"{bt}[{i}]")
+                iterate_data(itr, f"{bt}[{i}]")
+        elif isinstance(data, dict):
+            check_size(data, f"{bt}")
+            for key, itr in data.items():
+                iterate_data(itr, f'{bt}["{key}"]')
+
+    # start recursive check over entire JSON dict
+    iterate_data(input_data, "input_data")
+
+
+def test_timestamps(input_data):
+    """Verify starting timestamps are less than ending timestamps"""
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    cb_start = {}
+    cb_end = {}
+    for titr in ["hsa_api_traces"]:
+        for itr in sdk_data["callback_records"][titr]:
+            cid = itr["correlation_id"]["internal"]
+            phase = itr["phase"]
+            if phase == 1:
+                cb_start[cid] = itr["timestamp"]
+            elif phase == 2:
+                cb_end[cid] = itr["timestamp"]
+                assert cb_start[cid] <= itr["timestamp"]
+            else:
+                assert phase == 1 or phase == 2
+
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["start_timestamp"] <= itr["end_timestamp"]
+
+    for titr in ["memory_allocations"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["start_timestamp"] < itr["end_timestamp"], f"[{titr}] {itr}"
+            assert itr["correlation_id"]["internal"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["init_time"] < itr["start_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["init_time"] < itr["end_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["fini_time"] > itr["start_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["fini_time"] > itr["end_timestamp"]
+            ), f"[{titr}] {itr}"
+
+            api_start = cb_start[itr["correlation_id"]["internal"]]
+            # api_end = cb_end[itr["correlation_id"]["internal"]]
+            assert api_start < itr["start_timestamp"], f"[{titr}] {itr}"
+            # assert api_end <= itr["end_timestamp"], f"[{titr}] {itr}"
+
+
+def test_internal_correlation_ids(input_data):
+    """Assure correlation ids are unique"""
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    api_corr_ids = []
+    for titr in ["hsa_api_traces"]:
+        for itr in sdk_data["callback_records"][titr]:
+            api_corr_ids.append(itr["correlation_id"]["internal"])
+
+        for itr in sdk_data["buffer_records"][titr]:
+            api_corr_ids.append(itr["correlation_id"]["internal"])
+
+    api_corr_ids_sorted = sorted(api_corr_ids)
+    api_corr_ids_unique = list(set(api_corr_ids))
+
+    for itr in sdk_data["buffer_records"]["memory_allocations"]:
+        assert itr["correlation_id"]["internal"] in api_corr_ids_unique
+
+    len_corr_id_unq = len(api_corr_ids_unique)
+    assert len(api_corr_ids) != len_corr_id_unq
+    assert max(api_corr_ids_sorted) == len_corr_id_unq
+
+
+def test_external_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    extern_corr_ids = []
+    for titr in ["hsa_api_traces"]:
+        for itr in sdk_data["callback_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0
+            assert itr["thread_id"] == itr["correlation_id"]["external"]
+            extern_corr_ids.append(itr["correlation_id"]["external"])
+
+    extern_corr_ids = list(set(sorted(extern_corr_ids)))
+    for titr in ["hsa_api_traces"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert (
+                itr["thread_id"] == itr["correlation_id"]["external"]
+            ), f"[{titr}] {itr}"
+            assert itr["thread_id"] in extern_corr_ids, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
+
+    for titr in ["memory_allocations"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
+
+        for itr in sdk_data["callback_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
+
+
+def test_memory_alloc_sizes(input_data):
+    """Ensure trace file memory allocation operations match up with the memory allocation operations performed in hsa-memory-allocation"""
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    # Op values:
+    #   0 == ??? (unknown)
+    #   1 == hsa_memory_allocate
+    #   2 == hsa_amd_vmem_handle_create
+    #   3 == hsa_memory_free
+    #   4 == hsa_amd_vmem_handle_release
+    memory_alloc_cnt = dict(
+        [
+            (idx, {"agent": set(), "starting_addr": set(), "size": set(), "count": 0})
+            for idx in range(1, 5)
+        ]
+    )
+    for itr in sdk_data["buffer_records"]["memory_allocations"]:
+        op_id = itr["operation"]
+        assert op_id > 0 and op_id <= 5, f"{itr}"
+        memory_alloc_cnt[op_id]["count"] += 1
+        memory_alloc_cnt[op_id]["starting_addr"].add(itr.address)
+        memory_alloc_cnt[op_id]["size"].add(itr.allocation_size)
+        memory_alloc_cnt[op_id]["agent"].add(itr.agent_id.handle)
+
+    for itr in sdk_data["callback_records"]["memory_copies"]:
+        op_id = itr.operation
+        assert op_id > 0 and op_id <= 5, f"{itr}"
+        memory_alloc_cnt[op_id]["count"] += 1
+
+        phase = itr.phase
+        pitr = itr.payload
+
+        assert phase is not None, f"{itr}"
+        assert pitr is not None, f"{itr}"
+
+        if phase == 1:
+            assert pitr.start_timestamp == 0, f"{itr}"
+            assert pitr.end_timestamp == 0, f"{itr}"
+        elif phase == 2:
+            assert pitr.start_timestamp > 0, f"{itr}"
+            assert pitr.end_timestamp > 0, f"{itr}"
+            assert pitr.end_timestamp >= pitr.start_timestamp, f"{itr}"
+
+            memory_alloc_cnt[op_id]["starting_addr"].add(pitr.address)
+            memory_alloc_cnt[op_id]["size"].add(pitr.allocation_size)
+            memory_alloc_cnt[op_id]["agent"].add(pitr.agent_id.handle)
+        else:
+            assert phase == 1 or phase == 2, f"{itr}"
+
+    # In the memory allocation test which generates this file
+    # 6 hsa_memory_allocation calls with 1024 bytes were called
+    # and 9 hsa_amd_memory_pool_allocations with 2048 bytes
+    # were called
+    assert memory_alloc_cnt[1]["count"] == 15
+    assert memory_alloc_cnt[3]["count"] == 15
+    # assert memory_alloc_cnt[3]["count"] == 3
+    assert len(memory_alloc_cnt[1]["starting_addr"]) == len(
+        memory_alloc_cnt[3]["starting_addr"]
+    )
+
+    # assert len(memory_alloc_cnt[3]["starting_addr"]) == 3
+    assert len(memory_alloc_cnt[1]["size"]) == 2
+    # assert len(memory_alloc_cnt[3]["size"]) == 1
+    assert 1024 in memory_alloc_cnt[1]["size"]
+    assert 2048 in memory_alloc_cnt[1]["size"]
+    assert len(memory_alloc_cnt[1]["agent"]) == 2
+    # assert len(memory_alloc_cnt[3]["agent"]) == 1
+
+
+def test_retired_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    def _sort_dict(inp):
+        return dict(sorted(inp.items()))
+
+    api_corr_ids = {}
+    for titr in ["hsa_api_traces"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            corr_id = itr["correlation_id"]["internal"]
+            assert corr_id not in api_corr_ids.keys()
+            api_corr_ids[corr_id] = itr
+
+    alloc_corr_ids = {}
+    for titr in ["memory_allocations"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            corr_id = itr["correlation_id"]["internal"]
+            assert corr_id not in alloc_corr_ids.keys()
+            alloc_corr_ids[corr_id] = itr
+
+    retired_corr_ids = {}
+    for itr in sdk_data["buffer_records"]["retired_correlation_ids"]:
+        corr_id = itr["internal_correlation_id"]
+        assert corr_id not in retired_corr_ids.keys()
+        retired_corr_ids[corr_id] = itr
+
+    api_corr_ids = _sort_dict(api_corr_ids)
+    alloc_corr_ids = _sort_dict(alloc_corr_ids)
+    retired_corr_ids = _sort_dict(retired_corr_ids)
+
+    for cid, itr in alloc_corr_ids.items():
+        assert cid in retired_corr_ids.keys()
+        retired_ts = retired_corr_ids[cid]["timestamp"]
+        end_ts = itr["end_timestamp"]
+        assert (retired_ts - end_ts) > 0, f"correlation-id: {cid}, data: {itr}"
+
+    for cid, itr in api_corr_ids.items():
+        assert cid in retired_corr_ids.keys()
+        retired_ts = retired_corr_ids[cid]["timestamp"]
+        end_ts = itr["end_timestamp"]
+        assert (retired_ts - end_ts) > 0, f"correlation-id: {cid}, data: {itr}"
+
+    assert len(api_corr_ids.keys()) == (len(retired_corr_ids.keys()))
+
+
+if __name__ == "__main__":
+    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
+    sys.exit(exit_code)

projects/rocprofiler-sdk/tests/kernel-tracing/CMakeLists.txt

@@ -0,0 +1,67 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(
+    rocprofiler-sdk-tests-kernel-tracing
+    LANGUAGES CXX
+    VERSION 0.0.0)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+set(PYTEST_ARGS)
+if(ROCPROFILER_MEMCHECK MATCHES "(Address|Thread|UndefinedBehavior)Sanitizer"
+   OR ROCPROFILER_BUILD_CODECOV
+   OR ROCPROFILER_DISABLE_UNSTABLE_CTESTS)
+    set(PYTEST_ARGS -k "not test_total_runtime")
+endif()
+
+if(ROCPROFILER_MEMCHECK_PRELOAD_ENV)
+    set(PRELOAD_ENV
+        "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}:$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+else()
+    set(PRELOAD_ENV "LD_PRELOAD=$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+endif()
+
+add_test(NAME test-kernel-tracing-execute COMMAND $<TARGET_FILE:reproducible-runtime>)
+
+set(kernel-tracing-env
+    "${PRELOAD_ENV}"
+    "ROCPROFILER_TOOL_OUTPUT_FILE=kernel-tracing-test.json"
+    "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:rocprofiler-sdk::rocprofiler-sdk-shared-library>:$ENV{LD_LIBRARY_PATH}"
+    )
+
+set_tests_properties(
+    test-kernel-tracing-execute
+    PROPERTIES TIMEOUT
+               100
+               LABELS
+               "integration-tests"
+               ENVIRONMENT
+               "${kernel-tracing-env}"
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               FIXTURES_SETUP
+               kernel-tracing)
+
+# copy to binary directory
+rocprofiler_configure_pytest_files(COPY validate.py conftest.py CONFIG pytest.ini)
+
+add_test(
+    NAME test-kernel-tracing-validate
+    COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py ${PYTEST_ARGS}
+            --input ${CMAKE_CURRENT_BINARY_DIR}/kernel-tracing-test.json)
+
+set_tests_properties(
+    test-kernel-tracing-validate
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "integration-tests"
+               DEPENDS
+               test-kernel-tracing-execute
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               FIXTURES_REQUIRED
+               kernel-tracing)

projects/rocprofiler-sdk/tests/kernel-tracing/conftest.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import json
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--input",
+        action="store",
+        default="kernel-tracing-test.json",
+        help="Input JSON",
+    )
+
+
+@pytest.fixture
+def input_data(request):
+    filename = request.config.getoption("--input")
+    with open(filename, "r") as inp:
+        return json.load(inp)

projects/rocprofiler-sdk/tests/kernel-tracing/pytest.ini

@@ -0,0 +1,5 @@
+
+[pytest]
+addopts = --durations=20 -rA -s -vv
+testpaths = validate.py
+pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages

projects/rocprofiler-sdk/tests/kernel-tracing/validate.py

@@ -0,0 +1,356 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import pytest
+
+test_api_traces = [
+    "hsa_api_traces",
+    "marker_api_traces",
+    "hip_api_traces",
+    "rccl_api_traces",
+    "scratch_memory_traces",
+]
+
+
+# helper function
+def node_exists(name, data, min_len=1):
+    assert name in data
+    assert data[name] is not None
+    if isinstance(data[name], (list, tuple, dict, set)):
+        assert len(data[name]) >= min_len
+
+
+def test_data_structure(input_data):
+    """verify minimum amount of expected data is present"""
+    data = input_data
+
+    node_exists("rocprofiler-sdk-json-tool", data)
+
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    node_exists("metadata", sdk_data)
+    node_exists("pid", sdk_data["metadata"])
+    node_exists("main_tid", sdk_data["metadata"])
+    node_exists("init_time", sdk_data["metadata"])
+    node_exists("fini_time", sdk_data["metadata"])
+
+    node_exists("agents", sdk_data)
+    node_exists("call_stack", sdk_data)
+    node_exists("callback_records", sdk_data)
+    node_exists("buffer_records", sdk_data)
+
+    node_exists("names", sdk_data["callback_records"])
+    node_exists("code_objects", sdk_data["callback_records"])
+    node_exists("kernel_symbols", sdk_data["callback_records"])
+    node_exists("host_functions", sdk_data["callback_records"])
+    node_exists("hsa_api_traces", sdk_data["callback_records"])
+    node_exists("hip_api_traces", sdk_data["callback_records"], 0)
+    node_exists("marker_api_traces", sdk_data["callback_records"])
+    node_exists("kernel_dispatch", sdk_data["callback_records"])
+
+    node_exists("names", sdk_data["buffer_records"])
+    node_exists("kernel_dispatch", sdk_data["buffer_records"])
+    node_exists("memory_copies", sdk_data["buffer_records"], 0)
+    node_exists("hsa_api_traces", sdk_data["buffer_records"])
+    node_exists("hip_api_traces", sdk_data["buffer_records"], 0)
+    node_exists("marker_api_traces", sdk_data["buffer_records"])
+    node_exists("retired_correlation_ids", sdk_data["buffer_records"])
+
+
+def test_size_entries(input_data):
+    # check that size fields are > 0 but account for function arguments
+    # which are named "size"
+    def check_size(data, bt):
+        if "size" in data.keys():
+            if isinstance(data["size"], str) and bt.endswith('["args"]'):
+                pass
+            else:
+                assert data["size"] > 0, f"origin: {bt}"
+
+    # recursively check the entire data structure
+    def iterate_data(data, bt):
+        if isinstance(data, (list, tuple)):
+            for i, itr in enumerate(data):
+                if isinstance(itr, dict):
+                    check_size(itr, f"{bt}[{i}]")
+                iterate_data(itr, f"{bt}[{i}]")
+        elif isinstance(data, dict):
+            check_size(data, f"{bt}")
+            for key, itr in data.items():
+                iterate_data(itr, f'{bt}["{key}"]')
+
+    # start recursive check over entire JSON dict
+    iterate_data(input_data, "input_data")
+
+
+def test_timestamps(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    cb_start = {}
+    cb_end = {}
+    for titr in test_api_traces:
+        for itr in sdk_data["callback_records"][titr]:
+            cid = itr["correlation_id"]["internal"]
+            phase = itr["phase"]
+            if phase == 1:
+                cb_start[cid] = itr["timestamp"]
+            elif phase == 2:
+                cb_end[cid] = itr["timestamp"]
+                assert cb_start[cid] <= itr["timestamp"]
+            else:
+                assert phase == 1 or phase == 2
+
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["start_timestamp"] <= itr["end_timestamp"]
+
+    for titr in ["kernel_dispatch", "memory_copies"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["start_timestamp"] < itr["end_timestamp"], f"[{titr}] {itr}"
+            assert itr["correlation_id"]["internal"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["init_time"] < itr["start_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["init_time"] < itr["end_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["fini_time"] > itr["start_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["fini_time"] > itr["end_timestamp"]
+            ), f"[{titr}] {itr}"
+
+            api_start = cb_start[itr["correlation_id"]["internal"]]
+            # api_end = cb_end[itr["correlation_id"]["internal"]]
+            assert api_start < itr["start_timestamp"], f"[{titr}] {itr}"
+            # assert api_end <= itr["end_timestamp"], f"[{titr}] {itr}"
+
+
+def test_total_runtime(input_data):
+    sdk_data = input_data["rocprofiler-sdk-json-tool"]
+
+    runtime_data = []
+    for itr in sdk_data["buffer_records"]["kernel_dispatch"]:
+        elapsed = itr["end_timestamp"] - itr["start_timestamp"]
+        runtime_data.append(elapsed)  # in nanoseconds
+
+    expected_runtime = 1.0e3  # one second in milliseconds
+
+    assert (sum(runtime_data) * 1.0e-6) >= (0.8 * expected_runtime)
+    assert (sum(runtime_data) * 1.0e-6) <= (1.2 * expected_runtime)
+
+
+def test_internal_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    api_corr_ids = []
+    for titr in test_api_traces:
+        for itr in sdk_data["callback_records"][titr]:
+            api_corr_ids.append(itr["correlation_id"]["internal"])
+
+        for itr in sdk_data["buffer_records"][titr]:
+            api_corr_ids.append(itr["correlation_id"]["internal"])
+
+    api_corr_ids_sorted = sorted(api_corr_ids)
+    api_corr_ids_unique = list(set(api_corr_ids))
+
+    for itr in sdk_data["buffer_records"]["kernel_dispatch"]:
+        assert itr["correlation_id"]["internal"] in api_corr_ids_unique
+
+    for itr in sdk_data["buffer_records"]["memory_copies"]:
+        assert itr["correlation_id"]["internal"] in api_corr_ids_unique
+
+    len_corr_id_unq = len(api_corr_ids_unique)
+    assert len(api_corr_ids) != len_corr_id_unq
+    assert max(api_corr_ids_sorted) == len_corr_id_unq
+
+
+def test_retired_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    def _sort_dict(inp):
+        return dict(sorted(inp.items()))
+
+    api_corr_ids = {}
+    for titr in test_api_traces:
+        for itr in sdk_data["buffer_records"][titr]:
+            corr_id = itr["correlation_id"]["internal"]
+            assert corr_id not in api_corr_ids.keys()
+            api_corr_ids[corr_id] = itr
+
+    async_corr_ids = {}
+    for titr in ["kernel_dispatch", "memory_copies"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            corr_id = itr["correlation_id"]["internal"]
+            assert corr_id not in async_corr_ids.keys()
+            async_corr_ids[corr_id] = itr
+
+    retired_corr_ids = {}
+    for itr in sdk_data["buffer_records"]["retired_correlation_ids"]:
+        corr_id = itr["internal_correlation_id"]
+        assert corr_id not in retired_corr_ids.keys()
+        retired_corr_ids[corr_id] = itr
+
+    api_corr_ids = _sort_dict(api_corr_ids)
+    async_corr_ids = _sort_dict(async_corr_ids)
+    retired_corr_ids = _sort_dict(retired_corr_ids)
+
+    for cid, itr in async_corr_ids.items():
+        assert cid in retired_corr_ids.keys()
+        retired_ts = retired_corr_ids[cid]["timestamp"]
+        end_ts = itr["end_timestamp"]
+        assert (retired_ts - end_ts) > 0, f"correlation-id: {cid}, data: {itr}"
+
+    for cid, itr in api_corr_ids.items():
+        assert cid in retired_corr_ids.keys()
+        retired_ts = retired_corr_ids[cid]["timestamp"]
+        end_ts = itr["end_timestamp"]
+        assert (retired_ts - end_ts) > 0, f"correlation-id: {cid}, data: {itr}"
+
+    assert len(api_corr_ids.keys()) == (len(retired_corr_ids.keys()))
+
+
+def test_external_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    extern_corr_ids = []
+    for titr in test_api_traces:
+        for itr in sdk_data["callback_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0
+            assert itr["thread_id"] == itr["correlation_id"]["external"]
+            extern_corr_ids.append(itr["correlation_id"]["external"])
+
+    extern_corr_ids = list(set(sorted(extern_corr_ids)))
+    for titr in test_api_traces:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert (
+                itr["thread_id"] == itr["correlation_id"]["external"]
+            ), f"[{titr}] {itr}"
+            assert itr["thread_id"] in extern_corr_ids, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
+
+    for titr in ["kernel_dispatch", "memory_copies"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
+
+
+def test_kernel_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    symbol_info = {}
+    for itr in sdk_data["callback_records"]["kernel_symbols"]:
+        phase = itr["phase"]
+        payload = itr["payload"]
+        kern_id = payload["kernel_id"]
+
+        assert phase == 1 or phase == 2
+        assert kern_id > 0
+        if phase == 1:
+            assert len(payload["kernel_name"]) > 0
+            symbol_info[kern_id] = payload
+        elif phase == 2:
+            assert payload["kernel_id"] in symbol_info.keys()
+            assert payload["kernel_name"] == symbol_info[kern_id]["kernel_name"]
+
+    for itr in sdk_data["buffer_records"]["kernel_dispatch"]:
+        assert itr["dispatch_info"]["kernel_id"] in symbol_info.keys()
+
+    for itr in sdk_data["callback_records"]["kernel_dispatch"]:
+        assert itr["payload"]["dispatch_info"]["kernel_id"] in symbol_info.keys()
+
+
+def test_kernel_dispatch_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    num_dispatches = len(sdk_data["buffer_records"]["kernel_dispatch"])
+    num_cb_dispatches = len(sdk_data["callback_records"]["kernel_dispatch"])
+
+    assert num_cb_dispatches == (3 * num_dispatches)
+
+    bf_seq_ids = []
+    for itr in sdk_data["buffer_records"]["kernel_dispatch"]:
+        bf_seq_ids.append(itr["dispatch_info"]["dispatch_id"])
+
+    cb_seq_ids = []
+    for itr in sdk_data["callback_records"]["kernel_dispatch"]:
+        cb_seq_ids.append(itr["payload"]["dispatch_info"]["dispatch_id"])
+
+    bf_seq_ids = sorted(bf_seq_ids)
+    cb_seq_ids = sorted(cb_seq_ids)
+
+    assert (3 * len(bf_seq_ids)) == len(cb_seq_ids)
+
+    assert bf_seq_ids[0] == cb_seq_ids[0]
+    assert bf_seq_ids[-1] == cb_seq_ids[-1]
+
+    def get_uniq(data):
+        return list(set(data))
+
+    bf_seq_ids_uniq = get_uniq(bf_seq_ids)
+    cb_seq_ids_uniq = get_uniq(cb_seq_ids)
+
+    assert bf_seq_ids == bf_seq_ids_uniq
+    assert len(cb_seq_ids) == (3 * len(cb_seq_ids_uniq))
+    assert len(bf_seq_ids) == num_dispatches
+    assert len(bf_seq_ids_uniq) == num_dispatches
+    assert len(cb_seq_ids_uniq) == num_dispatches
+
+
+def test_async_copy_direction(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    # Direction values:
+    #   0 == ??? (unknown)
+    #   1 == H2H (host to host)
+    #   2 == H2D (host to device)
+    #   3 == D2H (device to host)
+    #   4 == D2D (device to device)
+    async_dir_cnt = dict([(idx, 0) for idx in range(0, 5)])
+    for itr in sdk_data["buffer_records"]["memory_copies"]:
+        op_id = itr["operation"]
+        async_dir_cnt[op_id] += 1
+
+    # in the reproducible-runtime test which generates the input file,
+    # we don't expect any async memory copy operations
+    assert async_dir_cnt[0] == 0
+    assert async_dir_cnt[1] == 0
+    assert async_dir_cnt[2] == 0
+    assert async_dir_cnt[3] == 0
+    assert async_dir_cnt[4] == 0
+
+
+if __name__ == "__main__":
+    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
+    sys.exit(exit_code)

projects/rocprofiler-sdk/tests/lib/CMakeLists.txt

@@ -0,0 +1,18 @@
+#
+# Integration test application libraries
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(rocprofiler-sdk-tests-lib LANGUAGES C CXX)
+
+set(CMAKE_BUILD_RPATH "\$ORIGIN:\$ORIGIN/../lib")
+
+# libraries used by integration test apps which DO NOT link to rocprofiler-sdk-roctx
+add_subdirectory(vector-operations)
+
+set(CMAKE_BUILD_RPATH
+    "\$ORIGIN:\$ORIGIN/../lib:$<TARGET_FILE_DIR:rocprofiler-sdk-roctx::rocprofiler-sdk-roctx-shared-library>"
+    )
+
+# libraries used by integration test apps which DO link to rocprofiler-sdk-roctx
+add_subdirectory(transpose)

projects/rocprofiler-sdk/tests/lib/transpose/CMakeLists.txt

@@ -0,0 +1,56 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-lib-transpose-shared-library LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+option(TRANSPOSE_USE_MPI "Enable MPI support in transpose-shared-library exe" OFF)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(transpose.cpp PROPERTIES LANGUAGE HIP)
+add_library(transpose-shared-library SHARED)
+target_sources(transpose-shared-library PRIVATE transpose.cpp)
+target_compile_options(transpose-shared-library PRIVATE -W -Wall -Wextra -Wpedantic
+                                                        -Wshadow -Werror)
+target_include_directories(transpose-shared-library PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+set_target_properties(transpose-shared-library PROPERTIES OUTPUT_NAME transpose)
+
+find_package(Threads REQUIRED)
+target_link_libraries(transpose-shared-library PRIVATE Threads::Threads)
+
+find_package(rocprofiler-sdk-roctx REQUIRED)
+target_link_libraries(transpose-shared-library
+                      PRIVATE rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
+
+if(TRANSPOSE_USE_MPI)
+    find_package(MPI REQUIRED)
+    target_compile_definitions(transpose-shared-library PRIVATE USE_MPI)
+    target_link_libraries(transpose-shared-library PRIVATE MPI::MPI_C)
+endif()

projects/rocprofiler-sdk/tests/lib/transpose/transpose.cpp

@@ -0,0 +1,238 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "hip/hip_runtime.h"
+#include "rocprofiler-sdk-roctx/roctx.h"
+
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <mutex>
+#include <random>
+#include <stdexcept>
+#include <thread>
+
+#if defined(USE_MPI)
+#    include <mpi.h>
+#endif
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t                      = std::unique_lock<std::mutex>;
+auto               print_lock          = std::mutex{};
+constexpr unsigned shared_mem_tile_dim = 32;
+
+void
+check_hip_error(void);
+
+void
+verify(int* in, int* out, int M, int N);
+
+__global__ void
+transpose(const int* in, int* out, int M, int N);
+
+void
+run_transpose_impl(int rank, int tid, int ndevice, size_t nitr, size_t nsync);
+
+__global__ void
+transpose(const int* in, int* out, int M, int N)
+{
+    __shared__ int tile[shared_mem_tile_dim][shared_mem_tile_dim];
+
+    int idx = (blockIdx.y * blockDim.y + threadIdx.y) * M + blockIdx.x * blockDim.x + threadIdx.x;
+    tile[threadIdx.y][threadIdx.x] = in[idx];
+    __syncthreads();
+    idx      = (blockIdx.x * blockDim.x + threadIdx.y) * N + blockIdx.y * blockDim.y + threadIdx.x;
+    out[idx] = tile[threadIdx.x][threadIdx.y];
+}
+
+void
+run_transpose_impl(int rank, int tid, int devid, size_t nitr, size_t nsync)
+{
+    roctxRangePush("run_transpose_impl");
+
+    constexpr unsigned int M = 4960 * 2;
+    constexpr unsigned int N = 4960 * 2;
+
+    hipStream_t stream = {};
+
+    printf("[transpose] Rank %i, thread %i assigned to device %i\n", rank, tid, devid);
+    HIP_API_CALL(hipSetDevice(devid));
+    HIP_API_CALL(hipStreamCreate(&stream));
+
+    auto_lock_t _lk{print_lock};
+    std::cout << "[transpose][" << rank << "][" << tid << "] M: " << M << " N: " << N << std::endl;
+    _lk.unlock();
+
+    std::default_random_engine         _engine{std::random_device{}() * (rank + 1) * (tid + 1)};
+    std::uniform_int_distribution<int> _dist{0, 1000};
+
+    size_t size       = sizeof(int) * M * N;
+    int*   inp_matrix = new int[size];
+    int*   out_matrix = new int[size];
+    for(size_t i = 0; i < M * N; i++)
+    {
+        inp_matrix[i] = _dist(_engine);
+        out_matrix[i] = 0;
+    }
+    int* in  = nullptr;
+    int* out = nullptr;
+
+    HIP_API_CALL(hipMalloc(&in, size));
+    HIP_API_CALL(hipMalloc(&out, size));
+    HIP_API_CALL(hipMemsetAsync(in, 0, size, stream));
+    HIP_API_CALL(hipMemsetAsync(out, 0, size, stream));
+    HIP_API_CALL(hipMemcpyAsync(in, inp_matrix, size, hipMemcpyHostToDevice, stream));
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    dim3 grid(M / 32, N / 32, 1);
+    dim3 block(32, 32, 1);  // transpose
+
+    auto t1 = std::chrono::high_resolution_clock::now();
+    for(size_t i = 0; i < nitr; ++i)
+    {
+        transpose<<<grid, block, 0, stream>>>(in, out, M, N);
+        check_hip_error();
+        if(i % nsync == (nsync - 1)) HIP_API_CALL(hipStreamSynchronize(stream));
+    }
+    auto t2 = std::chrono::high_resolution_clock::now();
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipMemcpyAsync(out_matrix, out, size, hipMemcpyDeviceToHost, stream));
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+    float  GB   = (float) size * nitr * 2 / (1 << 30);
+
+    print_lock.lock();
+    std::cout << "[transpose][" << rank << "][" << tid << "] Runtime of transpose is " << time
+              << " sec\n";
+    std::cout << "[transpose][" << rank << "][" << tid
+              << "] The average performance of transpose is " << GB / time << " GBytes/sec"
+              << std::endl;
+    print_lock.unlock();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipStreamDestroy(stream));
+
+    // cpu_transpose(matrix, out_matrix, M, N);
+    verify(inp_matrix, out_matrix, M, N);
+
+    HIP_API_CALL(hipFree(in));
+    HIP_API_CALL(hipFree(out));
+
+    delete[] inp_matrix;
+    delete[] out_matrix;
+
+    roctxRangePop();
+}
+
+void
+check_hip_error(void)
+{
+    hipError_t err = hipGetLastError();
+    if(err != hipSuccess)
+    {
+        auto_lock_t _lk{print_lock};
+        std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
+        throw std::runtime_error("hip_api_call");
+    }
+}
+
+void
+verify(int* in, int* out, int M, int N)
+{
+    for(int i = 0; i < 10; i++)
+    {
+        int row = rand() % M;
+        int col = rand() % N;
+        if(in[row * N + col] != out[col * M + row])
+        {
+            auto_lock_t _lk{print_lock};
+            std::cout << "mismatch: " << row << ", " << col << " : " << in[row * N + col] << " | "
+                      << out[col * M + row] << "\n";
+        }
+    }
+}
+}  // namespace
+
+void
+run_transpose(size_t nthreads, size_t nitr, size_t nsync)
+{
+    auto range_id = roctxRangeStart("run_transpose");
+
+    int rank = 0;
+    int size = 1;
+
+    printf("[transpose] Number of threads: %zu\n", nthreads);
+    printf("[transpose] Number of iterations: %zu\n", nitr);
+    printf("[transpose] Syncing every %zu iterations\n", nsync);
+
+#if defined(USE_MPI)
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+#else
+    (void) size;
+#endif
+    // this is a temporary workaround in omnitrace when HIP + MPI is enabled
+    int ndevice = 0;
+    HIP_API_CALL(hipGetDeviceCount(&ndevice));
+    printf("[transpose] Number of devices found: %i\n", ndevice);
+    auto devids = std::vector<int>{};
+    devids.resize(size * nthreads, 0);
+    int devid = 0;
+    for(size_t i = 0; i < nthreads; ++i)
+    {
+        for(int j = 0; j < size; ++j)
+        {
+            auto idx       = (j * nthreads) + i;
+            devids.at(idx) = devid++ % ndevice;
+        }
+    }
+    auto devid_offset = (rank * nthreads);
+    auto _threads     = std::vector<std::thread>{};
+    for(size_t i = 1; i < nthreads; ++i)
+        _threads.emplace_back(
+            run_transpose_impl, rank, i, devids.at(devid_offset + i), nitr, nsync);
+    run_transpose_impl(rank, 0, devids.at(devid_offset + 0), nitr, nsync);
+    for(auto& itr : _threads)
+        itr.join();
+
+#if defined(USE_MPI)
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+    roctxRangeStop(range_id);
+}

projects/rocprofiler-sdk/tests/lib/transpose/transpose.hpp

@@ -0,0 +1,28 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#pragma once
+
+#include <cstddef>
+
+void
+run_transpose(size_t nthreads, size_t nitr, size_t nsync);

projects/rocprofiler-sdk/tests/lib/vector-operations/CMakeLists.txt

@@ -0,0 +1,45 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-tests-lib-vector-operations LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_HIP_STANDARD 17)
+set(CMAKE_HIP_EXTENSIONS OFF)
+set(CMAKE_HIP_STANDARD_REQUIRED ON)
+
+set_source_files_properties(vector-ops.cpp PROPERTIES LANGUAGE HIP)
+add_library(vector-ops-shared-library SHARED)
+target_sources(vector-ops-shared-library PRIVATE vector-ops.cpp)
+target_compile_options(vector-ops-shared-library PRIVATE -W -Wall -Wextra -Wpedantic
+                                                         -Wshadow -Werror)
+target_include_directories(vector-ops-shared-library PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+set_target_properties(vector-ops-shared-library PROPERTIES OUTPUT_NAME vector-ops)
+
+find_package(Threads REQUIRED)
+target_link_libraries(vector-ops-shared-library
+                      PRIVATE Threads::Threads rocprofiler-sdk::tests-common-library)

projects/rocprofiler-sdk/tests/lib/vector-operations/vector-ops.cpp

@@ -0,0 +1,294 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include <assert.h>
+#include <hip/hip_runtime.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <algorithm>
+#include <iostream>
+#include <mutex>
+#include <thread>
+#include <vector>
+
+#include "common/defines.hpp"
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t = std::unique_lock<std::mutex>;
+auto print_lock   = std::mutex{};
+
+constexpr auto WIDTH  = (1 << 12);  // 4096
+constexpr auto HEIGHT = (1 << 11);  // 2048
+constexpr auto DEPTH  = (1 << 0);   // 1
+constexpr auto NUM    = (WIDTH * HEIGHT * DEPTH);
+
+struct dimensions
+{
+    int x = 1;
+    int y = 1;
+    int z = 1;
+};
+
+constexpr auto threads_per_block = dimensions{64, 1, 1};
+
+// Computes vectorAdd with matrix-multiply
+template <typename Tp>
+__global__ void
+addition_kernel(Tp* __restrict__ a,
+                const Tp* __restrict__ b,
+                const Tp* __restrict__ c,
+                int width,
+                int /*height*/)
+{
+    // printf("addition kernel\n");
+    int x = blockDim.x * blockIdx.x + threadIdx.x;
+    int y = blockDim.y * blockIdx.y + threadIdx.y;
+
+    if(x >= WIDTH || y >= HEIGHT) return;
+    int index = y * width + x;
+
+    a[index] = b[index] + c[index];
+}
+
+template <typename Tp>
+__global__ void
+subtract_kernel(Tp* __restrict__ a,
+                const Tp* __restrict__ b,
+                const Tp* __restrict__ c,
+                int width,
+                int /*height*/)
+{
+    // printf("subtract kernel\n");
+    int x = blockDim.x * blockIdx.x + threadIdx.x;
+    int y = blockDim.y * blockIdx.y + threadIdx.y;
+
+    if(x >= WIDTH || y >= HEIGHT) return;
+    int index = y * width + x;
+
+    a[index] = abs(b[index] - c[index]);
+}
+
+template <typename Tp>
+__global__ void
+multiply_kernel(Tp* __restrict__ a,
+                const Tp* __restrict__ b,
+                const Tp* __restrict__ c,
+                int width,
+                int /*height*/)
+{
+    // printf("multiply kernel\n");
+    int x = blockDim.x * blockIdx.x + threadIdx.x;
+    int y = blockDim.y * blockIdx.y + threadIdx.y;
+
+    if(x >= WIDTH || y >= HEIGHT) return;
+    int index = y * width + x;
+
+    a[index] = (b[index] - 1) * (c[index] - 1) + 1;
+}
+
+template <typename Tp>
+__global__ void
+divide_kernel(Tp* __restrict__ a,
+              const Tp* __restrict__ b,
+              const Tp* __restrict__ c,
+              int width,
+              int /*height*/)
+{
+    // printf("divide kernel\n");
+    int x = blockDim.x * blockIdx.x + threadIdx.x;
+    int y = blockDim.y * blockIdx.y + threadIdx.y;
+
+    if(x >= WIDTH || y >= HEIGHT) return;
+    int index = y * width + x;
+
+    a[index] = (b[index] - c[index]) / abs(c[index] + b[index]) + 1;
+}
+
+void
+run_vector_ops_impl(int num_queue, int device_id)
+{
+    auto t1 = std::chrono::high_resolution_clock::now();
+
+    HIP_API_CALL(hipSetDevice(device_id));
+
+    std::vector<float*> hostA(num_queue);
+    std::vector<float*> hostB(num_queue);
+    std::vector<float*> hostC(num_queue);
+
+    std::vector<float*> deviceA(num_queue);
+    std::vector<float*> deviceB(num_queue);
+    std::vector<float*> deviceC(num_queue);
+
+    std::vector<hipStream_t> streams(num_queue);
+
+    auto sync_stream = [num_queue, &streams](int q) {
+        if(q < 0 || q >= num_queue)
+            throw std::runtime_error{std::string{"invalid stream id: "} + std::to_string(q)};
+
+        HIP_API_CALL(hipStreamSynchronize(streams.at(q)));
+    };
+
+    auto sync_streams = [num_queue, sync_stream]() {
+        for(int i = 0; i < num_queue; ++i)
+            sync_stream(i);
+    };
+
+    for(int q = 0; q < num_queue; q++)
+    {
+        HIP_API_CALL(hipStreamCreateWithFlags(&streams[q], hipStreamNonBlocking));
+
+        HIP_API_CALL(HIP_HOST_ALLOC_FUNC(&hostA[q], NUM * sizeof(float), 0));
+        HIP_API_CALL(HIP_HOST_ALLOC_FUNC(&hostB[q], NUM * sizeof(float), 0));
+        HIP_API_CALL(HIP_HOST_ALLOC_FUNC(&hostC[q], NUM * sizeof(float), 0));
+
+        // initialize the input data
+        for(int i = 0; i < NUM; i++)
+        {
+            hostB[q][i] = static_cast<float>(i);
+            hostC[q][i] = static_cast<float>(i * 100.0f);
+        }
+
+        HIP_API_CALL(hipMallocAsync(&deviceA[q], NUM * sizeof(float), streams[q]));
+        HIP_API_CALL(hipMallocAsync(&deviceB[q], NUM * sizeof(float), streams[q]));
+        HIP_API_CALL(hipMallocAsync(&deviceC[q], NUM * sizeof(float), streams[q]));
+
+        HIP_API_CALL(hipMemcpyAsync(
+            deviceB[q], hostB[q], NUM * sizeof(float), hipMemcpyHostToDevice, streams[q]));
+        HIP_API_CALL(hipMemcpyAsync(
+            deviceC[q], hostC[q], NUM * sizeof(float), hipMemcpyHostToDevice, streams[q]));
+    }
+
+    sync_streams();
+
+    for(int q = 0; q < num_queue; q++)
+    {
+        hipLaunchKernelGGL(addition_kernel,
+                           dim3(WIDTH / threads_per_block.x, HEIGHT / threads_per_block.y),
+                           dim3(threads_per_block.x, threads_per_block.y),
+                           0,
+                           streams[q],
+                           deviceA[q],
+                           deviceB[q],
+                           deviceC[q],
+                           WIDTH,
+                           HEIGHT);
+
+        hipLaunchKernelGGL(subtract_kernel,
+                           dim3(WIDTH / threads_per_block.x, HEIGHT / threads_per_block.y),
+                           dim3(threads_per_block.x, threads_per_block.y),
+                           0,
+                           streams[q],
+                           deviceA[q],
+                           deviceB[q],
+                           deviceC[q],
+                           WIDTH,
+                           HEIGHT);
+
+        hipLaunchKernelGGL(multiply_kernel,
+                           dim3(WIDTH / threads_per_block.x, HEIGHT / threads_per_block.y),
+                           dim3(threads_per_block.x, threads_per_block.y),
+                           0,
+                           streams[q],
+                           deviceA[q],
+                           deviceB[q],
+                           deviceC[q],
+                           WIDTH,
+                           HEIGHT);
+
+        hipLaunchKernelGGL(divide_kernel,
+                           dim3(WIDTH / threads_per_block.x, HEIGHT / threads_per_block.y),
+                           dim3(threads_per_block.x, threads_per_block.y),
+                           0,
+                           streams[q],
+                           deviceB[q],
+                           deviceA[q],
+                           deviceC[q],
+                           WIDTH,
+                           HEIGHT);
+    }
+
+    sync_streams();
+
+    for(int q = 0; q < num_queue; q++)
+    {
+        HIP_API_CALL(hipMemcpyAsync(
+            hostA[q], deviceA[q], NUM * sizeof(float), hipMemcpyDeviceToHost, streams[q]));
+
+        sync_stream(q);
+
+        HIP_API_CALL(hipFree(deviceA[q]));
+        HIP_API_CALL(hipFree(deviceB[q]));
+        HIP_API_CALL(hipFree(deviceC[q]));
+
+        HIP_API_CALL(HIP_HOST_FREE_FUNC(hostA[q]));
+        HIP_API_CALL(HIP_HOST_FREE_FUNC(hostB[q]));
+        HIP_API_CALL(HIP_HOST_FREE_FUNC(hostC[q]));
+
+        HIP_API_CALL(hipStreamDestroy(streams[q]));
+    }
+
+    auto   t2   = std::chrono::high_resolution_clock::now();
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+
+    print_lock.lock();
+    std::cout << "[vector-ops] Runtime of vector-ops is " << time << " sec\n";
+    print_lock.unlock();
+}
+}  // namespace
+
+void
+run_vector_ops(int num_threads, int num_queue)
+{
+    int device_count = 0;
+    HIP_API_CALL(hipGetDeviceCount(&device_count));
+
+    if(device_count == 0) throw std::runtime_error{"No HIP devices found"};
+
+    num_threads = std::max<int>(num_threads, 1);
+    num_queue   = std::max<int>(num_queue, 1);
+
+    auto _threads = std::vector<std::thread>{};
+    _threads.reserve(num_threads);
+
+    for(int i = 0; i < num_threads; ++i)
+        _threads.emplace_back(run_vector_ops_impl, num_queue, i % device_count);
+
+    for(auto& itr : _threads)
+        itr.join();
+}

projects/rocprofiler-sdk/tests/lib/vector-operations/vector-ops.hpp

@@ -0,0 +1,26 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#pragma once
+
+void
+run_vector_ops(int num_threads, int num_queue);

projects/rocprofiler-sdk/tests/openmp-tools/CMakeLists.txt

@@ -0,0 +1,84 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(
+    rocprofiler-sdk-tests-openmp-tools
+    LANGUAGES CXX
+    VERSION 0.0.0)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+set(PYTEST_ARGS)
+if(ROCPROFILER_MEMCHECK MATCHES "(Address|Thread)Sanitizer" OR ROCPROFILER_BUILD_CODECOV)
+    set(PYTEST_ARGS -k "not test_total_runtime")
+endif()
+
+if(ROCPROFILER_MEMCHECK_PRELOAD_ENV)
+    set(PRELOAD_ENV
+        "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}:$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+else()
+    set(PRELOAD_ENV "LD_PRELOAD=$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+endif()
+
+set(ROCPROFILER_MEMCHECK_TYPES "ThreadSanitizer" "AddressSanitizer"
+                               "UndefinedBehaviorSanitizer")
+
+if(ROCPROFILER_MEMCHECK AND ROCPROFILER_MEMCHECK IN_LIST ROCPROFILER_MEMCHECK_TYPES)
+    set(IS_DISABLED ON)
+else()
+    set(IS_DISABLED OFF)
+endif()
+
+# disable when GPU-0 is navi2, navi3, and navi4
+list(GET rocprofiler-sdk-tests-gfx-info 0 openmp-tools-gpu-0-gfx-info)
+if("${openmp-tools-gpu-0-gfx-info}" MATCHES "^gfx(10|11|12)[0-9][0-9]$")
+    set(IS_DISABLED ON)
+endif()
+
+add_test(NAME test-openmp-tools-execute COMMAND $<TARGET_FILE:openmp-target>)
+
+set(openmp-tools-env
+    "${PRELOAD_ENV}"
+    "OMP_NUM_THREADS=2"
+    "OMP_DISPLAY_ENV=1"
+    "OMP_TARGET_OFFLOAD=mandatory"
+    "ROCR_VISIBLE_DEVICES=0"
+    "ROCPROFILER_TOOL_OUTPUT_FILE=openmp-tools-test.json"
+    "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:rocprofiler-sdk::rocprofiler-sdk-shared-library>:$ENV{LD_LIBRARY_PATH}"
+    )
+
+set_tests_properties(
+    test-openmp-tools-execute
+    PROPERTIES TIMEOUT
+               100
+               LABELS
+               "integration-tests;openmp-target"
+               ENVIRONMENT
+               "${openmp-tools-env}"
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               DISABLED
+               "${IS_DISABLED}")
+
+# copy to binary directory
+rocprofiler_configure_pytest_files(COPY validate.py conftest.py CONFIG pytest.ini)
+
+add_test(
+    NAME test-openmp-tools-validate
+    COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py ${PYTEST_ARGS}
+            --input ${CMAKE_CURRENT_BINARY_DIR}/openmp-tools-test.json)
+
+set_tests_properties(
+    test-openmp-tools-validate
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "integration-tests;openmp-target"
+               DEPENDS
+               test-openmp-tools-execute
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               DISABLED
+               "${IS_DISABLED}")

projects/rocprofiler-sdk/tests/openmp-tools/conftest.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import json
+import pytest
+
+from rocprofiler_sdk.pytest_utils.dotdict import dotdict
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--input",
+        action="store",
+        default="openmp-tools-test.json",
+        help="Input JSON",
+    )
+
+
+@pytest.fixture
+def input_data(request):
+    filename = request.config.getoption("--input")
+    with open(filename, "r") as inp:
+        return dotdict(json.load(inp))

projects/rocprofiler-sdk/tests/openmp-tools/pytest.ini

@@ -0,0 +1,5 @@
+
+[pytest]
+addopts = --durations=20 -rA -s -vv
+testpaths = validate.py
+pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages

projects/rocprofiler-sdk/tests/openmp-tools/validate.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import pytest
+
+
+# helper function
+def node_exists(name, data, min_len=1):
+    assert name in data
+    assert data[name] is not None
+    if isinstance(data[name], (list, tuple, dict, set)):
+        assert len(data[name]) >= min_len
+
+
+def test_data_structure(input_data):
+    """verify minimum amount of expected data is present"""
+    data = input_data
+
+    node_exists("rocprofiler-sdk-json-tool", data)
+
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    node_exists("metadata", sdk_data)
+    node_exists("pid", sdk_data["metadata"])
+    node_exists("main_tid", sdk_data["metadata"])
+    node_exists("init_time", sdk_data["metadata"])
+    node_exists("fini_time", sdk_data["metadata"])
+
+    node_exists("agents", sdk_data)
+    node_exists("call_stack", sdk_data)
+    node_exists("callback_records", sdk_data)
+    node_exists("buffer_records", sdk_data)
+
+    node_exists("names", sdk_data.callback_records)
+    node_exists("code_objects", sdk_data.callback_records)
+    node_exists("kernel_symbols", sdk_data.callback_records)
+    node_exists("hsa_api_traces", sdk_data.callback_records)
+    node_exists("hip_api_traces", sdk_data.callback_records, 0)
+    node_exists("marker_api_traces", sdk_data.callback_records)
+    node_exists("rccl_api_traces", sdk_data.callback_records, 0)
+    node_exists("ompt_traces", sdk_data.callback_records)
+    node_exists("kernel_dispatch", sdk_data.callback_records)
+
+    node_exists("names", sdk_data.buffer_records)
+    node_exists("kernel_dispatch", sdk_data.buffer_records)
+    node_exists("memory_copies", sdk_data.buffer_records, 0)
+    node_exists("hsa_api_traces", sdk_data.buffer_records)
+    node_exists("hip_api_traces", sdk_data.buffer_records, 0)
+    node_exists("marker_api_traces", sdk_data.buffer_records)
+    node_exists("rccl_api_traces", sdk_data.buffer_records, 0)
+    node_exists("ompt_traces", sdk_data.buffer_records)
+    node_exists("retired_correlation_ids", sdk_data.buffer_records)
+
+
+def test_size_entries(input_data):
+    # check that size fields are > 0 but account for function arguments
+    # which are named "size"
+    def check_size(data, bt):
+        if "size" in data.keys():
+            if isinstance(data.size, str) and bt.endswith('["args"]'):
+                pass
+            else:
+                assert data.size > 0, f"origin: {bt}"
+
+    # recursively check the entire data structure
+    def iterate_data(data, bt):
+        if isinstance(data, (list, tuple)):
+            for i, itr in enumerate(data):
+                if isinstance(itr, dict):
+                    check_size(itr, f"{bt}[{i}]")
+                iterate_data(itr, f"{bt}[{i}]")
+        elif isinstance(data, dict):
+            check_size(data, f"{bt}")
+            for key, itr in data.items():
+                iterate_data(itr, f'{bt}["{key}"]')
+
+    # start recursive check over entire JSON dict
+    iterate_data(input_data, "input_data")
+
+
+def test_timestamps(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    cb_start = {}
+    cb_end = {}
+    for titr in [
+        "hsa_api_traces",
+        "marker_api_traces",
+        "hip_api_traces",
+        "rccl_api_traces",
+        "ompt_traces",
+    ]:
+        for itr in sdk_data.callback_records[titr]:
+            cid = itr.correlation_id.internal
+            phase = itr.phase
+            if phase == 1:
+                cb_start[cid] = itr.timestamp
+            elif phase == 2:
+                cb_end[cid] = itr.timestamp
+                assert cb_start[cid] <= itr.timestamp
+
+        for itr in sdk_data.buffer_records[titr]:
+            assert itr.start_timestamp <= itr.end_timestamp
+
+    for titr in ["kernel_dispatch", "memory_copies"]:
+        for itr in sdk_data.buffer_records[titr]:
+            assert itr.start_timestamp < itr.end_timestamp, f"[{titr}] {itr}"
+            assert itr.correlation_id.internal > 0, f"[{titr}] {itr}"
+            assert itr.correlation_id.external > 0, f"[{titr}] {itr}"
+            assert sdk_data.metadata.init_time < itr.start_timestamp, f"[{titr}] {itr}"
+            assert sdk_data.metadata.init_time < itr.end_timestamp, f"[{titr}] {itr}"
+            assert sdk_data.metadata.fini_time > itr.start_timestamp, f"[{titr}] {itr}"
+            assert sdk_data.metadata.fini_time > itr.end_timestamp, f"[{titr}] {itr}"
+
+            api_start = cb_start[itr.correlation_id.internal]
+            # api_end = cb_end[itr.correlation_id.internal]
+            assert api_start < itr.start_timestamp, f"[{titr}] {itr}"
+            # assert api_end <= itr.end_timestamp, f"[{titr}] {itr}"
+
+
+def test_total_runtime(input_data):
+    sdk_data = input_data["rocprofiler-sdk-json-tool"]
+
+    runtime_data = []
+    for itr in sdk_data.buffer_records.kernel_dispatch:
+        elapsed = itr.end_timestamp - itr.start_timestamp
+        runtime_data.append(elapsed)  # in nanoseconds
+
+    expected_runtime = 1.0e-6  # one millisecond
+
+    assert sum(runtime_data) >= expected_runtime
+
+
+def test_internal_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    api_corr_ids = []
+    for titr in [
+        "hsa_api_traces",
+        "marker_api_traces",
+        "hip_api_traces",
+        "rccl_api_traces",
+        "ompt_traces",
+    ]:
+        for itr in sdk_data.callback_records[titr]:
+            api_corr_ids.append(itr.correlation_id.internal)
+
+        for itr in sdk_data.buffer_records[titr]:
+            api_corr_ids.append(itr.correlation_id.internal)
+
+    api_corr_ids_sorted = sorted(api_corr_ids)
+    api_corr_ids_unique = list(set(api_corr_ids))
+
+    for itr in sdk_data.buffer_records.kernel_dispatch:
+        assert itr.correlation_id.internal in api_corr_ids_unique
+
+    for itr in sdk_data.buffer_records.memory_copies:
+        assert itr.correlation_id.internal in api_corr_ids_unique
+
+    len_corr_id_unq = len(api_corr_ids_unique)
+    assert len(api_corr_ids) != len_corr_id_unq
+    assert max(api_corr_ids_sorted) == len_corr_id_unq
+
+
+def test_retired_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    def _sort_dict(inp):
+        return dict(sorted(inp.items()))
+
+    api_corr_ids = {}
+    for titr in [
+        "hsa_api_traces",
+        "marker_api_traces",
+        "hip_api_traces",
+        "rccl_api_traces",
+        "ompt_traces",
+    ]:
+        for itr in sdk_data.buffer_records[titr]:
+            corr_id = itr.correlation_id.internal
+            assert corr_id not in api_corr_ids.keys()
+            api_corr_ids[corr_id] = itr
+
+    async_corr_ids = {}
+    for titr in ["kernel_dispatch", "memory_copies"]:
+        for itr in sdk_data.buffer_records[titr]:
+            corr_id = itr.correlation_id.internal
+            assert corr_id not in async_corr_ids.keys()
+            async_corr_ids[corr_id] = itr
+
+    retired_corr_ids = {}
+    for itr in sdk_data.buffer_records.retired_correlation_ids:
+        corr_id = itr.internal_correlation_id
+        assert corr_id not in retired_corr_ids.keys()
+        retired_corr_ids[corr_id] = itr
+
+    api_corr_ids = _sort_dict(api_corr_ids)
+    async_corr_ids = _sort_dict(async_corr_ids)
+    retired_corr_ids = _sort_dict(retired_corr_ids)
+
+    for cid, itr in async_corr_ids.items():
+        assert cid in retired_corr_ids.keys()
+        retired_ts = retired_corr_ids[cid].timestamp
+        end_ts = itr.end_timestamp
+        assert (retired_ts - end_ts) > 0, f"correlation-id: {cid}, data: {itr}"
+
+    for cid, itr in api_corr_ids.items():
+        assert cid in retired_corr_ids.keys()
+        retired_ts = retired_corr_ids[cid].timestamp
+        end_ts = itr.end_timestamp
+        assert (retired_ts - end_ts) > 0, f"correlation-id: {cid}, data: {itr}"
+
+    assert len(api_corr_ids.keys()) == (len(retired_corr_ids.keys()))
+
+
+def test_external_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    extern_corr_ids = []
+    for titr in [
+        "hsa_api_traces",
+        "marker_api_traces",
+        "hip_api_traces",
+        "rccl_api_traces",
+        "ompt_traces",
+    ]:
+        for itr in sdk_data.callback_records[titr]:
+            assert itr.correlation_id.external > 0
+            assert itr.thread_id == itr.correlation_id.external
+            extern_corr_ids.append(itr.correlation_id.external)
+
+    extern_corr_ids = list(set(sorted(extern_corr_ids)))
+    for titr in [
+        "hsa_api_traces",
+        "marker_api_traces",
+        "hip_api_traces",
+        "rccl_api_traces",
+        "ompt_traces",
+    ]:
+        for itr in sdk_data.buffer_records[titr]:
+            assert itr.correlation_id.external > 0, f"[{titr}] {itr}"
+            assert itr.thread_id == itr.correlation_id.external, f"[{titr}] {itr}"
+            assert itr.thread_id in extern_corr_ids, f"[{titr}] {itr}"
+            assert itr.correlation_id.external in extern_corr_ids, f"[{titr}] {itr}"
+
+    for titr in ["kernel_dispatch", "memory_copies"]:
+        for itr in sdk_data.buffer_records[titr]:
+            assert itr.correlation_id.external > 0, f"[{titr}] {itr}"
+            assert itr.correlation_id.external in extern_corr_ids, f"[{titr}] {itr}"
+
+
+def test_kernel_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    symbol_info = {}
+    for itr in sdk_data.callback_records.kernel_symbols:
+        phase = itr.phase
+        payload = itr.payload
+        kern_id = payload.kernel_id
+
+        assert phase == 1 or phase == 2
+        assert kern_id > 0
+        if phase == 1:
+            assert len(payload.kernel_name) > 0
+            symbol_info[kern_id] = payload
+        elif phase == 2:
+            assert payload.kernel_id in symbol_info.keys()
+            assert payload.kernel_name == symbol_info[kern_id].kernel_name
+
+    for itr in sdk_data.buffer_records.kernel_dispatch:
+        assert itr.dispatch_info.kernel_id in symbol_info.keys()
+
+    for itr in sdk_data.callback_records.kernel_dispatch:
+        assert itr.payload.dispatch_info.kernel_id in symbol_info.keys()
+
+
+def test_kernel_dispatch_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    num_dispatches = len(sdk_data.buffer_records.kernel_dispatch)
+    num_cb_dispatches = len(sdk_data.callback_records.kernel_dispatch)
+
+    assert num_cb_dispatches == (3 * num_dispatches)
+
+    bf_seq_ids = []
+    for itr in sdk_data.buffer_records.kernel_dispatch:
+        bf_seq_ids.append(itr.dispatch_info.dispatch_id)
+
+    cb_seq_ids = []
+    for itr in sdk_data.callback_records.kernel_dispatch:
+        cb_seq_ids.append(itr.payload.dispatch_info.dispatch_id)
+
+    bf_seq_ids = sorted(bf_seq_ids)
+    cb_seq_ids = sorted(cb_seq_ids)
+
+    assert (3 * len(bf_seq_ids)) == len(cb_seq_ids)
+
+    assert bf_seq_ids[0] == cb_seq_ids[0]
+    assert bf_seq_ids[-1] == cb_seq_ids[-1]
+
+    def get_uniq(data):
+        return list(set(data))
+
+    bf_seq_ids_uniq = get_uniq(bf_seq_ids)
+    cb_seq_ids_uniq = get_uniq(cb_seq_ids)
+
+    assert bf_seq_ids == bf_seq_ids_uniq
+    assert len(cb_seq_ids) == (3 * len(cb_seq_ids_uniq))
+    assert len(bf_seq_ids) == num_dispatches
+    assert len(bf_seq_ids_uniq) == num_dispatches
+    assert len(cb_seq_ids_uniq) == num_dispatches
+
+
+if __name__ == "__main__":
+    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
+    sys.exit(exit_code)

projects/rocprofiler-sdk/tests/pc_sampling/CMakeLists.txt

@@ -0,0 +1,85 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-samples-pc-sampling-integration-test LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+find_package(rocprofiler-sdk REQUIRED)
+
+add_library(pc-sampling-integration-test-client SHARED)
+target_sources(
+    pc-sampling-integration-test-client
+    PRIVATE address_translation.cpp
+            address_translation.hpp
+            client.cpp
+            client.hpp
+            cid_retirement.cpp
+            cid_retirement.hpp
+            codeobj.cpp
+            codeobj.hpp
+            external_cid.cpp
+            external_cid.hpp
+            kernel_tracing.cpp
+            kernel_tracing.hpp
+            pcs.hpp
+            pcs.cpp
+            utils.hpp
+            utils.cpp)
+target_link_libraries(
+    pc-sampling-integration-test-client
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::tests-build-flags
+            rocprofiler-sdk::tests-common-library)
+
+set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
+find_package(Threads REQUIRED)
+
+add_executable(pc-sampling-integration-test)
+target_sources(pc-sampling-integration-test PRIVATE main.cpp)
+target_link_libraries(
+    pc-sampling-integration-test
+    PRIVATE pc-sampling-integration-test-client Threads::Threads
+            rocprofiler-sdk::tests-build-flags)
+
+# Check if PC sampling is disabled and whether we should disable the test
+rocprofiler_sdk_pc_sampling_disabled(IS_PC_SAMPLING_DISABLED)
+
+add_test(NAME pc-sampling-integration-test
+         COMMAND $<TARGET_FILE:pc-sampling-integration-test>)
+
+set(pc-sampling-integration-test-env "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")
+
+set_tests_properties(
+    pc-sampling-integration-test
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "integration-tests;pc-sampling"
+               SKIP_REGULAR_EXPRESSION
+               "PC sampling unavailable"
+               ENVIRONMENT
+               "${pc-sampling-integration-test-env}"
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               DISABLED
+               "${IS_PC_SAMPLING_DISABLED}")

projects/rocprofiler-sdk/tests/pc_sampling/address_translation.cpp

@@ -0,0 +1,205 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+#include "address_translation.hpp"
+#include "pcs.hpp"
+#include "utils.hpp"
+
+#include <cassert>
+#include <cstdio>
+#include <iostream>
+#include <memory>
+#include <sstream>
+#include <unordered_set>
+
+namespace client
+{
+namespace address_translation
+{
+namespace
+{
+struct FlatProfiler
+{
+    FlatProfiler()  = default;
+    ~FlatProfiler() = default;
+
+    CodeobjAddressTranslate translator        = {};
+    KernelObjectMap         kernel_object_map = {};
+    FlatProfile             flat_profile      = {};
+    std::mutex              global_mut        = {};
+};
+}  // namespace
+
+// Raw pointer to prevent early destruction of static objects
+FlatProfiler* flat_profiler = nullptr;
+
+void
+init()
+{
+    flat_profiler = new FlatProfiler();
+}
+
+void
+fini()
+{
+    delete flat_profiler;
+    flat_profiler = nullptr;
+}
+
+CodeobjAddressTranslate&
+get_address_translator()
+{
+    return flat_profiler->translator;
+}
+
+KernelObjectMap&
+get_kernel_object_map()
+{
+    return flat_profiler->kernel_object_map;
+}
+
+FlatProfile&
+get_flat_profile()
+{
+    return flat_profiler->flat_profile;
+}
+
+std::mutex&
+get_global_mutex()
+{
+    return flat_profiler->global_mut;
+}
+
+KernelObject::KernelObject(uint64_t    code_object_id,
+                           std::string kernel_name,
+                           uint64_t    begin_address,
+                           uint64_t    end_address)
+: code_object_id_(code_object_id)
+, kernel_name_(kernel_name)
+, begin_address_(begin_address)
+, end_address_(end_address)
+{
+    auto&    translator = get_address_translator();
+    uint64_t vaddr      = begin_address;
+    while(vaddr < end_address)
+    {
+        auto inst = translator.get(code_object_id, vaddr);
+        vaddr += inst->size;
+        this->add_instruction(std::move(inst));
+    }
+}
+
+void
+dump_flat_profile()
+{
+    // It seems that an instruction can be part of multiple
+    // instances of the same kernel loaded on two different devices.
+    // We need to prevent counting the same instruction multiple times.
+    std::unordered_set<Instruction*> visited_instructions;
+
+    const auto& kernel_object_map = get_kernel_object_map();
+    const auto& flat_profile      = get_flat_profile();
+
+    std::stringstream ss;
+    uint64_t          samples_num = 0;
+    kernel_object_map.iterate_kernel_objects([&](const KernelObject* kernel_obj) {
+        ss << "\n====================================";
+        ss << "The kernel: " << kernel_obj->kernel_name()
+           << " with the begin address: " << kernel_obj->begin_address()
+           << " from code object with id: " << kernel_obj->code_object_id() << std::endl;
+        kernel_obj->iterate_instrunctions([&](const Instruction& inst) {
+            ss << "\t";
+            ss << inst.inst << "\t";
+            ss << inst.comment << "\t";
+            ss << "samples: ";
+            const auto* _sample_instruction = flat_profile.get_sample_instruction(inst);
+            if(_sample_instruction == nullptr)
+                ss << "0";
+            else
+            {
+                _sample_instruction->process([&](const SampleInstruction& sample_instruction) {
+                    ss << sample_instruction.sample_count();
+                    // Each instruction should be visited exactly once.
+                    // Otherwise, code object loading/unloading and relocations
+                    // are not handled properly.
+                    assert(visited_instructions.count(sample_instruction.inst()) == 0);
+                    // Assure that each instruction is counted once.
+                    if(visited_instructions.count(sample_instruction.inst()) == 0)
+                    {
+                        samples_num += sample_instruction.sample_count();
+                        visited_instructions.insert(sample_instruction.inst());
+                    }
+
+                    if(sample_instruction.exec_mask_counts().size() <= 1)
+                    {
+                        ss << ", exec_mask: " << std::hex;
+                        ss << sample_instruction.exec_mask_counts().begin()->first;
+                        ss << std::dec;
+                        assert(sample_instruction.sample_count() ==
+                               sample_instruction.exec_mask_counts().begin()->second);
+                    }
+                    else
+                    {
+                        uint64_t num_samples_sum = 0;
+                        // More than one exec_mask
+                        for(auto& [exec_mask, samples_per_exec] :
+                            sample_instruction.exec_mask_counts())
+                        {
+                            ss << std::endl;
+                            ss << "\t\t"
+                               << "exec_mask: " << std::hex << exec_mask;
+                            ss << "\t"
+                               << "samples: " << std::dec << samples_per_exec;
+                            num_samples_sum += samples_per_exec;
+                            ss << std::endl;
+                        }
+                        assert(sample_instruction.sample_count() == num_samples_sum);
+                    }
+                });
+            }
+            ss << std::endl;
+        });
+        ss << "====================================\n" << std::endl;
+    });
+
+    ss << "The total number of valid decoded samples: "
+       << flat_profile.get_valid_decoded_samples_num() << std::endl;
+    ss << "The total number of invalid samples      : " << flat_profile.get_invalid_samples_num()
+       << std::endl;
+
+    *utils::get_output_stream() << ss.str() << std::endl;
+
+    utils::pcs_assert(
+        samples_num == flat_profile.get_valid_decoded_samples_num(),
+        "Number of collected valid samples different than the number of decoded samples.");
+    utils::pcs_assert(samples_num > 0, "No valid samples collected/decoded.");
+    utils::pcs_assert(flat_profile.more_valid_decoded_samples_expected(),
+                      "More invalid samples observed.");
+}
+
+}  // namespace address_translation
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/address_translation.hpp

@@ -0,0 +1,309 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <rocprofiler-sdk/cxx/codeobj/code_printing.hpp>
+
+#include <algorithm>
+#include <atomic>
+#include <cassert>
+#include <functional>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <shared_mutex>
+#include <string>
+#include <vector>
+
+namespace client
+{
+namespace address_translation
+{
+using Instruction             = rocprofiler::sdk::codeobj::disassembly::Instruction;
+using CodeobjAddressTranslate = rocprofiler::sdk::codeobj::disassembly::CodeobjAddressTranslate;
+using marker_id_t             = rocprofiler::sdk::codeobj::disassembly::marker_id_t;
+
+/**
+ * @brief Pair (code_object_id, pc_addr) uniquely identifies an instruction.
+ */
+struct inst_id_t
+{
+    marker_id_t code_object_id = 0;
+    uint64_t    pc_addr        = 0;
+
+    bool operator==(const inst_id_t& b) const
+    {
+        return this->pc_addr == b.pc_addr && this->code_object_id == b.code_object_id;
+    };
+
+    bool operator<(const inst_id_t& b) const
+    {
+        if(this->code_object_id == b.code_object_id) return this->pc_addr < b.pc_addr;
+        return this->code_object_id < b.code_object_id;
+    };
+};
+
+class KernelObject
+{
+private:
+    using process_inst_fn = std::function<void(const Instruction&)>;
+
+public:
+    KernelObject() = default;
+    KernelObject(uint64_t    code_object_id,
+                 std::string kernel_name,
+                 uint64_t    begin_address,
+                 uint64_t    end_address);
+
+    // write lock required
+    void add_instruction(std::unique_ptr<Instruction> instruction)
+    {
+        auto lock = std::unique_lock{mut};
+
+        instructions_.push_back(std::move(instruction));
+    }
+
+    // read lock required
+    void iterate_instrunctions(process_inst_fn fn) const
+    {
+        auto lock = std::shared_lock{mut};
+
+        for(const auto& inst : this->instructions_)
+            fn(*inst);
+    }
+
+    uint64_t    code_object_id() const { return code_object_id_; };
+    std::string kernel_name() const { return kernel_name_; };
+    uint64_t    begin_address() const { return begin_address_; };
+    uint64_t    end_address() const { return end_address_; };
+
+private:
+    mutable std::shared_mutex                 mut             = {};
+    uint64_t                                  code_object_id_ = 0;
+    std::string                               kernel_name_    = {};
+    uint64_t                                  begin_address_  = 0;
+    uint64_t                                  end_address_    = 0;
+    std::vector<std::unique_ptr<Instruction>> instructions_   = {};
+};
+
+class KernelObjectMap
+{
+private:
+    using process_kernel_fn = std::function<void(const KernelObject*)>;
+
+public:
+    KernelObjectMap() = default;
+
+    // write lock required
+    void add_kernel(uint64_t    code_object_id,
+                    std::string name,
+                    uint64_t    begin_address,
+                    uint64_t    end_address)
+    {
+        auto lock = std::unique_lock{mut};
+
+        auto key = form_key(code_object_id, name, begin_address);
+        auto it  = kernel_object_map.find(key);
+        assert(it == kernel_object_map.end());
+        kernel_object_map.insert(
+            {key,
+             std::make_unique<KernelObject>(code_object_id, name, begin_address, end_address)});
+    }
+
+#if 0
+    // read lock required
+    KernelObject* get_kernel(uint64_t code_object_id, std::string name)
+    {
+        auto lock = std::shared_lock{mut};
+
+        auto key = form_key(code_object_id, name);
+        auto it = kernel_object_map.find(key);
+        if(it == kernel_object_map.end())
+        {
+            return nullptr;
+        }
+
+        return it->second.get();
+    }
+#endif
+
+    // read lock required
+    void iterate_kernel_objects(process_kernel_fn fn) const
+    {
+        auto lock = std::shared_lock{mut};
+
+        for(auto& [_, kernel_obj] : kernel_object_map)
+            fn(kernel_obj.get());
+    }
+
+private:
+    std::unordered_map<std::string, std::unique_ptr<KernelObject>> kernel_object_map = {};
+    mutable std::shared_mutex                                      mut               = {};
+
+    std::string form_key(uint64_t code_object_id, std::string kernel_name, uint64_t begin_address)
+    {
+        return std::to_string(code_object_id) + "_" + kernel_name + "_" +
+               std::to_string(begin_address);
+    }
+};
+
+class SampleInstruction
+{
+private:
+    using proces_sample_inst_fn = std::function<void(const SampleInstruction&)>;
+
+public:
+    SampleInstruction() = default;
+    SampleInstruction(std::unique_ptr<Instruction> inst)
+    : inst_(std::move(inst))
+    {}
+
+    // write lock required
+    void add_sample(uint64_t exec_mask)
+    {
+        auto lock = std::unique_lock{mut};
+
+        if(exec_mask_counts_.find(exec_mask) == exec_mask_counts_.end())
+        {
+            exec_mask_counts_[exec_mask] = 0;
+        }
+        exec_mask_counts_[exec_mask]++;
+        sample_count_++;
+    }
+
+    // read lock required
+    void process(proces_sample_inst_fn fn) const
+    {
+        auto lock = std::shared_lock{mut};
+
+        fn(*this);
+    }
+
+    Instruction* inst() const { return inst_.get(); };
+    // In case an instruction is samples with different exec masks,
+    // keep track of how many time each exec_mask was observed.
+    const std::map<uint64_t, uint64_t>& exec_mask_counts() const { return exec_mask_counts_; }
+    // How many time this instruction is samples
+    uint64_t sample_count() const { return sample_count_; };
+
+private:
+    mutable std::shared_mutex mut = {};
+
+    // FIXME: prevent direct access of the following fields.
+    // The following fields should be accessible only from within `process` function.
+    std::unique_ptr<Instruction> inst_ = {};
+    // In case an instruction is samples with different exec masks,
+    // keep track of how many time each exec_mask was observed.
+    std::map<uint64_t, uint64_t> exec_mask_counts_ = {};
+    // How many time this instruction is samples
+    uint64_t sample_count_ = 0;
+};
+
+class FlatProfile
+{
+public:
+    FlatProfile() = default;
+
+    // write lock required
+    void add_sample(std::unique_ptr<Instruction> instruction, uint64_t exec_mask)
+    {
+        // counting valid decoded samples
+        valid_decoded_samples_num++;
+        auto lock = std::unique_lock{mut};
+
+        inst_id_t inst_id = {.code_object_id = instruction->codeobj_id,
+                             .pc_addr        = instruction->ld_addr};
+        auto      itr     = samples.find(inst_id);
+        if(itr == samples.end())
+        {
+            // Add new instruction
+            samples.insert({inst_id, std::make_unique<SampleInstruction>(std::move(instruction))});
+            itr = samples.find(inst_id);
+        }
+
+        auto* sample_instruction = itr->second.get();
+        sample_instruction->add_sample(exec_mask);
+    }
+
+    // read lock required
+    const SampleInstruction* get_sample_instruction(const Instruction& inst) const
+    {
+        auto lock = std::shared_lock{mut};
+
+        // TODO: Avoid creating a new instance of `inst_id_t` whenever querying
+        // sampled instructions.
+        inst_id_t inst_id = {.code_object_id = inst.codeobj_id, .pc_addr = inst.ld_addr};
+        auto      itr     = samples.find(inst_id);
+        if(itr == samples.end()) return nullptr;
+        return itr->second.get();
+        return nullptr;
+    }
+
+    void add_invalid_sample()
+    {
+        // counting invalid samples
+        invalid_decoded_samples_num++;
+    }
+
+    /**
+     * @brief Verify that more valid decoded samples is generated.
+     */
+    bool more_valid_decoded_samples_expected() const
+    {
+        return valid_decoded_samples_num > invalid_decoded_samples_num;
+    }
+
+    uint64_t get_valid_decoded_samples_num() const { return valid_decoded_samples_num; }
+
+    uint64_t get_invalid_samples_num() const { return invalid_decoded_samples_num; }
+
+private:
+    // TODO: optimize to use unordered_map
+    std::map<inst_id_t, std::unique_ptr<SampleInstruction>> samples                     = {};
+    std::atomic<uint64_t>                                   valid_decoded_samples_num   = {};
+    std::atomic<uint64_t>                                   invalid_decoded_samples_num = {};
+    mutable std::shared_mutex                               mut                         = {};
+};
+
+std::mutex&
+get_global_mutex();
+
+CodeobjAddressTranslate&
+get_address_translator();
+
+KernelObjectMap&
+get_kernel_object_map();
+
+FlatProfile&
+get_flat_profile();
+
+void
+dump_flat_profile();
+
+void
+init();
+
+void
+fini();
+}  // namespace address_translation
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/cid_retirement.cpp

@@ -0,0 +1,129 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+/**
+ * @file samples/pc_sampling_library/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include "utils.hpp"
+
+#include <rocprofiler-sdk/buffer.h>
+#include <rocprofiler-sdk/external_correlation.h>
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/hip/runtime_api_id.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <iostream>
+#include <memory>
+#include <sstream>
+
+namespace client
+{
+namespace cid_retirement
+{
+constexpr size_t BUFFER_SIZE_BYTES = 8192;
+constexpr size_t WATERMARK         = (BUFFER_SIZE_BYTES / 4);
+
+rocprofiler_buffer_id_t cid_retirement_buffer;
+
+void
+cid_retirement_tracing_buffered(rocprofiler_context_id_t /*context*/,
+                                rocprofiler_buffer_id_t /*buffer_id*/,
+                                rocprofiler_record_header_t** headers,
+                                size_t                        num_headers,
+                                void* /*user_data*/,
+                                uint64_t /*drop_count*/)
+{
+    std::stringstream ss;
+
+    for(size_t i = 0; i < num_headers; ++i)
+    {
+        auto* header = headers[i];
+
+        if(header == nullptr)
+        {
+            throw std::runtime_error{
+                "rocprofiler provided a null pointer to header. this should never happen"};
+        }
+        else if(header->hash !=
+                rocprofiler_record_header_compute_hash(header->category, header->kind))
+        {
+            throw std::runtime_error{"rocprofiler_record_header_t (category | kind) != hash"};
+        }
+        else if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING)
+        {
+            if(header->kind == ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT)
+            {
+                auto* cid_record =
+                    static_cast<rocprofiler_buffer_tracing_correlation_id_retirement_record_t*>(
+                        header->payload);
+                ss << "... The retired internal correlation id is: "
+                   << cid_record->internal_correlation_id;
+                ss << ", the timestamp is: " << cid_record->timestamp;
+                ss << std::endl;
+                // TODO: assert that the retiring timestamp is greater than
+                // the greatest timestamp of PC samples matching the retired CID.
+            }
+        }
+    }
+
+    *utils::get_output_stream() << ss.str();
+}
+
+void
+configure_cid_retirement_tracing(rocprofiler_context_id_t context)
+{
+    ROCPROFILER_CALL(rocprofiler_create_buffer(context,
+                                               BUFFER_SIZE_BYTES,
+                                               WATERMARK,
+                                               ROCPROFILER_BUFFER_POLICY_LOSSLESS,
+                                               cid_retirement_tracing_buffered,
+                                               nullptr,
+                                               &cid_retirement_buffer),
+                     "buffer creation");
+
+    ROCPROFILER_CALL(rocprofiler_configure_buffer_tracing_service(
+                         context,
+                         ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT,
+                         nullptr,
+                         0,
+                         cid_retirement_buffer),
+                     "buffer tracing service for cid retirement configure");
+}
+
+void
+flush_retired_cids()
+{
+    ROCPROFILER_CALL(rocprofiler_flush_buffer(cid_retirement_buffer),
+                     "Cannot flush retired CIDs buffer");
+    *utils::get_output_stream() << "Retired CIDs flushed..." << std::endl;
+}
+
+}  // namespace cid_retirement
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/cid_retirement.hpp

@@ -0,0 +1,38 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+namespace client
+{
+namespace cid_retirement
+{
+void
+configure_cid_retirement_tracing(rocprofiler_context_id_t context);
+
+void
+flush_retired_cids();
+}  // namespace cid_retirement
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/client.cpp

@@ -0,0 +1,225 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+/**
+ * @file samples/pc_sampling_library/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include "client.hpp"
+
+#include "address_translation.hpp"
+#include "cid_retirement.hpp"
+#include "codeobj.hpp"
+#include "external_cid.hpp"
+#include "kernel_tracing.hpp"
+#include "pcs.hpp"
+#include "utils.hpp"
+
+#include <rocprofiler-sdk/buffer.h>
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/internal_threading.h>
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <regex>
+#include <string>
+#include <vector>
+
+namespace client
+{
+namespace
+{
+rocprofiler_client_id_t*      client_id        = nullptr;
+rocprofiler_client_finalize_t client_fini_func = nullptr;
+rocprofiler_context_id_t      client_ctx{0};
+
+int
+tool_init(rocprofiler_client_finalize_t fini_func, void* /*tool_data*/)
+{
+    client_fini_func = fini_func;
+
+    address_translation::init();
+    external_cid::init();
+    pcs::init();
+
+    ROCPROFILER_CALL(rocprofiler_create_context(&client_ctx), "Cannot create context\n");
+
+    pcs::configure_pc_sampling_on_all_agents(client_ctx);
+
+    // Enable code object tracing service, to match PC samples to corresponding code object
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_tracing_service(client_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT,
+                                                       nullptr,
+                                                       0,
+                                                       client::codeobj::codeobj_tracing_callback,
+                                                       nullptr),
+        "code object tracing service configure");
+
+    cid_retirement::configure_cid_retirement_tracing(client_ctx);
+    // Kernel tracing service need for external correlation service.
+    kernel_tracing::configure_kernel_tracing_service(client_ctx);
+    external_cid::configure_external_correlation_service(client_ctx);
+
+    int valid_ctx = 0;
+    ROCPROFILER_CALL(rocprofiler_context_is_valid(client_ctx, &valid_ctx),
+                     "failure checking context validity");
+    if(valid_ctx == 0)
+    {
+        // notify rocprofiler that initialization failed
+        // and all the contexts, buffers, etc. created
+        // should be ignored
+        return -1;
+    }
+
+    ROCPROFILER_CALL(rocprofiler_start_context(client_ctx), "rocprofiler context start failed");
+
+    return 0;
+}
+
+void
+tool_fini(void* /*tool_data*/)
+{
+    // Drain all retired correlation IDs
+    client::sync();
+
+    if(client_id)
+    {
+        // Assert the context is inactive.
+        int state = -1;
+        ROCPROFILER_CALL(rocprofiler_context_is_active(client_ctx, &state),
+                         "Cannot inspect the stat of the context.")
+        assert(state == 0);
+
+        // No need to stop the context, since it has been stopped implicitly by the rocprofiler-SDK.
+
+        // Flush remaining PC samples
+        pcs::flush_and_destroy_buffers();
+    }
+
+    address_translation::dump_flat_profile();
+    // deallocation
+    address_translation::fini();
+    external_cid::fini();
+    pcs::fini();
+}
+
+}  // namespace
+
+// forward declaration
+void
+setup();
+
+void
+setup()
+{
+    // Do not force configuration
+    if(int status = 0;
+       rocprofiler_is_initialized(&status) == ROCPROFILER_STATUS_SUCCESS && status == 0)
+    {
+        *utils::get_output_stream() << "Client forces rocprofiler configuration.\n" << std::endl;
+        ROCPROFILER_CALL(rocprofiler_force_configure(&rocprofiler_configure),
+                         "failed to force configuration");
+    }
+}
+
+void
+shutdown()
+{}
+
+void
+sync()
+{
+    // Flush rocprofiler-SDK's buffers containing PC samples.
+    pcs::flush_buffers();
+
+    // Flush retired correlation IDs.
+    cid_retirement::flush_retired_cids();
+}
+
+}  // namespace client
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // only activate if main tool
+    if(priority > 0) return nullptr;
+
+    // set the client name
+    id->name = "PCSamplingExampleTool";
+
+    // store client info
+    client::client_id = id;
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " is using rocprofiler v" << major << "." << minor << "." << patch << " ("
+         << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    std::ostream* output_stream = nullptr;
+    std::string   filename      = "pc_sampling_integration_test.log";
+    if(auto* outfile = getenv("ROCPROFILER_SAMPLE_OUTPUT_FILE"); outfile) filename = outfile;
+    if(filename == "stdout")
+        output_stream = &std::cout;
+    else if(filename == "stderr")
+        output_stream = &std::cerr;
+    else
+        output_stream = new std::ofstream{filename};
+
+    client::utils::get_output_stream() = output_stream;
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &client::tool_init,
+                                            &client::tool_fini,
+                                            static_cast<void*>(output_stream)};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/tests/pc_sampling/client.hpp

@@ -0,0 +1,44 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#ifdef pc_sampling_code_obj_tracing_client_EXPORTS
+#    define CLIENT_API __attribute__((visibility("default")))
+#else
+#    define CLIENT_API
+#endif
+
+#define USE_CLIENT_SHUTDOWN_EXPLICITLY 1
+
+namespace client
+{
+void
+setup() CLIENT_API;
+
+void
+shutdown() CLIENT_API;
+
+void
+sync() CLIENT_API;
+
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/codeobj.cpp

@@ -0,0 +1,261 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+/**
+ * @file samples/pc_sampling_library/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include "address_translation.hpp"
+#include "client.hpp"
+#include "pcs.hpp"
+#include "utils.hpp"
+
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <cxxabi.h>
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <regex>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace client
+{
+namespace codeobj
+{
+#define CODEOBJ_DEBUG 0
+
+constexpr bool COPY_MEMORY_CODEOBJ = true;
+
+std::string
+cxa_demangle(std::string_view _mangled_name, int* _status)
+{
+    constexpr size_t buffer_len = 4096;
+    // return the mangled since there is no buffer
+    if(_mangled_name.empty())
+    {
+        *_status = -2;
+        return std::string{};
+    }
+
+    auto _demangled_name = std::string{_mangled_name};
+
+    // PARAMETERS to __cxa_demangle
+    //  mangled_name:
+    //      A NULL-terminated character string containing the name to be demangled.
+    //  buffer:
+    //      A region of memory, allocated with malloc, of *length bytes, into which the
+    //      demangled name is stored. If output_buffer is not long enough, it is expanded
+    //      using realloc. output_buffer may instead be NULL; in that case, the demangled
+    //      name is placed in a region of memory allocated with malloc.
+    //  _buflen:
+    //      If length is non-NULL, the length of the buffer containing the demangled name
+    //      is placed in *length.
+    //  status:
+    //      *status is set to one of the following values
+    size_t _demang_len = 0;
+    char*  _demang = abi::__cxa_demangle(_demangled_name.c_str(), nullptr, &_demang_len, _status);
+    switch(*_status)
+    {
+        //  0 : The demangling operation succeeded.
+        // -1 : A memory allocation failure occurred.
+        // -2 : mangled_name is not a valid name under the C++ ABI mangling rules.
+        // -3 : One of the arguments is invalid.
+        case 0:
+        {
+            if(_demang) _demangled_name = std::string{_demang};
+            break;
+        }
+        case -1:
+        {
+            char _msg[buffer_len];
+            ::memset(_msg, '\0', buffer_len * sizeof(char));
+            ::snprintf(_msg,
+                       buffer_len,
+                       "memory allocation failure occurred demangling %s",
+                       _demangled_name.c_str());
+            ::perror(_msg);
+            break;
+        }
+        case -2: break;
+        case -3:
+        {
+            char _msg[buffer_len];
+            ::memset(_msg, '\0', buffer_len * sizeof(char));
+            ::snprintf(_msg,
+                       buffer_len,
+                       "Invalid argument in: (\"%s\", nullptr, nullptr, %p)",
+                       _demangled_name.c_str(),
+                       (void*) _status);
+            ::perror(_msg);
+            break;
+        }
+        default: break;
+    };
+
+    // if it "demangled" but the length is zero, set the status to -2
+    if(_demang_len == 0 && *_status == 0) *_status = -2;
+
+    // free allocated buffer
+    ::free(_demang);
+    return _demangled_name;
+}
+
+template <typename Tp>
+std::string
+as_hex(Tp _v, size_t _width = 16)
+{
+    auto _ss = std::stringstream{};
+    _ss.fill('0');
+    _ss << "0x" << std::hex << std::setw(_width) << _v;
+    return _ss.str();
+}
+
+void
+codeobj_tracing_callback(rocprofiler_callback_tracing_record_t record,
+                         rocprofiler_user_data_t* /*user_data*/,
+                         void* /*callback_data*/)
+{
+    std::stringstream info;
+
+    info << "-----------------------------\n";
+    if(record.kind == ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT &&
+       record.operation == ROCPROFILER_CODE_OBJECT_LOAD)
+    {
+        auto* data =
+            static_cast<rocprofiler_callback_tracing_code_object_load_data_t*>(record.payload);
+
+        if(record.phase == ROCPROFILER_CALLBACK_PHASE_LOAD)
+        {
+            auto& global_mut = address_translation::get_global_mutex();
+            {
+                auto lock = std::unique_lock{global_mut};
+
+                auto& translator = client::address_translation::get_address_translator();
+                // register code object inside the decoder
+                if(std::string_view(data->uri).find("file:///") == 0)
+                {
+                    translator.addDecoder(
+                        data->uri, data->code_object_id, data->load_delta, data->load_size);
+                }
+                else if(COPY_MEMORY_CODEOBJ)
+                {
+                    translator.addDecoder(reinterpret_cast<const void*>(data->memory_base),
+                                          data->memory_size,
+                                          data->code_object_id,
+                                          data->load_delta,
+                                          data->load_size);
+                }
+                else
+                {
+                    return;
+                }
+
+                // extract symbols from code object
+                auto& kernel_object_map = client::address_translation::get_kernel_object_map();
+                auto  symbolmap         = translator.getSymbolMap(data->code_object_id);
+                for(auto& [vaddr, symbol] : symbolmap)
+                {
+                    kernel_object_map.add_kernel(
+                        data->code_object_id, symbol.name, vaddr, vaddr + symbol.mem_size);
+                }
+            }
+
+            info << "code object load :: ";
+        }
+        else if(record.phase == ROCPROFILER_CALLBACK_PHASE_UNLOAD)
+        {
+            // Ensure all PC samples of the unloaded code object are decoded,
+            // prior to removing the decoder.
+            client::sync();
+            auto& global_mut = address_translation::get_global_mutex();
+            {
+                auto  lock       = std::unique_lock{global_mut};
+                auto& translator = client::address_translation::get_address_translator();
+                translator.removeDecoder(data->code_object_id, data->load_delta);
+            }
+
+            info << "code object unload :: ";
+        }
+
+        info << "code_object_id=" << data->code_object_id
+             << ", rocp_agent=" << data->rocp_agent.handle << ", uri=" << data->uri
+             << ", load_base=" << as_hex(data->load_base) << ", load_size=" << data->load_size
+             << ", load_delta=" << as_hex(data->load_delta);
+        if(data->storage_type == ROCPROFILER_CODE_OBJECT_STORAGE_TYPE_FILE)
+            info << ", storage_file_descr=" << data->storage_file;
+        else if(data->storage_type == ROCPROFILER_CODE_OBJECT_STORAGE_TYPE_MEMORY)
+            info << ", storage_memory_base=" << as_hex(data->memory_base)
+                 << ", storage_memory_size=" << data->memory_size;
+
+        info << std::endl;
+    }
+    if(record.kind == ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT &&
+       record.operation == ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER)
+    {
+        auto* data =
+            static_cast<rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t*>(
+                record.payload);
+
+        if(record.phase == ROCPROFILER_CALLBACK_PHASE_LOAD)
+        {
+            info << "kernel symbol load :: ";
+        }
+        else if(record.phase == ROCPROFILER_CALLBACK_PHASE_UNLOAD)
+        {
+            info << "kernel symbol unload :: ";
+            // client_kernels.erase(data->kernel_id);
+        }
+
+        auto kernel_name     = std::regex_replace(data->kernel_name, std::regex{"(\\.kd)$"}, "");
+        int  demangle_status = 0;
+        kernel_name          = cxa_demangle(kernel_name, &demangle_status);
+
+        info << "code_object_id=" << data->code_object_id << ", kernel_id=" << data->kernel_id
+             << ", kernel_object=" << as_hex(data->kernel_object)
+             << ", kernarg_segment_size=" << data->kernarg_segment_size
+             << ", kernarg_segment_alignment=" << data->kernarg_segment_alignment
+             << ", group_segment_size=" << data->group_segment_size
+             << ", private_segment_size=" << data->private_segment_size
+             << ", kernel_name=" << kernel_name;
+
+        info << std::endl;
+    }
+
+    *utils::get_output_stream() << info.str() << std::endl;
+}
+
+}  // namespace codeobj
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/codeobj.hpp

@@ -0,0 +1,38 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+namespace client
+{
+namespace codeobj
+{
+void
+codeobj_tracing_callback(rocprofiler_callback_tracing_record_t record,
+                         rocprofiler_user_data_t*              user_data,
+                         void*                                 callback_data);
+
+}  // namespace codeobj
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/external_cid.cpp

@@ -0,0 +1,110 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+/**
+ * @file samples/pc_sampling_library/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include "utils.hpp"
+
+#include <rocprofiler-sdk/buffer.h>
+#include <rocprofiler-sdk/external_correlation.h>
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/hip/runtime_api_id.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <stdint.h>
+#include <atomic>
+#include <iostream>
+#include <memory>
+#include <sstream>
+
+namespace client
+{
+namespace external_cid
+{
+namespace
+{
+template <typename Arg, typename... Args>
+auto
+make_array(Arg arg, Args&&... args)
+{
+    constexpr auto N = 1 + sizeof...(Args);
+    return std::array<Arg, N>{std::forward<Arg>(arg), std::forward<Args>(args)...};
+}
+}  // namespace
+
+/**
+ * @brief Must be called at the beginning of the `tool_ini`.
+ */
+void
+init()
+{}
+
+/**
+ * @brief Should be called at the of the `tool_fini`
+ */
+void
+fini()
+{}
+
+int
+set_external_correlation_id(rocprofiler_thread_id_t /*thr_id*/,
+                            rocprofiler_context_id_t /*ctx_id*/,
+                            rocprofiler_external_correlation_id_request_kind_t /*kind*/,
+                            rocprofiler_tracing_operation_t /*op*/,
+                            uint64_t                 internal_corr_id,
+                            rocprofiler_user_data_t* external_corr_id,
+                            void* /*user_data*/)
+{
+    // In multi-queues (devices) scenario, incrementing external correlation IDs
+    // might not always match with incrementing internal correlation IDs.
+    // Thus, use the value of internal correlation ID and verify that both
+    // externall correlation IDs and internal correlation IDs are the same
+    // in delivered PC samples.
+    external_corr_id->value = internal_corr_id;
+    return 0;
+}
+
+void
+configure_external_correlation_service(rocprofiler_context_id_t context)
+{
+    auto external_corr_id_request_kinds =
+        make_array(ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_KERNEL_DISPATCH);
+
+    ROCPROFILER_CHECK(rocprofiler_configure_external_correlation_id_request_service(
+        context,
+        external_corr_id_request_kinds.data(),
+        external_corr_id_request_kinds.size(),
+        set_external_correlation_id,
+        nullptr));
+}
+
+}  // namespace external_cid
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/external_cid.hpp

@@ -0,0 +1,42 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <rocprofiler-sdk/external_correlation.h>
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+namespace client
+{
+namespace external_cid
+{
+void
+configure_external_correlation_service(rocprofiler_context_id_t context);
+
+void
+init();
+
+void
+fini();
+}  // namespace external_cid
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/kernel_tracing.cpp

@@ -0,0 +1,78 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+/**
+ * @file samples/pc_sampling_library/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include "utils.hpp"
+
+#include <rocprofiler-sdk/external_correlation.h>
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <atomic>
+#include <iostream>
+#include <memory>
+
+namespace client
+{
+namespace kernel_tracing
+{
+constexpr size_t BUFFER_SIZE_BYTES = 8192;
+constexpr size_t WATERMARK         = (BUFFER_SIZE_BYTES / 4);
+
+rocprofiler_buffer_id_t kernel_tracing_buffer;
+
+void
+kernel_tracing_buffered(rocprofiler_context_id_t /*context*/,
+                        rocprofiler_buffer_id_t /*buffer_id*/,
+                        rocprofiler_record_header_t** /*headers*/,
+                        size_t /*num_headers*/,
+                        void* /*user_data*/,
+                        uint64_t /*drop_count*/)
+{}
+
+void
+configure_kernel_tracing_service(rocprofiler_context_id_t context)
+{
+    ROCPROFILER_CHECK(rocprofiler_create_buffer(context,
+                                                BUFFER_SIZE_BYTES,
+                                                WATERMARK,
+                                                ROCPROFILER_BUFFER_POLICY_LOSSLESS,
+                                                kernel_tracing_buffered,
+                                                nullptr,
+                                                &kernel_tracing_buffer));
+
+    ROCPROFILER_CHECK(rocprofiler_configure_buffer_tracing_service(
+        context, ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH, nullptr, 0, kernel_tracing_buffer));
+}
+
+}  // namespace kernel_tracing
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/kernel_tracing.hpp

@@ -0,0 +1,41 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+namespace client
+{
+namespace kernel_tracing
+{
+void
+kernel_tracing_callback(rocprofiler_callback_tracing_record_t record,
+                        rocprofiler_user_data_t*              user_data,
+                        void*                                 callback_data);
+
+void
+configure_kernel_tracing_service(rocprofiler_context_id_t context);
+
+}  // namespace kernel_tracing
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/main.cpp

@@ -0,0 +1,224 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include <hip/hip_runtime.h>
+
+#include <stdio.h>
+#include <cassert>
+#include <iostream>
+#include <random>
+
+namespace
+{
+#define M                          8192
+#define N                          8192
+#define K                          8192
+#define TileSize                   16
+#define BLOCK_SIZE_X               16
+#define BLOCK_SIZE_Y               16
+#define GRID_SIZE_X                (M + BLOCK_SIZE_X - 1) / BLOCK_SIZE_X
+#define GRID_SIZE_Y                (N + BLOCK_SIZE_Y - 1) / BLOCK_SIZE_Y
+#define WAVES_PER_BLOCK_MI200_PLUS (BLOCK_SIZE_X * BLOCK_SIZE_Y) / 64
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+}  // namespace
+
+namespace
+{
+void
+check_hip_error(void);
+}  // namespace
+
+__global__ void
+matrix_multiply(float* A, float* B, float* Out, int /*m*/, int n, int k)
+{
+    int gid_x = blockDim.x * blockIdx.x + threadIdx.x;
+    int gid_y = blockDim.y * blockIdx.y + threadIdx.y;
+
+    if(gid_x < N && gid_y < M)
+    {
+        float sum = 0;
+        for(int i = 0; i < k; ++i)
+        {
+            sum += A[gid_y * k + i] * B[i * n + gid_x];
+        }
+
+        Out[gid_y * n + gid_x] = sum;
+    }
+}
+
+#if 1
+__global__ void
+matrix_multiply_tile(float* A, float* B, float* Out, int m, int n, int k)
+{
+    __shared__ float subTileM[TileSize][TileSize];
+    __shared__ float subTileN[TileSize][TileSize];
+
+    int bx = blockIdx.x;
+    int by = blockIdx.y;
+    int tx = threadIdx.x;
+    int ty = threadIdx.y;
+
+    int row = by * TileSize + ty;
+    int col = bx * TileSize + tx;
+
+    float sum = 0;
+    for(int i = 0; i < ((k - 1) / TileSize + 1); i++)
+    {
+        int curr_l = row * k + i * TileSize + tx;
+        int curr_r = (i * TileSize + ty) * n + col;
+
+        if(i * TileSize + tx < k && row < m)
+        {
+            subTileM[ty][tx] = A[curr_l];
+        }
+        else
+        {
+            subTileM[ty][tx] = 0.0;
+        }
+
+        if(i * TileSize + ty < k && col < n)
+        {
+            subTileN[ty][tx] = B[curr_r];
+        }
+        else
+        {
+            subTileN[ty][tx] = 0.0;
+        }
+
+        __syncthreads();
+
+        for(int j = 0; j < TileSize; j++)
+        {
+            if(j + TileSize * i < k)
+            {
+                sum += subTileM[ty][j] * subTileN[j][tx];
+            }
+        }
+
+        __syncthreads();
+    }
+
+    if(row < m && col < n)
+    {
+        Out[row * n + col] = sum;
+    }
+}
+#endif
+
+void
+run_hip_app()
+{
+    std::vector<float> A(M * K);
+    std::vector<float> B(K * N);
+    std::vector<float> Out(M * N);
+
+    // Randomly initialize the matrices
+    for(int i = 0; i < M * K; ++i)
+    {
+        A[i] = (float) rand() / (float) RAND_MAX;
+    }
+
+    for(int i = 0; i < K * N; ++i)
+    {
+        B[i] = (float) rand() / (float) RAND_MAX;
+    }
+
+    // Allocate GPU Memory
+    float *d_A, *d_B, *d_Out;
+    HIP_API_CALL(hipMalloc(&d_A, sizeof(float) * M * K));
+    HIP_API_CALL(hipMalloc(&d_B, sizeof(float) * K * N));
+    HIP_API_CALL(hipMalloc(&d_Out, sizeof(float) * M * N));
+
+    // Copy data to GPU
+    HIP_API_CALL(hipMemcpy(d_A, A.data(), sizeof(float) * M * K, hipMemcpyHostToDevice));
+    HIP_API_CALL(hipMemcpy(d_B, B.data(), sizeof(float) * K * N, hipMemcpyHostToDevice));
+
+    // Run the kernel
+    dim3 block_size(BLOCK_SIZE_X, BLOCK_SIZE_Y);
+    dim3 grid_size((M + block_size.x - 1) / block_size.x, (N + block_size.y - 1) / block_size.y);
+    matrix_multiply<<<grid_size, block_size>>>(d_A, d_B, d_Out, M, N, K);
+    check_hip_error();
+    matrix_multiply_tile<<<grid_size, block_size>>>(d_A, d_B, d_Out, M, N, K);
+    check_hip_error();
+
+    // Copy data back to CPU
+    HIP_API_CALL(hipMemcpy(Out.data(), d_Out, sizeof(float) * M * N, hipMemcpyDeviceToHost));
+
+    // Free GPU Memory
+    HIP_API_CALL(hipFree(d_A));
+    HIP_API_CALL(hipFree(d_B));
+    HIP_API_CALL(hipFree(d_Out));
+}
+
+#define DEVICE_ID 0
+
+int
+main(int /*argc*/, char** /*argv*/)
+{
+    int deviceId = DEVICE_ID;
+
+    auto status = hipSetDevice(deviceId);
+    assert(status == hipSuccess);
+    HIP_API_CALL(status);
+
+    int currDeviceId = -1;
+    status           = hipGetDevice(&currDeviceId);
+    HIP_API_CALL(status);
+    assert(status == hipSuccess);
+    assert(deviceId == currDeviceId);
+
+    for(int i = 0; i < 1; i++)
+    {
+        std::cout << "<<< MatMul starts" << std::endl;
+        run_hip_app();
+        std::cout << ">>> MatMul ends" << std::endl;
+    }
+
+    return 0;
+}
+
+namespace
+{
+void
+check_hip_error(void)
+{
+    hipError_t err = hipGetLastError();
+    if(err != hipSuccess)
+    {
+        std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
+        throw std::runtime_error("hip_api_call");
+    }
+}
+}  // namespace

projects/rocprofiler-sdk/tests/pc_sampling/pcs.cpp

@@ -0,0 +1,580 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+#include "pcs.hpp"
+#include "address_translation.hpp"
+#include "codeobj.hpp"
+#include "external_cid.hpp"
+#include "utils.hpp"
+
+#include <cassert>
+#include <cstdio>
+#include <iomanip>
+#include <memory>
+#include <sstream>
+#include <unordered_set>
+
+namespace client
+{
+namespace pcs
+{
+namespace
+{
+constexpr int    MAX_FAILURES      = 10;
+constexpr size_t BUFFER_SIZE_BYTES = 8192;
+constexpr size_t WATERMARK         = (BUFFER_SIZE_BYTES / 4);
+
+struct tool_agent_info;
+using avail_configs_vec_t         = std::vector<rocprofiler_pc_sampling_configuration_t>;
+using tool_agent_info_vec_t       = std::vector<std::unique_ptr<tool_agent_info>>;
+using pc_sampling_buffer_id_vec_t = std::vector<rocprofiler_buffer_id_t>;
+
+namespace
+{
+constexpr uint64_t stochastic_interval = 1048576;  // 2 ^ 20 cycles
+}  // namespace
+
+struct tool_agent_info
+{
+    rocprofiler_agent_id_t               agent_id;
+    std::unique_ptr<avail_configs_vec_t> avail_configs;
+    const rocprofiler_agent_t*           agent;
+};
+
+struct PCSampler
+{
+private:
+    using code_object_id_t     = uint64_t;
+    using code_object_id_set_t = std::unordered_set<code_object_id_t>;
+
+public:
+    PCSampler() = default;
+
+    ~PCSampler()
+    {
+        // Assert that `active_code_objects` is empty.
+        // For more information, refer to the comments above.
+        assert(active_code_objects.empty());
+        // Clear the data
+        buffer_ids.clear();
+    }
+
+    // GPU agents supporting PC sampling
+    tool_agent_info_vec_t gpu_agents = {};
+    // ROCProfiler-SDK PC sampling buffers
+    pc_sampling_buffer_id_vec_t buffer_ids = {};
+    // The set that keeps track of reported code object loading/unloading events.
+    // At the end of the test, the sets needs to be empty.
+    // Namely, each loading event will insert a code object id into the set,
+    // while each unloading event will delete a code ojbect id from the set.
+    code_object_id_set_t active_code_objects = {};
+};
+
+// The reason for using raw pointers is the following.
+// Sometimes, statically created objects of the client::pcs
+// namespace might be freed prior to the `tool_fini`,
+// meaning objects of `pcs` namespace become unusable inside `tool_fini`.
+// Instead, use raw pointers to control objects deallocation time.
+PCSampler* pc_sampler = nullptr;
+
+// forward declaration
+bool
+query_avail_configs_for_agent(tool_agent_info* agent_info);
+
+rocprofiler_status_t
+find_all_gpu_agents_supporting_pc_sampling_impl(rocprofiler_agent_version_t version,
+                                                const void**                agents,
+                                                size_t                      num_agents,
+                                                void*                       user_data)
+{
+    assert(version == ROCPROFILER_AGENT_INFO_VERSION_0);
+    // user_data represent the pointer to the array where gpu_agent will be stored
+    if(!user_data) return ROCPROFILER_STATUS_ERROR;
+
+    std::stringstream ss;
+
+    auto* _out_agents = static_cast<tool_agent_info_vec_t*>(user_data);
+    auto* _agents     = reinterpret_cast<const rocprofiler_agent_t**>(agents);
+    for(size_t i = 0; i < num_agents; i++)
+    {
+        if(_agents[i]->type == ROCPROFILER_AGENT_TYPE_GPU)
+        {
+            // Instantiate the tool_agent_info.
+            // Store pointer to the rocprofiler_agent_t and instatiate a vector of
+            // available configurations.
+            // Move the ownership to the _out_agents
+            auto tool_gpu_agent           = std::make_unique<tool_agent_info>();
+            tool_gpu_agent->agent_id      = _agents[i]->id;
+            tool_gpu_agent->avail_configs = std::make_unique<avail_configs_vec_t>();
+            tool_gpu_agent->agent         = _agents[i];
+            // Check if the GPU agent supports PC sampling. If so, add it to the
+            // output list `_out_agents`.
+            if(query_avail_configs_for_agent(tool_gpu_agent.get()))
+                _out_agents->push_back(std::move(tool_gpu_agent));
+        }
+
+        ss << "[" << __FUNCTION__ << "] " << _agents[i]->name << " :: "
+           << "id=" << _agents[i]->id.handle << ", "
+           << "type=" << _agents[i]->type << "\n";
+    }
+
+    *utils::get_output_stream() << ss.str() << "\n";
+
+    return ROCPROFILER_STATUS_SUCCESS;
+}
+
+void
+find_all_gpu_agents_supporting_pc_sampling()
+{
+    // This function returns the all gpu agents supporting some kind of PC sampling
+    ROCPROFILER_CALL(
+        rocprofiler_query_available_agents(ROCPROFILER_AGENT_INFO_VERSION_0,
+                                           &find_all_gpu_agents_supporting_pc_sampling_impl,
+                                           sizeof(rocprofiler_agent_t),
+                                           static_cast<void*>(&pc_sampler->gpu_agents)),
+        "Failed to find GPU agents");
+}
+
+/**
+ * @brief The function queries available PC sampling configurations.
+ * If there is at least one available configuration, it returns true.
+ * Otherwise, this function returns false to indicate the agent does
+ * not support PC sampling.
+ */
+bool
+query_avail_configs_for_agent(tool_agent_info* agent_info)
+{
+    // Clear the available configurations vector
+    agent_info->avail_configs->clear();
+
+    auto cb = [](const rocprofiler_pc_sampling_configuration_t* configs,
+                 size_t                                         num_config,
+                 void*                                          user_data) {
+        auto* avail_configs = static_cast<avail_configs_vec_t*>(user_data);
+        for(size_t i = 0; i < num_config; i++)
+        {
+            avail_configs->emplace_back(configs[i]);
+        }
+        return ROCPROFILER_STATUS_SUCCESS;
+    };
+
+    auto status = rocprofiler_query_pc_sampling_agent_configurations(
+        agent_info->agent_id, cb, agent_info->avail_configs.get());
+
+    std::stringstream ss;
+
+    if(status != ROCPROFILER_STATUS_SUCCESS)
+    {
+        // The query operation failed, so consider the PC sampling is unsupported at the agent.
+        // This can happen if the PC sampling service is invoked within the ROCgdb.
+        ss << "Querying PC sampling capabilities failed with status: " << status << "\n";
+        *utils::get_output_stream() << ss.str() << "\n";
+        return false;
+    }
+    else if(agent_info->avail_configs->size() == 0)
+    {
+        // No available configuration at the moment, so mark the PC sampling as unsupported.
+        return false;
+    }
+
+    ss << "The agent with the id: " << agent_info->agent_id.handle << " supports the "
+       << agent_info->avail_configs->size() << " configurations: "
+       << "\n";
+    size_t ind = 0;
+    for(auto& cfg : *agent_info->avail_configs)
+    {
+        ss << "(" << ++ind << ".) "
+           << "method: " << cfg.method << ", "
+           << "unit: " << cfg.unit << ", "
+           << "min_interval: " << cfg.min_interval << ", "
+           << "max_interval: " << cfg.max_interval << ", "
+           << "flags: " << std::hex << cfg.flags << std::dec
+           << ((cfg.flags == ROCPROFILER_PC_SAMPLING_CONFIGURATION_FLAGS_INTERVAL_POW2)
+                   ? " (an interval value must be power of 2)"
+                   : "")
+           << "\n";
+    }
+
+    *utils::get_output_stream() << ss.str() << std::flush;
+
+    return true;
+}
+
+void
+configure_pc_sampling_prefer_stochastic(tool_agent_info*         agent_info,
+                                        rocprofiler_context_id_t context_id,
+                                        rocprofiler_buffer_id_t  buffer_id)
+{
+    auto   stochastic_picked = false;
+    int    failures          = MAX_FAILURES;
+    size_t interval          = 0;
+    do
+    {
+        // Update the list of available configurations
+        auto success = query_avail_configs_for_agent(agent_info);
+        if(!success)
+        {
+            // An error occured while querying PC sampling capabilities,
+            // so avoid trying configuring PC sampling service.
+            // Instead return false to indicated a failure.
+            ROCPROFILER_CALL(ROCPROFILER_STATUS_ERROR,
+                             "Could not configuring PC sampling service due to failure with query "
+                             "capabilities.");
+        }
+
+        const rocprofiler_pc_sampling_configuration_t* first_host_trap_config  = nullptr;
+        const rocprofiler_pc_sampling_configuration_t* first_stochastic_config = nullptr;
+        // Search until encountering on the stochastic configuration, if any.
+        // Otherwise, use the host trap config
+        for(auto const& cfg : *agent_info->avail_configs)
+        {
+            if(cfg.method == ROCPROFILER_PC_SAMPLING_METHOD_STOCHASTIC)
+            {
+                first_stochastic_config = &cfg;
+                stochastic_picked       = true;
+                break;
+            }
+            else if(!first_host_trap_config &&
+                    cfg.method == ROCPROFILER_PC_SAMPLING_METHOD_HOST_TRAP)
+            {
+                first_host_trap_config = &cfg;
+            }
+        }
+
+        // Check if the stochastic config is found. Use host trap config otherwise.
+        const rocprofiler_pc_sampling_configuration_t* picked_cfg =
+            (first_stochastic_config != nullptr) ? first_stochastic_config : first_host_trap_config;
+
+        interval = (stochastic_picked) ? stochastic_interval : picked_cfg->min_interval;
+
+        auto status = rocprofiler_configure_pc_sampling_service(context_id,
+                                                                agent_info->agent_id,
+                                                                picked_cfg->method,
+                                                                picked_cfg->unit,
+                                                                interval,
+                                                                buffer_id,
+                                                                0);
+        if(status == ROCPROFILER_STATUS_SUCCESS)
+        {
+            *utils::get_output_stream()
+                << ">>> We chose " << (stochastic_picked ? "stochastic" : "Host-Trap")
+                << " PC sampling with the interval: " << interval << " "
+                << (stochastic_picked ? "clock-cycles" : "micro seconds")
+                << " on the agent: " << agent_info->agent->id.handle << "\n";
+            return;
+        }
+        else if(status != ROCPROFILER_STATUS_ERROR_NOT_AVAILABLE)
+        {
+            ROCPROFILER_CALL(status, "Failed to configure PC sampling");
+        }
+        // status ==  ROCPROFILER_STATUS_ERROR_NOT_AVAILABLE
+        // means another process P2 already configured PC sampling.
+        // Query available configurations again and receive the configurations picked by P2.
+        // However, if P2 destroys PC sampling service after query function finished,
+        // but before the `rocprofiler_configure_pc_sampling_service` is called,
+        // then the `rocprofiler_configure_pc_sampling_service` will fail again.
+        // The process P1 executing this loop can spin wait (starve) if it is unlucky enough
+        // to always be interuppted by some other process P2 that creates/destroys
+        // PC sampling service on the same device while P1 is executing the code
+        // after the `query_avail_configs_for_agent` and
+        // before the `rocprofiler_configure_pc_sampling_service`.
+        // This should happen very rarely, but just to be sure, we introduce a counter `failures`
+        // that will allow certain amount of failures to process P1.
+    } while(--failures);
+
+    // The process failed too many times configuring PC sampling,
+    // report this to user;
+    ROCPROFILER_CALL(ROCPROFILER_STATUS_ERROR,
+                     "Failed too many times configuring PC sampling service");
+}
+
+template <typename PcSamplingRecordT>
+void
+print_sample_common_fields(std::ostream& os, const PcSamplingRecordT* pc_sample)
+{
+    os << "(code_obj_id, offset): (" << pc_sample->pc.code_object_id << ", 0x" << std::hex
+       << pc_sample->pc.code_object_offset << "), "
+       << "timestamp: " << std::dec << pc_sample->timestamp << ", "
+       << "exec: " << std::hex << std::setw(16) << pc_sample->exec_mask << ", "
+       << "workgroup_id_(x=" << std::dec << std::setw(5) << pc_sample->workgroup_id.x << ", "
+       << "y=" << std::setw(5) << pc_sample->workgroup_id.y << ", "
+       << "z=" << std::setw(5) << pc_sample->workgroup_id.z << "), "
+       << "wave_in_group: " << std::setw(2) << static_cast<unsigned int>(pc_sample->wave_in_group)
+       << ", "
+       << "chiplet: " << std::setw(2) << static_cast<unsigned int>(pc_sample->hw_id.chiplet) << ", "
+       << "dispatch_id: " << std::setw(7) << pc_sample->dispatch_id << ","
+       << "correlation: {internal=" << std::setw(7) << pc_sample->correlation_id.internal << ", "
+       << "external=" << std::setw(5) << pc_sample->correlation_id.external.value << "}, ";
+}
+
+void
+print_sample(std::ostream& os, const rocprofiler_pc_sampling_record_host_trap_v0_t* sample)
+{
+    print_sample_common_fields(os, sample);
+    os << "\n";
+}
+
+void
+print_sample(std::ostream& os, const rocprofiler_pc_sampling_record_stochastic_v0_t* sample)
+{
+    print_sample_common_fields(os, sample);
+
+    if(sample->wave_issued)
+    {
+        auto* inst_c_str = rocprofiler_get_pc_sampling_instruction_type_name(
+            static_cast<rocprofiler_pc_sampling_instruction_type_t>(sample->inst_type));
+        utils::pcs_assert(inst_c_str != nullptr, "Invalid instruction type");
+        os << "wave issued " << std::string(inst_c_str) << " instruction, ";
+    }
+    else
+    {
+        auto* reason_c_str = rocprofiler_get_pc_sampling_instruction_not_issued_reason_name(
+            static_cast<rocprofiler_pc_sampling_instruction_not_issued_reason_t>(
+                sample->snapshot.reason_not_issued));
+        utils::pcs_assert(reason_c_str != nullptr, "Invalid not issued reason");
+        os << "wave is stalled due to: " << std::string(reason_c_str) << " reason, ";
+    }
+
+    auto snapshot = sample->snapshot;
+    os << "two VALU instructions issued: " << static_cast<unsigned int>(snapshot.dual_issue_valu)
+       << ", ";
+
+    os << "arbiter state: {pipe issued: ("
+       << "VALU: " << static_cast<unsigned int>(snapshot.arb_state_issue_valu) << ", "
+       << "MATRIX: " << static_cast<unsigned int>(snapshot.arb_state_issue_matrix) << ", "
+       << "LDS: " << static_cast<unsigned int>(snapshot.arb_state_issue_lds) << ", "
+       << "LDS_DIRECT: " << static_cast<unsigned int>(snapshot.arb_state_issue_lds_direct) << ", "
+       << "SCALAR: " << static_cast<unsigned int>(snapshot.arb_state_issue_scalar) << ", "
+       << "TEX: " << static_cast<unsigned int>(snapshot.arb_state_issue_vmem_tex) << ", "
+       << "FLAT: " << static_cast<unsigned int>(snapshot.arb_state_issue_flat) << ", "
+       << "EXPORT: " << static_cast<unsigned int>(snapshot.arb_state_issue_exp) << ", "
+       << "MISC: " << static_cast<unsigned int>(snapshot.arb_state_issue_misc) << "), "
+       << "pipe stalled: ("
+       << "VALU: " << static_cast<unsigned int>(snapshot.arb_state_stall_valu) << ", "
+       << "MATRIX: " << static_cast<unsigned int>(snapshot.arb_state_stall_matrix) << ", "
+       << "LDS: " << static_cast<unsigned int>(snapshot.arb_state_stall_lds) << ", "
+       << "LDS_DIRECT: " << static_cast<unsigned int>(snapshot.arb_state_stall_lds_direct) << ", "
+       << "SCALAR: " << static_cast<unsigned int>(snapshot.arb_state_stall_scalar) << ", "
+       << "TEX: " << static_cast<unsigned int>(snapshot.arb_state_stall_vmem_tex) << ", "
+       << "FLAT: " << static_cast<unsigned int>(snapshot.arb_state_stall_flat) << ", "
+       << "EXPORT: " << static_cast<unsigned int>(snapshot.arb_state_stall_exp) << ", "
+       << "MISC: " << static_cast<unsigned int>(snapshot.arb_state_stall_misc) << ")}";
+
+    os << "\n";
+}
+
+template <typename PcSamplingRecordT>
+static inline void
+process_sample(const PcSamplingRecordT*                      pc_sample,
+               address_translation::CodeobjAddressTranslate& translator,
+               address_translation::FlatProfile&             flat_profile)
+{
+    // Ignore samples from blit kernels or self-modifying code.
+    if(pc_sample->correlation_id.internal == ROCPROFILER_CORRELATION_ID_INTERNAL_NONE) return;
+
+    auto corr_id = pc_sample->correlation_id;
+    // Internal correlation IDs are generated by the ROCProfiler-SDK for
+    // kernel dispatches only. Similarly, the test tool generate external
+    // correlation IDs for the kernel dispatches only.
+    // Thus, we should expect them to be equal.
+    assert(corr_id.internal == corr_id.external.value);
+    assert(corr_id.external.value > 0);
+
+    // Decoding the PC
+    auto inst = translator.get(pc_sample->pc.code_object_id, pc_sample->pc.code_object_offset);
+    flat_profile.add_sample(std::move(inst), pc_sample->exec_mask);
+
+    // TODO: introduce checks specific to stochastic sampling
+    // TODO: print an instruction inside print_sample
+}
+
+void
+rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
+                                 rocprofiler_buffer_id_t /*buffer_id*/,
+                                 rocprofiler_record_header_t** headers,
+                                 size_t                        num_headers,
+                                 void* /*data*/,
+                                 uint64_t drop_count)
+{
+    std::stringstream ss;
+    ss << "The number of delivered samples is: " << num_headers << ", "
+       << "while the number of dropped samples is: " << drop_count << "\n";
+
+    auto& flat_profile = client::address_translation::get_flat_profile();
+    auto& translator   = client::address_translation::get_address_translator();
+    auto& global_mut   = address_translation::get_global_mutex();
+
+    {
+        auto lock = std::unique_lock{global_mut};
+
+        for(size_t i = 0; i < num_headers; i++)
+        {
+            auto* cur_header = headers[i];
+
+            if(cur_header == nullptr)
+            {
+                throw std::runtime_error{
+                    "rocprofiler provided a null pointer to header. this should never happen"};
+            }
+            else if(cur_header->hash !=
+                    rocprofiler_record_header_compute_hash(cur_header->category, cur_header->kind))
+            {
+                throw std::runtime_error{"rocprofiler_record_header_t (category | kind) != hash"};
+            }
+            else if(cur_header->category == ROCPROFILER_BUFFER_CATEGORY_PC_SAMPLING)
+            {
+                if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_HOST_TRAP_V0_SAMPLE)
+                {
+                    auto* pc_sample = static_cast<rocprofiler_pc_sampling_record_host_trap_v0_t*>(
+                        cur_header->payload);
+                    print_sample(ss, pc_sample);
+                    process_sample(pc_sample, translator, flat_profile);
+                }
+                else if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_STOCHASTIC_V0_SAMPLE)
+                {
+                    auto* pc_sample = static_cast<rocprofiler_pc_sampling_record_stochastic_v0_t*>(
+                        cur_header->payload);
+                    print_sample(ss, pc_sample);
+                    process_sample(pc_sample, translator, flat_profile);
+                }
+                else if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_INVALID_SAMPLE)
+                {
+                    // tracking number of invalid samples
+                    flat_profile.add_invalid_sample();
+                }
+                else
+                {
+                    std::cerr << "Unexpected kind of PC sampling record: " << cur_header->kind
+                              << "\n";
+                    exit(-1);
+                }
+            }
+            else
+            {
+                throw std::runtime_error{"unexpected rocprofiler_record_header_t category + kind"};
+            }
+        }
+
+        // TODO: do we need some sync here?
+        *utils::get_output_stream() << ss.str() << "\n";
+    }
+}
+}  // namespace
+
+void
+init()
+{
+    pc_sampler = new PCSampler();
+}
+
+void
+fini()
+{
+    delete pc_sampler;
+    pc_sampler = nullptr;
+}
+
+void
+configure_pc_sampling_on_all_agents(rocprofiler_context_id_t context)
+{
+    find_all_gpu_agents_supporting_pc_sampling();
+
+    if(pc_sampler->gpu_agents.empty())
+    {
+        *utils::get_output_stream() << "No availabe gpu agents supporting PC sampling"
+                                    << "\n";
+        // Emit the message to skip the test.
+        std::cerr << "PC sampling unavailable"
+                  << "\n";
+        // Exit with no error if none of the GPUs support PC sampling.
+        exit(0);
+    }
+
+    auto& buff_ids_vec = pc_sampler->buffer_ids;
+
+    for(auto& gpu_agent : pc_sampler->gpu_agents)
+    {
+        // creating a buffer that will hold pc sampling information
+        rocprofiler_buffer_policy_t drop_buffer_action = ROCPROFILER_BUFFER_POLICY_LOSSLESS;
+        auto                        buffer_id          = rocprofiler_buffer_id_t{};
+        ROCPROFILER_CALL(rocprofiler_create_buffer(context,
+                                                   client::pcs::BUFFER_SIZE_BYTES,
+                                                   client::pcs::WATERMARK,
+                                                   drop_buffer_action,
+                                                   client::pcs::rocprofiler_pc_sampling_callback,
+                                                   nullptr,
+                                                   &buffer_id),
+                         "Cannot create pc sampling buffer");
+
+        client::pcs::configure_pc_sampling_prefer_stochastic(gpu_agent.get(), context, buffer_id);
+
+        // One helper thread per GPU agent's buffer.
+        auto client_agent_thread = rocprofiler_callback_thread_t{};
+        ROCPROFILER_CALL(rocprofiler_create_callback_thread(&client_agent_thread),
+                         "failure creating callback thread");
+
+        ROCPROFILER_CALL(rocprofiler_assign_callback_thread(buffer_id, client_agent_thread),
+                         "failed to assign thread for buffer");
+
+        buff_ids_vec.emplace_back(buffer_id);
+    }
+}
+
+void
+flush_buffers()
+{
+    // Flush rocproifler-SDK's buffers containing PC samples.
+    for(const auto& buff_id : pc_sampler->buffer_ids)
+    {
+        // Flush the buffer explicitly
+        ROCPROFILER_CALL(rocprofiler_flush_buffer(buff_id), "Failure flushing buffer");
+    }
+}
+
+void
+flush_and_destroy_buffers()
+{
+    for(const auto& buff_id : pc_sampler->buffer_ids)
+    {
+        // Flush the buffer explicitly
+        ROCPROFILER_CALL(rocprofiler_flush_buffer(buff_id), "Failure flushing buffer");
+        // Destroying the buffer
+        rocprofiler_status_t status = rocprofiler_destroy_buffer(buff_id);
+        if(status == ROCPROFILER_STATUS_ERROR_BUFFER_BUSY)
+        {
+            *utils::get_output_stream()
+                << "The buffer is busy, so we cannot destroy it at the moment."
+                << "\n";
+        }
+        else
+        {
+            ROCPROFILER_CALL(status, "Cannot destroy buffer");
+        }
+    }
+}
+}  // namespace pcs
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/pcs.hpp

@@ -0,0 +1,52 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <atomic>
+#include <vector>
+
+namespace client
+{
+namespace pcs
+{
+// Must be called first (prior to any other function from this namespace)
+void
+init();
+
+// Must be called at the end of the `tool_fini`
+void
+fini();
+
+void
+configure_pc_sampling_on_all_agents(rocprofiler_context_id_t context);
+
+void
+flush_buffers();
+
+void
+flush_and_destroy_buffers();
+}  // namespace pcs
+}  // namespace client

projects/rocprofiler-sdk/tests/pc_sampling/utils.cpp

@@ -0,0 +1,51 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "utils.hpp"
+
+namespace client
+{
+namespace utils
+{
+std::ostream*&
+get_output_stream()
+{
+    // The output strea is initially unitialized
+    static std::ostream* _v = nullptr;
+    return _v;
+}
+
+/**
+ * @brief Shows @p error_msg and aborts if @p condition is false.
+ *
+ */
+void
+pcs_assert(bool condition, std::string_view error_msg)
+{
+    if(!condition)
+    {
+        std::cerr << "PC Sampling Assertion Error: " << error_msg << "\n";
+        abort();
+    }
+}
+}  // namespace utils
+}  // namespace client