Add 'projects/rocprofiler-sdk/' from commit 'bf0fad1d5406fbc51403ba1aa9621a9d4a9bce2b'

git-subtree-dir: projects/rocprofiler-sdk
git-subtree-mainline: 50a90550e9
git-subtree-split: bf0fad1d54

projects/rocprofiler-sdk/samples/CMakeLists.txt

@@ -0,0 +1,49 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(rocprofiler-sdk-samples LANGUAGES C CXX)
+
+set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "samples")
+if(CMAKE_BUILD_TYPE STREQUAL "")
+    set(CMAKE_BUILD_TYPE
+        "RelWithDebInfo"
+        CACHE STRING "Build type" FORCE)
+endif()
+
+if(NOT PROJECT_IS_TOP_LEVEL)
+    set(CMAKE_MESSAGE_INDENT "[${PROJECT_NAME}] ")
+endif()
+
+include(GNUInstallDirs)
+
+# always use lib instead of lib64
+set(CMAKE_INSTALL_LIBDIR "lib")
+
+# this should be defaulted to OFF by ROCm 7.0.1 or 7.1 this should only used to disable
+# sample tests in extreme circumstances
+option(ROCPROFILER_DISABLE_UNSTABLE_CTESTS "Disable unstable tests" ON)
+
+enable_testing()
+include(CTest)
+
+# generally needed
+find_package(rocprofiler-sdk REQUIRED)
+
+# get the gfx architectures that are present on the system
+rocprofiler_sdk_get_gfx_architectures(rocprofiler-sdk-samples-gfx-info ECHO)
+
+# common utilities for samples
+add_subdirectory(common)
+
+# actual samples
+add_subdirectory(api_callback_tracing)
+add_subdirectory(api_buffered_tracing)
+add_subdirectory(code_object_tracing)
+add_subdirectory(counter_collection)
+add_subdirectory(intercept_table)
+add_subdirectory(code_object_isa_decode)
+add_subdirectory(external_correlation_id_request)
+add_subdirectory(pc_sampling)
+add_subdirectory(openmp_target)

projects/rocprofiler-sdk/samples/api_buffered_tracing/CMakeLists.txt

@@ -0,0 +1,57 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-samples-buffered-api-tracing LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+find_package(rocprofiler-sdk REQUIRED)
+
+add_library(buffered-api-tracing-client SHARED)
+target_sources(buffered-api-tracing-client PRIVATE client.cpp client.hpp)
+target_link_libraries(
+    buffered-api-tracing-client
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::samples-build-flags
+            rocprofiler-sdk::samples-common-library)
+
+set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
+find_package(Threads REQUIRED)
+
+add_executable(buffered-api-tracing)
+target_sources(buffered-api-tracing PRIVATE main.cpp)
+target_link_libraries(
+    buffered-api-tracing
+    PRIVATE buffered-api-tracing-client Threads::Threads
+            rocprofiler-sdk::samples-build-flags rocprofiler-sdk::samples-common-library)
+
+rocprofiler_samples_get_preload_env(PRELOAD_ENV buffered-api-tracing-client)
+rocprofiler_samples_get_ld_library_path_env(LIBRARY_PATH_ENV)
+
+set(buffered-api-tracing-env ${PRELOAD_ENV} ${LIBRARY_PATH_ENV})
+
+add_test(NAME buffered-api-tracing COMMAND $<TARGET_FILE:buffered-api-tracing>)
+
+set_tests_properties(
+    buffered-api-tracing
+    PROPERTIES TIMEOUT 45 LABELS "samples" ENVIRONMENT "${buffered-api-tracing-env}"
+               FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")

projects/rocprofiler-sdk/samples/api_buffered_tracing/README.md

@@ -0,0 +1,18 @@
+# API Buffer Tracing Sample
+
+## Services
+
+- Code object callback tracing for mapping kernel IDs to kernel names
+- HSA API (Core, AMD Ext)
+- HIP API (Runtime)
+- Kernel dispatch
+- Memory copy
+- Page Migration
+- Scratch Memory
+
+## Properties
+
+- Buffer size of 4096 bytes which is automatically flushed once >= 87.5% of buffer is filled (3584 bytes)
+- Creation of dedicated thread for buffer callback delivery
+- Push external correlation IDs once per thread (value is thread ID)
+- Receives notifications for internal thread creation

projects/rocprofiler-sdk/samples/api_buffered_tracing/client.cpp

@@ -0,0 +1,589 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+/**
+ * @file samples/api_buffered_tracing/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include "client.hpp"
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include "common/call_stack.hpp"
+#include "common/defines.hpp"
+#include "common/filesystem.hpp"
+#include "common/name_info.hpp"
+
+#include <atomic>
+#include <cassert>
+#include <chrono>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <thread>
+#include <unordered_set>
+#include <vector>
+
+namespace client
+{
+namespace
+{
+using common::buffer_name_info;
+using common::call_stack_t;
+using common::source_location;
+
+using kernel_symbol_data_t = rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t;
+using kernel_symbol_map_t  = std::unordered_map<rocprofiler_kernel_id_t, kernel_symbol_data_t>;
+
+rocprofiler_client_id_t*      client_id        = nullptr;
+rocprofiler_client_finalize_t client_fini_func = nullptr;
+rocprofiler_context_id_t      client_ctx       = {0};
+rocprofiler_buffer_id_t       client_buffer    = {};
+buffer_name_info              client_name_info = {};
+kernel_symbol_map_t           client_kernels   = {};
+
+template <typename Tp>
+std::string
+as_hex(Tp _v, size_t _width = 16)
+{
+    uintptr_t _vp = 0;
+    if constexpr(std::is_pointer<Tp>::value)
+        _vp = reinterpret_cast<uintptr_t>(_v);
+    else
+        _vp = _v;
+
+    auto _ss = std::stringstream{};
+    _ss.fill('0');
+    _ss << "0x" << std::hex << std::setw(_width) << _vp;
+    return _ss.str();
+}
+
+void
+print_call_stack(const call_stack_t& _call_stack)
+{
+    common::print_call_stack("api_buffered_trace.log", _call_stack);
+}
+
+void
+tool_code_object_callback(rocprofiler_callback_tracing_record_t record,
+                          rocprofiler_user_data_t*              user_data,
+                          void*                                 callback_data)
+{
+    if(record.kind == ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT &&
+       record.operation == ROCPROFILER_CODE_OBJECT_LOAD)
+    {
+        if(record.phase == ROCPROFILER_CALLBACK_PHASE_UNLOAD)
+        {
+            // flush the buffer to ensure that any lookups for the client kernel names for the code
+            // object are completed
+            auto flush_status = rocprofiler_flush_buffer(client_buffer);
+            if(flush_status != ROCPROFILER_STATUS_ERROR_BUFFER_BUSY)
+                ROCPROFILER_CALL(flush_status, "buffer flush");
+        }
+    }
+    else if(record.kind == ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT &&
+            record.operation == ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER)
+    {
+        auto* data = static_cast<kernel_symbol_data_t*>(record.payload);
+        if(record.phase == ROCPROFILER_CALLBACK_PHASE_LOAD)
+        {
+            client_kernels.emplace(data->kernel_id, *data);
+        }
+        else if(record.phase == ROCPROFILER_CALLBACK_PHASE_UNLOAD)
+        {
+            // do not erase just in case a buffer callback needs this
+            // client_kernels.erase(data->kernel_id);
+        }
+    }
+
+    (void) user_data;
+    (void) callback_data;
+}
+
+void
+tool_tracing_callback(rocprofiler_context_id_t      context,
+                      rocprofiler_buffer_id_t       buffer_id,
+                      rocprofiler_record_header_t** headers,
+                      size_t                        num_headers,
+                      void*                         user_data,
+                      uint64_t                      drop_count)
+{
+    assert(user_data != nullptr);
+    assert(drop_count == 0 && "drop count should be zero for lossless policy");
+
+    if(num_headers == 0)
+        throw std::runtime_error{
+            "rocprofiler invoked a buffer callback with no headers. this should never happen"};
+    else if(headers == nullptr)
+        throw std::runtime_error{"rocprofiler invoked a buffer callback with a null pointer to the "
+                                 "array of headers. this should never happen"};
+
+    for(size_t i = 0; i < num_headers; ++i)
+    {
+        auto* header = headers[i];
+
+        auto kind_name = std::string{};
+        if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING)
+        {
+            const char* _name = nullptr;
+            auto        _kind = static_cast<rocprofiler_buffer_tracing_kind_t>(header->kind);
+            ROCPROFILER_CALL(rocprofiler_query_buffer_tracing_kind_name(_kind, &_name, nullptr),
+                             "query buffer tracing kind name");
+            if(_name)
+            {
+                static size_t len = 15;
+
+                kind_name = std::string{_name};
+                len       = std::max(len, kind_name.length());
+                kind_name.resize(len, ' ');
+                kind_name += " :: ";
+            }
+        }
+
+        auto get_name = [](const auto* _record) -> std::string_view {
+            try
+            {
+                return client_name_info.at(_record->kind, _record->operation);
+            } catch(std::exception& e)
+            {
+                std::cerr << __FUNCTION__
+                          << " threw an exception for buffer tracing kind=" << _record->kind
+                          << ", operation=" << _record->operation << "\nException: " << e.what()
+                          << std::flush;
+                abort();
+            }
+            return std::string_view{"??"};
+        };
+
+        if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING &&
+           (header->kind == ROCPROFILER_BUFFER_TRACING_HSA_CORE_API ||
+            header->kind == ROCPROFILER_BUFFER_TRACING_HSA_AMD_EXT_API ||
+            header->kind == ROCPROFILER_BUFFER_TRACING_HSA_IMAGE_EXT_API ||
+            header->kind == ROCPROFILER_BUFFER_TRACING_HSA_FINALIZE_EXT_API))
+        {
+            auto* record =
+                static_cast<rocprofiler_buffer_tracing_hsa_api_record_t*>(header->payload);
+            auto info = std::stringstream{};
+            info << "tid=" << record->thread_id << ", context=" << context.handle
+                 << ", buffer_id=" << buffer_id.handle
+                 << ", cid=" << record->correlation_id.internal
+                 << ", extern_cid=" << record->correlation_id.external.value
+                 << ", kind=" << record->kind << ", operation=" << record->operation
+                 << ", start=" << record->start_timestamp << ", stop=" << record->end_timestamp
+                 << ", name=" << get_name(record);
+
+            if(record->start_timestamp > record->end_timestamp)
+            {
+                auto msg = std::stringstream{};
+                msg << "hsa api: start > end (" << record->start_timestamp << " > "
+                    << record->end_timestamp
+                    << "). diff = " << (record->start_timestamp - record->end_timestamp);
+                std::cerr << "threw an exception " << msg.str() << "\n" << std::flush;
+                // throw std::runtime_error{msg.str()};
+            }
+
+            static_cast<call_stack_t*>(user_data)->emplace_back(
+                source_location{__FUNCTION__, __FILE__, __LINE__, kind_name + info.str()});
+        }
+        else if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING &&
+                header->kind == ROCPROFILER_BUFFER_TRACING_HIP_RUNTIME_API)
+        {
+            auto* record =
+                static_cast<rocprofiler_buffer_tracing_hip_api_record_t*>(header->payload);
+            auto info = std::stringstream{};
+            info << "tid=" << record->thread_id << ", context=" << context.handle
+                 << ", buffer_id=" << buffer_id.handle
+                 << ", cid=" << record->correlation_id.internal
+                 << ", extern_cid=" << record->correlation_id.external.value
+                 << ", kind=" << record->kind << ", operation=" << record->operation
+                 << ", start=" << record->start_timestamp << ", stop=" << record->end_timestamp
+                 << ", name=" << client_name_info[record->kind][record->operation];
+
+            if(record->start_timestamp > record->end_timestamp)
+            {
+                auto msg = std::stringstream{};
+                msg << "hip api: start > end (" << record->start_timestamp << " > "
+                    << record->end_timestamp
+                    << "). diff = " << (record->start_timestamp - record->end_timestamp);
+                std::cerr << "threw an exception " << msg.str() << "\n" << std::flush;
+                // throw std::runtime_error{msg.str()};
+            }
+
+            static_cast<call_stack_t*>(user_data)->emplace_back(
+                source_location{__FUNCTION__, __FILE__, __LINE__, kind_name + info.str()});
+        }
+        else if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING &&
+                header->kind == ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH)
+        {
+            auto* record =
+                static_cast<rocprofiler_buffer_tracing_kernel_dispatch_record_t*>(header->payload);
+
+            auto info = std::stringstream{};
+
+            auto kernel_id   = record->dispatch_info.kernel_id;
+            auto kernel_name = (client_kernels.count(kernel_id) > 0)
+                                   ? std::string_view{client_kernels.at(kernel_id).kernel_name}
+                                   : std::string_view{"??"};
+
+            info << "tid=" << record->thread_id << ", context=" << context.handle
+                 << ", buffer_id=" << buffer_id.handle
+                 << ", cid=" << record->correlation_id.internal
+                 << ", extern_cid=" << record->correlation_id.external.value
+                 << ", kind=" << record->kind << ", operation=" << record->operation
+                 << ", agent_id=" << record->dispatch_info.agent_id.handle
+                 << ", queue_id=" << record->dispatch_info.queue_id.handle
+                 << ", kernel_id=" << record->dispatch_info.kernel_id << ", kernel=" << kernel_name
+                 << ", start=" << record->start_timestamp << ", stop=" << record->end_timestamp
+                 << ", private_segment_size=" << record->dispatch_info.private_segment_size
+                 << ", group_segment_size=" << record->dispatch_info.group_segment_size
+                 << ", workgroup_size=(" << record->dispatch_info.workgroup_size.x << ","
+                 << record->dispatch_info.workgroup_size.y << ","
+                 << record->dispatch_info.workgroup_size.z << "), grid_size=("
+                 << record->dispatch_info.grid_size.x << "," << record->dispatch_info.grid_size.y
+                 << "," << record->dispatch_info.grid_size.z << ")";
+
+            if(record->start_timestamp > record->end_timestamp)
+                throw std::runtime_error("kernel dispatch: start > end");
+
+            static_cast<call_stack_t*>(user_data)->emplace_back(
+                source_location{__FUNCTION__, __FILE__, __LINE__, kind_name + info.str()});
+        }
+        else if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING &&
+                header->kind == ROCPROFILER_BUFFER_TRACING_MEMORY_COPY)
+        {
+            auto* record =
+                static_cast<rocprofiler_buffer_tracing_memory_copy_record_t*>(header->payload);
+
+            auto info = std::stringstream{};
+
+            info << "tid=" << record->thread_id << ", context=" << context.handle
+                 << ", buffer_id=" << buffer_id.handle
+                 << ", cid=" << record->correlation_id.internal
+                 << ", extern_cid=" << record->correlation_id.external.value
+                 << ", kind=" << record->kind << ", operation=" << record->operation
+                 << ", src_agent_id=" << record->src_agent_id.handle
+                 << ", dst_agent_id=" << record->dst_agent_id.handle
+                 << ", direction=" << record->operation << ", start=" << record->start_timestamp
+                 << ", stop=" << record->end_timestamp << ", name=" << get_name(record);
+
+            if(record->start_timestamp > record->end_timestamp)
+                throw std::runtime_error("memory copy: start > end");
+
+            static_cast<call_stack_t*>(user_data)->emplace_back(
+                source_location{__FUNCTION__, __FILE__, __LINE__, kind_name + info.str()});
+        }
+        else if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING &&
+                header->kind == ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY)
+        {
+            auto* record =
+                static_cast<rocprofiler_buffer_tracing_scratch_memory_record_t*>(header->payload);
+
+            auto info = std::stringstream{};
+
+            auto _elapsed =
+                std::chrono::duration_cast<std::chrono::duration<double, std::micro>>(
+                    std::chrono::nanoseconds{record->end_timestamp - record->start_timestamp})
+                    .count();
+
+            info << "tid=" << record->thread_id << ", context=" << context.handle
+                 << ", buffer_id=" << buffer_id.handle
+                 << ", cid=" << record->correlation_id.internal
+                 << ", extern_cid=" << record->correlation_id.external.value
+                 << ", kind=" << record->kind << ", operation=" << record->operation
+                 << ", agent_id=" << record->agent_id.handle
+                 << ", queue_id=" << record->queue_id.handle << ", thread_id=" << record->thread_id
+                 << ", elapsed=" << std::setprecision(3) << std::fixed << _elapsed
+                 << " usec, flags=" << record->flags << ", name=" << get_name(record);
+
+            static_cast<call_stack_t*>(user_data)->emplace_back(
+                source_location{__FUNCTION__, __FILE__, __LINE__, kind_name + info.str()});
+        }
+        else
+        {
+            auto _msg = std::stringstream{};
+            _msg << "unexpected rocprofiler_record_header_t category + kind: (" << header->category
+                 << " + " << header->kind << ")";
+            throw std::runtime_error{_msg.str()};
+        }
+    }
+}
+
+void
+thread_precreate(rocprofiler_runtime_library_t lib, void* tool_data)
+{
+    static_cast<call_stack_t*>(tool_data)->emplace_back(
+        source_location{__FUNCTION__,
+                        __FILE__,
+                        __LINE__,
+                        std::string{"internal thread about to be created by rocprofiler (lib="} +
+                            std::to_string(lib) + ")"});
+}
+
+void
+thread_postcreate(rocprofiler_runtime_library_t lib, void* tool_data)
+{
+    static_cast<call_stack_t*>(tool_data)->emplace_back(
+        source_location{__FUNCTION__,
+                        __FILE__,
+                        __LINE__,
+                        std::string{"internal thread was created by rocprofiler (lib="} +
+                            std::to_string(lib) + ")"});
+}
+
+int
+tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
+{
+    assert(tool_data != nullptr);
+
+    auto* call_stack_v = static_cast<call_stack_t*>(tool_data);
+
+    call_stack_v->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, ""});
+
+    client_name_info = common::get_buffer_tracing_names();
+
+    for(const auto& itr : client_name_info)
+    {
+        auto name_idx = std::stringstream{};
+        name_idx << " [" << std::setw(3) << itr.value << "]";
+        call_stack_v->emplace_back(
+            source_location{"rocprofiler_buffer_tracing_kind_names          " + name_idx.str(),
+                            __FILE__,
+                            __LINE__,
+                            std::string{itr.name}});
+
+        for(auto [didx, ditr] : itr.items())
+        {
+            auto operation_idx = std::stringstream{};
+            operation_idx << " [" << std::setw(3) << didx << "]";
+            call_stack_v->emplace_back(source_location{
+                "rocprofiler_buffer_tracing_kind_operation_names" + operation_idx.str(),
+                __FILE__,
+                __LINE__,
+                std::string{"- "} + std::string{*ditr}});
+        }
+    }
+
+    client_fini_func = fini_func;
+
+    ROCPROFILER_CALL(rocprofiler_create_context(&client_ctx), "context creation");
+
+    auto code_object_ops = std::vector<rocprofiler_tracing_operation_t>{
+        ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER};
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_tracing_service(client_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT,
+                                                       code_object_ops.data(),
+                                                       code_object_ops.size(),
+                                                       tool_code_object_callback,
+                                                       nullptr),
+        "code object tracing service configure");
+
+    constexpr auto buffer_size_bytes      = 4096;
+    constexpr auto buffer_watermark_bytes = buffer_size_bytes - (buffer_size_bytes / 8);
+
+    ROCPROFILER_CALL(rocprofiler_create_buffer(client_ctx,
+                                               buffer_size_bytes,
+                                               buffer_watermark_bytes,
+                                               ROCPROFILER_BUFFER_POLICY_LOSSLESS,
+                                               tool_tracing_callback,
+                                               tool_data,
+                                               &client_buffer),
+                     "buffer creation");
+
+    for(auto itr :
+        {ROCPROFILER_BUFFER_TRACING_HSA_CORE_API, ROCPROFILER_BUFFER_TRACING_HSA_AMD_EXT_API})
+    {
+        ROCPROFILER_CALL(rocprofiler_configure_buffer_tracing_service(
+                             client_ctx, itr, nullptr, 0, client_buffer),
+                         "buffer tracing service configure");
+    }
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_buffer_tracing_service(
+            client_ctx, ROCPROFILER_BUFFER_TRACING_HIP_RUNTIME_API, nullptr, 0, client_buffer),
+        "buffer tracing service configure");
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_buffer_tracing_service(
+            client_ctx, ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH, nullptr, 0, client_buffer),
+        "buffer tracing service for kernel dispatch configure");
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_buffer_tracing_service(
+            client_ctx, ROCPROFILER_BUFFER_TRACING_MEMORY_COPY, nullptr, 0, client_buffer),
+        "buffer tracing service for memory copy configure");
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_buffer_tracing_service(
+            client_ctx, ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY, nullptr, 0, client_buffer),
+        "buffer tracing service for scratch memory configure");
+
+    auto client_thread = rocprofiler_callback_thread_t{};
+    ROCPROFILER_CALL(rocprofiler_create_callback_thread(&client_thread),
+                     "creating callback thread");
+
+    ROCPROFILER_CALL(rocprofiler_assign_callback_thread(client_buffer, client_thread),
+                     "assignment of thread for buffer");
+
+    int valid_ctx = 0;
+    ROCPROFILER_CALL(rocprofiler_context_is_valid(client_ctx, &valid_ctx),
+                     "context validity check");
+    if(valid_ctx == 0)
+    {
+        // notify rocprofiler that initialization failed
+        // and all the contexts, buffers, etc. created
+        // should be ignored
+        return -1;
+    }
+
+    ROCPROFILER_CALL(rocprofiler_start_context(client_ctx), "rocprofiler context start");
+
+    // no errors
+    return 0;
+}
+
+void
+tool_fini(void* tool_data)
+{
+    assert(tool_data != nullptr);
+
+    auto* _call_stack = static_cast<call_stack_t*>(tool_data);
+    _call_stack->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, ""});
+
+    print_call_stack(*_call_stack);
+
+    delete _call_stack;
+}
+}  // namespace
+
+void
+setup()
+{
+    if(int status = 0;
+       rocprofiler_is_initialized(&status) == ROCPROFILER_STATUS_SUCCESS && status == 0)
+    {
+        ROCPROFILER_CALL(rocprofiler_force_configure(&rocprofiler_configure),
+                         "force configuration");
+    }
+}
+
+void
+shutdown()
+{
+    if(client_id)
+    {
+        ROCPROFILER_CALL(rocprofiler_flush_buffer(client_buffer), "buffer flush");
+        client_fini_func(*client_id);
+    }
+}
+
+void
+start()
+{
+    ROCPROFILER_CALL(rocprofiler_start_context(client_ctx), "context start");
+}
+
+void
+identify(uint64_t val)
+{
+    auto _tid = rocprofiler_thread_id_t{};
+    rocprofiler_get_thread_id(&_tid);
+    rocprofiler_user_data_t user_data = {};
+    user_data.value                   = val;
+    rocprofiler_push_external_correlation_id(client_ctx, _tid, user_data);
+}
+
+void
+stop()
+{
+    ROCPROFILER_CALL(rocprofiler_stop_context(client_ctx), "context stop");
+}
+}  // namespace client
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name = "ExampleTool";
+
+    // store client info
+    client::client_id = id;
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " (priority=" << priority << ") is using rocprofiler-sdk v" << major << "."
+         << minor << "." << patch << " (" << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    auto* client_tool_data = new std::vector<client::source_location>{};
+
+    client_tool_data->emplace_back(
+        client::source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+
+    ROCPROFILER_CALL(rocprofiler_at_internal_thread_create(
+                         client::thread_precreate,
+                         client::thread_postcreate,
+                         ROCPROFILER_LIBRARY | ROCPROFILER_HSA_LIBRARY | ROCPROFILER_HIP_LIBRARY |
+                             ROCPROFILER_MARKER_LIBRARY,
+                         static_cast<void*>(client_tool_data)),
+                     "registration for thread creation notifications");
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &client::tool_init,
+                                            &client::tool_fini,
+                                            static_cast<void*>(client_tool_data)};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/api_buffered_tracing/client.hpp

@@ -0,0 +1,49 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#ifdef buffered_api_tracing_client_EXPORTS
+#    define CLIENT_API __attribute__((visibility("default")))
+#else
+#    define CLIENT_API
+#endif
+
+#include <cstdint>
+
+namespace client
+{
+void
+setup() CLIENT_API;
+
+void
+shutdown() CLIENT_API;
+
+void
+start() CLIENT_API;
+
+void
+stop() CLIENT_API;
+
+void
+identify(uint64_t corr_id) CLIENT_API;
+}  // namespace client

projects/rocprofiler-sdk/samples/api_buffered_tracing/main.cpp

@@ -0,0 +1,426 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "client.hpp"
+
+#include "common/defines.hpp"
+#include "hip/hip_runtime.h"
+
+#include <libgen.h>
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <mutex>
+#include <random>
+#include <sstream>
+#include <stdexcept>
+#include <thread>
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t                      = std::unique_lock<std::mutex>;
+auto               print_lock          = std::mutex{};
+size_t             nthread_per_device  = 2;
+size_t             nitr                = 500;
+size_t             nsync               = 10;
+constexpr unsigned shared_mem_tile_dim = 32;
+
+void
+check_hip_error(void);
+
+void
+verify(int* in, int* out, int M, int N);
+}  // namespace
+
+__global__ void
+transpose(const int* in, int* out, int M, int N);
+
+void
+run(int rank, int tid, int devid, int argc, char** argv);
+
+void
+run_transpose(int rank, int tid, hipStream_t stream, int argc, char** argv);
+
+void
+run_migrate(int rank, int tid, hipStream_t stream, int, char** argv);
+
+void
+run_scratch(int rank, int tid, hipStream_t stream, int argc, char** argv);
+
+int
+main(int argc, char** argv)
+{
+    client::setup();  // forces rocprofiler to configure/initialize
+    client::start();  // starts context before any API tables are available
+    client::identify(1);
+
+    auto* exe_name = basename(argv[0]);
+
+    int rank = 0;
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{argv[i]};
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            fprintf(stderr,
+                    "usage: %s [NUM_THREADS_PER_DEVICE (%zu)] [NUM_ITERATION (%zu)] "
+                    "[SYNC_EVERY_N_ITERATIONS (%zu)]\n",
+                    exe_name,
+                    nthread_per_device,
+                    nitr,
+                    nsync);
+            exit(EXIT_SUCCESS);
+        }
+    }
+    if(argc > 1) nthread_per_device = atoll(argv[1]);
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    int ndevice = 0;
+    HIP_API_CALL(hipGetDeviceCount(&ndevice));
+
+    auto nthreads = (ndevice * nthread_per_device);
+
+    printf("[%s] Number of devices found: %i\n", exe_name, ndevice);
+    printf("[%s] Number of threads (per device): %zu\n", exe_name, nthread_per_device);
+    printf("[%s] Number of threads (total): %zu\n", exe_name, nthreads);
+    printf("[%s] Number of iterations: %zu\n", exe_name, nitr);
+    printf("[%s] Syncing every %zu iterations\n", exe_name, nsync);
+
+    {
+        auto _threads = std::vector<std::thread>{};
+        for(size_t i = 0; i < nthreads; ++i)
+            _threads.emplace_back(run, rank, i, i % ndevice, argc, argv);
+        for(auto& itr : _threads)
+            itr.join();
+    }
+
+    HIP_API_CALL(hipDeviceSynchronize());
+    HIP_API_CALL(hipDeviceReset());
+
+    client::stop();
+    client::shutdown();
+
+    return 0;
+}
+
+__global__ void
+transpose(const int* in, int* out, int M, int N)
+{
+    __shared__ int tile[shared_mem_tile_dim][shared_mem_tile_dim];
+
+    int idx = (blockIdx.y * blockDim.y + threadIdx.y) * M + blockIdx.x * blockDim.x + threadIdx.x;
+    tile[threadIdx.y][threadIdx.x] = in[idx];
+    __syncthreads();
+    idx      = (blockIdx.x * blockDim.x + threadIdx.y) * N + blockIdx.y * blockDim.y + threadIdx.x;
+    out[idx] = tile[threadIdx.x][threadIdx.y];
+}
+
+template <typename Tp>
+__global__ void
+test_page_migrate(Tp* data, Tp val)
+{
+    int idx = (blockIdx.x * blockDim.x) + threadIdx.x;
+    data[idx] += val;
+}
+
+__global__ void
+test_kern_large(uint64_t* output)
+{
+    uint64_t result = 0;
+    int      test[4000];
+    memset(test, 5, 4000);
+    for(int& i : test)
+    {
+        i = i + 7;
+        *output += i;
+        result += i;
+    }
+    *output ^= result;
+    *output ^= result;
+}
+
+__global__ void
+test_kern_medium(uint64_t* output)
+{
+    uint64_t result = 0;
+    int      test[175];
+    memset(test, 5, 175);
+    for(int& i : test)
+    {
+        i = i + 7;
+        *output += i;
+        result += i;
+    }
+    *output ^= result;
+    *output ^= result;
+}
+
+__global__ void
+test_kern_small(uint64_t* output)
+{
+    uint64_t result = 0;
+    int      test[2];
+    for(int& i : test)
+    {
+        i = i + 7;
+        *output += i;
+        result += i;
+    }
+    *output ^= result;
+    *output ^= result;
+}
+
+void
+run(int rank, int tid, int devid, int argc, char** argv)
+{
+    client::identify(tid + 1);
+
+    auto* stream = hipStream_t{};
+    HIP_API_CALL(hipSetDevice(devid));
+    HIP_API_CALL(hipStreamCreate(&stream));
+
+    run_migrate(rank, tid, stream, argc, argv);
+    run_scratch(rank, tid, stream, argc, argv);
+    run_transpose(rank, tid, stream, argc, argv);
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipStreamDestroy(stream));
+}
+
+void
+run_transpose(int rank, int tid, hipStream_t stream, int argc, char** argv)
+{
+    auto* exe_name = basename(argv[0]);
+
+    unsigned int M = 4960 * 2;
+    unsigned int N = 4960 * 2;
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    auto_lock_t _lk{print_lock};
+    std::cout << "[" << exe_name << "][transpose][" << rank << "][" << tid << "] M: " << M
+              << " N: " << N << std::endl;
+    _lk.unlock();
+
+    std::default_random_engine         _engine{std::random_device{}() * (rank + 1) * (tid + 1)};
+    std::uniform_int_distribution<int> _dist{0, 1000};
+
+    size_t size       = sizeof(int) * M * N;
+    int*   inp_matrix = new int[size];
+    int*   out_matrix = new int[size];
+    for(size_t i = 0; i < M * N; i++)
+    {
+        inp_matrix[i] = _dist(_engine);
+        out_matrix[i] = 0;
+    }
+    int* in  = nullptr;
+    int* out = nullptr;
+
+    HIP_API_CALL(hipMalloc(&in, size));
+    HIP_API_CALL(hipMalloc(&out, size));
+    HIP_API_CALL(hipMemsetAsync(in, 0, size, stream));
+    HIP_API_CALL(hipMemsetAsync(out, 0, size, stream));
+    HIP_API_CALL(hipMemcpyAsync(in, inp_matrix, size, hipMemcpyHostToDevice, stream));
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    dim3 grid(M / 32, N / 32, 1);
+    dim3 block(32, 32, 1);  // transpose
+
+    print_lock.lock();
+    printf("[%s][transpose][%i][%i] grid=(%i,%i,%i), block=(%i,%i,%i)\n",
+           exe_name,
+           rank,
+           tid,
+           grid.x,
+           grid.y,
+           grid.z,
+           block.x,
+           block.y,
+           block.z);
+    print_lock.unlock();
+
+    auto t1 = std::chrono::high_resolution_clock::now();
+    for(size_t i = 0; i < nitr; ++i)
+    {
+        transpose<<<grid, block, 0, stream>>>(in, out, M, N);
+        check_hip_error();
+        if(i % nsync == (nsync - 1)) HIP_API_CALL(hipStreamSynchronize(stream));
+    }
+    auto t2 = std::chrono::high_resolution_clock::now();
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipMemcpyAsync(out_matrix, out, size, hipMemcpyDeviceToHost, stream));
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+    float  GB   = (float) size * nitr * 2 / (1 << 30);
+
+    print_lock.lock();
+    std::cout << "[" << exe_name << "][transpose][" << rank << "][" << tid
+              << "] Runtime of transpose is " << time << " sec\n";
+    std::cout << "[" << exe_name << "][transpose][" << rank << "][" << tid
+              << "] The average performance of transpose is " << GB / time << " GBytes/sec"
+              << std::endl;
+    print_lock.unlock();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    // cpu_transpose(matrix, out_matrix, M, N);
+    verify(inp_matrix, out_matrix, M, N);
+
+    HIP_API_CALL(hipFree(in));
+    HIP_API_CALL(hipFree(out));
+
+    delete[] inp_matrix;
+    delete[] out_matrix;
+}
+
+void
+run_scratch(int rank, int tid, hipStream_t stream, int, char** argv)
+{
+    auto t1 = std::chrono::high_resolution_clock::now();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    const auto* exe_name = basename(argv[0]);
+
+    uint64_t* data_ptr = nullptr;
+    HIP_API_CALL(HIP_HOST_ALLOC_FUNC(&data_ptr, sizeof(uint64_t), 0));
+    *data_ptr = 0;
+
+    test_kern_small<<<1000, 1, 0, stream>>>(data_ptr);
+    test_kern_medium<<<1000, 1, 0, stream>>>(data_ptr);
+    test_kern_small<<<1000, 1, 0, stream>>>(data_ptr);
+    test_kern_large<<<1100, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    test_kern_small<<<1000, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    test_kern_medium<<<1000, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    test_kern_small<<<1000, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    test_kern_large<<<1100, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    auto   t2   = std::chrono::high_resolution_clock::now();
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+
+    print_lock.lock();
+    std::cout << "[" << exe_name << "][scratch][" << rank << "][" << tid
+              << "] Runtime of scratch is " << time << " sec\n";
+    print_lock.unlock();
+}
+
+void
+run_migrate(int rank, int tid, hipStream_t stream, int, char** argv)
+{
+    using data_type            = uint64_t;
+    constexpr data_type init_v = 1;
+    constexpr data_type incr_v = 1;
+
+    auto t1 = std::chrono::high_resolution_clock::now();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    const auto* exe_name  = basename(argv[0]);
+    auto        page_data = std::vector<data_type>(1024, 0);
+
+    HIP_API_CALL(hipHostRegister(
+        page_data.data(), page_data.size() * sizeof(data_type), hipHostRegisterDefault));
+
+    for(auto& itr : page_data)
+        itr = init_v;
+
+    test_page_migrate<<<1, 1024, 0, stream>>>(page_data.data(), incr_v);
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    for(auto& itr : page_data)
+    {
+        auto diff = (itr - incr_v);
+        if(diff != init_v)
+        {
+            auto msg = std::stringstream{};
+            msg << "invalid diff: " << diff << ". expected: " << init_v;
+            throw std::runtime_error{msg.str()};
+        }
+    }
+
+    HIP_API_CALL(hipHostUnregister(page_data.data()));
+
+    auto   t2   = std::chrono::high_resolution_clock::now();
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+
+    print_lock.lock();
+    std::cout << "[" << exe_name << "][migrate][" << rank << "][" << tid
+              << "] Runtime of migrate is " << time << " sec\n";
+    print_lock.unlock();
+}
+
+namespace
+{
+void
+check_hip_error(void)
+{
+    hipError_t err = hipGetLastError();
+    if(err != hipSuccess)
+    {
+        auto_lock_t _lk{print_lock};
+        std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
+        throw std::runtime_error("hip_api_call");
+    }
+}
+
+void
+verify(int* in, int* out, int M, int N)
+{
+    for(int i = 0; i < 10; i++)
+    {
+        int row = rand() % M;
+        int col = rand() % N;
+        if(in[row * N + col] != out[col * M + row])
+        {
+            auto_lock_t _lk{print_lock};
+            std::cout << "mismatch: " << row << ", " << col << " : " << in[row * N + col] << " | "
+                      << out[col * M + row] << "\n";
+        }
+    }
+}
+}  // namespace

projects/rocprofiler-sdk/samples/api_callback_tracing/CMakeLists.txt

@@ -0,0 +1,61 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-samples-callback-api-tracing LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+find_package(rocprofiler-sdk REQUIRED)
+
+add_library(callback-api-tracing-client SHARED)
+target_sources(callback-api-tracing-client PRIVATE client.cpp client.hpp)
+target_link_libraries(
+    callback-api-tracing-client
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::samples-build-flags
+            rocprofiler-sdk::samples-common-library)
+
+set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
+
+find_package(Threads REQUIRED)
+find_package(rocprofiler-sdk-roctx REQUIRED)
+
+add_executable(callback-api-tracing)
+target_sources(callback-api-tracing PRIVATE main.cpp)
+target_link_libraries(
+    callback-api-tracing
+    PRIVATE callback-api-tracing-client Threads::Threads
+            rocprofiler-sdk-roctx::rocprofiler-sdk-roctx
+            rocprofiler-sdk::samples-build-flags)
+
+rocprofiler_samples_get_preload_env(PRELOAD_ENV callback-api-tracing-client)
+rocprofiler_samples_get_ld_library_path_env(
+    LIBRARY_PATH_ENV rocprofiler-sdk-roctx::rocprofiler-sdk-roctx-shared-library)
+
+set(callback-api-tracing-env ${PRELOAD_ENV} ${LIBRARY_PATH_ENV})
+
+add_test(NAME callback-api-tracing COMMAND $<TARGET_FILE:callback-api-tracing>)
+
+set_tests_properties(
+    callback-api-tracing
+    PROPERTIES TIMEOUT 45 LABELS "samples" ENVIRONMENT "${callback-api-tracing-env}"
+               FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")

projects/rocprofiler-sdk/samples/api_callback_tracing/README.md

@@ -0,0 +1,14 @@
+# API Callback Tracing Sample
+
+## Services
+
+- Captures events like API calls using callbacks.
+- HSA API (Core, AMD Ext)
+- HIP API (Runtime)
+- Marker API (Core, Name)
+
+## Properties
+
+- Handles roctxProfilerPause and roctxProfilerResume operations using a control context.
+- Captures API calls and logs details like thread ID, operation type, and duration.
+- Provides a detailed trace of all function calls and events for debugging.

projects/rocprofiler-sdk/samples/api_callback_tracing/client.cpp

@@ -0,0 +1,376 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+/**
+ * @file samples/api_callback_tracing/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include "client.hpp"
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+#include <rocprofiler-sdk/cxx/operators.hpp>
+#include <rocprofiler-sdk/cxx/version.hpp>
+
+#include "common/call_stack.hpp"
+#include "common/defines.hpp"
+#include "common/filesystem.hpp"
+#include "common/name_info.hpp"
+
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <ratio>
+#include <string>
+#include <string_view>
+#include <unordered_set>
+#include <vector>
+namespace client
+{
+namespace
+{
+using common::call_stack_t;
+using common::callback_name_info;
+using common::source_location;
+
+rocprofiler_client_id_t*      client_id        = nullptr;
+rocprofiler_client_finalize_t client_fini_func = nullptr;
+rocprofiler_context_id_t      client_ctx       = {0};
+
+void
+print_call_stack(const call_stack_t& _call_stack)
+{
+    common::print_call_stack("api_callback_trace.log", _call_stack);
+}
+
+void
+tool_tracing_ctrl_callback(rocprofiler_callback_tracing_record_t record,
+                           rocprofiler_user_data_t*,
+                           void* client_data)
+{
+    auto* ctx = static_cast<rocprofiler_context_id_t*>(client_data);
+
+    if(record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER &&
+       record.kind == ROCPROFILER_CALLBACK_TRACING_MARKER_CONTROL_API &&
+       record.operation == ROCPROFILER_MARKER_CONTROL_API_ID_roctxProfilerPause)
+    {
+        ROCPROFILER_CALL(rocprofiler_stop_context(*ctx), "pausing client context");
+    }
+    else if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT &&
+            record.kind == ROCPROFILER_CALLBACK_TRACING_MARKER_CONTROL_API &&
+            record.operation == ROCPROFILER_MARKER_CONTROL_API_ID_roctxProfilerResume)
+    {
+        ROCPROFILER_CALL(rocprofiler_start_context(*ctx), "resuming client context");
+    }
+}
+
+void
+tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
+                      rocprofiler_user_data_t*              user_data,
+                      void*                                 callback_data)
+{
+    assert(callback_data != nullptr);
+
+    auto     now = std::chrono::steady_clock::now().time_since_epoch().count();
+    uint64_t dt  = 0;
+    if(record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER)
+        user_data->value = now;
+    else
+        dt = (now - user_data->value);
+
+    auto info = std::stringstream{};
+    info << std::left << "tid=" << record.thread_id << ", cid=" << std::setw(3)
+         << record.correlation_id.internal << ", kind=" << record.kind
+         << ", operation=" << std::setw(3) << record.operation << ", phase=" << record.phase
+         << ", dt_nsec=" << std::setw(6) << dt;
+
+    auto info_data_cb = [](rocprofiler_callback_tracing_kind_t,
+                           rocprofiler_tracing_operation_t,
+                           uint32_t          arg_num,
+                           const void* const arg_value_addr,
+                           int32_t           indirection_count,
+                           const char*       arg_type,
+                           const char*       arg_name,
+                           const char*       arg_value_str,
+                           int32_t           dereference_count,
+                           void*             cb_data) -> int {
+        auto& dss = *static_cast<std::stringstream*>(cb_data);
+        dss << ((arg_num == 0) ? "(" : ", ");
+        dss << arg_num << ": " << arg_name << "=" << arg_value_str;
+        (void) arg_value_addr;
+        (void) arg_type;
+        (void) indirection_count;
+        (void) dereference_count;
+        return 0;
+    };
+
+    int32_t max_deref = (record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER) ? 1 : 2;
+    auto    info_data = std::stringstream{};
+    ROCPROFILER_CALL(rocprofiler_iterate_callback_tracing_kind_operation_args(
+                         record, info_data_cb, max_deref, static_cast<void*>(&info_data)),
+                     "Failure iterating trace operation args");
+
+    auto info_data_str = info_data.str();
+    if(!info_data_str.empty()) info << " " << info_data_str << ")";
+
+    static auto _mutex = std::mutex{};
+    _mutex.lock();
+    static_cast<call_stack_t*>(callback_data)
+        ->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+    _mutex.unlock();
+}
+
+void
+tool_control_init(rocprofiler_context_id_t& primary_ctx)
+{
+    // Create a specialized (throw-away) context for handling ROCTx profiler pause and resume.
+    // A separate context is used because if the context that is associated with roctxProfilerPause
+    // disabled that same context, a call to roctxProfilerResume would be ignored because the
+    // context that enables the callback for that API call is disabled.
+    auto cntrl_ctx = rocprofiler_context_id_t{0};
+    ROCPROFILER_CALL(rocprofiler_create_context(&cntrl_ctx), "control context creation failed");
+
+    // enable callback marker tracing with only the pause/resume operations
+    ROCPROFILER_CALL(rocprofiler_configure_callback_tracing_service(
+                         cntrl_ctx,
+                         ROCPROFILER_CALLBACK_TRACING_MARKER_CONTROL_API,
+                         nullptr,
+                         0,
+                         tool_tracing_ctrl_callback,
+                         &primary_ctx),
+                     "callback tracing service failed to configure");
+
+    // start the context so that it is always active
+    ROCPROFILER_CALL(rocprofiler_start_context(cntrl_ctx), "start of control context");
+}
+
+int
+tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
+{
+    assert(tool_data != nullptr);
+
+    auto* call_stack_v = static_cast<call_stack_t*>(tool_data);
+
+    call_stack_v->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, ""});
+
+    callback_name_info name_info = common::get_callback_tracing_names();
+
+    for(const auto& itr : name_info)
+    {
+        auto name_idx = std::stringstream{};
+        name_idx << " [" << std::setw(3) << itr.value << "]";
+        call_stack_v->emplace_back(
+            source_location{"rocprofiler_callback_tracing_kind_names          " + name_idx.str(),
+                            __FILE__,
+                            __LINE__,
+                            std::string{itr.name}});
+
+        for(auto [didx, ditr] : itr.items())
+        {
+            auto operation_idx = std::stringstream{};
+            operation_idx << " [" << std::setw(3) << didx << "]";
+            call_stack_v->emplace_back(source_location{
+                "rocprofiler_callback_tracing_kind_operation_names" + operation_idx.str(),
+                __FILE__,
+                __LINE__,
+                std::string{"- "} + std::string{*ditr}});
+        }
+    }
+
+    client_fini_func = fini_func;
+
+    ROCPROFILER_CALL(rocprofiler_create_context(&client_ctx), "context creation failed");
+
+    // enable the control
+    tool_control_init(client_ctx);
+
+    for(auto itr : {ROCPROFILER_CALLBACK_TRACING_HSA_CORE_API,
+                    ROCPROFILER_CALLBACK_TRACING_HSA_AMD_EXT_API,
+                    ROCPROFILER_CALLBACK_TRACING_HSA_IMAGE_EXT_API,
+                    ROCPROFILER_CALLBACK_TRACING_HSA_FINALIZE_EXT_API})
+    {
+        ROCPROFILER_CALL(rocprofiler_configure_callback_tracing_service(
+                             client_ctx, itr, nullptr, 0, tool_tracing_callback, tool_data),
+                         "callback tracing service failed to configure");
+    }
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_tracing_service(client_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_HIP_RUNTIME_API,
+                                                       nullptr,
+                                                       0,
+                                                       tool_tracing_callback,
+                                                       tool_data),
+        "callback tracing service failed to configure");
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_tracing_service(client_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_MARKER_CORE_API,
+                                                       nullptr,
+                                                       0,
+                                                       tool_tracing_callback,
+                                                       tool_data),
+        "callback tracing service failed to configure");
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_tracing_service(client_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_MARKER_NAME_API,
+                                                       nullptr,
+                                                       0,
+                                                       tool_tracing_callback,
+                                                       tool_data),
+        "callback tracing service failed to configure");
+
+    int valid_ctx = 0;
+    ROCPROFILER_CALL(rocprofiler_context_is_valid(client_ctx, &valid_ctx),
+                     "failure checking context validity");
+    if(valid_ctx == 0)
+    {
+        // notify rocprofiler that initialization failed
+        // and all the contexts, buffers, etc. created
+        // should be ignored
+        return -1;
+    }
+
+    ROCPROFILER_CALL(rocprofiler_start_context(client_ctx), "rocprofiler context start failed");
+
+    // no errors
+    return 0;
+}
+
+void
+tool_fini(void* tool_data)
+{
+    assert(tool_data != nullptr);
+
+    auto* _call_stack = static_cast<call_stack_t*>(tool_data);
+    _call_stack->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, ""});
+
+    print_call_stack(*_call_stack);
+
+    delete _call_stack;
+}
+}  // namespace
+
+void
+setup()
+{}
+
+void
+shutdown()
+{
+    if(client_id) client_fini_func(*client_id);
+}
+
+void
+start()
+{
+    ROCPROFILER_CALL(rocprofiler_start_context(client_ctx), "rocprofiler context start failed");
+}
+
+void
+stop()
+{
+    int status = 0;
+    ROCPROFILER_CALL(rocprofiler_is_initialized(&status), "failed to retrieve init status");
+    if(status != 0)
+    {
+        ROCPROFILER_CALL(rocprofiler_stop_context(client_ctx), "rocprofiler context stop failed");
+    }
+}
+}  // namespace client
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              version_string,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name = "ExampleTool";
+
+    // store client info
+    client::client_id = id;
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " (priority=" << priority << ") is using rocprofiler-sdk v" << major << "."
+         << minor << "." << patch << " (" << version_string << ")";
+
+    std::clog << info.str() << std::endl;
+
+    // demonstration of alternative way to get the version info
+    {
+        auto runtime_version = rocprofiler_version_triplet_t{};
+        ROCPROFILER_CALL(rocprofiler_get_version_triplet(&runtime_version),
+                         "failed to get version info");
+
+        if(rocprofiler_version_triplet_t{major, minor, patch} != runtime_version)
+        {
+            throw std::runtime_error{"version info mismatch"};
+        }
+
+        if(rocprofiler_version_triplet_t{major, minor, patch} !=
+           rocprofiler::sdk::version::compute_version_triplet<100>(version))
+        {
+            throw std::runtime_error{"version triplet incorrectly calculated"};
+        }
+    }
+
+    // data passed around all the callbacks
+    auto* client_tool_data = new std::vector<client::source_location>{};
+
+    // add first entry
+    client_tool_data->emplace_back(
+        client::source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &client::tool_init,
+                                            &client::tool_fini,
+                                            static_cast<void*>(client_tool_data)};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/api_callback_tracing/client.hpp

@@ -0,0 +1,44 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#ifdef callback_api_tracing_client_EXPORTS
+#    define CLIENT_API __attribute__((visibility("default")))
+#else
+#    define CLIENT_API
+#endif
+
+namespace client
+{
+void
+setup() CLIENT_API;
+
+void
+shutdown() CLIENT_API;
+
+void
+start() CLIENT_API;
+
+void
+stop() CLIENT_API;
+}  // namespace client

projects/rocprofiler-sdk/samples/api_callback_tracing/main.cpp

@@ -0,0 +1,273 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "client.hpp"
+
+#include <hip/hip_runtime.h>
+#include <rocprofiler-sdk-roctx/roctx.h>
+
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <mutex>
+#include <random>
+#include <stdexcept>
+#include <thread>
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error %i: %s\n",                                                 \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    (int) error_,                                                                  \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t                      = std::unique_lock<std::mutex>;
+auto               print_lock          = std::mutex{};
+size_t             nthreads            = 2;
+size_t             nitr                = 500;
+size_t             nsync               = 10;
+constexpr unsigned shared_mem_tile_dim = 32;
+
+void
+check_hip_error(void);
+
+void
+verify(int* in, int* out, int M, int N);
+}  // namespace
+
+__global__ void
+transpose_a(const int* in, int* out, int M, int N);
+
+void
+run(int rank, int tid, hipStream_t stream, int argc, char** argv);
+
+int
+main(int argc, char** argv)
+{
+    client::setup();  // currently does nothing
+    // client::start(); // currently will fail
+
+    auto range_id = roctxRangeStart("main");
+
+    int rank = 0;
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{argv[i]};
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            fprintf(stderr,
+                    "usage: transpose [NUM_THREADS (%zu)] [NUM_ITERATION (%zu)] "
+                    "[SYNC_EVERY_N_ITERATIONS (%zu)]\n",
+                    nthreads,
+                    nitr,
+                    nsync);
+            exit(EXIT_SUCCESS);
+        }
+    }
+    if(argc > 1) nthreads = atoll(argv[1]);
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    printf("[transpose] Number of threads: %zu\n", nthreads);
+    printf("[transpose] Number of iterations: %zu\n", nitr);
+    printf("[transpose] Syncing every %zu iterations\n", nsync);
+
+    // this is a temporary workaround in omnitrace when HIP + MPI is enabled
+    int ndevice = 0;
+    int devid   = rank;
+    HIP_API_CALL(hipGetDeviceCount(&ndevice));
+    printf("[transpose] Number of devices found: %i\n", ndevice);
+    if(ndevice > 0)
+    {
+        devid = rank % ndevice;
+        HIP_API_CALL(hipSetDevice(devid));
+        printf("[transpose] Rank %i assigned to device %i\n", rank, devid);
+    }
+    if(rank == devid && rank < ndevice)
+    {
+        std::vector<std::thread> _threads{};
+        std::vector<hipStream_t> _streams(nthreads);
+        roctxMark("stream creation");
+        for(size_t i = 0; i < nthreads; ++i)
+            HIP_API_CALL(hipStreamCreate(&_streams.at(i)));
+        roctxMark("thread creation");
+        for(size_t i = 1; i < nthreads; ++i)
+            _threads.emplace_back(run, rank, i, _streams.at(i), argc, argv);
+        run(rank, 0, _streams.at(0), argc, argv);
+        roctxMark("thread sync");
+        for(auto& itr : _threads)
+            itr.join();
+        roctxMark("stream destroy");
+        for(size_t i = 0; i < nthreads; ++i)
+            HIP_API_CALL(hipStreamDestroy(_streams.at(i)));
+    }
+
+    HIP_API_CALL(hipDeviceSynchronize());
+
+    auto tid = roctx_thread_id_t{};
+    // get the thread id recognized by rocprofiler-sdk from roctx
+    roctxGetThreadId(&tid);
+    // pause API tracing
+    roctxProfilerPause(tid);
+    // would not expect below to show up in profiler (depends on tool)
+    HIP_API_CALL(hipDeviceReset());
+    // resume API tracing
+    roctxProfilerResume(tid);
+
+    roctxRangeStop(range_id);
+
+    client::stop();
+    client::shutdown();
+
+    return 0;
+}
+
+__global__ void
+transpose_a(const int* in, int* out, int M, int N)
+{
+    __shared__ int tile[shared_mem_tile_dim][shared_mem_tile_dim];
+
+    int idx = (blockIdx.y * blockDim.y + threadIdx.y) * M + blockIdx.x * blockDim.x + threadIdx.x;
+    tile[threadIdx.y][threadIdx.x] = in[idx];
+    __syncthreads();
+    idx      = (blockIdx.x * blockDim.x + threadIdx.y) * N + blockIdx.y * blockDim.y + threadIdx.x;
+    out[idx] = tile[threadIdx.x][threadIdx.y];
+}
+
+void
+run(int rank, int tid, hipStream_t stream, int argc, char** argv)
+{
+    auto run_name = std::stringstream{};
+    run_name << __FUNCTION__ << "(" << rank << ", " << tid << ")";
+    roctxRangePush(run_name.str().c_str());
+
+    unsigned int M = 4960 * 2;
+    unsigned int N = 4960 * 2;
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    auto_lock_t _lk{print_lock};
+    std::cout << "[transpose][" << rank << "][" << tid << "] M: " << M << " N: " << N << std::endl;
+    _lk.unlock();
+
+    auto _seed   = std::random_device{}() * (rank + 1) * (tid + 1);
+    auto _engine = std::default_random_engine{_seed};
+    auto _dist   = std::uniform_int_distribution<int>{0, 1000};
+
+    size_t size       = sizeof(int) * M * N;
+    int*   inp_matrix = new int[size];
+    int*   out_matrix = new int[size];
+    for(size_t i = 0; i < M * N; i++)
+    {
+        inp_matrix[i] = _dist(_engine);
+        out_matrix[i] = 0;
+    }
+    int* in  = nullptr;
+    int* out = nullptr;
+
+    HIP_API_CALL(hipMalloc(&in, size));
+    HIP_API_CALL(hipMalloc(&out, size));
+    HIP_API_CALL(hipMemsetAsync(in, 0, size, stream));
+    HIP_API_CALL(hipMemsetAsync(out, 0, size, stream));
+    HIP_API_CALL(hipMemcpyAsync(in, inp_matrix, size, hipMemcpyHostToDevice, stream));
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    dim3 grid(M / 32, N / 32, 1);
+    dim3 block(32, 32, 1);  // transpose_a
+
+    auto t1 = std::chrono::high_resolution_clock::now();
+    for(size_t i = 0; i < nitr; ++i)
+    {
+        transpose_a<<<grid, block, 0, stream>>>(in, out, M, N);
+        check_hip_error();
+        if(i % nsync == (nsync - 1)) HIP_API_CALL(hipStreamSynchronize(stream));
+    }
+    auto t2 = std::chrono::high_resolution_clock::now();
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipMemcpyAsync(out_matrix, out, size, hipMemcpyDeviceToHost, stream));
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+    float  GB   = (float) size * nitr * 2 / (1 << 30);
+
+    print_lock.lock();
+    std::cout << "[transpose][" << rank << "][" << tid << "] Runtime of transpose is " << time
+              << " sec\n";
+    std::cout << "[transpose][" << rank << "][" << tid
+              << "] The average performance of transpose is " << GB / time << " GBytes/sec"
+              << std::endl;
+    print_lock.unlock();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    // cpu_transpose(matrix, out_matrix, M, N);
+    verify(inp_matrix, out_matrix, M, N);
+
+    HIP_API_CALL(hipFree(in));
+    HIP_API_CALL(hipFree(out));
+
+    delete[] inp_matrix;
+    delete[] out_matrix;
+
+    roctxRangePop();
+}
+
+namespace
+{
+void
+check_hip_error(void)
+{
+    hipError_t err = hipGetLastError();
+    if(err != hipSuccess)
+    {
+        auto_lock_t _lk{print_lock};
+        std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
+        throw std::runtime_error("hip_api_call");
+    }
+}
+
+void
+verify(int* in, int* out, int M, int N)
+{
+    for(int i = 0; i < 10; i++)
+    {
+        int row = rand() % M;
+        int col = rand() % N;
+        if(in[row * N + col] != out[col * M + row])
+        {
+            auto_lock_t _lk{print_lock};
+            std::cout << "mismatch: " << row << ", " << col << " : " << in[row * N + col] << " | "
+                      << out[col * M + row] << "\n";
+        }
+    }
+}
+}  // namespace

projects/rocprofiler-sdk/samples/code_object_isa_decode/CMakeLists.txt

@@ -0,0 +1,57 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-samples-code-object-isa-decode LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+find_package(rocprofiler-sdk REQUIRED)
+find_package(
+    amd_comgr
+    REQUIRED
+    CONFIG
+    HINTS
+    ${rocm_version_DIR}
+    ${ROCM_PATH}
+    PATHS
+    ${rocm_version_DIR}
+    ${ROCM_PATH})
+
+set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
+set_source_files_properties(main.cpp PROPERTIES COMPILE_FLAGS "-g")
+
+add_executable(code-object-isa-decode)
+target_sources(code-object-isa-decode PRIVATE main.cpp client.cpp)
+target_link_libraries(
+    code-object-isa-decode
+    PRIVATE rocprofiler-sdk::samples-common-library rocprofiler-sdk::rocprofiler-sdk
+            amd_comgr rocprofiler-sdk::samples-build-flags)
+
+rocprofiler_samples_get_preload_env(PRELOAD_ENV)
+
+add_test(NAME code-object-isa-decode COMMAND $<TARGET_FILE:code-object-isa-decode>)
+
+set_tests_properties(
+    code-object-isa-decode
+    PROPERTIES TIMEOUT 45 LABELS "samples" ENVIRONMENT "${PRELOAD_ENV}"
+               FAIL_REGULAR_EXPRESSION "threw an exception")

projects/rocprofiler-sdk/samples/code_object_isa_decode/README.md

@@ -0,0 +1,13 @@
+# CodeObject tracing
+
+## Services
+
+- code object tracing.
+  
+## Properties
+
+- This tool hooks into ROCProfiler's callback and buffer tracing mechanisms to:
+  - Decode and analyze GPU code objects.
+  - Three kernel variants are used in sample; simple transpose, in-place LDS swap and LDS no bank conflicts.
+  - Trace kernel symbols and instructions.
+  - Log disassembly and statistics for debugging or performance analysis.

projects/rocprofiler-sdk/samples/code_object_isa_decode/client.cpp

@@ -0,0 +1,272 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+#define OUTPUT_OFSTREAM "code_obj_isa_decode.log"
+
+/**
+ * @file samples/code_object_isa_decode/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+#include <rocprofiler-sdk/cxx/codeobj/code_printing.hpp>
+
+#include "common/defines.hpp"
+#include "common/filesystem.hpp"
+
+#include <cxxabi.h>
+#include <atomic>
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <regex>
+#include <string>
+#include <string_view>
+#include <thread>
+#include <vector>
+
+constexpr bool COPY_MEMORY_CODEOBJ = true;
+
+namespace client
+{
+std::ostream&
+output_stream()
+{
+    static std::ofstream file(OUTPUT_OFSTREAM);
+
+    static bool file_is_open_check = [&]() {
+        if(!file.is_open())
+            std::cout << "Could not open log file: " << OUTPUT_OFSTREAM << ", writing to stdout\n";
+        else
+            std::cout << "Writing code-object-isa-decode log to: " << OUTPUT_OFSTREAM << std::endl;
+        return file.is_open();
+    }();
+
+    if(!file_is_open_check) return std::cout;
+    return file;
+};
+
+namespace
+{
+using code_obj_load_data_t = rocprofiler_callback_tracing_code_object_load_data_t;
+using kernel_symbol_data_t = rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t;
+using kernel_symbol_map_t  = std::unordered_map<std::string, std::pair<uint64_t, size_t>>;
+
+using Instruction             = rocprofiler::sdk::codeobj::disassembly::Instruction;
+using CodeobjAddressTranslate = rocprofiler::sdk::codeobj::disassembly::CodeobjAddressTranslate;
+
+rocprofiler_client_id_t*      client_id          = nullptr;
+rocprofiler_client_finalize_t client_fini_func   = nullptr;
+rocprofiler_context_id_t      client_ctx         = {0};
+kernel_symbol_map_t           registered_kernels = {};
+
+CodeobjAddressTranslate codeobjTranslate;
+
+void
+tool_codeobj_tracing_callback(rocprofiler_callback_tracing_record_t record,
+                              rocprofiler_user_data_t*              user_data,
+                              void*                                 callback_data)
+{
+    if(record.kind != ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT) return;
+    if(record.phase != ROCPROFILER_CALLBACK_PHASE_LOAD) return;
+
+    if(record.operation == ROCPROFILER_CODE_OBJECT_LOAD)
+    {
+        auto* data = static_cast<code_obj_load_data_t*>(record.payload);
+
+        if(std::string_view(data->uri).find("file:///") == 0)
+        {
+            codeobjTranslate.addDecoder(
+                data->uri, data->code_object_id, data->load_delta, data->load_size);
+        }
+        else if(COPY_MEMORY_CODEOBJ)
+        {
+            codeobjTranslate.addDecoder(reinterpret_cast<const void*>(data->memory_base),
+                                        data->memory_size,
+                                        data->code_object_id,
+                                        data->load_delta,
+                                        data->load_size);
+        }
+        else
+        {
+            return;
+        }
+
+        auto symbolmap = codeobjTranslate.getSymbolMap();
+        for(auto& [vaddr, symbol] : symbolmap)
+            registered_kernels.insert({symbol.name, {vaddr, vaddr + symbol.mem_size}});
+    }
+    else if(record.operation == ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER)
+    {
+        output_stream() << std::hex;
+        auto* data        = static_cast<kernel_symbol_data_t*>(record.payload);
+        auto  kernel_name = std::regex_replace(data->kernel_name, std::regex{"(\\.kd)$"}, "");
+
+        if(registered_kernels.find(kernel_name) == registered_kernels.end())
+        {
+            output_stream() << "Not Found: " << kernel_name << " in codeobj." << std::endl;
+            return;
+        }
+
+        auto& begin_end = registered_kernels.at(kernel_name);
+
+        output_stream() << std::hex << "Found: " << kernel_name << " at addr: 0x" << begin_end.first
+                        << std::dec << ". Printing first 64 bytes:" << std::endl;
+
+        std::unordered_set<std::string> references{};
+
+        int num_waitcnts = 0;
+        int num_scalar   = 0;
+        int num_vector   = 0;
+        int num_other    = 0;
+
+        size_t vaddr = begin_end.first;
+        while(vaddr < begin_end.second)
+        {
+            auto inst = codeobjTranslate.get(vaddr);
+            assert(inst != nullptr);
+            if(inst->comment.size())
+            {
+                std::string_view source = inst->comment;
+                if(source.rfind('/') < source.size()) source = source.substr(source.rfind('/'));
+                if(vaddr < begin_end.first + 64) output_stream() << '\t' << inst->inst << '\n';
+
+                if(source.rfind(':') < source.size()) source = source.substr(0, source.rfind(':'));
+
+                references.insert(std::string(source));
+            }
+            if(inst->inst.find("v_") == 0)
+                num_vector++;
+            else if(inst->inst.find("s_waitcnt") == 0)
+                num_waitcnts++;
+            else if(inst->inst.find("s_") == 0)
+                num_scalar++;
+            else
+                num_other++;
+
+            vaddr += inst->size;
+        }
+
+        output_stream() << "  --- Num Scalar: " << num_scalar
+                        << "\n  --- Num Vector: " << num_vector
+                        << "\n  --- Num Waitcnts: " << num_waitcnts
+                        << "\n  --- Other instructions: " << num_other
+                        << "\nKernel has source references to: " << std::endl;
+        for(auto& ref : references)
+            output_stream() << '\t' << ref << std::endl;
+    }
+
+    (void) user_data;
+    (void) callback_data;
+}
+
+int
+tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
+{
+    client_fini_func = fini_func;
+
+    ROCPROFILER_CALL(rocprofiler_create_context(&client_ctx), "context creation");
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_tracing_service(client_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT,
+                                                       nullptr,
+                                                       0,
+                                                       tool_codeobj_tracing_callback,
+                                                       tool_data),
+        "code object tracing service configure");
+
+    int valid_ctx = 0;
+    ROCPROFILER_CALL(rocprofiler_context_is_valid(client_ctx, &valid_ctx),
+                     "context validity check");
+    if(valid_ctx == 0)
+    {
+        // notify rocprofiler that initialization failed
+        // and all the contexts, buffers, etc. created
+        // should be ignored
+        return -1;
+    }
+
+    ROCPROFILER_CALL(rocprofiler_start_context(client_ctx), "context start");
+
+    // no errors
+    return 0;
+}
+
+void
+tool_fini(void* /* tool_data */)
+{}
+
+}  // namespace
+
+}  // namespace client
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name = "ExampleTool";
+
+    // store client info
+    client::client_id = id;
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " (priority=" << priority << ") is using rocprofiler-sdk v" << major << "."
+         << minor << "." << patch << " (" << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &client::tool_init,
+                                            &client::tool_fini,
+                                            nullptr};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/code_object_isa_decode/main.cpp

@@ -0,0 +1,249 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "hip/hip_runtime.h"
+
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <random>
+#include <stdexcept>
+#include <vector>
+#include "transpose_kernels.hpp"
+
+#define PRINT_ALIGN 36
+
+namespace
+{
+using lock_guard_t = std::lock_guard<std::mutex>;
+auto print_lock    = std::mutex{};
+}  // namespace
+
+enum TransposeType
+{
+    TRANSPOSE_NAIVE,
+    TRANSPOSE_INPLACE_LDS,
+    TRANSPOSE_NO_BANK_CONFLICTS
+};
+
+class ITranspose
+{
+public:
+    virtual void run(TransposeType ttype, int numThreadsY, int num_iter) = 0;
+    virtual ~ITranspose(){};
+};
+
+template <typename T>
+class Transpose : public ITranspose
+{
+public:
+    Transpose(int dev, size_t _M)
+    : devID(dev)
+    , M(_M)
+    , databytes(_M * _M * sizeof(T))
+    {
+        HIP_API_CALL(hipSetDevice(devID));
+        HIP_API_CALL(hipStreamCreate(&stream));
+
+        std::default_random_engine         _engine{std::random_device{}() * rand()};
+        std::uniform_int_distribution<int> _dist{0, 1000};
+
+        inp_matrix = new T[M * M];
+        out_matrix = new T[M * M];
+
+        for(size_t i = 0; i < M * M; i++)
+            inp_matrix[i] = static_cast<T>(_dist(_engine));
+        memset(out_matrix, 0, databytes);
+
+        HIP_API_CALL(hipMalloc(&in, databytes));
+        HIP_API_CALL(hipMalloc(&out, databytes));
+        HIP_API_CALL(hipMemsetAsync(in, 0, databytes, stream));
+        HIP_API_CALL(hipMemsetAsync(out, 0, databytes, stream));
+        HIP_API_CALL(hipMemcpyAsync(in, inp_matrix, databytes, hipMemcpyDefault, stream));
+
+        HIP_API_CALL(hipEventCreate(&start));
+        HIP_API_CALL(hipEventCreate(&stop));
+    }
+
+    void run(TransposeType ttype, int numThreadsY, int num_iter) override
+    {
+        HIP_API_CALL(hipSetDevice(devID));
+        dim3 grid(M / TILE_DIM, M / TILE_DIM, 1);
+        dim3 block(TILE_DIM, numThreadsY, 1);
+
+        auto        Kernel     = transposeNaive<T>;
+        std::string KernelName = "transposeNaive";
+        if(ttype == TransposeType::TRANSPOSE_NO_BANK_CONFLICTS)
+        {
+            Kernel     = transposeLdsNoBankConflicts<T>;
+            KernelName = "transposeLdsNoBankConflicts";
+        }
+        else if(ttype == TransposeType::TRANSPOSE_INPLACE_LDS)
+        {
+            Kernel     = transposeLdsSwapInplace<T>;
+            KernelName = "transposeLdsSwapInplace";
+        }
+
+        {
+            std::string functypeid = __PRETTY_FUNCTION__;
+            auto        it_beg     = functypeid.rfind("[T = ");
+            auto        it_end     = functypeid.rfind(']');
+
+            if(it_beg != std::string::npos) it_beg += std::string("[T = ").size();
+
+            if(it_beg < it_end && it_end != std::string::npos)
+                KernelName += '<' + functypeid.substr(it_beg, it_end - it_beg) + '>';
+        }
+
+        HIP_API_CALL(hipStreamSynchronize(stream));
+        HIP_API_CALL(hipEventRecord(start, stream));
+
+        for(int i = 0; i < num_iter; i++)
+        {
+            Kernel<<<grid, block, 0, stream>>>(out, in, M);
+            HIP_API_CALL(hipGetLastError());
+        }
+
+        HIP_API_CALL(hipEventRecord(stop, stream));
+        HIP_API_CALL(hipMemcpyAsync(out_matrix, out, databytes, hipMemcpyDefault, stream));
+        HIP_API_CALL(hipEventSynchronize(stop));
+
+        float time;
+        HIP_API_CALL(hipEventElapsedTime(&time, start, stop));
+        float GB = databytes * num_iter * 2 / float(1 << 30);
+
+        {
+            lock_guard_t _lk{print_lock};
+            std::cout << "The average performance of " << std::setw(38) << KernelName << " : "
+                      << (1000 * GB / time) << " GB/s" << std::endl;
+        }
+
+        verify();
+    }
+
+    void verify() const
+    {
+        HIP_API_CALL(hipStreamSynchronize(stream));
+        for(int i = 0; i < 10; i++)
+        {
+            int row = rand() % M;
+            int col = rand() % M;
+            if(inp_matrix[row * M + col] != out_matrix[col * M + row])
+            {
+                lock_guard_t _lk{print_lock};
+                std::cout << "mismatch: " << row << ", " << col << " : "
+                          << inp_matrix[row * M + col] << " | " << out_matrix[col * M + row]
+                          << std::endl;
+            }
+        }
+    }
+
+    virtual ~Transpose()
+    {
+        HIP_API_CALL(hipSetDevice(devID));
+        HIP_API_CALL(hipEventDestroy(start));
+        HIP_API_CALL(hipEventDestroy(stop));
+
+        HIP_API_CALL(hipFree(in));
+        HIP_API_CALL(hipFree(out));
+        HIP_API_CALL(hipStreamDestroy(stream));
+
+        delete[] inp_matrix;
+        delete[] out_matrix;
+    }
+
+    const int    devID;
+    const size_t M;
+    const size_t databytes;
+
+    hipStream_t stream;
+    hipEvent_t  start, stop;
+
+    T* inp_matrix = nullptr;
+    T* out_matrix = nullptr;
+
+    T* in  = nullptr;
+    T* out = nullptr;
+};
+
+int
+main(int argc, char** argv)
+{
+    int deviceId  = 0;
+    int blockDimY = 8;
+    int num_iter  = 3;
+    int mat_size  = 8192;
+
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{argv[i]};
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            std::cout << "usage: transpose "
+                      << "[MatrixSize (" << mat_size << ")] "
+                      << "[numIter (" << num_iter << ")] "
+                      << "[blockDimY (" << blockDimY << ")] "
+                      << "[DEVICE_ID (" << deviceId << ")] " << std::endl;
+            exit(EXIT_SUCCESS);
+        }
+    }
+    if(argc > 1) mat_size = atoll(argv[1]);
+    if(argc > 2) num_iter = atoll(argv[2]);
+    if(argc > 3) blockDimY = atoll(argv[3]);
+    if(argc > 4) deviceId = atoll(argv[4]);
+
+    printf("[transpose] Matrix size: %d, device ID: %d, num iter: %d, blockDimY: %d\n",
+           mat_size,
+           deviceId,
+           num_iter,
+           blockDimY);
+
+    int ndevice = 0;
+    HIP_API_CALL(hipGetDeviceCount(&ndevice));
+    printf("[transpose] Number of devices found: %i\n", ndevice);
+    assert(ndevice > 0);
+
+    if(deviceId >= ndevice) exit(EXIT_FAILURE);
+
+    {
+        std::vector<std::unique_ptr<ITranspose>> kernels;
+        kernels.push_back(std::make_unique<Transpose<int>>(deviceId, mat_size));
+        kernels.push_back(std::make_unique<Transpose<float>>(deviceId, mat_size));
+        kernels.push_back(std::make_unique<Transpose<double>>(deviceId, mat_size));
+
+        for(auto& kernel : kernels)
+        {
+            kernel->run(TransposeType::TRANSPOSE_NAIVE, blockDimY, num_iter);
+            kernel->run(TransposeType::TRANSPOSE_INPLACE_LDS, blockDimY, num_iter);
+            kernel->run(TransposeType::TRANSPOSE_NO_BANK_CONFLICTS, blockDimY, num_iter);
+        }
+    }
+
+    HIP_API_CALL(hipDeviceSynchronize());
+    HIP_API_CALL(hipDeviceReset());
+
+    return 0;
+}

projects/rocprofiler-sdk/samples/code_object_isa_decode/transpose_kernels.hpp

@@ -0,0 +1,106 @@
+// MIT License
+//
+// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#pragma once
+
+#include "hip/hip_runtime.h"
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            lock_guard_t _hip_api_print_lk{print_lock};                                            \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            exit(EXIT_FAILURE);                                                                    \
+        }                                                                                          \
+    }
+
+#define TILE_DIM 64
+
+template <typename T>
+__global__ void
+transposeNaive(T* odata, const T* idata, size_t size)
+{
+    size_t idx        = blockIdx.x * TILE_DIM + threadIdx.x;
+    size_t block_posy = blockIdx.y * TILE_DIM;
+
+    for(size_t idy = threadIdx.y; idy < TILE_DIM; idy += blockDim.y)
+        odata[size * idx + block_posy + idy] = idata[idx + (block_posy + idy) * size];
+}
+
+template <typename T>
+__global__ void
+transposeLdsNoBankConflicts(T* odata, const T* idata, size_t size)
+{
+    __shared__ T tile[TILE_DIM][TILE_DIM + 1];
+
+    size_t idx_in   = blockIdx.x * TILE_DIM + threadIdx.x;
+    size_t idy_in   = blockIdx.y * TILE_DIM + threadIdx.y;
+    size_t index_in = idx_in + idy_in * size;
+
+    size_t idx_out   = blockIdx.y * TILE_DIM + threadIdx.x;
+    size_t idy_out   = blockIdx.x * TILE_DIM + threadIdx.y;
+    size_t index_out = idx_out + idy_out * size;
+
+    for(size_t y = 0; y < TILE_DIM; y += blockDim.y)
+        tile[threadIdx.y + y][threadIdx.x] = idata[index_in + y * size];
+
+    __syncthreads();
+
+    for(size_t y = 0; y < TILE_DIM; y += blockDim.y)
+        odata[index_out + y * size] = tile[threadIdx.x][threadIdx.y + y];
+}
+
+// Generates more interesting ISA
+template <typename T>
+__global__ void
+transposeLdsSwapInplace(T* odata, const T* idata, size_t size)
+{
+    __shared__ T tile[TILE_DIM][TILE_DIM];
+
+    const size_t idx_in = blockIdx.x * TILE_DIM + threadIdx.x;
+
+    for(size_t idy = threadIdx.y; idy < TILE_DIM; idy += blockDim.y)
+        tile[idy][threadIdx.x] = idata[idx_in + (idy + blockIdx.y * TILE_DIM) * size];
+
+    __syncthreads();
+
+    for(size_t idy = threadIdx.y; idy < TILE_DIM; idy += blockDim.y)
+        if(idy < threadIdx.x)
+        {
+            T temp                 = tile[idy][threadIdx.x];
+            tile[idy][threadIdx.x] = tile[threadIdx.x][idy];
+            tile[threadIdx.x][idy] = temp;
+        }
+
+    __syncthreads();
+
+    const size_t idx_out = blockIdx.y * TILE_DIM + threadIdx.x;
+
+    for(size_t idy = threadIdx.y; idy < TILE_DIM; idy += blockDim.y)
+        odata[(blockIdx.x * TILE_DIM + idy) * size + idx_out] = tile[idy][threadIdx.x];
+}

projects/rocprofiler-sdk/samples/code_object_tracing/CMakeLists.txt

@@ -0,0 +1,52 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-samples-code-object-tracing LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+find_package(rocprofiler-sdk REQUIRED)
+
+add_library(code-object-tracing-client SHARED)
+target_sources(code-object-tracing-client PRIVATE client.cpp)
+target_link_libraries(
+    code-object-tracing-client
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::samples-build-flags
+            rocprofiler-sdk::samples-common-library)
+
+set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
+find_package(Threads REQUIRED)
+
+add_executable(code-object-tracing)
+target_sources(code-object-tracing PRIVATE main.cpp)
+target_link_libraries(
+    code-object-tracing PRIVATE code-object-tracing-client Threads::Threads
+                                rocprofiler-sdk::samples-build-flags)
+
+add_test(NAME code-object-tracing COMMAND $<TARGET_FILE:code-object-tracing>)
+
+set_tests_properties(
+    code-object-tracing
+    PROPERTIES TIMEOUT 45 LABELS "samples" ENVIRONMENT
+               "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}" FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}")

projects/rocprofiler-sdk/samples/code_object_tracing/README.md

@@ -0,0 +1,11 @@
+# CodeObject Tracing
+
+## Services
+
+Trace and analyze the execution of GPU code objects and kernel symbols.
+
+## Properties
+
+- This tool is designed to capture and log information about code object loading/unloading and kernel symbol registration/un-registration events during the execution of GPU programs.
+
+- Whenever a relevant event occurs, such as a code object being loaded/unloaded or a kernel symbol being registered/unregistered. The function processes the event data, formats it into a human-readable string, and appends it to the call stack.

projects/rocprofiler-sdk/samples/code_object_tracing/client.cpp

@@ -0,0 +1,387 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+/**
+ * @file samples/code_object_tracing/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include "common/defines.hpp"
+#include "common/filesystem.hpp"
+
+#include <cxxabi.h>
+#include <atomic>
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <regex>
+#include <string>
+#include <string_view>
+#include <thread>
+#include <vector>
+
+namespace client
+{
+namespace
+{
+struct source_location
+{
+    std::string function = {};
+    std::string file     = {};
+    uint32_t    line     = 0;
+    std::string context  = {};
+};
+
+using call_stack_t         = std::vector<source_location>;
+using code_obj_load_data_t = rocprofiler_callback_tracing_code_object_load_data_t;
+using kernel_symbol_data_t = rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t;
+using kernel_symbol_map_t  = std::unordered_map<rocprofiler_kernel_id_t, kernel_symbol_data_t>;
+
+rocprofiler_client_id_t*      client_id        = nullptr;
+rocprofiler_client_finalize_t client_fini_func = nullptr;
+rocprofiler_context_id_t      client_ctx       = {0};
+kernel_symbol_map_t*          client_kernels   = nullptr;
+
+std::string
+cxa_demangle(std::string_view _mangled_name, int* _status)
+{
+    constexpr size_t buffer_len = 4096;
+    // return the mangled since there is no buffer
+    if(_mangled_name.empty())
+    {
+        *_status = -2;
+        return std::string{};
+    }
+
+    auto _demangled_name = std::string{_mangled_name};
+
+    // PARAMETERS to __cxa_demangle
+    //  mangled_name:
+    //      A NULL-terminated character string containing the name to be demangled.
+    //  buffer:
+    //      A region of memory, allocated with malloc, of *length bytes, into which the
+    //      demangled name is stored. If output_buffer is not long enough, it is expanded
+    //      using realloc. output_buffer may instead be NULL; in that case, the demangled
+    //      name is placed in a region of memory allocated with malloc.
+    //  _buflen:
+    //      If length is non-NULL, the length of the buffer containing the demangled name
+    //      is placed in *length.
+    //  status:
+    //      *status is set to one of the following values
+    size_t _demang_len = 0;
+    char*  _demang = abi::__cxa_demangle(_demangled_name.c_str(), nullptr, &_demang_len, _status);
+    switch(*_status)
+    {
+        //  0 : The demangling operation succeeded.
+        // -1 : A memory allocation failure occurred.
+        // -2 : mangled_name is not a valid name under the C++ ABI mangling rules.
+        // -3 : One of the arguments is invalid.
+        case 0:
+        {
+            if(_demang) _demangled_name = std::string{_demang};
+            break;
+        }
+        case -1:
+        {
+            char _msg[buffer_len];
+            ::memset(_msg, '\0', buffer_len * sizeof(char));
+            ::snprintf(_msg,
+                       buffer_len,
+                       "memory allocation failure occurred demangling %s",
+                       _demangled_name.c_str());
+            ::perror(_msg);
+            break;
+        }
+        case -2: break;
+        case -3:
+        {
+            char _msg[buffer_len];
+            ::memset(_msg, '\0', buffer_len * sizeof(char));
+            ::snprintf(_msg,
+                       buffer_len,
+                       "Invalid argument in: (\"%s\", nullptr, nullptr, %p)",
+                       _demangled_name.c_str(),
+                       (void*) _status);
+            ::perror(_msg);
+            break;
+        }
+        default: break;
+    };
+
+    // if it "demangled" but the length is zero, set the status to -2
+    if(_demang_len == 0 && *_status == 0) *_status = -2;
+
+    // free allocated buffer
+    ::free(_demang);
+    return _demangled_name;
+}
+
+void
+print_call_stack(const call_stack_t& _call_stack)
+{
+    auto ofname = std::string{"code_object_trace.log"};
+    if(auto* eofname = getenv("ROCPROFILER_SAMPLE_OUTPUT_FILE")) ofname = eofname;
+
+    std::ostream* ofs     = nullptr;
+    auto          cleanup = std::function<void(std::ostream*&)>{};
+
+    if(ofname == "stdout")
+        ofs = &std::cout;
+    else if(ofname == "stderr")
+        ofs = &std::cerr;
+    else
+    {
+        ofs = new std::ofstream{ofname};
+        if(ofs && *ofs)
+            cleanup = [](std::ostream*& _os) { delete _os; };
+        else
+        {
+            std::cerr << "Error outputting to " << ofname << ". Redirecting to stderr...\n";
+            ofname = "stderr";
+            ofs    = &std::cerr;
+        }
+    }
+
+    std::cout << "Outputting collected data to " << ofname << "...\n" << std::flush;
+
+    size_t n = 0;
+    for(const auto& itr : _call_stack)
+    {
+        *ofs << std::left << std::setw(2) << ++n << "/" << std::setw(2) << _call_stack.size()
+             << " [" << common::fs::path{itr.file}.filename() << ":" << itr.line << "] "
+             << std::setw(20) << itr.function;
+        if(!itr.context.empty()) *ofs << " :: " << itr.context;
+        *ofs << "\n";
+    }
+
+    *ofs << std::flush;
+
+    if(cleanup) cleanup(ofs);
+}
+
+template <typename Tp>
+std::string
+as_hex(Tp _v, size_t _width = 16)
+{
+    auto _ss = std::stringstream{};
+    _ss.fill('0');
+    _ss << "0x" << std::hex << std::setw(_width) << _v;
+    return _ss.str();
+}
+
+void
+tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
+                      rocprofiler_user_data_t*              user_data,
+                      void*                                 callback_data)
+{
+    if(record.kind == ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT &&
+       record.operation == ROCPROFILER_CODE_OBJECT_LOAD)
+    {
+        auto* data         = static_cast<code_obj_load_data_t*>(record.payload);
+        auto* call_stack_v = static_cast<call_stack_t*>(callback_data);
+        auto  info         = std::stringstream{};
+
+        if(record.phase == ROCPROFILER_CALLBACK_PHASE_LOAD)
+        {
+            info << "code object load :: ";
+        }
+        else if(record.phase == ROCPROFILER_CALLBACK_PHASE_UNLOAD)
+        {
+            info << "code object unload :: ";
+        }
+
+        info << "code_object_id=" << data->code_object_id
+             << ", rocp_agent=" << data->rocp_agent.handle << ", uri=" << data->uri
+             << ", load_base=" << as_hex(data->load_base) << ", load_size=" << data->load_size
+             << ", load_delta=" << as_hex(data->load_delta);
+        if(data->storage_type == ROCPROFILER_CODE_OBJECT_STORAGE_TYPE_FILE)
+            info << ", storage_file_descr=" << data->storage_file;
+        else if(data->storage_type == ROCPROFILER_CODE_OBJECT_STORAGE_TYPE_MEMORY)
+            info << ", storage_memory_base=" << as_hex(data->memory_base)
+                 << ", storage_memory_size=" << data->memory_size;
+
+        call_stack_v->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+    }
+    if(record.kind == ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT &&
+       record.operation == ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER)
+    {
+        auto* data         = static_cast<kernel_symbol_data_t*>(record.payload);
+        auto* call_stack_v = static_cast<call_stack_t*>(callback_data);
+        auto  info         = std::stringstream{};
+
+        if(record.phase == ROCPROFILER_CALLBACK_PHASE_LOAD)
+        {
+            info << "kernel symbol load :: ";
+            client_kernels->emplace(data->kernel_id, *data);
+        }
+        else if(record.phase == ROCPROFILER_CALLBACK_PHASE_UNLOAD)
+        {
+            info << "kernel symbol unload :: ";
+            client_kernels->erase(data->kernel_id);
+        }
+
+        auto kernel_name     = std::regex_replace(data->kernel_name, std::regex{"(\\.kd)$"}, "");
+        int  demangle_status = 0;
+        kernel_name          = cxa_demangle(kernel_name, &demangle_status);
+
+        info << "code_object_id=" << data->code_object_id << ", kernel_id=" << data->kernel_id
+             << ", kernel_object=" << as_hex(data->kernel_object)
+             << ", kernarg_segment_size=" << data->kernarg_segment_size
+             << ", kernarg_segment_alignment=" << data->kernarg_segment_alignment
+             << ", group_segment_size=" << data->group_segment_size
+             << ", private_segment_size=" << data->private_segment_size
+             << ", kernel_name=" << kernel_name;
+
+        call_stack_v->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+    }
+
+    (void) user_data;
+}
+
+int
+tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
+{
+    client_kernels = new kernel_symbol_map_t{};
+
+    assert(tool_data != nullptr);
+
+    auto* call_stack_v = static_cast<call_stack_t*>(tool_data);
+
+    call_stack_v->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, ""});
+
+    client_fini_func = fini_func;
+
+    ROCPROFILER_CALL(rocprofiler_create_context(&client_ctx), "context creation");
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_tracing_service(client_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT,
+                                                       nullptr,
+                                                       0,
+                                                       tool_tracing_callback,
+                                                       tool_data),
+        "code object tracing service configure");
+
+    int valid_ctx = 0;
+    ROCPROFILER_CALL(rocprofiler_context_is_valid(client_ctx, &valid_ctx),
+                     "context validity check");
+    if(valid_ctx == 0)
+    {
+        // notify rocprofiler that initialization failed
+        // and all the contexts, buffers, etc. created
+        // should be ignored
+        return -1;
+    }
+
+    ROCPROFILER_CALL(rocprofiler_start_context(client_ctx), "context start");
+
+    // no errors
+    return 0;
+}
+
+void
+tool_fini(void* tool_data)
+{
+    assert(tool_data != nullptr);
+
+    auto* _call_stack = static_cast<call_stack_t*>(tool_data);
+    _call_stack->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, ""});
+
+    print_call_stack(*_call_stack);
+
+    delete _call_stack;
+    delete client_kernels;
+}
+
+void
+setup()
+{
+    if(int status = 0;
+       rocprofiler_is_initialized(&status) == ROCPROFILER_STATUS_SUCCESS && status == 0)
+    {
+        ROCPROFILER_CALL(rocprofiler_force_configure(&rocprofiler_configure),
+                         "force configuration");
+    }
+}
+}  // namespace
+
+// force configuration when library is loaded
+bool cfg_on_load = (client::setup(), true);
+}  // namespace client
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name = "ExampleTool";
+
+    // store client info
+    client::client_id = id;
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " (priority=" << priority << ") is using rocprofiler-sdk v" << major << "."
+         << minor << "." << patch << " (" << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    auto* client_tool_data = new std::vector<client::source_location>{};
+
+    client_tool_data->emplace_back(
+        client::source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &client::tool_init,
+                                            &client::tool_fini,
+                                            static_cast<void*>(client_tool_data)};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/code_object_tracing/main.cpp

@@ -0,0 +1,250 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "hip/hip_runtime.h"
+
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <mutex>
+#include <random>
+#include <stdexcept>
+#include <thread>
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t                      = std::unique_lock<std::mutex>;
+auto               print_lock          = std::mutex{};
+size_t             nthreads            = 2;
+size_t             nitr                = 500;
+size_t             nsync               = 10;
+constexpr unsigned shared_mem_tile_dim = 32;
+
+void
+check_hip_error(void);
+
+void
+verify(int* in, int* out, int M, int N);
+}  // namespace
+
+__global__ void
+transpose_a(const int* in, int* out, int M, int N);
+
+void
+run(int rank, int tid, hipStream_t stream, int argc, char** argv);
+
+int
+main(int argc, char** argv)
+{
+    int rank = 0;
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{argv[i]};
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            fprintf(stderr,
+                    "usage: transpose [NUM_THREADS (%zu)] [NUM_ITERATION (%zu)] "
+                    "[SYNC_EVERY_N_ITERATIONS (%zu)]\n",
+                    nthreads,
+                    nitr,
+                    nsync);
+            exit(EXIT_SUCCESS);
+        }
+    }
+    if(argc > 1) nthreads = atoll(argv[1]);
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    printf("[transpose] Number of threads: %zu\n", nthreads);
+    printf("[transpose] Number of iterations: %zu\n", nitr);
+    printf("[transpose] Syncing every %zu iterations\n", nsync);
+
+    // this is a temporary workaround in omnitrace when HIP + MPI is enabled
+    int ndevice = 0;
+    int devid   = rank;
+    HIP_API_CALL(hipGetDeviceCount(&ndevice));
+    printf("[transpose] Number of devices found: %i\n", ndevice);
+    if(ndevice > 0)
+    {
+        devid = rank % ndevice;
+        HIP_API_CALL(hipSetDevice(devid));
+        printf("[transpose] Rank %i assigned to device %i\n", rank, devid);
+    }
+    if(rank == devid && rank < ndevice)
+    {
+        std::vector<std::thread> _threads{};
+        std::vector<hipStream_t> _streams(nthreads);
+        for(size_t i = 0; i < nthreads; ++i)
+            HIP_API_CALL(hipStreamCreate(&_streams.at(i)));
+        for(size_t i = 1; i < nthreads; ++i)
+            _threads.emplace_back(run, rank, i, _streams.at(i), argc, argv);
+        run(rank, 0, _streams.at(0), argc, argv);
+        for(auto& itr : _threads)
+            itr.join();
+        for(size_t i = 0; i < nthreads; ++i)
+            HIP_API_CALL(hipStreamDestroy(_streams.at(i)));
+    }
+    HIP_API_CALL(hipDeviceSynchronize());
+    HIP_API_CALL(hipDeviceReset());
+
+    return 0;
+}
+
+__global__ void
+transpose_a(const int* in, int* out, int M, int N)
+{
+    __shared__ int tile[shared_mem_tile_dim][shared_mem_tile_dim];
+
+    int idx = (blockIdx.y * blockDim.y + threadIdx.y) * M + blockIdx.x * blockDim.x + threadIdx.x;
+    tile[threadIdx.y][threadIdx.x] = in[idx];
+    __syncthreads();
+    idx      = (blockIdx.x * blockDim.x + threadIdx.y) * N + blockIdx.y * blockDim.y + threadIdx.x;
+    out[idx] = tile[threadIdx.x][threadIdx.y];
+}
+
+void
+run(int rank, int tid, hipStream_t stream, int argc, char** argv)
+{
+    unsigned int M = 4960 * 2;
+    unsigned int N = 4960 * 2;
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    auto_lock_t _lk{print_lock};
+    std::cout << "[transpose][" << rank << "][" << tid << "] M: " << M << " N: " << N << std::endl;
+    _lk.unlock();
+
+    std::default_random_engine         _engine{std::random_device{}() * (rank + 1) * (tid + 1)};
+    std::uniform_int_distribution<int> _dist{0, 1000};
+
+    size_t size       = sizeof(int) * M * N;
+    int*   inp_matrix = new int[size];
+    int*   out_matrix = new int[size];
+    for(size_t i = 0; i < M * N; i++)
+    {
+        inp_matrix[i] = _dist(_engine);
+        out_matrix[i] = 0;
+    }
+    int* in  = nullptr;
+    int* out = nullptr;
+
+    HIP_API_CALL(hipMalloc(&in, size));
+    HIP_API_CALL(hipMalloc(&out, size));
+    HIP_API_CALL(hipMemsetAsync(in, 0, size, stream));
+    HIP_API_CALL(hipMemsetAsync(out, 0, size, stream));
+    HIP_API_CALL(hipMemcpyAsync(in, inp_matrix, size, hipMemcpyHostToDevice, stream));
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    dim3 grid(M / 32, N / 32, 1);
+    dim3 block(32, 32, 1);  // transpose_a
+
+    print_lock.lock();
+    printf("[transpose][%i][%i] grid=(%i,%i,%i), block=(%i,%i,%i)\n",
+           rank,
+           tid,
+           grid.x,
+           grid.y,
+           grid.z,
+           block.x,
+           block.y,
+           block.z);
+    print_lock.unlock();
+
+    auto t1 = std::chrono::high_resolution_clock::now();
+    for(size_t i = 0; i < nitr; ++i)
+    {
+        transpose_a<<<grid, block, 0, stream>>>(in, out, M, N);
+        check_hip_error();
+        if(i % nsync == (nsync - 1)) HIP_API_CALL(hipStreamSynchronize(stream));
+    }
+    auto t2 = std::chrono::high_resolution_clock::now();
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipMemcpyAsync(out_matrix, out, size, hipMemcpyDeviceToHost, stream));
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+    float  GB   = (float) size * nitr * 2 / (1 << 30);
+
+    print_lock.lock();
+    std::cout << "[transpose][" << rank << "][" << tid << "] Runtime of transpose is " << time
+              << " sec\n";
+    std::cout << "[transpose][" << rank << "][" << tid
+              << "] The average performance of transpose is " << GB / time << " GBytes/sec"
+              << std::endl;
+    print_lock.unlock();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    // cpu_transpose(matrix, out_matrix, M, N);
+    verify(inp_matrix, out_matrix, M, N);
+
+    HIP_API_CALL(hipFree(in));
+    HIP_API_CALL(hipFree(out));
+
+    delete[] inp_matrix;
+    delete[] out_matrix;
+}
+
+namespace
+{
+void
+check_hip_error(void)
+{
+    hipError_t err = hipGetLastError();
+    if(err != hipSuccess)
+    {
+        auto_lock_t _lk{print_lock};
+        std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
+        throw std::runtime_error("hip_api_call");
+    }
+}
+
+void
+verify(int* in, int* out, int M, int N)
+{
+    for(int i = 0; i < 10; i++)
+    {
+        int row = rand() % M;
+        int col = rand() % N;
+        if(in[row * N + col] != out[col * M + row])
+        {
+            auto_lock_t _lk{print_lock};
+            std::cout << "mismatch: " << row << ", " << col << " : " << in[row * N + col] << " | "
+                      << out[col * M + row] << "\n";
+        }
+    }
+}
+}  // namespace

projects/rocprofiler-sdk/samples/common/CMakeLists.txt

@@ -0,0 +1,93 @@
+#
+# common utilities for samples
+#
+
+find_package(rocprofiler-sdk REQUIRED)
+
+# rocprofiler-sdk provides a Findlibdw.cmake
+find_package(libdw REQUIRED)
+
+# default FAIL_REGULAR_EXPRESSION for tests
+set(ROCPROFILER_DEFAULT_FAIL_REGEX
+    "threw an exception|Permission denied|Could not create logging file|failed with error code|Subprocess aborted"
+    CACHE INTERNAL "Default FAIL_REGULAR_EXPRESSION for tests")
+
+# build flags
+add_library(rocprofiler-sdk-samples-build-flags INTERFACE)
+add_library(rocprofiler-sdk::samples-build-flags ALIAS
+            rocprofiler-sdk-samples-build-flags)
+target_compile_options(rocprofiler-sdk-samples-build-flags INTERFACE -W -Wall -Wextra
+                                                                     -Wshadow)
+target_compile_features(rocprofiler-sdk-samples-build-flags INTERFACE cxx_std_17)
+
+if(ROCPROFILER_BUILD_CI OR ROCPROFILER_BUILD_WERROR)
+    target_compile_options(rocprofiler-sdk-samples-build-flags INTERFACE -Werror)
+endif()
+
+# common utilities
+cmake_path(GET CMAKE_CURRENT_SOURCE_DIR PARENT_PATH COMMON_LIBRARY_INCLUDE_DIR)
+
+add_library(rocprofiler-sdk-samples-common-library INTERFACE)
+add_library(rocprofiler-sdk::samples-common-library ALIAS
+            rocprofiler-sdk-samples-common-library)
+target_link_libraries(rocprofiler-sdk-samples-common-library
+                      INTERFACE rocprofiler-sdk::samples-build-flags libdw::libdw)
+target_compile_features(rocprofiler-sdk-samples-common-library INTERFACE cxx_std_17)
+target_include_directories(rocprofiler-sdk-samples-common-library
+                           INTERFACE ${COMMON_LIBRARY_INCLUDE_DIR})
+
+set(EXTERNAL_SUBMODULE_DIR "${PROJECT_SOURCE_DIR}/../external")
+cmake_path(ABSOLUTE_PATH EXTERNAL_SUBMODULE_DIR NORMALIZE)
+
+if(EXISTS ${EXTERNAL_SUBMODULE_DIR}/filesystem/include/ghc/filesystem.hpp)
+    target_compile_definitions(
+        rocprofiler-sdk-samples-common-library
+        INTERFACE $<BUILD_INTERFACE:ROCPROFILER_SAMPLES_HAS_GHC_LIB_FILESYSTEM=1>)
+    target_include_directories(
+        rocprofiler-sdk-samples-common-library SYSTEM
+        INTERFACE $<BUILD_INTERFACE:${EXTERNAL_SUBMODULE_DIR}/filesystem/include>)
+endif()
+
+# function for getting the LD_PRELOAD environment variable
+function(rocprofiler_samples_get_preload_env _VAR)
+    set(_PRELOAD_ENV_LIBS ${ROCPROFILER_MEMCHECK_PRELOAD_ENV_VALUE} $ENV{LD_PRELOAD})
+
+    foreach(_TARG ${ARGN})
+        if(NOT TARGET ${_TARG})
+            message(
+                FATAL_ERROR
+                    "rocprofiler_samples_get_preload_env: '${_TARG}' is not a valid target"
+                )
+        endif()
+        list(APPEND _PRELOAD_ENV_LIBS $<TARGET_FILE:${_TARG}>)
+    endforeach()
+
+    if(_PRELOAD_ENV_LIBS)
+        string(REPLACE ";" ":" _PRELOAD_ENV "LD_PRELOAD=${_PRELOAD_ENV_LIBS}")
+    endif()
+
+    set(${_VAR}
+        "${_PRELOAD_ENV}"
+        PARENT_SCOPE)
+endfunction()
+
+# function for getting the LD_LIBRARY_PATH environment variable
+function(rocprofiler_samples_get_ld_library_path_env _VAR)
+
+    set(_LDLIB_PATH "LD_LIBRARY_PATH=")
+    foreach(_TARG ${ARGN})
+        if(NOT TARGET ${_TARG})
+            message(
+                FATAL_ERROR
+                    "rocprofiler_samples_get_ld_library_path_env: '${_TARG}' is not a valid target"
+                )
+        endif()
+        string(APPEND _LDLIB_PATH "$<TARGET_FILE_DIR:${_TARG}>:")
+    endforeach()
+    # append the environments current LD_LIBRARY_PATH
+    string(APPEND _LDLIB_PATH "$ENV{LD_LIBRARY_PATH}")
+
+    set(${_VAR}
+        "${_LDLIB_PATH}"
+        PARENT_SCOPE)
+endfunction()

projects/rocprofiler-sdk/samples/common/call_stack.hpp

@@ -0,0 +1,90 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "filesystem.hpp"
+
+#include <cstdint>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace common
+{
+struct source_location
+{
+    std::string function = {};
+    std::string file     = {};
+    uint32_t    line     = 0;
+    std::string context  = {};
+};
+
+using call_stack_t = std::vector<source_location>;
+
+inline void
+print_call_stack(std::string         ofname,
+                 const call_stack_t& _call_stack,
+                 const char*         env_variable = "ROCPROFILER_SAMPLE_OUTPUT_FILE")
+{
+    if(auto* eofname = getenv(env_variable)) ofname = eofname;
+
+    std::ostream* ofs     = nullptr;
+    auto          cleanup = std::function<void(std::ostream*&)>{};
+
+    if(ofname == "stdout")
+        ofs = &std::cout;
+    else if(ofname == "stderr")
+        ofs = &std::cerr;
+    else
+    {
+        ofs = new std::ofstream{ofname};
+        if(ofs && *ofs)
+            cleanup = [](std::ostream*& _os) { delete _os; };
+        else
+        {
+            std::cerr << "Error outputting to " << ofname << ". Redirecting to stderr...\n";
+            ofname = "stderr";
+            ofs    = &std::cerr;
+        }
+    }
+
+    std::cout << "Outputting collected data to " << ofname << "...\n" << std::flush;
+
+    size_t n = 0;
+    for(const auto& itr : _call_stack)
+    {
+        *ofs << std::left << std::setw(2) << ++n << "/" << std::setw(2) << _call_stack.size()
+             << " [" << common::fs::path{itr.file}.filename() << ":" << itr.line << "] "
+             << std::setw(20) << itr.function;
+        if(!itr.context.empty()) *ofs << " :: " << itr.context;
+        *ofs << "\n";
+    }
+
+    *ofs << std::flush;
+
+    if(cleanup) cleanup(ofs);
+}
+}  // namespace common

projects/rocprofiler-sdk/samples/common/defines.hpp

@@ -0,0 +1,79 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#define ROCPROFILER_VAR_NAME_COMBINE(X, Y) X##Y
+#define ROCPROFILER_VARIABLE(X, Y)         ROCPROFILER_VAR_NAME_COMBINE(X, Y)
+
+#define ROCPROFILER_WARN(result)                                                                   \
+    {                                                                                              \
+        rocprofiler_status_t ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__) = result;                 \
+        if(ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__) != ROCPROFILER_STATUS_SUCCESS)              \
+        {                                                                                          \
+            std::string status_msg =                                                               \
+                rocprofiler_get_status_string(ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__));        \
+            std::cerr << "[" << __FILE__ << ":" << __LINE__ << "] " << #result                     \
+                      << " returned error code " << ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__)    \
+                      << ": " << status_msg << ". This is just a warning!" << std::endl;           \
+        }                                                                                          \
+    }
+
+#define ROCPROFILER_CHECK(result)                                                                  \
+    {                                                                                              \
+        rocprofiler_status_t ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__) = result;                 \
+        if(ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__) != ROCPROFILER_STATUS_SUCCESS)              \
+        {                                                                                          \
+            std::string status_msg =                                                               \
+                rocprofiler_get_status_string(ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__));        \
+            std::stringstream errmsg{};                                                            \
+            errmsg << "[" << __FILE__ << ":" << __LINE__ << "] " << #result                        \
+                   << " failed with error code " << ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__)    \
+                   << " :: " << status_msg;                                                        \
+            throw std::runtime_error(errmsg.str());                                                \
+        }                                                                                          \
+    }
+
+#define ROCPROFILER_CALL(result, msg)                                                              \
+    {                                                                                              \
+        rocprofiler_status_t ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__) = result;                 \
+        if(ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__) != ROCPROFILER_STATUS_SUCCESS)              \
+        {                                                                                          \
+            std::string status_msg =                                                               \
+                rocprofiler_get_status_string(ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__));        \
+            std::cerr << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg            \
+                      << " failed with error code " << ROCPROFILER_VARIABLE(CHECKSTATUS, __LINE__) \
+                      << ": " << status_msg << std::endl;                                          \
+            std::stringstream errmsg{};                                                            \
+            errmsg << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg " failure ("  \
+                   << status_msg << ")";                                                           \
+            throw std::runtime_error(errmsg.str());                                                \
+        }                                                                                          \
+    }
+
+#if HIP_VERSION >= 60300000
+#    define HIP_HOST_ALLOC_FUNC hipHostMalloc
+#    define HIP_HOST_FREE_FUNC  hipHostFree
+#else
+#    define HIP_HOST_ALLOC_FUNC hipHostMalloc
+#    define HIP_HOST_FREE_FUNC  hipHostFree
+#endif

projects/rocprofiler-sdk/samples/common/filesystem.hpp

@@ -0,0 +1,77 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#if !defined(ROCPROFILER_SAMPLES_HAS_GHC_LIB_FILESYSTEM)
+#    if defined __has_include
+#        if __has_include(<ghc/filesystem.hpp>)
+#            define ROCPROFILER_SAMPLES_HAS_GHC_LIB_FILESYSTEM 1
+#        else
+#            define ROCPROFILER_SAMPLES_HAS_GHC_LIB_FILESYSTEM 0
+#        endif
+#    else
+#        define ROCPROFILER_SAMPLES_HAS_GHC_LIB_FILESYSTEM 0
+#    endif
+#endif
+
+#if ROCPROFILER_SAMPLES_HAS_GHC_LIB_FILESYSTEM == 0
+#    if defined __has_include
+#        if __has_include(<version>)
+#            include <version>
+#        endif
+#    endif
+
+#    if defined(__cpp_lib_filesystem)
+#        define ROCPROFILER_SAMPLES_HAS_CPP_LIB_FILESYSTEM 1
+#    else
+#        if defined __has_include
+#            if __has_include(<filesystem>)
+#                define ROCPROFILER_SAMPLES_HAS_CPP_LIB_FILESYSTEM 1
+#            endif
+#        endif
+#    endif
+#endif
+
+// include the correct filesystem header
+#if defined(ROCPROFILER_SAMPLES_HAS_GHC_LIB_FILESYSTEM) &&                                         \
+    ROCPROFILER_SAMPLES_HAS_GHC_LIB_FILESYSTEM > 0
+#    include <ghc/filesystem.hpp>
+#elif defined(ROCPROFILER_SAMPLES_HAS_CPP_LIB_FILESYSTEM) &&                                       \
+    ROCPROFILER_SAMPLES_HAS_CPP_LIB_FILESYSTEM > 0
+#    include <filesystem>
+#else
+#    include <experimental/filesystem>
+#endif
+
+namespace common
+{
+#if defined(ROCPROFILER_SAMPLES_HAS_GHC_LIB_FILESYSTEM) &&                                         \
+    ROCPROFILER_SAMPLES_HAS_GHC_LIB_FILESYSTEM > 0
+namespace fs = ::ghc::filesystem;  // NOLINT(misc-unused-alias-decls)
+#elif defined(ROCPROFILER_SAMPLES_HAS_CPP_LIB_FILESYSTEM) &&                                       \
+    ROCPROFILER_SAMPLES_HAS_CPP_LIB_FILESYSTEM > 0
+namespace fs = ::std::filesystem;  // NOLINT(misc-unused-alias-decls)
+#else
+namespace fs = ::std::experimental::filesystem;  // NOLINT(misc-unused-alias-decls)
+#endif
+}  // namespace common

projects/rocprofiler-sdk/samples/common/name_info.hpp

@@ -0,0 +1,55 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+#include <rocprofiler-sdk/cxx/name_info.hpp>
+
+#include "defines.hpp"
+
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+namespace common
+{
+using callback_name_info = rocprofiler::sdk::callback_name_info;
+using buffer_name_info   = rocprofiler::sdk::buffer_name_info;
+
+inline auto
+get_buffer_tracing_names()
+{
+    return rocprofiler::sdk::get_buffer_tracing_names();
+}
+
+inline auto
+get_callback_tracing_names()
+{
+    return rocprofiler::sdk::get_callback_tracing_names();
+}
+}  // namespace common

projects/rocprofiler-sdk/samples/counter_collection/CMakeLists.txt

@@ -0,0 +1,219 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-samples-counter-collection LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+if(ROCPROFILER_MEMCHECK STREQUAL "ThreadSanitizer")
+    set(IS_THREAD_SANITIZER ON)
+else()
+    set(IS_THREAD_SANITIZER OFF)
+endif()
+
+find_package(rocprofiler-sdk REQUIRED)
+
+add_library(counter-collection-buffer-client SHARED)
+target_sources(counter-collection-buffer-client PRIVATE buffered_client.cpp client.hpp)
+target_link_libraries(
+    counter-collection-buffer-client
+    PUBLIC rocprofiler-sdk::samples-build-flags
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::samples-common-library)
+
+set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
+add_executable(counter-collection-buffer)
+target_sources(counter-collection-buffer PRIVATE main.cpp)
+target_link_libraries(counter-collection-buffer PRIVATE counter-collection-buffer-client
+                                                        Threads::Threads)
+
+rocprofiler_samples_get_ld_library_path_env(LIBRARY_PATH_ENV)
+rocprofiler_samples_get_preload_env(PRELOAD_ENV counter-collection-buffer-client)
+
+set(counter-collection-buffer-env "${PRELOAD_ENV}" "${LIBRARY_PATH_ENV}")
+
+add_test(NAME counter-collection-buffer COMMAND $<TARGET_FILE:counter-collection-buffer>)
+
+set_tests_properties(
+    counter-collection-buffer
+    PROPERTIES TIMEOUT
+               120
+               LABELS
+               "samples"
+               ENVIRONMENT
+               "${counter-collection-buffer-env}"
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               DISABLED
+               "${ROCPROFILER_DISABLE_UNSTABLE_CTESTS}")
+
+set_source_files_properties(device_serialized_main.cpp PROPERTIES LANGUAGE HIP)
+add_executable(counter-collection-buffer-device-serialization)
+target_sources(counter-collection-buffer-device-serialization
+               PRIVATE device_serialized_main.cpp)
+target_link_libraries(counter-collection-buffer-device-serialization
+                      PRIVATE counter-collection-buffer-client Threads::Threads)
+
+rocprofiler_samples_get_ld_library_path_env(LIBRARY_PATH_ENV)
+rocprofiler_samples_get_preload_env(PRELOAD_ENV counter-collection-buffer-client)
+
+set(counter-collection-buffer-device-serialization-env "${PRELOAD_ENV}"
+                                                       "${LIBRARY_PATH_ENV}")
+
+add_test(NAME counter-collection-buffer-device-serialization
+         COMMAND $<TARGET_FILE:counter-collection-buffer-device-serialization>)
+
+set_tests_properties(
+    counter-collection-buffer-device-serialization
+    PROPERTIES TIMEOUT 120 LABELS "samples" ENVIRONMENT
+               "${counter-collection-buffer-device-serialization-env}"
+               FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
+
+add_library(counter-collection-callback-client SHARED)
+target_sources(counter-collection-callback-client PRIVATE callback_client.cpp client.hpp)
+target_link_libraries(
+    counter-collection-callback-client
+    PUBLIC rocprofiler-sdk::samples-build-flags
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::samples-common-library)
+
+set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
+add_executable(counter-collection-callback)
+target_sources(counter-collection-callback PRIVATE main.cpp)
+target_link_libraries(counter-collection-callback
+                      PRIVATE counter-collection-callback-client Threads::Threads)
+
+rocprofiler_samples_get_preload_env(PRELOAD_ENV counter-collection-callback-client)
+
+set(counter-collection-callback-env "${PRELOAD_ENV}" "${LIBRARY_PATH_ENV}")
+
+add_test(NAME counter-collection-callback
+         COMMAND $<TARGET_FILE:counter-collection-callback>)
+
+set_tests_properties(
+    counter-collection-callback
+    PROPERTIES TIMEOUT
+               120
+               LABELS
+               "samples"
+               ENVIRONMENT
+               "${counter-collection-callback-env}"
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               DISABLED
+               "${ROCPROFILER_DISABLE_UNSTABLE_CTESTS}")
+
+add_library(counter-collection-functional-counter-client SHARED)
+target_sources(counter-collection-functional-counter-client
+               PRIVATE print_functional_counters_client.cpp client.hpp)
+target_link_libraries(
+    counter-collection-functional-counter-client
+    PUBLIC rocprofiler-sdk::samples-build-flags
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::samples-common-library)
+
+add_executable(counter-collection-print-functional-counters)
+target_sources(counter-collection-print-functional-counters PRIVATE main.cpp)
+target_link_libraries(
+    counter-collection-print-functional-counters
+    PRIVATE counter-collection-functional-counter-client Threads::Threads)
+
+rocprofiler_samples_get_preload_env(PRELOAD_ENV
+                                    counter-collection-functional-counter-client)
+
+set(counter-collection-functional-counter-env "${PRELOAD_ENV}" "${LIBRARY_PATH_ENV}")
+
+add_test(NAME counter-collection-print-functional-counters
+         COMMAND $<TARGET_FILE:counter-collection-print-functional-counters>)
+
+set_tests_properties(
+    counter-collection-print-functional-counters
+    PROPERTIES TIMEOUT 120 LABELS "samples" ENVIRONMENT
+               "${counter-collection-functional-counter-env}" FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
+
+add_library(counter-collection-device-profiling-client SHARED)
+target_sources(counter-collection-device-profiling-client
+               PRIVATE device_counting_async_client.cpp client.hpp)
+target_link_libraries(
+    counter-collection-device-profiling-client
+    PUBLIC rocprofiler-sdk::samples-build-flags
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::samples-common-library)
+
+add_executable(counter-collection-device-profiling)
+target_sources(counter-collection-device-profiling PRIVATE main.cpp)
+target_link_libraries(counter-collection-device-profiling
+                      PRIVATE counter-collection-device-profiling-client Threads::Threads)
+
+rocprofiler_samples_get_preload_env(PRELOAD_ENV
+                                    counter-collection-device-profiling-client)
+
+set(counter-collection-functional-counter-env "${PRELOAD_ENV}" "${LIBRARY_PATH_ENV}")
+
+add_test(NAME counter-collection-device-profiling
+         COMMAND $<TARGET_FILE:counter-collection-device-profiling>)
+
+set_tests_properties(
+    counter-collection-device-profiling
+    PROPERTIES TIMEOUT
+               120
+               LABELS
+               "samples"
+               ENVIRONMENT
+               "${counter-collection-functional-counter-env}"
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               DISABLED
+               "${IS_THREAD_SANITIZER}")
+
+add_library(counter-collection-device-profiling-sync-client SHARED)
+target_sources(counter-collection-device-profiling-sync-client
+               PRIVATE device_counting_sync_client.cpp client.hpp)
+target_link_libraries(
+    counter-collection-device-profiling-sync-client
+    PUBLIC rocprofiler-sdk::samples-build-flags
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::samples-common-library)
+
+add_executable(counter-collection-device-profiling-sync)
+target_sources(counter-collection-device-profiling-sync PRIVATE main.cpp)
+target_link_libraries(
+    counter-collection-device-profiling-sync
+    PRIVATE counter-collection-device-profiling-sync-client Threads::Threads)
+
+rocprofiler_samples_get_preload_env(PRELOAD_ENV
+                                    counter-collection-device-profiling-sync-client)
+
+set(counter-collection-functional-counter-env "${PRELOAD_ENV}" "${LIBRARY_PATH_ENV}")
+
+add_test(NAME counter-collection-device-profiling-sync
+         COMMAND $<TARGET_FILE:counter-collection-device-profiling-sync>)
+
+set_tests_properties(
+    counter-collection-device-profiling-sync
+    PROPERTIES TIMEOUT
+               120
+               LABELS
+               "samples"
+               ENVIRONMENT
+               "${counter-collection-functional-counter-env}"
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               DISABLED
+               "${IS_THREAD_SANITIZER}")

projects/rocprofiler-sdk/samples/counter_collection/README.md

@@ -0,0 +1,21 @@
+# Counter collection
+
+## Services
+
+- Dispatch counting
+- Device Counting async
+- Device Counting sync
+  
+## Properties
+
+- Initializes tool and setup for counting service.
+- Create a collection profile for the counters.
+- Outputs counters mentioned during profiler creation.
+- Usage of enum ROCPROFILER_BUFFER_CATEGORY_COUNTERS.
+- Buffered_callback
+  - This sample shows the usage of buffered approach when collecting counters. buffered callback is called when the buffer is full (or when the buffer is flushed). The callback is responsible for processing the records in the buffer.
+
+- Dispatch callback
+  - This sample creates a profile to collect the counter SQ_WAVES for all kernel dispatch packets.
+  
+- Prints all functional counters.

projects/rocprofiler-sdk/samples/counter_collection/buffered_client.cpp

@@ -0,0 +1,445 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "client.hpp"
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <cstdint>
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <mutex>
+#include <set>
+#include <shared_mutex>
+#include <sstream>
+#include <unordered_map>
+#include <vector>
+
+#define ROCPROFILER_CALL(result, msg)                                                              \
+    {                                                                                              \
+        rocprofiler_status_t CHECKSTATUS = result;                                                 \
+        if(CHECKSTATUS != ROCPROFILER_STATUS_SUCCESS)                                              \
+        {                                                                                          \
+            std::string status_msg = rocprofiler_get_status_string(CHECKSTATUS);                   \
+            std::cerr << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg            \
+                      << " failed with error code " << CHECKSTATUS << ": " << status_msg           \
+                      << std::endl;                                                                \
+            std::stringstream errmsg{};                                                            \
+            errmsg << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg " failure ("  \
+                   << status_msg << ")";                                                           \
+            throw std::runtime_error(errmsg.str());                                                \
+        }                                                                                          \
+    }
+
+int
+start()
+{
+    return 1;
+}
+
+namespace
+{
+rocprofiler_context_id_t&
+get_client_ctx()
+{
+    static rocprofiler_context_id_t ctx{0};
+    return ctx;
+}
+
+rocprofiler_buffer_id_t&
+get_buffer()
+{
+    static rocprofiler_buffer_id_t buf = {};
+    return buf;
+}
+
+std::unordered_map<uint64_t, std::vector<rocprofiler_counter_record_dimension_info_t>>**
+dimension_cache()
+{
+    static std::unordered_map<uint64_t, std::vector<rocprofiler_counter_record_dimension_info_t>>*
+        cache;
+    return &cache;
+}
+
+/**
+ * For a given counter, query the dimensions that it has. Typically you will
+ * want to call this function once to get the dimensions and cache them.
+ */
+std::vector<rocprofiler_counter_record_dimension_info_t>
+counter_dimensions(rocprofiler_counter_id_t counter)
+{
+    if(*dimension_cache() == nullptr) return {};
+
+    if((*dimension_cache())->count(counter.handle) > 0)
+    {
+        return (*dimension_cache())->at(counter.handle);
+    }
+
+    return {};
+}
+
+void
+fill_dimension_cache(rocprofiler_counter_id_t counter)
+{
+    assert(*dimension_cache() != nullptr);
+    std::vector<rocprofiler_counter_record_dimension_info_t> dims;
+    rocprofiler_counter_info_v1_t                            info;
+    ROCPROFILER_CALL(rocprofiler_query_counter_info(
+                         counter, ROCPROFILER_COUNTER_INFO_VERSION_1, static_cast<void*>(&info)),
+                     "Could not query info for counter");
+
+    (*dimension_cache())
+        ->emplace(counter.handle,
+                  std::vector<rocprofiler_counter_record_dimension_info_t>{
+                      *info.dimensions, *info.dimensions + info.dimensions_count});
+}
+
+/**
+ * buffered_callback (set in rocprofiler_create_buffer in tool_init) is called when the
+ * buffer is full (or when the buffer is flushed). The callback is responsible for processing
+ * the records in the buffer. The records are returned in the headers array. The headers
+ * can contain counter records as well as other records (such as tracing). These
+ * records need to be filtered based on the category type. For counter collection,
+ * they should be filtered by category == ROCPROFILER_BUFFER_CATEGORY_COUNTERS.
+ */
+void
+buffered_callback(rocprofiler_context_id_t,
+                  rocprofiler_buffer_id_t,
+                  rocprofiler_record_header_t** headers,
+                  size_t                        num_headers,
+                  void*                         user_data,
+                  uint64_t)
+{
+    std::stringstream ss;
+    // Iterate through the returned records
+    for(size_t i = 0; i < num_headers; ++i)
+    {
+        auto* header = headers[i];
+        if(header->category == ROCPROFILER_BUFFER_CATEGORY_COUNTERS &&
+           header->kind == ROCPROFILER_COUNTER_RECORD_PROFILE_COUNTING_DISPATCH_HEADER)
+        {
+            // Print the returned counter data.
+            auto* record =
+                static_cast<rocprofiler_dispatch_counting_service_record_t*>(header->payload);
+            ss << "[Dispatch_Id: " << record->dispatch_info.dispatch_id
+               << " Kernel_ID: " << record->dispatch_info.kernel_id
+               << " Corr_Id: " << record->correlation_id.internal << ")]\n";
+        }
+        else if(header->category == ROCPROFILER_BUFFER_CATEGORY_COUNTERS &&
+                header->kind == ROCPROFILER_COUNTER_RECORD_VALUE)
+        {
+            // Print the returned counter data.
+            auto* record = static_cast<rocprofiler_counter_record_t*>(header->payload);
+            rocprofiler_counter_id_t counter_id = {.handle = 0};
+
+            rocprofiler_query_record_counter_id(record->id, &counter_id);
+
+            ss << "  (Dispatch_Id: " << record->dispatch_id << " Counter_Id: " << counter_id.handle
+               << " Record_Id: " << record->id << " Dimensions: [";
+
+            for(auto& dim : counter_dimensions(counter_id))
+            {
+                size_t pos = 0;
+                rocprofiler_query_record_dimension_position(record->id, dim.id, &pos);
+                ss << "{" << dim.name << ": " << pos << "},";
+            }
+            ss << "] Value [D]: " << record->counter_value << "),";
+        }
+    }
+
+    auto* output_stream = static_cast<std::ostream*>(user_data);
+    if(!output_stream) throw std::runtime_error{"nullptr to output stream"};
+
+    *output_stream << "[" << __FUNCTION__ << "] " << ss.str() << "\n";
+}
+
+/**
+ * Cache to store the profile configs for each agent. This is used to prevent
+ * constructing the same profile config multiple times. Used by dispatch_callback
+ * to select the profile config (and in turn counters) to use when a kernel dispatch
+ * is received.
+ */
+std::unordered_map<uint64_t, rocprofiler_counter_config_id_t>&
+get_profile_cache()
+{
+    static std::unordered_map<uint64_t, rocprofiler_counter_config_id_t> profile_cache;
+    return profile_cache;
+}
+
+/**
+ * Callback from rocprofiler when an kernel dispatch is enqueued into the HSA queue.
+ * rocprofiler_counter_config_id_t* is a return to specify what counters to collect
+ * for this dispatch (dispatch_packet). This example function creates a profile
+ * to collect the counter SQ_WAVES for all kernel dispatch packets.
+ */
+void
+dispatch_callback(rocprofiler_dispatch_counting_service_data_t dispatch_data,
+                  rocprofiler_counter_config_id_t*             config,
+                  rocprofiler_user_data_t* /*user_data*/,
+                  void* /*callback_data_args*/)
+{
+    /**
+     * This simple example uses the same profile counter set for all agents.
+     * We store this in a cache to prevent constructing many identical profile counter
+     * sets.
+     */
+    auto search_cache = [&]() {
+        if(auto pos = get_profile_cache().find(dispatch_data.dispatch_info.agent_id.handle);
+           pos != get_profile_cache().end())
+        {
+            *config = pos->second;
+            return true;
+        }
+        return false;
+    };
+
+    if(!search_cache())
+    {
+        std::cerr << "No profile for agent found in cache\n";
+        exit(-1);
+    }
+}
+
+/**
+ * Construct a profile config for an agent. This function takes an agent (obtained from
+ * get_gpu_device_agents()) and a set of counter names to collect. It returns a profile
+ * that can be used when a dispatch is received for the agent to collect the specified
+ * counters. Note: while you can dynamically create these profiles, it is more efficient
+ * to consturct them once in advance (i.e. in tool_init()) since there are non-trivial
+ * costs associated with constructing the profile.
+ */
+rocprofiler_counter_config_id_t
+build_profile_for_agent(rocprofiler_agent_id_t       agent,
+                        const std::set<std::string>& counters_to_collect)
+{
+    std::vector<rocprofiler_counter_id_t> gpu_counters;
+
+    // Iterate all the counters on the agent and store them in gpu_counters.
+    ROCPROFILER_CALL(rocprofiler_iterate_agent_supported_counters(
+                         agent,
+                         [](rocprofiler_agent_id_t,
+                            rocprofiler_counter_id_t* counters,
+                            size_t                    num_counters,
+                            void*                     user_data) {
+                             std::vector<rocprofiler_counter_id_t>* vec =
+                                 static_cast<std::vector<rocprofiler_counter_id_t>*>(user_data);
+                             for(size_t i = 0; i < num_counters; i++)
+                             {
+                                 vec->push_back(counters[i]);
+                             }
+                             return ROCPROFILER_STATUS_SUCCESS;
+                         },
+                         static_cast<void*>(&gpu_counters)),
+                     "Could not fetch supported counters");
+
+    // Find the counters we actually want to collect (i.e. those in counters_to_collect)
+    std::vector<rocprofiler_counter_id_t> collect_counters;
+    for(auto& counter : gpu_counters)
+    {
+        rocprofiler_counter_info_v0_t info;
+        ROCPROFILER_CALL(
+            rocprofiler_query_counter_info(
+                counter, ROCPROFILER_COUNTER_INFO_VERSION_0, static_cast<void*>(&info)),
+            "Could not query info for counter");
+        if(counters_to_collect.count(std::string(info.name)) > 0)
+        {
+            std::clog << "Counter: " << counter.handle << " " << info.name << "\n";
+            collect_counters.push_back(counter);
+            fill_dimension_cache(counter);
+        }
+    }
+
+    // Create and return the profile
+    rocprofiler_counter_config_id_t profile = {.handle = 0};
+    ROCPROFILER_CALL(rocprofiler_create_counter_config(
+                         agent, collect_counters.data(), collect_counters.size(), &profile),
+                     "Could not construct profile cfg");
+
+    return profile;
+}
+
+/**
+ * Returns all GPU agents visible to rocprofiler on the system
+ */
+std::vector<rocprofiler_agent_v0_t>
+get_gpu_device_agents()
+{
+    std::vector<rocprofiler_agent_v0_t> agents;
+
+    // Callback used by rocprofiler_query_available_agents to return
+    // agents on the device. This can include CPU agents as well. We
+    // select GPU agents only (i.e. type == ROCPROFILER_AGENT_TYPE_GPU)
+    rocprofiler_query_available_agents_cb_t iterate_cb = [](rocprofiler_agent_version_t agents_ver,
+                                                            const void**                agents_arr,
+                                                            size_t                      num_agents,
+                                                            void*                       udata) {
+        if(agents_ver != ROCPROFILER_AGENT_INFO_VERSION_0)
+            throw std::runtime_error{"unexpected rocprofiler agent version"};
+        auto* agents_v = static_cast<std::vector<rocprofiler_agent_v0_t>*>(udata);
+        for(size_t i = 0; i < num_agents; ++i)
+        {
+            const auto* agent = static_cast<const rocprofiler_agent_v0_t*>(agents_arr[i]);
+            if(agent->type == ROCPROFILER_AGENT_TYPE_GPU) agents_v->emplace_back(*agent);
+        }
+        return ROCPROFILER_STATUS_SUCCESS;
+    };
+
+    // Query the agents, only a single callback is made that contains a vector
+    // of all agents.
+    ROCPROFILER_CALL(
+        rocprofiler_query_available_agents(ROCPROFILER_AGENT_INFO_VERSION_0,
+                                           iterate_cb,
+                                           sizeof(rocprofiler_agent_t),
+                                           const_cast<void*>(static_cast<const void*>(&agents))),
+        "query available agents");
+    return agents;
+}
+
+/**
+ * Initialize the tool. This function is called once when the tool is loaded.
+ * The function is responsible for creating the context, buffer, profile configs
+ * (details counters to collect on each agent), configuring the dispatch profile
+ * counting service, and starting the context.
+ */
+int
+tool_init(rocprofiler_client_finalize_t, void* user_data)
+{
+    ROCPROFILER_CALL(rocprofiler_create_context(&get_client_ctx()), "context creation failed");
+    ROCPROFILER_CALL(rocprofiler_create_buffer(get_client_ctx(),
+                                               4096,
+                                               2048,
+                                               ROCPROFILER_BUFFER_POLICY_LOSSLESS,
+                                               buffered_callback,
+                                               user_data,
+                                               &get_buffer()),
+                     "buffer creation failed");
+
+    // Get a vector of all GPU devices on the system.
+    auto agents = get_gpu_device_agents();
+
+    if(agents.empty())
+    {
+        std::cerr << "No agents found" << std::endl;
+        return 1;
+    }
+
+    // Construct the profiles in advance for each agent that is a GPU
+    for(const auto& agent : agents)
+    {
+        // get_profile_cache() is a map that can be accessed by dispatch_callback
+        // below to select the profile config to use when a kernel dispatch is
+        // recieved.
+        get_profile_cache().emplace(
+            agent.id.handle, build_profile_for_agent(agent.id, std::set<std::string>{"TCC_HIT"}));
+    }
+
+    auto client_thread = rocprofiler_callback_thread_t{};
+    // Create the callback thread
+    ROCPROFILER_CALL(rocprofiler_create_callback_thread(&client_thread),
+                     "failure creating callback thread");
+    // Create the buffer and assign the callback thread to the buffer, when the buffer is full
+    // a callback will be issued (to client_thread)
+    ROCPROFILER_CALL(rocprofiler_assign_callback_thread(get_buffer(), client_thread),
+                     "failed to assign thread for buffer");
+
+    // Setup the dispatch profile counting service. This service will trigger the dispatch_callback
+    // when a kernel dispatch is enqueued into the HSA queue. The callback will specify what
+    // counters to collect by returning a profile config id. In this example, we create the profile
+    // configs above and store them in the map get_profile_cache() so we can look them up at
+    // dispatch.
+    ROCPROFILER_CALL(rocprofiler_configure_buffer_dispatch_counting_service(
+                         get_client_ctx(), get_buffer(), dispatch_callback, nullptr),
+                     "Could not setup buffered service");
+
+    // Start the context (start intercepting kernel dispatches).
+    ROCPROFILER_CALL(rocprofiler_start_context(get_client_ctx()), "start context");
+
+    // no errors
+    return 0;
+}
+
+void
+tool_fini(void* user_data)
+{
+    std::clog << "In tool fini\n";
+
+    // Flush the buffer and stop the context
+    ROCPROFILER_CALL(rocprofiler_flush_buffer(get_buffer()), "buffer flush");
+    rocprofiler_stop_context(get_client_ctx());
+
+    auto* output_stream = static_cast<std::ostream*>(user_data);
+    *output_stream << std::flush;
+    if(output_stream != &std::cout && output_stream != &std::cerr) delete output_stream;
+
+    auto* tmp_ptr      = *dimension_cache();
+    *dimension_cache() = nullptr;
+    delete tmp_ptr;
+}
+}  // namespace
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name = "CounterClientSample";
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " (priority=" << priority << ") is using rocprofiler-sdk v" << major << "."
+         << minor << "." << patch << " (" << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    std::ostream* output_stream = nullptr;
+    std::string   filename      = "counter_collection.log";
+    if(auto* outfile = getenv("ROCPROFILER_SAMPLE_OUTPUT_FILE"); outfile) filename = outfile;
+    if(filename == "stdout")
+        output_stream = &std::cout;
+    else if(filename == "stderr")
+        output_stream = &std::cerr;
+    else
+        output_stream = new std::ofstream{filename};
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &tool_init,
+                                            &tool_fini,
+                                            static_cast<void*>(output_stream)};
+
+    *dimension_cache() =
+        new std::unordered_map<uint64_t,
+                               std::vector<rocprofiler_counter_record_dimension_info_t>>();
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/counter_collection/callback_client.cpp

@@ -0,0 +1,264 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "client.hpp"
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <mutex>
+#include <set>
+#include <shared_mutex>
+#include <sstream>
+#include <unordered_map>
+#include <vector>
+
+#define ROCPROFILER_CALL(result, msg)                                                              \
+    {                                                                                              \
+        rocprofiler_status_t CHECKSTATUS = result;                                                 \
+        if(CHECKSTATUS != ROCPROFILER_STATUS_SUCCESS)                                              \
+        {                                                                                          \
+            std::string status_msg = rocprofiler_get_status_string(CHECKSTATUS);                   \
+            std::cerr << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg            \
+                      << " failed with error code " << CHECKSTATUS << ": " << status_msg           \
+                      << std::endl;                                                                \
+            std::stringstream errmsg{};                                                            \
+            errmsg << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg " failure ("  \
+                   << status_msg << ")";                                                           \
+            throw std::runtime_error(errmsg.str());                                                \
+        }                                                                                          \
+    }
+
+int
+start()
+{
+    return 1;
+}
+
+namespace
+{
+struct tool_data_t
+{
+    std::mutex    mut{};
+    std::ostream* output_stream{nullptr};
+};
+
+rocprofiler_context_id_t&
+get_client_ctx()
+{
+    static rocprofiler_context_id_t ctx{0};
+    return ctx;
+}
+
+void
+record_callback(rocprofiler_dispatch_counting_service_data_t dispatch_data,
+                rocprofiler_counter_record_t*                record_data,
+                size_t                                       record_count,
+                rocprofiler_user_data_t /* user_data */,
+                void* callback_data_args)
+{
+    std::stringstream ss;
+    ss << "Dispatch_Id=" << dispatch_data.dispatch_info.dispatch_id
+       << ", Kernel_id=" << dispatch_data.dispatch_info.kernel_id
+       << ", Corr_Id=" << dispatch_data.correlation_id.internal << ": ";
+    for(size_t i = 0; i < record_count; ++i)
+        ss << "(Id: " << record_data[i].id << " Value [D]: " << record_data[i].counter_value
+           << "),";
+
+    auto* tool = static_cast<tool_data_t*>(callback_data_args);
+    if(!tool || !tool->output_stream) throw std::runtime_error{"nullptr to output stream"};
+
+    auto _lk = std::unique_lock{tool->mut};
+    *tool->output_stream << "[" << __FUNCTION__ << "] " << ss.str() << "\n";
+}
+
+/**
+ * Callback from rocprofiler when an kernel dispatch is enqueued into the HSA queue.
+ * rocprofiler_counter_config_id_t* is a return to specify what counters to collect
+ * for this dispatch (dispatch_packet). This example function creates a profile
+ * to collect the counter SQ_WAVES for all kernel dispatch packets.
+ */
+void
+dispatch_callback(rocprofiler_dispatch_counting_service_data_t dispatch_data,
+                  rocprofiler_counter_config_id_t*             config,
+                  rocprofiler_user_data_t* /*user_data*/,
+                  void* /*callback_data_args*/)
+{
+    /**
+     * This simple example uses the same profile counter set for all agents.
+     * We store this in a cache to prevent constructing many identical profile counter
+     * sets. We first check the cache to see if we have already constructed a counter"
+     * set for the agent. If we have, return it. Otherwise, construct a new profile counter
+     * set.
+     */
+    static std::shared_mutex                                             m_mutex       = {};
+    static std::unordered_map<uint64_t, rocprofiler_counter_config_id_t> profile_cache = {};
+
+    auto search_cache = [&]() {
+        if(auto pos = profile_cache.find(dispatch_data.dispatch_info.agent_id.handle);
+           pos != profile_cache.end())
+        {
+            *config = pos->second;
+            return true;
+        }
+        return false;
+    };
+
+    {
+        auto rlock = std::shared_lock{m_mutex};
+        if(search_cache()) return;
+    }
+
+    auto wlock = std::unique_lock{m_mutex};
+    if(search_cache()) return;
+
+    // Counters we want to collect (here its SQ_WAVES)
+    std::set<std::string> counters_to_collect = {"SQ_WAVES"};
+    // GPU Counter IDs
+    std::vector<rocprofiler_counter_id_t> gpu_counters;
+
+    // Iterate through the agents and get the counters available on that agent
+    ROCPROFILER_CALL(rocprofiler_iterate_agent_supported_counters(
+                         dispatch_data.dispatch_info.agent_id,
+                         [](rocprofiler_agent_id_t,
+                            rocprofiler_counter_id_t* counters,
+                            size_t                    num_counters,
+                            void*                     user_data) {
+                             std::vector<rocprofiler_counter_id_t>* vec =
+                                 static_cast<std::vector<rocprofiler_counter_id_t>*>(user_data);
+                             for(size_t i = 0; i < num_counters; i++)
+                             {
+                                 vec->push_back(counters[i]);
+                             }
+                             return ROCPROFILER_STATUS_SUCCESS;
+                         },
+                         static_cast<void*>(&gpu_counters)),
+                     "Could not fetch supported counters");
+
+    std::vector<rocprofiler_counter_id_t> collect_counters;
+    // Look for the counters contained in counters_to_collect in gpu_counters
+    for(auto& counter : gpu_counters)
+    {
+        rocprofiler_counter_info_v0_t info;
+        ROCPROFILER_CALL(
+            rocprofiler_query_counter_info(
+                counter, ROCPROFILER_COUNTER_INFO_VERSION_0, static_cast<void*>(&info)),
+            "Could not query info");
+        if(counters_to_collect.count(std::string(info.name)) > 0)
+        {
+            std::clog << "Counter: " << counter.handle << " " << info.name << "\n";
+            collect_counters.push_back(counter);
+        }
+    }
+
+    // Create a colleciton profile for the counters
+    rocprofiler_counter_config_id_t profile = {.handle = 0};
+    ROCPROFILER_CALL(rocprofiler_create_counter_config(dispatch_data.dispatch_info.agent_id,
+                                                       collect_counters.data(),
+                                                       collect_counters.size(),
+                                                       &profile),
+                     "Could not construct profile cfg");
+
+    profile_cache.emplace(dispatch_data.dispatch_info.agent_id.handle, profile);
+    // Return the profile to collect those counters for this dispatch
+    *config = profile;
+}
+
+int
+tool_init(rocprofiler_client_finalize_t, void* user_data)
+{
+    ROCPROFILER_CALL(rocprofiler_create_context(&get_client_ctx()), "context creation failed");
+
+    ROCPROFILER_CALL(rocprofiler_configure_callback_dispatch_counting_service(
+                         get_client_ctx(), dispatch_callback, nullptr, record_callback, user_data),
+                     "Could not setup counting service");
+    ROCPROFILER_CALL(rocprofiler_start_context(get_client_ctx()), "start context");
+
+    // no errors
+    return 0;
+}
+
+void
+tool_fini(void* user_data)
+{
+    assert(user_data);
+    std::clog << "In tool fini\n";
+    rocprofiler_stop_context(get_client_ctx());
+    auto* tool_data = static_cast<tool_data_t*>(user_data);
+
+    {
+        auto  _lk           = std::unique_lock{tool_data->mut};
+        auto* output_stream = tool_data->output_stream;
+
+        *output_stream << std::flush;
+        if(output_stream != &std::cout && output_stream != &std::cerr) delete output_stream;
+    }
+
+    delete tool_data;
+}
+}  // namespace
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name = "CounterClientSample";
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " (priority=" << priority << ") is using rocprofiler-sdk v" << major << "."
+         << minor << "." << patch << " (" << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    auto* tool_data = new tool_data_t{};
+
+    std::string filename = "counter_collection.log";
+    if(auto* outfile = getenv("ROCPROFILER_SAMPLE_OUTPUT_FILE"); outfile) filename = outfile;
+    if(filename == "stdout")
+        tool_data->output_stream = &std::cout;
+    else if(filename == "stderr")
+        tool_data->output_stream = &std::cerr;
+    else
+        tool_data->output_stream = new std::ofstream{filename};
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &tool_init,
+                                            &tool_fini,
+                                            static_cast<void*>(tool_data)};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/counter_collection/client.hpp

@@ -0,0 +1,30 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <iostream>
+
+#define CLIENT_API __attribute__((visibility("default")))
+
+int
+start() CLIENT_API;

projects/rocprofiler-sdk/samples/counter_collection/device_counting_async_client.cpp

@@ -0,0 +1,367 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "client.hpp"
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <atomic>
+#include <chrono>
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <mutex>
+#include <set>
+#include <shared_mutex>
+#include <sstream>
+#include <stdexcept>
+#include <thread>
+#include <unordered_map>
+#include <vector>
+
+#define ROCPROFILER_CALL(result, msg)                                                              \
+    {                                                                                              \
+        rocprofiler_status_t CHECKSTATUS = result;                                                 \
+        if(CHECKSTATUS != ROCPROFILER_STATUS_SUCCESS)                                              \
+        {                                                                                          \
+            std::string status_msg = rocprofiler_get_status_string(CHECKSTATUS);                   \
+            std::cerr << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg            \
+                      << " failed with error code " << CHECKSTATUS << ": " << status_msg           \
+                      << std::endl;                                                                \
+            std::stringstream errmsg{};                                                            \
+            errmsg << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg " failure ("  \
+                   << status_msg << ")";                                                           \
+            throw std::runtime_error(errmsg.str());                                                \
+        }                                                                                          \
+    }
+
+int
+start()
+{
+    return 1;
+}
+
+namespace
+{
+rocprofiler_agent_id_t&
+expected_agent()
+{
+    static rocprofiler_agent_id_t expected_agent = {.handle = 0};
+    return expected_agent;
+}
+rocprofiler_context_id_t&
+get_client_ctx()
+{
+    static rocprofiler_context_id_t ctx{0};
+    return ctx;
+}
+
+rocprofiler_buffer_id_t&
+get_buffer()
+{
+    static rocprofiler_buffer_id_t buf = {};
+    return buf;
+}
+
+/**
+ * Buffer callback called when the buffer is full. rocprofiler_record_header_t
+ * can contain counter records as well as other records (such as tracing). These
+ * records need to be filtered based on the category type.
+ */
+void
+buffered_callback(rocprofiler_context_id_t,
+                  rocprofiler_buffer_id_t,
+                  rocprofiler_record_header_t** headers,
+                  size_t                        num_headers,
+                  void*                         user_data,
+                  uint64_t)
+{
+    std::stringstream ss;
+    // Iterate through the returned records
+    for(size_t i = 0; i < num_headers; ++i)
+    {
+        auto* header = headers[i];
+        if(header->category == ROCPROFILER_BUFFER_CATEGORY_COUNTERS &&
+           header->kind == ROCPROFILER_COUNTER_RECORD_PROFILE_COUNTING_DISPATCH_HEADER)
+        {}
+        else if(header->category == ROCPROFILER_BUFFER_CATEGORY_COUNTERS &&
+                header->kind == ROCPROFILER_COUNTER_RECORD_VALUE)
+        {
+            // Print the returned counter data.
+            auto* record = static_cast<rocprofiler_counter_record_t*>(header->payload);
+            ss << "  (Id: " << record->id << " Value [D]: " << record->counter_value << ","
+               << " user_data: " << record->user_data.value << "),";
+
+            // Check that the agent is what we expect
+            if(record->agent_id.handle != expected_agent().handle)
+            {
+                throw std::runtime_error("Unexpected agent - " +
+                                         std::to_string(record->agent_id.handle) + " " +
+                                         std::to_string(expected_agent().handle));
+            }
+        }
+    }
+
+    auto* output_stream = static_cast<std::ostream*>(user_data);
+    if(!output_stream) throw std::runtime_error{"nullptr to output stream"};
+
+    *output_stream << "[" << __FUNCTION__ << "] " << ss.str() << "\n";
+}
+
+std::unordered_map<uint64_t, rocprofiler_counter_config_id_t>&
+get_profile_cache()
+{
+    static std::unordered_map<uint64_t, rocprofiler_counter_config_id_t> profile_cache;
+    return profile_cache;
+}
+
+/**
+ * Callback from rocprofiler when an kernel dispatch is enqueued into the HSA queue.
+ * rocprofiler_counter_config_id_t* is a return to specify what counters to collect
+ * for this dispatch (dispatch_packet). This example function creates a profile
+ * to collect the counter SQ_WAVES for all kernel dispatch packets.
+ */
+void
+set_profile(rocprofiler_context_id_t               context_id,
+            rocprofiler_agent_id_t                 agent,
+            rocprofiler_device_counting_agent_cb_t set_config,
+            void*)
+{
+    /**
+     * This simple example uses the same profile counter set for all agents.
+     * We store this in a cache to prevent constructing many identical profile counter
+     * sets. We first check the cache to see if we have already constructed a counter"
+     * set for the agent. If we have, return it. Otherwise, construct a new profile counter
+     * set.
+     */
+    auto search_cache = [&]() {
+        if(auto pos = get_profile_cache().find(agent.handle); pos != get_profile_cache().end())
+        {
+            set_config(context_id, pos->second);
+            return true;
+        }
+        return false;
+    };
+
+    if(!search_cache())
+    {
+        std::cerr << "No profile for agent found in cache\n";
+        exit(-1);
+    }
+}
+
+rocprofiler_counter_config_id_t
+build_profile_for_agent(rocprofiler_agent_id_t agent)
+{
+    std::set<std::string>                 counters_to_collect = {"SQ_WAVES"};
+    std::vector<rocprofiler_counter_id_t> gpu_counters;
+
+    ROCPROFILER_CALL(rocprofiler_iterate_agent_supported_counters(
+                         agent,
+                         [](rocprofiler_agent_id_t,
+                            rocprofiler_counter_id_t* counters,
+                            size_t                    num_counters,
+                            void*                     user_data) {
+                             std::vector<rocprofiler_counter_id_t>* vec =
+                                 static_cast<std::vector<rocprofiler_counter_id_t>*>(user_data);
+                             for(size_t i = 0; i < num_counters; i++)
+                             {
+                                 vec->push_back(counters[i]);
+                             }
+                             return ROCPROFILER_STATUS_SUCCESS;
+                         },
+                         static_cast<void*>(&gpu_counters)),
+                     "Could not fetch supported counters");
+
+    std::vector<rocprofiler_counter_id_t> collect_counters;
+    for(auto& counter : gpu_counters)
+    {
+        rocprofiler_counter_info_v0_t info;
+        ROCPROFILER_CALL(
+            rocprofiler_query_counter_info(
+                counter, ROCPROFILER_COUNTER_INFO_VERSION_0, static_cast<void*>(&info)),
+            "Could not query info for counter");
+        if(counters_to_collect.count(std::string(info.name)) > 0)
+        {
+            std::clog << "Counter: " << counter.handle << " " << info.name << "\n";
+            collect_counters.push_back(counter);
+        }
+    }
+
+    rocprofiler_counter_config_id_t profile = {.handle = 0};
+    ROCPROFILER_CALL(rocprofiler_create_counter_config(
+                         agent, collect_counters.data(), collect_counters.size(), &profile),
+                     "Could not construct profile cfg");
+
+    return profile;
+}
+
+std::atomic<bool>&
+exit_toggle()
+{
+    static std::atomic<bool> exit_toggle = false;
+    return exit_toggle;
+}
+
+int
+tool_init(rocprofiler_client_finalize_t, void* user_data)
+{
+    ROCPROFILER_CALL(rocprofiler_create_context(&get_client_ctx()), "context creation failed");
+
+    ROCPROFILER_CALL(rocprofiler_create_buffer(get_client_ctx(),
+                                               4096,
+                                               2048,
+                                               ROCPROFILER_BUFFER_POLICY_LOSSLESS,
+                                               buffered_callback,
+                                               user_data,
+                                               &get_buffer()),
+                     "buffer creation failed");
+
+    std::vector<rocprofiler_agent_v0_t>     agents;
+    rocprofiler_query_available_agents_cb_t iterate_cb = [](rocprofiler_agent_version_t agents_ver,
+                                                            const void**                agents_arr,
+                                                            size_t                      num_agents,
+                                                            void*                       udata) {
+        if(agents_ver != ROCPROFILER_AGENT_INFO_VERSION_0)
+            throw std::runtime_error{"unexpected rocprofiler agent version"};
+        auto* agents_v = static_cast<std::vector<rocprofiler_agent_v0_t>*>(udata);
+        for(size_t i = 0; i < num_agents; ++i)
+            agents_v->emplace_back(*static_cast<const rocprofiler_agent_v0_t*>(agents_arr[i]));
+        return ROCPROFILER_STATUS_SUCCESS;
+    };
+
+    ROCPROFILER_CALL(
+        rocprofiler_query_available_agents(ROCPROFILER_AGENT_INFO_VERSION_0,
+                                           iterate_cb,
+                                           sizeof(rocprofiler_agent_t),
+                                           const_cast<void*>(static_cast<const void*>(&agents))),
+        "query available agents");
+
+    auto client_thread = rocprofiler_callback_thread_t{};
+    ROCPROFILER_CALL(rocprofiler_create_callback_thread(&client_thread),
+                     "failure creating callback thread");
+    ROCPROFILER_CALL(rocprofiler_assign_callback_thread(get_buffer(), client_thread),
+                     "failed to assign thread for buffer");
+
+    // Construct the profiles in advance for each agent that is a GPU
+    for(const auto& agent : agents)
+    {
+        if(agent.type == ROCPROFILER_AGENT_TYPE_GPU)
+        {
+            get_profile_cache().emplace(agent.id.handle, build_profile_for_agent(agent.id));
+            expected_agent() = agent.id;
+            break;
+        }
+    }
+
+    if(agents.empty())
+    {
+        std::cerr << "No agents found" << std::endl;
+        return 1;
+    }
+
+    ROCPROFILER_CALL(rocprofiler_configure_device_counting_service(
+                         get_client_ctx(), get_buffer(), expected_agent(), set_profile, nullptr),
+                     "Could not setup buffered service");
+
+    std::thread([=]() {
+        size_t count = 1;
+        rocprofiler_start_context(get_client_ctx());
+        while(exit_toggle().load() == false)
+        {
+            rocprofiler_sample_device_counting_service(get_client_ctx(),
+                                                       {.value = count},
+                                                       ROCPROFILER_COUNTER_FLAG_NONE,
+                                                       nullptr,
+                                                       nullptr);
+            count++;
+            std::this_thread::sleep_for(std::chrono::milliseconds(50));
+        }
+        exit_toggle().store(false);
+    }).detach();
+
+    // no errors
+    return 0;
+}
+
+void
+tool_fini(void* user_data)
+{
+    std::clog << "In tool fini\n" << std::flush;
+
+    exit_toggle().store(true);
+    while(exit_toggle().load() == true)
+    {};
+
+    rocprofiler_stop_context(get_client_ctx());
+    ROCPROFILER_CALL(rocprofiler_flush_buffer(get_buffer()), "buffer flush");
+
+    auto* output_stream = static_cast<std::ostream*>(user_data);
+    *output_stream << std::flush;
+    if(output_stream != &std::cout && output_stream != &std::cerr) delete output_stream;
+
+    std::clog << "Completed tool fini\n" << std::flush;
+}
+}  // namespace
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name = "CounterClientSample";
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " (priority=" << priority << ") is using rocprofiler-sdk v" << major << "."
+         << minor << "." << patch << " (" << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    std::ostream* output_stream = nullptr;
+    std::string   filename      = "counter_collection.log";
+    if(auto* outfile = getenv("ROCPROFILER_SAMPLE_OUTPUT_FILE"); outfile) filename = outfile;
+    if(filename == "stdout")
+        output_stream = &std::cout;
+    else if(filename == "stderr")
+        output_stream = &std::cerr;
+    else
+        output_stream = new std::ofstream{filename};
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &tool_init,
+                                            &tool_fini,
+                                            static_cast<void*>(output_stream)};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/counter_collection/device_counting_sync_client.cpp

@@ -0,0 +1,488 @@
+// MIT License
+//
+// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "client.hpp"
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <atomic>
+#include <chrono>
+#include <cstdlib>
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <shared_mutex>
+#include <sstream>
+#include <stdexcept>
+#include <thread>
+#include <unordered_map>
+#include <vector>
+
+#define ROCPROFILER_CALL(result, msg)                                                              \
+    {                                                                                              \
+        rocprofiler_status_t CHECKSTATUS = result;                                                 \
+        if(CHECKSTATUS != ROCPROFILER_STATUS_SUCCESS)                                              \
+        {                                                                                          \
+            std::string status_msg = rocprofiler_get_status_string(CHECKSTATUS);                   \
+            std::cerr << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg            \
+                      << " failed with error code " << CHECKSTATUS << ": " << status_msg           \
+                      << std::endl;                                                                \
+            std::stringstream errmsg{};                                                            \
+            errmsg << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg " failure ("  \
+                   << status_msg << ")";                                                           \
+            throw std::runtime_error(errmsg.str());                                                \
+        }                                                                                          \
+    }
+
+int
+start()
+{
+    return 1;
+}
+
+namespace
+{
+// Class to sample counter values from the ROCProfiler API
+// This class is not thread safe and should not be shared between threads.
+// Only a single instance of this class should be created per agent.
+class counter_sampler
+{
+public:
+    // Setup system profiling for an agent
+    counter_sampler(rocprofiler_agent_id_t agent);
+
+    // Decode the counter name of a record
+    std::string decode_record_name(const rocprofiler_counter_record_t& rec) const;
+
+    // Get the dimensions of a record (what CU/SE/etc the counter is for). High cost operation
+    // should be cached if possible.
+    static std::unordered_map<std::string, size_t> get_record_dimensions(
+        const rocprofiler_counter_record_t& rec);
+
+    // Sample the counter values for a set of counters, returns the records in the out parameter.
+    rocprofiler_status_t sample_counter_values(const std::vector<std::string>&            counters,
+                                               std::vector<rocprofiler_counter_record_t>& out);
+
+    // Get the available agents on the system
+    static std::vector<rocprofiler_agent_v0_t> get_available_agents();
+
+    void flush() const { rocprofiler_flush_buffer(buf_); }
+    void stop() const { rocprofiler_stop_context(ctx_); }
+
+private:
+    rocprofiler_agent_id_t          agent_   = {};
+    rocprofiler_context_id_t        ctx_     = {};
+    rocprofiler_buffer_id_t         buf_     = {};
+    rocprofiler_counter_config_id_t profile_ = {.handle = 0};
+
+    std::map<std::vector<std::string>, rocprofiler_counter_config_id_t> cached_profiles_;
+    std::map<uint64_t, uint64_t>                                        profile_sizes_;
+    mutable std::map<uint64_t, std::string>                             id_to_name_;
+
+    // Internal function used to set the profile for the agent when start_context is called
+    void set_profile(rocprofiler_context_id_t ctx, rocprofiler_device_counting_agent_cb_t cb) const;
+
+    // Get the size of a counter in number of records
+    static size_t get_counter_size(rocprofiler_counter_id_t counter);
+
+    // Get the supported counters for an agent
+    static std::unordered_map<std::string, rocprofiler_counter_id_t> get_supported_counters(
+        rocprofiler_agent_id_t agent);
+
+    // Get the dimensions of a counter
+    static std::vector<rocprofiler_counter_record_dimension_info_t> get_counter_dimensions(
+        rocprofiler_counter_id_t counter);
+};
+
+counter_sampler::counter_sampler(rocprofiler_agent_id_t agent)
+: agent_(agent)
+{
+    // Setup context (should only be done once per agent)
+    auto client_thread = rocprofiler_callback_thread_t{};
+    ROCPROFILER_CALL(rocprofiler_create_context(&ctx_), "context creation failed");
+
+    ROCPROFILER_CALL(rocprofiler_create_buffer(
+                         ctx_,
+                         4096,
+                         2048,
+                         ROCPROFILER_BUFFER_POLICY_LOSSLESS,
+                         [](rocprofiler_context_id_t,
+                            rocprofiler_buffer_id_t,
+                            rocprofiler_record_header_t**,
+                            size_t,
+                            void*,
+                            uint64_t) {},
+                         nullptr,
+                         &buf_),
+                     "buffer creation failed");
+    ROCPROFILER_CALL(rocprofiler_create_callback_thread(&client_thread),
+                     "failure creating callback thread");
+    ROCPROFILER_CALL(rocprofiler_assign_callback_thread(buf_, client_thread),
+                     "failed to assign thread for buffer");
+
+    ROCPROFILER_CALL(rocprofiler_configure_device_counting_service(
+                         ctx_,
+                         buf_,
+                         agent,
+                         [](rocprofiler_context_id_t context_id,
+                            rocprofiler_agent_id_t,
+                            rocprofiler_device_counting_agent_cb_t set_config,
+                            void*                                  user_data) {
+                             if(user_data)
+                             {
+                                 auto* sampler = static_cast<counter_sampler*>(user_data);
+                                 sampler->set_profile(context_id, set_config);
+                             }
+                         },
+                         this),
+                     "Could not setup buffered service");
+}
+
+std::string
+counter_sampler::decode_record_name(const rocprofiler_counter_record_t& rec) const
+{
+    if(id_to_name_.empty())
+    {
+        auto name_to_id = counter_sampler::get_supported_counters(agent_);
+        for(const auto& [name, id] : name_to_id)
+        {
+            id_to_name_.emplace(id.handle, name);
+        }
+    }
+
+    rocprofiler_counter_id_t counter_id = {.handle = 0};
+    rocprofiler_query_record_counter_id(rec.id, &counter_id);
+    if(id_to_name_.find(counter_id.handle) == id_to_name_.end())
+    {
+        std::clog << "Unknown counter id = " << counter_id.handle << "\n";
+        return "UNKNOWN";
+    }
+    return id_to_name_.at(counter_id.handle);
+}
+
+std::unordered_map<std::string, size_t>
+counter_sampler::get_record_dimensions(const rocprofiler_counter_record_t& rec)
+{
+    std::unordered_map<std::string, size_t> out;
+    rocprofiler_counter_id_t                counter_id = {.handle = 0};
+    rocprofiler_query_record_counter_id(rec.id, &counter_id);
+    auto dims = get_counter_dimensions(counter_id);
+
+    for(auto& dim : dims)
+    {
+        size_t pos = 0;
+        rocprofiler_query_record_dimension_position(rec.id, dim.id, &pos);
+        out.emplace(dim.name, pos);
+    }
+    return out;
+}
+
+rocprofiler_status_t
+counter_sampler::sample_counter_values(const std::vector<std::string>&            counters,
+                                       std::vector<rocprofiler_counter_record_t>& out)
+{
+    auto profile_cached = cached_profiles_.find(counters);
+    if(profile_cached == cached_profiles_.end())
+    {
+        size_t                                expected_size = 0;
+        rocprofiler_counter_config_id_t       profile       = {};
+        std::vector<rocprofiler_counter_id_t> gpu_counters;
+        auto                                  roc_counters = get_supported_counters(agent_);
+        for(const auto& counter : counters)
+        {
+            auto it = roc_counters.find(counter);
+            if(it == roc_counters.end())
+            {
+                std::cerr << "Counter " << counter << " not found\n";
+                continue;
+            }
+            gpu_counters.push_back(it->second);
+            expected_size += get_counter_size(it->second);
+        }
+        ROCPROFILER_CALL(rocprofiler_create_counter_config(
+                             agent_, gpu_counters.data(), gpu_counters.size(), &profile),
+                         "Could not create profile");
+        cached_profiles_.emplace(counters, profile);
+        profile_sizes_.emplace(profile.handle, expected_size);
+        profile_cached = cached_profiles_.find(counters);
+    }
+    try
+    {
+        out.resize(profile_sizes_.at(profile_cached->second.handle));
+    } catch(const std::exception& e)
+    {
+        std::cerr << "Caught exception: " << e.what() << "\n";
+        return ROCPROFILER_STATUS_ERROR;
+    }
+    profile_ = profile_cached->second;
+    rocprofiler_start_context(ctx_);
+    std::this_thread::sleep_for(std::chrono::milliseconds(50));
+    size_t out_size = out.size();
+    auto   status   = rocprofiler_sample_device_counting_service(
+        ctx_, {}, ROCPROFILER_COUNTER_FLAG_NONE, out.data(), &out_size);
+    rocprofiler_stop_context(ctx_);
+    out.resize(out_size);
+    return status;
+}
+
+std::vector<rocprofiler_agent_v0_t>
+counter_sampler::get_available_agents()
+{
+    std::vector<rocprofiler_agent_v0_t>     agents;
+    rocprofiler_query_available_agents_cb_t iterate_cb = [](rocprofiler_agent_version_t agents_ver,
+                                                            const void**                agents_arr,
+                                                            size_t                      num_agents,
+                                                            void*                       udata) {
+        if(agents_ver != ROCPROFILER_AGENT_INFO_VERSION_0)
+            throw std::runtime_error{"unexpected rocprofiler agent version"};
+        auto* agents_v = static_cast<std::vector<rocprofiler_agent_v0_t>*>(udata);
+        for(size_t i = 0; i < num_agents; ++i)
+        {
+            const auto* rocp_agent = static_cast<const rocprofiler_agent_v0_t*>(agents_arr[i]);
+            if(rocp_agent->type == ROCPROFILER_AGENT_TYPE_GPU) agents_v->emplace_back(*rocp_agent);
+        }
+        return ROCPROFILER_STATUS_SUCCESS;
+    };
+
+    ROCPROFILER_CALL(
+        rocprofiler_query_available_agents(ROCPROFILER_AGENT_INFO_VERSION_0,
+                                           iterate_cb,
+                                           sizeof(rocprofiler_agent_t),
+                                           const_cast<void*>(static_cast<const void*>(&agents))),
+        "query available agents");
+    return agents;
+}
+
+void
+counter_sampler::set_profile(rocprofiler_context_id_t               ctx,
+                             rocprofiler_device_counting_agent_cb_t cb) const
+{
+    if(profile_.handle != 0)
+    {
+        cb(ctx, profile_);
+    }
+}
+
+size_t
+counter_sampler::get_counter_size(rocprofiler_counter_id_t counter)
+{
+    rocprofiler_counter_info_v1_t info;
+    ROCPROFILER_CALL(rocprofiler_query_counter_info(
+                         counter, ROCPROFILER_COUNTER_INFO_VERSION_1, static_cast<void*>(&info)),
+                     "Could not query info for counter");
+    return info.dimensions_instances_count;
+}
+
+std::unordered_map<std::string, rocprofiler_counter_id_t>
+counter_sampler::get_supported_counters(rocprofiler_agent_id_t agent)
+{
+    std::unordered_map<std::string, rocprofiler_counter_id_t> out;
+    std::vector<rocprofiler_counter_id_t>                     gpu_counters;
+
+    ROCPROFILER_CALL(rocprofiler_iterate_agent_supported_counters(
+                         agent,
+                         [](rocprofiler_agent_id_t,
+                            rocprofiler_counter_id_t* counters,
+                            size_t                    num_counters,
+                            void*                     user_data) {
+                             std::vector<rocprofiler_counter_id_t>* vec =
+                                 static_cast<std::vector<rocprofiler_counter_id_t>*>(user_data);
+                             for(size_t i = 0; i < num_counters; i++)
+                             {
+                                 vec->push_back(counters[i]);
+                             }
+                             return ROCPROFILER_STATUS_SUCCESS;
+                         },
+                         static_cast<void*>(&gpu_counters)),
+                     "Could not fetch supported counters");
+    for(auto& counter : gpu_counters)
+    {
+        rocprofiler_counter_info_v0_t info;
+        ROCPROFILER_CALL(
+            rocprofiler_query_counter_info(
+                counter, ROCPROFILER_COUNTER_INFO_VERSION_0, static_cast<void*>(&info)),
+            "Could not query info for counter");
+        out.emplace(info.name, counter);
+    }
+    return out;
+}
+
+std::vector<rocprofiler_counter_record_dimension_info_t>
+counter_sampler::get_counter_dimensions(rocprofiler_counter_id_t counter)
+{
+    rocprofiler_counter_info_v1_t info;
+    ROCPROFILER_CALL(rocprofiler_query_counter_info(
+                         counter, ROCPROFILER_COUNTER_INFO_VERSION_1, static_cast<void*>(&info)),
+                     "Could not query info for counter");
+    return std::vector<rocprofiler_counter_record_dimension_info_t>{
+        *info.dimensions, *info.dimensions + info.dimensions_count};
+}
+
+std::atomic<bool>&
+exit_toggle()
+{
+    static std::atomic<bool> exit_toggle = false;
+    return exit_toggle;
+}
+
+rocprofiler_client_finalize_t    finalize       = nullptr;
+rocprofiler_client_id_t*         client_id      = nullptr;
+std::shared_ptr<counter_sampler> sampler        = {};
+std::thread*                     sampler_thread = nullptr;
+}  // namespace
+
+int
+tool_init(rocprofiler_client_finalize_t fini_func, void*)
+{
+    finalize = fini_func;
+
+    std::atexit([]() {
+        if(client_id) finalize(*client_id);
+    });
+
+    // Get the agents available on the device
+    auto agents = counter_sampler::get_available_agents();
+    if(agents.empty())
+    {
+        std::cerr << "No agents found\n";
+        return -1;
+    }
+
+    // Use the first agent found
+    sampler = std::make_shared<counter_sampler>(agents[0].id);
+
+    sampler_thread = new std::thread{[=]() {
+        size_t                                    count = 1;
+        std::vector<rocprofiler_counter_record_t> records;
+        while(sampler && exit_toggle().load() == false)
+        {
+            auto status = sampler->sample_counter_values({"SQ_WAVES"}, records);
+            if(status == ROCPROFILER_STATUS_ERROR_HSA_NOT_LOADED)
+            {
+                std::clog << "HSA not loaded yet....\n";
+                std::this_thread::sleep_for(std::chrono::milliseconds(50));
+                continue;
+            }
+            std::clog << "Sample " << count << ":\n";
+            if(status == ROCPROFILER_STATUS_SUCCESS)
+            {
+                for(const auto& record : records)
+                {
+                    if(!sampler) break;
+                    auto recname = sampler->decode_record_name(record);
+                    std::clog << "\tCounter: " << record.id << " Name: " << recname
+                              << " Value: " << record.counter_value
+                              << " User data: " << record.user_data.value << "\n";
+                    if(count == 1)
+                    {
+                        if(!sampler) break;
+                        auto dims = sampler->get_record_dimensions(record);
+                        for(const auto& [name, pos] : dims)
+                        {
+                            std::clog << "\t\tDimension Name: " << name << ": " << pos << "\n";
+                        }
+                    }
+                }
+            }
+            count++;
+            std::this_thread::sleep_for(std::chrono::milliseconds(50));
+        }
+        exit_toggle().store(false);
+    }};
+
+    // no errors
+    return 0;
+}
+
+void
+tool_fini(void* user_data)
+{
+    std::clog << "In tool fini\n" << std::flush;
+
+    client_id = nullptr;
+
+    exit_toggle().store(true);
+    while(exit_toggle().load() == true)
+    {};
+
+    sampler->stop();
+    sampler->flush();
+
+    sampler_thread->join();
+
+    auto* output_stream = static_cast<std::ostream*>(user_data);
+    *output_stream << std::flush;
+    if(output_stream != &std::cout && output_stream != &std::cerr) delete output_stream;
+
+    sampler.reset();
+    delete sampler_thread;
+
+    std::clog << "Completed tool fini\n" << std::flush;
+}
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name  = "CounterClientSample";
+    client_id = id;
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " (priority=" << priority << ") is using rocprofiler-sdk v" << major << "."
+         << minor << "." << patch << " (" << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    std::ostream* output_stream = nullptr;
+    std::string   filename      = "counter_collection.log";
+    if(auto* outfile = getenv("ROCPROFILER_SAMPLE_OUTPUT_FILE"); outfile) filename = outfile;
+    if(filename == "stdout")
+        output_stream = &std::cout;
+    else if(filename == "stderr")
+        output_stream = &std::cerr;
+    else
+        output_stream = new std::ofstream{filename};
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &tool_init,
+                                            &tool_fini,
+                                            static_cast<void*>(output_stream)};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/counter_collection/device_serialized_main.cpp

@@ -0,0 +1,80 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include <hip/hip_runtime.h>
+
+#include "client.hpp"
+
+#define HIP_CALL(call)                                                                             \
+    do                                                                                             \
+    {                                                                                              \
+        hipError_t err = call;                                                                     \
+        if(err != hipSuccess)                                                                      \
+        {                                                                                          \
+            fprintf(stderr, "%s\n", hipGetErrorString(err));                                       \
+            abort();                                                                               \
+        }                                                                                          \
+    } while(0)
+
+__global__ void
+kernelA(int devid, volatile int* wait_on, int value, int* no_opt)
+{
+    printf("[device=%i][begin]  Wait on %i: %i (%i)\n", devid, value, *wait_on, *no_opt);
+    while(*wait_on != value)
+    {
+        (*no_opt)++;
+    };
+    printf("[device=%i][break]  Wait on %i: %i (%i)\n", devid, value, *wait_on, *no_opt);
+    (*wait_on)--;
+    printf("[device=%i][return] Wait on %i: %i (%i)\n", devid, value, *wait_on, *no_opt);
+}
+
+int
+main(int, char**)
+{
+    int ntotdevice = 0;
+    HIP_CALL(hipGetDeviceCount(&ntotdevice));
+    if(ntotdevice < 2) return 0;
+
+    start();
+    volatile int* check_value = nullptr;
+    int*          no_opt_0    = nullptr;
+    int*          no_opt_1    = nullptr;
+    HIP_CALL(hipMallocManaged(&check_value, sizeof(*check_value)));
+    HIP_CALL(hipMallocManaged(&no_opt_0, sizeof(*no_opt_0)));
+    HIP_CALL(hipMallocManaged(&no_opt_1, sizeof(*no_opt_1)));
+    *no_opt_0    = 0;
+    *no_opt_1    = 0;
+    *check_value = 1;
+
+    // Will hang if per-device serialization is not functional
+    HIP_CALL(hipSetDevice(0));
+    hipLaunchKernelGGL(kernelA, dim3(1), dim3(1), 0, 0, 0, check_value, 0, no_opt_0);
+
+    HIP_CALL(hipSetDevice(1));
+    hipLaunchKernelGGL(kernelA, dim3(1), dim3(1), 0, 0, 1, check_value, 1, no_opt_1);
+
+    HIP_CALL(hipSetDevice(0));
+    HIP_CALL(hipDeviceSynchronize());
+
+    std::cerr << "Run complete\n";
+}

projects/rocprofiler-sdk/samples/counter_collection/main.cpp

@@ -0,0 +1,159 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include <hip/hip_runtime.h>
+
+#include <libgen.h>
+#include "client.hpp"
+
+#define HIP_CALL(call)                                                                             \
+    do                                                                                             \
+    {                                                                                              \
+        hipError_t err = call;                                                                     \
+        if(err != hipSuccess)                                                                      \
+        {                                                                                          \
+            fprintf(stderr, "%s\n", hipGetErrorString(err));                                       \
+            abort();                                                                               \
+        }                                                                                          \
+    } while(0)
+
+__global__ void
+kernelA(int x, int y)
+{
+    x = x + y;
+}
+
+__global__ void
+kernelB(int x, int y)
+{
+    x = x + y;
+}
+
+template <typename T>
+__global__ void
+kernelC(T* C_d, const T* A_d, size_t N)
+{
+    size_t offset = (blockIdx.x * blockDim.x + threadIdx.x);
+    size_t stride = blockDim.x * gridDim.x;
+    for(size_t i = offset; i < N; i += stride)
+    {
+        C_d[i] = A_d[i] * A_d[i];
+    }
+}
+
+void
+launchKernels(const long NUM_LAUNCH, const long SYNC_INTERVAL, const int DEV_ID)
+{
+    // Normal HIP Calls
+    HIP_CALL(hipSetDevice(DEV_ID));
+    [[maybe_unused]] hipDeviceProp_t devProp;
+    HIP_CALL(hipGetDeviceProperties(&devProp, DEV_ID));
+
+    int* gpuMem = nullptr;
+    HIP_CALL(hipMalloc((void**) &gpuMem, 1 * sizeof(int)));
+
+    for(long i = 0; i < NUM_LAUNCH; i++)
+    {
+        // KernelA and KernelB to be profiled as part of the session
+        hipLaunchKernelGGL(kernelA, dim3(1), dim3(1), 0, 0, 1, 2);
+        hipLaunchKernelGGL(kernelB, dim3(1), dim3(1), 0, 0, 1, 2);
+        if(i % SYNC_INTERVAL == (SYNC_INTERVAL - 1)) HIP_CALL(hipDeviceSynchronize());
+    }
+
+    const int NElems = 512 * 512;
+    const int Nbytes = NElems * sizeof(int);
+    int *     A_d, *C_d;
+    int       A_h[NElems], C_h[NElems];
+
+    for(int i = 0; i < NElems; i++)
+    {
+        A_h[i] = i;
+    }
+
+    HIP_CALL(hipDeviceSynchronize());
+
+    HIP_CALL(hipMalloc(&A_d, Nbytes));
+    HIP_CALL(hipMalloc(&C_d, Nbytes));
+    HIP_CALL(hipMemcpy(A_d, A_h, Nbytes, hipMemcpyHostToDevice));
+    HIP_CALL(hipDeviceSynchronize());
+    const unsigned blocks          = 512;
+    const unsigned threadsPerBlock = 256;
+    for(long i = 0; i < NUM_LAUNCH; i++)
+    {
+        hipLaunchKernelGGL(kernelC, dim3(blocks), dim3(threadsPerBlock), 0, 0, C_d, A_d, NElems);
+        if(i % SYNC_INTERVAL == (SYNC_INTERVAL - 1)) HIP_CALL(hipDeviceSynchronize());
+    }
+    HIP_CALL(hipMemcpy(C_h, C_d, Nbytes, hipMemcpyDeviceToHost));
+    HIP_CALL(hipDeviceSynchronize());
+    HIP_CALL(hipFree(gpuMem));
+    HIP_CALL(hipFree(A_d));
+    HIP_CALL(hipFree(C_d));
+    HIP_CALL(hipDeviceReset());
+}
+
+int
+main(int argc, char** argv)
+{
+    auto* exe_name = basename(argv[0]);
+
+    int ntotdevice = 0;
+    HIP_CALL(hipGetDeviceCount(&ntotdevice));
+
+    long nitr    = 5000;
+    long nsync   = 50;
+    long ndevice = 0;
+
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{argv[i]};
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            fprintf(stderr,
+                    "usage: %s [NUM_ITERATION (%li)] [SYNC_EVERY_N_ITERATIONS (%li)] "
+                    "[NUMBER_OF_DEVICES (%li)]\n\n\tBy default, 0 for the number of devices means "
+                    "use all device available",
+                    exe_name,
+                    nitr,
+                    nsync,
+                    ndevice);
+            exit(EXIT_SUCCESS);
+        }
+    }
+
+    if(argc > 1) nitr = atol(argv[1]);
+    if(argc > 2) nsync = atoll(argv[2]);
+    if(argc > 3) ndevice = atol(argv[3]);
+
+    if(ndevice > ntotdevice) ndevice = ntotdevice;
+    if(ndevice < 1) ndevice = ntotdevice;
+
+    printf("[%s] Number of devices used: %li\n", exe_name, ndevice);
+    printf("[%s] Number of iterations: %li\n", exe_name, nitr);
+    printf("[%s] Syncing every %li iterations\n", exe_name, nsync);
+    std::cout << std::flush;
+
+    start();
+    for(long devid = 0; devid < ndevice; ++devid)
+        launchKernels(nitr, nsync, devid);
+
+    std::cerr << "Run complete\n" << std::flush;
+}

projects/rocprofiler-sdk/samples/counter_collection/print_functional_counters_client.cpp

@@ -0,0 +1,457 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "client.hpp"
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <unistd.h>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <shared_mutex>
+#include <sstream>
+#include <vector>
+
+#define PRINT_ONLY_FAILING false
+
+/**
+ * Tests the collection of all counters on the agent the test is run on.
+ */
+
+#define ROCPROFILER_CALL(result, msg)                                                              \
+    {                                                                                              \
+        rocprofiler_status_t CHECKSTATUS = result;                                                 \
+        if(CHECKSTATUS != ROCPROFILER_STATUS_SUCCESS)                                              \
+        {                                                                                          \
+            std::string status_msg = rocprofiler_get_status_string(CHECKSTATUS);                   \
+            std::cerr << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg            \
+                      << " failed with error code " << CHECKSTATUS << ": " << status_msg           \
+                      << std::endl;                                                                \
+            std::stringstream errmsg{};                                                            \
+            errmsg << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg " failure ("  \
+                   << status_msg << ")";                                                           \
+            throw std::runtime_error(errmsg.str());                                                \
+        }                                                                                          \
+    }
+
+int
+start()
+{
+    return 1;
+}
+
+namespace
+{
+rocprofiler_context_id_t&
+get_client_ctx()
+{
+    static rocprofiler_context_id_t ctx{0};
+    return ctx;
+}
+
+rocprofiler_buffer_id_t&
+get_buffer()
+{
+    static rocprofiler_buffer_id_t buf = {};
+    return buf;
+}
+
+// Struct to validate that all dimension values are present. Does
+// so by creating a tree of dimension values expected. If all are marked as
+// having values, then all values are present in the output.
+struct validate_dim_presence
+{
+    validate_dim_presence() {}
+
+    void maybe_forward(const rocprofiler_counter_record_dimension_info_t& dim)
+    {
+        if(sub_vectors.empty())
+        {
+            for(size_t i = 0; i < dim.instance_size; i++)
+            {
+                sub_vectors.emplace_back(std::make_unique<validate_dim_presence>());
+                sub_vectors.back()->vector_pos = std::make_pair(dim, i);
+            }
+        }
+        else
+        {
+            for(auto& vec : sub_vectors)
+            {
+                vec->maybe_forward(dim);
+            }
+        }
+    }
+
+    void mark_seen(const rocprofiler_counter_instance_id_t& id)
+    {
+        if(sub_vectors.empty())
+        {
+            has_value = true;
+            return;
+        }
+        size_t pos = 0;
+        ROCPROFILER_CALL(rocprofiler_query_record_dimension_position(
+                             id, sub_vectors.at(0)->vector_pos.first.id, &pos),
+                         "Could not query position");
+        sub_vectors.at(pos)->mark_seen(id);
+    }
+
+    bool check_seen(
+        std::stringstream&                                                           out,
+        std::vector<std::pair<rocprofiler_counter_record_dimension_info_t, size_t>>& pos_stack)
+    {
+        bool ret = true;
+        if(sub_vectors.empty())
+        {
+            if(!has_value)
+            {
+                ret = false;
+                out << "\tMissing Value at [";
+            }
+            else
+            {
+                out << "\tHas Value at [";
+            }
+            for(const auto& [dim, pos] : pos_stack)
+            {
+                out << dim.name << ":" << pos << ",";
+            }
+            out << "]\n";
+            return ret;
+        }
+
+        for(size_t i = 0; i < sub_vectors.size(); i++)
+        {
+            pos_stack.push_back(sub_vectors[i]->vector_pos);
+            if(!sub_vectors[i]->check_seen(out, pos_stack)) ret = false;
+            pos_stack.pop_back();
+        }
+        return ret;
+    }
+
+    std::pair<rocprofiler_counter_record_dimension_info_t, size_t> vector_pos;
+    std::vector<std::unique_ptr<validate_dim_presence>>            sub_vectors;
+    bool                                                           has_value{false};
+};
+
+struct CaptureRecords
+{
+    std::shared_mutex m_mutex{};
+    // <counter id handle, expected instances>
+    std::map<uint64_t, size_t> expected{};
+    // expected dims that we should see data for
+    std::map<uint64_t, validate_dim_presence> expected_data_dims{};
+    std::map<uint64_t, std::string>           expected_counter_names{};
+    std::vector<rocprofiler_counter_id_t>     remaining{};
+    // <counter_id handle, instances seen>
+    std::map<uint64_t, size_t> captured{};
+};
+
+CaptureRecords* REC = new CaptureRecords;
+
+CaptureRecords*
+get_capture()
+{
+    return REC;
+}
+
+void
+buffered_callback(rocprofiler_context_id_t,
+                  rocprofiler_buffer_id_t,
+                  rocprofiler_record_header_t** headers,
+                  size_t                        num_headers,
+                  void*,
+                  uint64_t)
+{
+    auto& cap   = *get_capture();
+    auto  wlock = std::unique_lock{cap.m_mutex};
+
+    std::map<uint64_t, size_t> seen_counters;
+    for(size_t i = 0; i < num_headers; ++i)
+    {
+        auto* header = headers[i];
+        if(header->category == ROCPROFILER_BUFFER_CATEGORY_COUNTERS &&
+           header->kind == ROCPROFILER_COUNTER_RECORD_VALUE)
+        {
+            // Record the counters we have in the buffer and the number of instances of
+            // the counter we have seen.
+            rocprofiler_counter_id_t counter;
+            auto* record = static_cast<rocprofiler_counter_record_t*>(header->payload);
+            rocprofiler_query_record_counter_id(record->id, &counter);
+            cap.expected_data_dims.at(counter.handle).mark_seen(record->id);
+            seen_counters.emplace(counter.handle, 0).first->second++;
+        }
+    }
+
+    // Store these counts for post execution comparison
+    for(const auto& [counter_id, instances] : seen_counters)
+    {
+        cap.captured.emplace(counter_id, 0).first->second += instances;
+    }
+}
+
+using agent_map_t = std::map<uint64_t, const rocprofiler_agent_v0_t*>;
+
+agent_map_t
+get_agent_info()
+{
+    auto iterate_cb = [](rocprofiler_agent_version_t agents_ver,
+                         const void**                agents_arr,
+                         size_t                      num_agents,
+                         void*                       user_data) {
+        if(agents_ver != ROCPROFILER_AGENT_INFO_VERSION_0)
+            throw std::runtime_error{"unexpected rocprofiler agent version"};
+
+        auto* agents_v = static_cast<agent_map_t*>(user_data);
+        for(size_t i = 0; i < num_agents; ++i)
+        {
+            const auto* itr = static_cast<const rocprofiler_agent_v0_t*>(agents_arr[i]);
+            agents_v->emplace(itr->id.handle, itr);
+        }
+        return ROCPROFILER_STATUS_SUCCESS;
+    };
+
+    auto _agents = agent_map_t{};
+    ROCPROFILER_CALL(
+        rocprofiler_query_available_agents(ROCPROFILER_AGENT_INFO_VERSION_0,
+                                           iterate_cb,
+                                           sizeof(rocprofiler_agent_t),
+                                           const_cast<void*>(static_cast<const void*>(&_agents))),
+        "query available agents");
+
+    return _agents;
+}
+
+void
+dispatch_callback(rocprofiler_dispatch_counting_service_data_t dispatch_data,
+                  rocprofiler_counter_config_id_t*             config,
+                  rocprofiler_user_data_t* /*user_data*/,
+                  void* /*callback_data_args*/)
+{
+    static auto agents = get_agent_info();
+
+    auto& cap   = *get_capture();
+    auto  wlock = std::unique_lock{cap.m_mutex};
+
+    /**
+     * Fetch all counters that are available for this agent if we haven't already.
+     * Each of these counters will be collected 1 by 1 for each dispatch until we
+     * have tried all counters. This requires the program to have at least counters
+     * number of kernel launches to test all counters.
+     */
+    if(cap.expected.empty())
+    {
+        std::vector<rocprofiler_counter_id_t> counters_needed;
+        ROCPROFILER_CALL(rocprofiler_iterate_agent_supported_counters(
+                             dispatch_data.dispatch_info.agent_id,
+                             [](rocprofiler_agent_id_t,
+                                rocprofiler_counter_id_t* counters,
+                                size_t                    num_counters,
+                                void*                     user_data) {
+                                 std::vector<rocprofiler_counter_id_t>* vec =
+                                     static_cast<std::vector<rocprofiler_counter_id_t>*>(user_data);
+                                 for(size_t i = 0; i < num_counters; i++)
+                                 {
+                                     vec->push_back(counters[i]);
+                                 }
+                                 return ROCPROFILER_STATUS_SUCCESS;
+                             },
+                             static_cast<void*>(&counters_needed)),
+                         "Could not fetch supported counters");
+
+        for(auto& found_counter : counters_needed)
+        {
+            rocprofiler_counter_info_v1_t info;
+
+            ROCPROFILER_CALL(
+                rocprofiler_query_counter_info(
+                    found_counter, ROCPROFILER_COUNTER_INFO_VERSION_1, static_cast<void*>(&info)),
+                "Could not query counter_id");
+            cap.expected_counter_names.emplace(found_counter.handle, std::string(info.name));
+            cap.remaining.push_back(found_counter);
+            cap.expected.emplace(found_counter.handle, info.dimensions_instances_count);
+
+            auto& info_vector =
+                cap.expected_data_dims.emplace(found_counter.handle, validate_dim_presence{})
+                    .first->second;
+
+            for(uint64_t i = 0; i < info.dimensions_count; i++)
+            {
+                info_vector.maybe_forward(*info.dimensions[i]);
+            }
+        }
+        if(cap.expected.empty())
+        {
+            std::clog << "No counters found for agent "
+                      << dispatch_data.dispatch_info.agent_id.handle << " ("
+                      << agents.at(dispatch_data.dispatch_info.agent_id.handle)->name << ")";
+        }
+    }
+    if(cap.remaining.empty()) return;
+
+    rocprofiler_counter_config_id_t profile = {.handle = 0};
+
+    // Select the next counter to collect.
+    if(rocprofiler_create_counter_config(
+           dispatch_data.dispatch_info.agent_id, &(cap.remaining.back()), 1, &profile) ==
+       ROCPROFILER_STATUS_SUCCESS)
+    {
+        *config = profile;
+        std::clog << "Attempting to read counter "
+                  << cap.expected_counter_names.at(cap.remaining.back().handle) << "\n";
+    }
+
+    cap.remaining.pop_back();
+}
+
+int
+tool_init(rocprofiler_client_finalize_t, void*)
+{
+    get_capture();
+    ROCPROFILER_CALL(rocprofiler_create_context(&get_client_ctx()), "context creation failed");
+
+    ROCPROFILER_CALL(rocprofiler_create_buffer(get_client_ctx(),
+                                               4096,
+                                               2048,
+                                               ROCPROFILER_BUFFER_POLICY_LOSSLESS,
+                                               buffered_callback,
+                                               nullptr,
+                                               &get_buffer()),
+                     "buffer creation failed");
+
+    auto client_thread = rocprofiler_callback_thread_t{};
+    ROCPROFILER_CALL(rocprofiler_create_callback_thread(&client_thread),
+                     "failure creating callback thread");
+
+    ROCPROFILER_CALL(rocprofiler_assign_callback_thread(get_buffer(), client_thread),
+                     "failed to assign thread for buffer");
+    ROCPROFILER_CALL(rocprofiler_configure_buffer_dispatch_counting_service(
+                         get_client_ctx(), get_buffer(), dispatch_callback, nullptr),
+                     "Could not setup buffered service");
+    rocprofiler_start_context(get_client_ctx());
+
+    // no errors
+    return 0;
+}
+
+void
+tool_fini(void*)
+{
+    rocprofiler_flush_buffer(get_buffer());
+    rocprofiler_stop_context(get_client_ctx());
+    // Flush buffer isn't waiting....
+    sleep(2);
+
+    std::clog << "In tool fini\n";
+
+    auto& cap   = *get_capture();
+    auto  wlock = std::unique_lock{cap.m_mutex};
+
+    // Print out errors in counters that were not collected or had differences in instance
+    // count information.
+    if(cap.captured.size() != cap.expected.size())
+    {
+        std::clog << "[ERROR] Expected " << cap.expected.size() << " counters collected but got "
+                  << cap.captured.size() << "\n";
+    }
+
+    for(const auto& [counter_id, expected] : cap.expected)
+    {
+        std::string name = "UNKNOWN";
+        if(auto pos = cap.expected_counter_names.find(counter_id);
+           pos != cap.expected_counter_names.end())
+        {
+            name = pos->second;
+        }
+
+        std::optional<size_t> actual_size;
+
+        if(auto pos = cap.captured.find(counter_id); pos != cap.captured.end())
+        {
+            actual_size = pos->second;
+        }
+
+        if(actual_size && *actual_size != expected)
+        {
+            std::clog << (*actual_size == expected ? "" : "[ERROR]") << "Counter ID: " << counter_id
+                      << " (" << name << ")"
+                      << " expected " << expected << " instances and got " << *actual_size << "\n";
+        }
+        else if(!actual_size)
+        {
+            std::clog << "[ERROR] Counter ID: " << counter_id << " (" << name
+                      << ") is missing from output\n";
+        }
+        else
+        {
+            // Counter collected OK
+            std::stringstream                                                           ss;
+            std::vector<std::pair<rocprofiler_counter_record_dimension_info_t, size_t>> stack;
+            bool passed = cap.expected_data_dims.at(counter_id).check_seen(ss, stack);
+            if(!PRINT_ONLY_FAILING || !passed)
+            {
+                std::clog << (passed ? "[OK] " : "[ERROR] ") << "Counter ID: " << counter_id << " ("
+                          << name << ")"
+                          << " Expected: " << expected << " Got: " << *actual_size << "\n";
+                std::clog << ss.str();
+            }
+        }
+    }
+}
+
+}  // namespace
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name = "CounterClientSample";
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " (priority=" << priority << ") is using rocprofiler-sdk v" << major << "."
+         << minor << "." << patch << " (" << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &tool_init,
+                                            &tool_fini,
+                                            static_cast<void*>(nullptr)};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/external_correlation_id_request/CMakeLists.txt

@@ -0,0 +1,72 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-samples-external-correlation-id-request LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+if(ROCPROFILER_DISABLE_UNSTABLE_CTESTS AND ROCPROFILER_MEMCHECK MATCHES "LeakSanitizer")
+    set(IS_DISABLED ON)
+else()
+    set(IS_DISABLED OFF)
+endif()
+
+find_package(rocprofiler-sdk REQUIRED)
+
+add_library(external-correlation-id-request-client SHARED)
+target_sources(external-correlation-id-request-client PRIVATE client.cpp client.hpp)
+target_link_libraries(
+    external-correlation-id-request-client
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::samples-build-flags
+            rocprofiler-sdk::samples-common-library)
+
+set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
+find_package(Threads REQUIRED)
+
+add_executable(external-correlation-id-request)
+target_sources(external-correlation-id-request PRIVATE main.cpp)
+target_link_libraries(
+    external-correlation-id-request
+    PRIVATE external-correlation-id-request-client Threads::Threads
+            rocprofiler-sdk::samples-build-flags rocprofiler-sdk::samples-common-library)
+
+rocprofiler_samples_get_preload_env(PRELOAD_ENV external-correlation-id-request-client)
+rocprofiler_samples_get_ld_library_path_env(LIBRARY_PATH_ENV)
+
+set(external-correlation-id-request-env ${PRELOAD_ENV} ${LIBRARY_PATH_ENV})
+
+add_test(NAME external-correlation-id-request
+         COMMAND $<TARGET_FILE:external-correlation-id-request>)
+
+set_tests_properties(
+    external-correlation-id-request
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "samples"
+               ENVIRONMENT
+               "${external-correlation-id-request-env}"
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               DISABLED
+               "${IS_DISABLED}")

projects/rocprofiler-sdk/samples/external_correlation_id_request/README.md

@@ -0,0 +1,24 @@
+# External Correlation ID Request Sample
+
+## Services
+
+- Code object callback tracing for mapping kernel IDs to kernel names
+- HIP Runtime API:
+  - hipLaunchKernel
+  - hipMemcpyAsync
+  - hipMemsetAsync
+  - hipMalloc
+- Kernel dispatch
+- Memory Copy
+- External correlation ID request:
+  - Kernel dispatch
+  - Memory copy
+- Correlation ID retirement
+
+## Properties
+
+- Subscribes to an external correlation ID request for all kernel dispatches and async memory copies
+- Generates an external correlation ID containing all the arguments passed to the request callback
+- Demonstrates that all external correlation IDs which are requested are passed back to tool in buffer callbacks
+- Demonstrates that all internal correlation IDs which are provided as an input argument to request are retired
+- Buffer size of 4096 bytes which is automatically flushed once >= 87.5% of buffer is filled (3584 bytes)

projects/rocprofiler-sdk/samples/external_correlation_id_request/client.cpp

@@ -0,0 +1,624 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+/**
+ * @file samples/api_buffered_tracing/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include "client.hpp"
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include "common/call_stack.hpp"
+#include "common/defines.hpp"
+#include "common/filesystem.hpp"
+#include "common/name_info.hpp"
+
+#include <atomic>
+#include <cassert>
+#include <chrono>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <shared_mutex>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <thread>
+#include <unordered_set>
+#include <vector>
+
+namespace client
+{
+namespace
+{
+struct external_corr_id_data;
+
+using common::buffer_name_info;
+using common::call_stack_t;
+using common::source_location;
+
+using kernel_symbol_data_t = rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t;
+using kernel_symbol_map_t  = std::unordered_map<rocprofiler_kernel_id_t, kernel_symbol_data_t>;
+using external_corr_id_set_t = std::unordered_set<external_corr_id_data*>;
+using retired_corr_id_set_t  = std::unordered_set<uint64_t>;
+
+rocprofiler_client_id_t*      client_id                = nullptr;
+rocprofiler_client_finalize_t client_fini_func         = nullptr;
+rocprofiler_context_id_t      client_ctx               = {0};
+rocprofiler_buffer_id_t       client_buffer            = {};
+buffer_name_info*             client_name_info         = new buffer_name_info{};
+kernel_symbol_map_t*          client_kernels           = new kernel_symbol_map_t{};
+auto                          client_mutex             = std::shared_mutex{};
+auto                          client_external_corr_ids = external_corr_id_set_t{};
+auto                          client_retired_corr_ids  = retired_corr_id_set_t{};
+
+void
+print_call_stack(const call_stack_t& _call_stack)
+{
+    common::print_call_stack("external_correlation_id_request.log", _call_stack);
+}
+
+void
+tool_code_object_callback(rocprofiler_callback_tracing_record_t record,
+                          rocprofiler_user_data_t*              user_data,
+                          void*                                 callback_data)
+{
+    if(record.kind == ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT &&
+       record.operation == ROCPROFILER_CODE_OBJECT_LOAD)
+    {
+        if(record.phase == ROCPROFILER_CALLBACK_PHASE_UNLOAD)
+        {
+            // flush the buffer to ensure that any lookups for the client kernel names for the code
+            // object are completed
+            auto flush_status = rocprofiler_flush_buffer(client_buffer);
+            if(flush_status != ROCPROFILER_STATUS_ERROR_BUFFER_BUSY)
+                ROCPROFILER_CHECK(flush_status);
+        }
+    }
+    else if(record.kind == ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT &&
+            record.operation == ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER)
+    {
+        auto* data = static_cast<kernel_symbol_data_t*>(record.payload);
+        if(record.phase == ROCPROFILER_CALLBACK_PHASE_LOAD)
+        {
+            client_kernels->emplace(data->kernel_id, *data);
+        }
+        else if(record.phase == ROCPROFILER_CALLBACK_PHASE_UNLOAD)
+        {
+            auto flush_status = rocprofiler_flush_buffer(client_buffer);
+            if(flush_status != ROCPROFILER_STATUS_ERROR_BUFFER_BUSY)
+                ROCPROFILER_CHECK(flush_status);
+
+            client_kernels->erase(data->kernel_id);
+        }
+    }
+
+    (void) user_data;
+    (void) callback_data;
+}
+
+struct external_corr_id_data
+{
+    using request_kind_t               = rocprofiler_external_correlation_id_request_kind_t;
+    static constexpr auto request_none = ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_NONE;
+
+    rocprofiler_thread_id_t         thread_id        = 0;
+    rocprofiler_context_id_t        context_id       = {.handle = 0};
+    request_kind_t                  kind             = request_none;
+    rocprofiler_tracing_operation_t operation        = 0;
+    uint64_t                        internal_corr_id = 0;
+    void*                           user_data        = nullptr;
+    uint64_t                        seen_count       = 0;
+
+    bool valid() const;
+
+    friend std::ostream& operator<<(std::ostream& os, external_corr_id_data data)
+    {
+        if(!data.valid()) return os;
+        auto ss = std::stringstream{};
+        ss << "seen=" << data.seen_count << ", thr_id=" << data.thread_id
+           << ", context_id=" << data.context_id.handle << ", kind=" << data.kind
+           << ", operation=" << data.operation << ", corr_id=" << data.internal_corr_id
+           << ", user_data=" << data.user_data;
+        return (os << ss.str());
+    }
+};
+
+bool
+operator==(external_corr_id_data lhs, external_corr_id_data rhs)
+{
+    return std::tie(lhs.thread_id,
+                    lhs.context_id.handle,
+                    lhs.kind,
+                    lhs.operation,
+                    lhs.internal_corr_id,
+                    lhs.user_data) == std::tie(rhs.thread_id,
+                                               rhs.context_id.handle,
+                                               rhs.kind,
+                                               rhs.operation,
+                                               rhs.internal_corr_id,
+                                               rhs.user_data);
+}
+
+bool
+operator!=(external_corr_id_data lhs, external_corr_id_data rhs)
+{
+    return !(lhs == rhs);
+}
+
+bool
+external_corr_id_data::valid() const
+{
+    static constexpr auto invalid_v = external_corr_id_data{};
+    return (*this != invalid_v);
+}
+
+int
+set_external_correlation_id(rocprofiler_thread_id_t                            thr_id,
+                            rocprofiler_context_id_t                           ctx_id,
+                            rocprofiler_external_correlation_id_request_kind_t kind,
+                            rocprofiler_tracing_operation_t                    op,
+                            uint64_t                                           internal_corr_id,
+                            rocprofiler_user_data_t*                           external_corr_id,
+                            void*                                              user_data)
+{
+    auto* _data =
+        new external_corr_id_data{thr_id, ctx_id, kind, op, internal_corr_id, user_data, 0};
+
+    {
+        static auto _mtx = std::mutex{};
+        auto        _lk  = std::unique_lock{_mtx};
+        client_external_corr_ids.emplace(_data);
+    }
+
+    external_corr_id->ptr = _data;
+
+    return 0;
+}
+
+void
+tool_tracing_callback(rocprofiler_context_id_t      context,
+                      rocprofiler_buffer_id_t       buffer_id,
+                      rocprofiler_record_header_t** headers,
+                      size_t                        num_headers,
+                      void*                         user_data,
+                      uint64_t /*drop_count*/)
+{
+    static const auto ensure_internal_correlation_id_retirement_ordering = [](uint64_t _corr_id) {
+        auto _lk = std::shared_lock<std::shared_mutex>{client_mutex};
+        // this correlation ID should not have reported as retired yet so
+        // we are demoing the expectation here
+        if(client_retired_corr_ids.count(_corr_id) > 0)
+        {
+            auto msg = std::stringstream{};
+            msg << "internal correlation id " << _corr_id << " was retired prematurely";
+            throw std::runtime_error{msg.str()};
+        }
+    };
+
+    for(size_t i = 0; i < num_headers; ++i)
+    {
+        auto* header = headers[i];
+
+        auto kind_name = std::string{};
+        if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING)
+        {
+            const char* _name = nullptr;
+            auto        _kind = static_cast<rocprofiler_buffer_tracing_kind_t>(header->kind);
+            ROCPROFILER_CHECK(rocprofiler_query_buffer_tracing_kind_name(_kind, &_name, nullptr));
+            if(_name)
+            {
+                static size_t len = 15;
+
+                kind_name = std::string{_name};
+                len       = std::max(len, kind_name.length());
+                kind_name.resize(len, ' ');
+                kind_name += " :: ";
+            }
+        }
+
+        if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING &&
+           header->kind == ROCPROFILER_BUFFER_TRACING_HIP_RUNTIME_API)
+        {
+            auto* record =
+                static_cast<rocprofiler_buffer_tracing_hip_api_record_t*>(header->payload);
+
+            // this should always be empty
+            auto _extern_corr_id = external_corr_id_data{};
+
+            // demonstrate reliability of correlation ID retirement ordering
+            ensure_internal_correlation_id_retirement_ordering(record->correlation_id.internal);
+
+            auto info = std::stringstream{};
+            info << "tid=" << record->thread_id << ", context=" << context.handle
+                 << ", buffer_id=" << buffer_id.handle
+                 << ", corr_id=" << record->correlation_id.internal << ", kind=" << record->kind
+                 << ", operation=" << record->operation
+                 << ", name=" << (*client_name_info)[record->kind][record->operation]
+                 << ", extern_corr_id={" << _extern_corr_id << "}";
+
+            static_cast<call_stack_t*>(user_data)->emplace_back(
+                source_location{__FUNCTION__, __FILE__, __LINE__, kind_name + info.str()});
+        }
+        else if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING &&
+                header->kind == ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH)
+        {
+            auto* record =
+                static_cast<rocprofiler_buffer_tracing_kernel_dispatch_record_t*>(header->payload);
+
+            // demonstrate reliability of correlation ID retirement ordering
+            ensure_internal_correlation_id_retirement_ordering(record->correlation_id.internal);
+
+            auto _extern_corr_id = external_corr_id_data{};
+            if(record->correlation_id.external.ptr)
+            {
+                auto* _extcid =
+                    static_cast<external_corr_id_data*>(record->correlation_id.external.ptr);
+                _extcid->seen_count++;
+                _extern_corr_id = *_extcid;
+                // demonstrate reliability of correlation ID retirement ordering
+                ensure_internal_correlation_id_retirement_ordering(_extcid->internal_corr_id);
+            }
+
+            auto info = std::stringstream{};
+
+            info << "tid=" << record->thread_id << ", context=" << context.handle
+                 << ", buffer_id=" << buffer_id.handle
+                 << ", corr_id=" << record->correlation_id.internal << ", kind=" << record->kind
+                 << ", operation=" << record->operation
+                 << ", agent_id=" << record->dispatch_info.agent_id.handle
+                 << ", queue_id=" << record->dispatch_info.queue_id.handle
+                 << ", dispatch_id=" << record->dispatch_info.dispatch_id
+                 << ", kernel_id=" << record->dispatch_info.kernel_id
+                 << ", kernel=" << client_kernels->at(record->dispatch_info.kernel_id).kernel_name
+                 << ", extern_corr_id={" << _extern_corr_id << "}";
+
+            static_cast<call_stack_t*>(user_data)->emplace_back(
+                source_location{__FUNCTION__, __FILE__, __LINE__, kind_name + info.str()});
+        }
+        else if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING &&
+                header->kind == ROCPROFILER_BUFFER_TRACING_MEMORY_COPY)
+        {
+            auto* record =
+                static_cast<rocprofiler_buffer_tracing_memory_copy_record_t*>(header->payload);
+
+            // demonstrate reliability of correlation ID retirement ordering
+            ensure_internal_correlation_id_retirement_ordering(record->correlation_id.internal);
+
+            auto _extern_corr_id = external_corr_id_data{};
+            if(record->correlation_id.external.ptr)
+            {
+                auto* _extcid =
+                    static_cast<external_corr_id_data*>(record->correlation_id.external.ptr);
+                _extcid->seen_count++;
+                _extern_corr_id = *_extcid;
+                // demonstrate reliability of correlation ID retirement ordering
+                ensure_internal_correlation_id_retirement_ordering(_extcid->internal_corr_id);
+            }
+
+            auto info = std::stringstream{};
+
+            info << "tid=" << record->thread_id << ", context=" << context.handle
+                 << ", buffer_id=" << buffer_id.handle
+                 << ", corr_id=" << record->correlation_id.internal << ", kind=" << record->kind
+                 << ", operation=" << record->operation
+                 << ", src_agent_id=" << record->src_agent_id.handle
+                 << ", dst_agent_id=" << record->dst_agent_id.handle
+                 << ", direction=" << record->operation << ", start=" << record->start_timestamp
+                 << ", stop=" << record->end_timestamp
+                 << ", name=" << client_name_info->at(record->kind, record->operation)
+                 << ", extern_corr_id={" << _extern_corr_id << "}";
+
+            static_cast<call_stack_t*>(user_data)->emplace_back(
+                source_location{__FUNCTION__, __FILE__, __LINE__, kind_name + info.str()});
+        }
+        else if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING &&
+                header->kind == ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT)
+        {
+            auto* record =
+                static_cast<rocprofiler_buffer_tracing_correlation_id_retirement_record_t*>(
+                    header->payload);
+
+            {
+                auto _lk = std::unique_lock<std::shared_mutex>{client_mutex};
+                client_retired_corr_ids.emplace(record->internal_correlation_id);
+            }
+
+            auto _extern_corr_id = external_corr_id_data{};
+            auto info            = std::stringstream{};
+
+            info << "context=" << context.handle << ", buffer_id=" << buffer_id.handle
+                 << ", corr_id=" << record->internal_correlation_id << ", kind=" << record->kind
+                 << ", timestamp=" << record->timestamp
+                 << ", name=" << client_name_info->at(record->kind) << ", extern_corr_id={"
+                 << _extern_corr_id << "}";
+
+            static_cast<call_stack_t*>(user_data)->emplace_back(
+                source_location{__FUNCTION__, __FILE__, __LINE__, kind_name + info.str()});
+        }
+        else
+        {
+            auto _msg = std::stringstream{};
+            _msg << "unexpected rocprofiler_record_header_t category + kind: (" << header->category
+                 << " + " << header->kind << ")";
+            throw std::runtime_error{_msg.str()};
+        }
+    }
+}
+
+template <typename Arg, typename... Args>
+auto
+make_array(Arg arg, Args&&... args)
+{
+    constexpr auto N = 1 + sizeof...(Args);
+    return std::array<Arg, N>{std::forward<Arg>(arg), std::forward<Args>(args)...};
+}
+
+int
+tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
+{
+    assert(tool_data != nullptr);
+
+    auto* call_stack_v = static_cast<call_stack_t*>(tool_data);
+
+    call_stack_v->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, ""});
+
+    *client_name_info = common::get_buffer_tracing_names();
+    client_fini_func  = fini_func;
+
+    ROCPROFILER_CHECK(rocprofiler_create_context(&client_ctx));
+
+    auto code_object_ops = std::vector<rocprofiler_tracing_operation_t>{
+        ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER};
+
+    ROCPROFILER_CHECK(
+        rocprofiler_configure_callback_tracing_service(client_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT,
+                                                       code_object_ops.data(),
+                                                       code_object_ops.size(),
+                                                       tool_code_object_callback,
+                                                       nullptr));
+
+    constexpr auto buffer_size_bytes      = 4096;
+    constexpr auto buffer_watermark_bytes = buffer_size_bytes - (buffer_size_bytes / 8);
+
+    ROCPROFILER_CHECK(rocprofiler_create_buffer(client_ctx,
+                                                buffer_size_bytes,
+                                                buffer_watermark_bytes,
+                                                ROCPROFILER_BUFFER_POLICY_LOSSLESS,
+                                                tool_tracing_callback,
+                                                tool_data,
+                                                &client_buffer));
+
+    auto external_corr_id_request_kinds =
+        make_array(ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_KERNEL_DISPATCH,
+                   ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_MEMORY_COPY);
+
+    ROCPROFILER_CHECK(rocprofiler_configure_external_correlation_id_request_service(
+        client_ctx,
+        external_corr_id_request_kinds.data(),
+        external_corr_id_request_kinds.size(),
+        set_external_correlation_id,
+        nullptr));
+
+    auto       hip_runtime_ops         = std::vector<rocprofiler_tracing_operation_t>{};
+    const auto desired_hip_runtime_ops = std::unordered_set<std::string_view>{
+        "hipLaunchKernel", "hipMemcpyAsync", "hipMemsetAsync", "hipMalloc"};
+    for(auto [idx, itr] : (*client_name_info)[ROCPROFILER_BUFFER_TRACING_HIP_RUNTIME_API].items())
+    {
+        if(desired_hip_runtime_ops.count(*itr) > 0) hip_runtime_ops.emplace_back(idx);
+    }
+
+    if(desired_hip_runtime_ops.size() != hip_runtime_ops.size())
+        throw std::runtime_error{"missing hip operations"};
+
+    ROCPROFILER_CHECK(
+        rocprofiler_configure_buffer_tracing_service(client_ctx,
+                                                     ROCPROFILER_BUFFER_TRACING_HIP_RUNTIME_API,
+                                                     hip_runtime_ops.data(),
+                                                     hip_runtime_ops.size(),
+                                                     client_buffer));
+
+    ROCPROFILER_CHECK(rocprofiler_configure_buffer_tracing_service(
+        client_ctx, ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH, nullptr, 0, client_buffer));
+
+    ROCPROFILER_CHECK(rocprofiler_configure_buffer_tracing_service(
+        client_ctx, ROCPROFILER_BUFFER_TRACING_MEMORY_COPY, nullptr, 0, client_buffer));
+
+    ROCPROFILER_CHECK(rocprofiler_configure_buffer_tracing_service(
+        client_ctx,
+        ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT,
+        nullptr,
+        0,
+        client_buffer));
+
+    int valid_ctx = 0;
+    ROCPROFILER_CHECK(rocprofiler_context_is_valid(client_ctx, &valid_ctx));
+    if(valid_ctx == 0)
+    {
+        // notify rocprofiler that initialization failed
+        // and all the contexts, buffers, etc. created
+        // should be ignored
+        return -1;
+    }
+
+    ROCPROFILER_CHECK(rocprofiler_start_context(client_ctx));
+
+    // no errors
+    return 0;
+}
+
+void
+tool_fini(void* tool_data)
+{
+    assert(tool_data != nullptr);
+    client_fini_func = nullptr;
+    client_id        = nullptr;
+
+    std::cout << "finalizing...\n" << std::flush;
+    rocprofiler_stop_context(client_ctx);
+    ROCPROFILER_CHECK(rocprofiler_flush_buffer(client_buffer));
+
+    auto* _call_stack = static_cast<call_stack_t*>(tool_data);
+    _call_stack->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, ""});
+
+    print_call_stack(*_call_stack);
+
+    size_t unretired = 0;
+    size_t unseen    = 0;
+    for(auto* itr : client_external_corr_ids)
+    {
+        if(itr->seen_count != 1)
+        {
+            std::cerr << "external correlation ID seen " << itr->seen_count << " times: {" << *itr
+                      << "}\n"
+                      << std::flush;
+            ++unseen;
+        }
+        if(client_retired_corr_ids.count(itr->internal_corr_id) != 1)
+        {
+            std::cerr << "internal correlation ID passed to external correlation ID request was "
+                         "not retired: {"
+                      << itr->internal_corr_id << "}\n"
+                      << std::flush;
+            ++unretired;
+        }
+
+        delete itr;
+    }
+
+    std::cerr << "external correlation IDs not seen   : " << unseen << "\n" << std::flush;
+    std::cerr << "internal correlation IDs not retired: " << unretired << "\n" << std::flush;
+
+    if(unseen > 0) throw std::runtime_error{"unseen external correlation id data"};
+    if(unretired > 0) throw std::runtime_error{"unretired internal correlation id values"};
+
+    delete _call_stack;
+}
+}  // namespace
+
+void
+setup()
+{
+    if(int status = 0;
+       rocprofiler_is_initialized(&status) == ROCPROFILER_STATUS_SUCCESS && status == 0)
+    {
+        ROCPROFILER_CHECK(rocprofiler_force_configure(&rocprofiler_configure));
+    }
+}
+
+void
+shutdown()
+{
+    if(client_id)
+    {
+        ROCPROFILER_CHECK(rocprofiler_flush_buffer(client_buffer));
+        client_fini_func(*client_id);
+    }
+}
+
+void
+start()
+{
+    ROCPROFILER_CHECK(rocprofiler_start_context(client_ctx));
+}
+
+void
+identify(uint64_t val)
+{
+    auto _tid = rocprofiler_thread_id_t{};
+    rocprofiler_get_thread_id(&_tid);
+    rocprofiler_user_data_t user_data = {};
+    user_data.value                   = val;
+    rocprofiler_push_external_correlation_id(client_ctx, _tid, user_data);
+}
+
+void
+stop()
+{
+    ROCPROFILER_CHECK(rocprofiler_stop_context(client_ctx));
+}
+}  // namespace client
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name = "ExampleTool";
+
+    // store client info
+    client::client_id = id;
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " (priority=" << priority << ") is using rocprofiler-sdk v" << major << "."
+         << minor << "." << patch << " (" << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    auto* client_tool_data = new std::vector<client::source_location>{};
+
+    client_tool_data->emplace_back(
+        client::source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+
+    std::atexit([]() {
+        std::cout << "atexit handler...\n" << std::flush;
+        if(client::client_fini_func && client::client_id)
+            client::client_fini_func(*client::client_id);
+    });
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &client::tool_init,
+                                            &client::tool_fini,
+                                            static_cast<void*>(client_tool_data)};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/external_correlation_id_request/client.hpp

@@ -0,0 +1,49 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#ifdef buffered_api_tracing_client_EXPORTS
+#    define CLIENT_API __attribute__((visibility("default")))
+#else
+#    define CLIENT_API
+#endif
+
+#include <cstdint>
+
+namespace client
+{
+void
+setup() CLIENT_API;
+
+void
+shutdown() CLIENT_API;
+
+void
+start() CLIENT_API;
+
+void
+stop() CLIENT_API;
+
+void
+identify(uint64_t corr_id) CLIENT_API;
+}  // namespace client

projects/rocprofiler-sdk/samples/external_correlation_id_request/main.cpp

@@ -0,0 +1,415 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "common/defines.hpp"
+#include "hip/hip_runtime.h"
+
+#include <libgen.h>
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <mutex>
+#include <random>
+#include <sstream>
+#include <stdexcept>
+#include <thread>
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t                      = std::unique_lock<std::mutex>;
+auto               print_lock          = std::mutex{};
+size_t             nthread_per_device  = 2;
+size_t             nitr                = 500;
+size_t             nsync               = 10;
+constexpr unsigned shared_mem_tile_dim = 32;
+
+void
+check_hip_error(void);
+
+void
+verify(int* in, int* out, int M, int N);
+}  // namespace
+
+__global__ void
+transpose(const int* in, int* out, int M, int N);
+
+void
+run(int rank, int tid, int devid, int argc, char** argv);
+
+void
+run_transpose(int rank, int tid, hipStream_t stream, int argc, char** argv);
+
+void
+run_migrate(int rank, int tid, hipStream_t stream, int, char** argv);
+
+void
+run_scratch(int rank, int tid, hipStream_t stream, int argc, char** argv);
+
+int
+main(int argc, char** argv)
+{
+    auto* exe_name = basename(argv[0]);
+
+    int rank = 0;
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{argv[i]};
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            fprintf(stderr,
+                    "usage: %s [NUM_THREADS_PER_DEVICE (%zu)] [NUM_ITERATION (%zu)] "
+                    "[SYNC_EVERY_N_ITERATIONS (%zu)]\n",
+                    exe_name,
+                    nthread_per_device,
+                    nitr,
+                    nsync);
+            exit(EXIT_SUCCESS);
+        }
+    }
+    if(argc > 1) nthread_per_device = atoll(argv[1]);
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    int ndevice = 0;
+    HIP_API_CALL(hipGetDeviceCount(&ndevice));
+
+    auto nthreads = (ndevice * nthread_per_device);
+
+    printf("[%s] Number of devices found: %i\n", exe_name, ndevice);
+    printf("[%s] Number of threads (per device): %zu\n", exe_name, nthread_per_device);
+    printf("[%s] Number of threads (total): %zu\n", exe_name, nthreads);
+    printf("[%s] Number of iterations: %zu\n", exe_name, nitr);
+    printf("[%s] Syncing every %zu iterations\n", exe_name, nsync);
+
+    {
+        auto _threads = std::vector<std::thread>{};
+        for(size_t i = 0; i < nthreads; ++i)
+            _threads.emplace_back(run, rank, i, i % ndevice, argc, argv);
+        for(auto& itr : _threads)
+            itr.join();
+    }
+
+    HIP_API_CALL(hipDeviceSynchronize());
+    HIP_API_CALL(hipDeviceReset());
+
+    return 0;
+}
+
+__global__ void
+transpose(const int* in, int* out, int M, int N)
+{
+    __shared__ int tile[shared_mem_tile_dim][shared_mem_tile_dim];
+
+    int idx = (blockIdx.y * blockDim.y + threadIdx.y) * M + blockIdx.x * blockDim.x + threadIdx.x;
+    tile[threadIdx.y][threadIdx.x] = in[idx];
+    __syncthreads();
+    idx      = (blockIdx.x * blockDim.x + threadIdx.y) * N + blockIdx.y * blockDim.y + threadIdx.x;
+    out[idx] = tile[threadIdx.x][threadIdx.y];
+}
+
+template <typename Tp>
+__global__ void
+test_page_migrate(Tp* data, Tp val)
+{
+    int idx = (blockIdx.x * blockDim.x) + threadIdx.x;
+    data[idx] += val;
+}
+
+__global__ void
+test_kern_large(uint64_t* output)
+{
+    uint64_t result = 0;
+    int      test[4000];
+    memset(test, 5, 4000);
+    for(int& i : test)
+    {
+        i = i + 7;
+        *output += i;
+        result += i;
+    }
+    *output ^= result;
+    *output ^= result;
+}
+
+__global__ void
+test_kern_medium(uint64_t* output)
+{
+    uint64_t result = 0;
+    int      test[175];
+    memset(test, 5, 175);
+    for(int& i : test)
+    {
+        i = i + 7;
+        *output += i;
+        result += i;
+    }
+    *output ^= result;
+    *output ^= result;
+}
+
+__global__ void
+test_kern_small(uint64_t* output)
+{
+    uint64_t result = 0;
+    int      test[2];
+    for(int& i : test)
+    {
+        i = i + 7;
+        *output += i;
+        result += i;
+    }
+    *output ^= result;
+    *output ^= result;
+}
+
+void
+run(int rank, int tid, int devid, int argc, char** argv)
+{
+    auto* stream = hipStream_t{};
+    HIP_API_CALL(hipSetDevice(devid));
+    HIP_API_CALL(hipStreamCreate(&stream));
+
+    run_migrate(rank, tid, stream, argc, argv);
+    run_scratch(rank, tid, stream, argc, argv);
+    run_transpose(rank, tid, stream, argc, argv);
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipStreamDestroy(stream));
+}
+
+void
+run_transpose(int rank, int tid, hipStream_t stream, int argc, char** argv)
+{
+    auto* exe_name = basename(argv[0]);
+
+    unsigned int M = 4960 * 2;
+    unsigned int N = 4960 * 2;
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    auto_lock_t _lk{print_lock};
+    std::cout << "[" << exe_name << "][transpose][" << rank << "][" << tid << "] M: " << M
+              << " N: " << N << std::endl;
+    _lk.unlock();
+
+    std::default_random_engine         _engine{std::random_device{}() * (rank + 1) * (tid + 1)};
+    std::uniform_int_distribution<int> _dist{0, 1000};
+
+    size_t size       = sizeof(int) * M * N;
+    int*   inp_matrix = new int[size];
+    int*   out_matrix = new int[size];
+    for(size_t i = 0; i < M * N; i++)
+    {
+        inp_matrix[i] = _dist(_engine);
+        out_matrix[i] = 0;
+    }
+    int* in  = nullptr;
+    int* out = nullptr;
+
+    HIP_API_CALL(hipMalloc(&in, size));
+    HIP_API_CALL(hipMalloc(&out, size));
+    HIP_API_CALL(hipMemsetAsync(in, 0, size, stream));
+    HIP_API_CALL(hipMemsetAsync(out, 0, size, stream));
+    HIP_API_CALL(hipMemcpyAsync(in, inp_matrix, size, hipMemcpyHostToDevice, stream));
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    dim3 grid(M / 32, N / 32, 1);
+    dim3 block(32, 32, 1);  // transpose
+
+    print_lock.lock();
+    printf("[%s][transpose][%i][%i] grid=(%i,%i,%i), block=(%i,%i,%i)\n",
+           exe_name,
+           rank,
+           tid,
+           grid.x,
+           grid.y,
+           grid.z,
+           block.x,
+           block.y,
+           block.z);
+    print_lock.unlock();
+
+    auto t1 = std::chrono::high_resolution_clock::now();
+    for(size_t i = 0; i < nitr; ++i)
+    {
+        transpose<<<grid, block, 0, stream>>>(in, out, M, N);
+        check_hip_error();
+        if(i % nsync == (nsync - 1)) HIP_API_CALL(hipStreamSynchronize(stream));
+    }
+    auto t2 = std::chrono::high_resolution_clock::now();
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipMemcpyAsync(out_matrix, out, size, hipMemcpyDeviceToHost, stream));
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+    float  GB   = (float) size * nitr * 2 / (1 << 30);
+
+    print_lock.lock();
+    std::cout << "[" << exe_name << "][transpose][" << rank << "][" << tid
+              << "] Runtime of transpose is " << time << " sec\n";
+    std::cout << "[" << exe_name << "][transpose][" << rank << "][" << tid
+              << "] The average performance of transpose is " << GB / time << " GBytes/sec"
+              << std::endl;
+    print_lock.unlock();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    // cpu_transpose(matrix, out_matrix, M, N);
+    verify(inp_matrix, out_matrix, M, N);
+
+    HIP_API_CALL(hipFree(in));
+    HIP_API_CALL(hipFree(out));
+
+    delete[] inp_matrix;
+    delete[] out_matrix;
+}
+
+void
+run_scratch(int rank, int tid, hipStream_t stream, int, char** argv)
+{
+    auto t1 = std::chrono::high_resolution_clock::now();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    const auto* exe_name = basename(argv[0]);
+
+    uint64_t* data_ptr = nullptr;
+    HIP_API_CALL(HIP_HOST_ALLOC_FUNC(&data_ptr, sizeof(uint64_t), 0));
+    *data_ptr = 0;
+
+    test_kern_small<<<1000, 1, 0, stream>>>(data_ptr);
+    test_kern_medium<<<1000, 1, 0, stream>>>(data_ptr);
+    test_kern_small<<<1000, 1, 0, stream>>>(data_ptr);
+    test_kern_large<<<1100, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    test_kern_small<<<1000, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    test_kern_medium<<<1000, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    test_kern_small<<<1000, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    test_kern_large<<<1100, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    auto   t2   = std::chrono::high_resolution_clock::now();
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+
+    print_lock.lock();
+    std::cout << "[" << exe_name << "][scratch][" << rank << "][" << tid
+              << "] Runtime of scratch is " << time << " sec\n";
+    print_lock.unlock();
+}
+
+void
+run_migrate(int rank, int tid, hipStream_t stream, int, char** argv)
+{
+    using data_type            = uint64_t;
+    constexpr data_type init_v = 1;
+    constexpr data_type incr_v = 1;
+
+    auto t1 = std::chrono::high_resolution_clock::now();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    const auto* exe_name  = basename(argv[0]);
+    auto        page_data = std::vector<data_type>(1024, 0);
+
+    HIP_API_CALL(hipHostRegister(
+        page_data.data(), page_data.size() * sizeof(data_type), hipHostRegisterDefault));
+
+    for(auto& itr : page_data)
+        itr = init_v;
+
+    test_page_migrate<<<1, 1024, 0, stream>>>(page_data.data(), incr_v);
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    for(auto& itr : page_data)
+    {
+        auto diff = (itr - incr_v);
+        if(diff != init_v)
+        {
+            auto msg = std::stringstream{};
+            msg << "invalid diff: " << diff << ". expected: " << init_v;
+            throw std::runtime_error{msg.str()};
+        }
+    }
+
+    HIP_API_CALL(hipHostUnregister(page_data.data()));
+
+    auto   t2   = std::chrono::high_resolution_clock::now();
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+
+    print_lock.lock();
+    std::cout << "[" << exe_name << "][migrate][" << rank << "][" << tid
+              << "] Runtime of migrate is " << time << " sec\n";
+    print_lock.unlock();
+}
+
+namespace
+{
+void
+check_hip_error(void)
+{
+    hipError_t err = hipGetLastError();
+    if(err != hipSuccess)
+    {
+        auto_lock_t _lk{print_lock};
+        std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
+        throw std::runtime_error("hip_api_call");
+    }
+}
+
+void
+verify(int* in, int* out, int M, int N)
+{
+    for(int i = 0; i < 10; i++)
+    {
+        int row = rand() % M;
+        int col = rand() % N;
+        if(in[row * N + col] != out[col * M + row])
+        {
+            auto_lock_t _lk{print_lock};
+            std::cout << "mismatch: " << row << ", " << col << " : " << in[row * N + col] << " | "
+                      << out[col * M + row] << "\n";
+        }
+    }
+}
+}  // namespace

projects/rocprofiler-sdk/samples/intercept_table/CMakeLists.txt

@@ -0,0 +1,58 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-samples-intercept-table LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+#
+# below is equivalent to: ``find_package(rocprofiler-sdk REQUIRED)``. Use COMPONENTS below
+# to demonstrate/test support for COMPONENTS
+find_package(rocprofiler-sdk REQUIRED COMPONENTS headers shared-library)
+
+add_library(intercept-table-client SHARED)
+target_sources(intercept-table-client PRIVATE client.cpp client.hpp)
+target_link_libraries(
+    intercept-table-client
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::samples-build-flags
+            rocprofiler-sdk::samples-common-library)
+
+set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
+find_package(Threads REQUIRED)
+
+add_executable(intercept-table)
+target_sources(intercept-table PRIVATE main.cpp)
+target_link_libraries(intercept-table PRIVATE intercept-table-client Threads::Threads
+                                              rocprofiler-sdk::samples-build-flags)
+
+rocprofiler_samples_get_preload_env(PRELOAD_ENV intercept-table-client)
+rocprofiler_samples_get_ld_library_path_env(LIBRARY_PATH_ENV)
+
+set(intercept-table-env ${PRELOAD_ENV} ${LIBRARY_PATH_ENV})
+
+add_test(NAME intercept-table COMMAND $<TARGET_FILE:intercept-table>)
+
+set_tests_properties(
+    intercept-table
+    PROPERTIES TIMEOUT 45 LABELS "samples" ENVIRONMENT "${intercept-table-env}"
+               FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")

projects/rocprofiler-sdk/samples/intercept_table/README.md

@@ -0,0 +1,10 @@
+# Runtime API Registration
+
+## Services
+
+- HIP runtime table registration
+
+## Properties
+
+- `api_registration_callback` function validates the type of library being intercepted, ensures there is only one instance of the HIP runtime library, and retrieves the dispatch table containing the API functions.
+- Collects a "call stack" of intercepted API calls.

projects/rocprofiler-sdk/samples/intercept_table/client.cpp

@@ -0,0 +1,322 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+/**
+ * @file samples/intercept_table/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include "client.hpp"
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include "common/defines.hpp"
+#include "common/filesystem.hpp"
+
+#include <hip/amd_detail/hip_api_trace.hpp>
+
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <ratio>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace client
+{
+namespace
+{
+struct source_location
+{
+    std::string function = {};
+    std::string file     = {};
+    uint32_t    line     = 0;
+    std::string context  = {};
+};
+
+using call_stack_t          = std::vector<source_location>;
+using callback_kind_names_t = std::map<rocprofiler_callback_tracing_kind_t, const char*>;
+using callback_kind_operation_names_t =
+    std::map<rocprofiler_callback_tracing_kind_t, std::map<uint32_t, const char*>>;
+using wrap_count_t = std::pair<source_location, size_t>;
+
+rocprofiler_client_id_t* client_id        = nullptr;
+auto*                    client_wrap_data = new std::map<size_t, wrap_count_t>{};
+size_t                   func_width       = 0;
+
+void
+print_call_stack(const call_stack_t& _call_stack)
+{
+    auto ofname = std::string{"intercept_table.log"};
+    if(auto* eofname = getenv("ROCPROFILER_SAMPLE_OUTPUT_FILE")) ofname = eofname;
+
+    std::ostream* ofs     = nullptr;
+    auto          cleanup = std::function<void(std::ostream*&)>{};
+
+    if(ofname == "stdout")
+        ofs = &std::cout;
+    else if(ofname == "stderr")
+        ofs = &std::cerr;
+    else
+    {
+        ofs = new std::ofstream{ofname};
+        if(ofs && *ofs)
+            cleanup = [](std::ostream*& _os) { delete _os; };
+        else
+        {
+            std::cerr << "Error outputting to " << ofname << ". Redirecting to stderr...\n";
+            ofname = "stderr";
+            ofs    = &std::cerr;
+        }
+    }
+
+    std::cout << "Outputting collected data to " << ofname << "...\n" << std::flush;
+
+    const size_t _func_width = std::min<size_t>(func_width, 60);
+    size_t       n           = 0;
+    *ofs << std::left;
+    for(const auto& itr : _call_stack)
+    {
+        *ofs << std::left << std::setw(2) << ++n << "/" << std::setw(2) << _call_stack.size()
+             << " [" << common::fs::path{itr.file}.filename() << ":" << itr.line << "] "
+             << std::setw(_func_width) << itr.function;
+        if(!itr.context.empty()) *ofs << " :: " << itr.context;
+        *ofs << "\n";
+    }
+
+    *ofs << std::flush;
+
+    if(cleanup) cleanup(ofs);
+}
+
+void
+tool_fini(void* tool_data)
+{
+    assert(tool_data != nullptr);
+
+    auto* _call_stack = static_cast<call_stack_t*>(tool_data);
+
+    size_t wrapped_count = 0;
+    for(const auto& itr : *client_wrap_data)
+    {
+        auto src_loc = itr.second.first;
+        src_loc.context += "call_count = " + std::to_string(itr.second.second);
+        _call_stack->emplace_back(std::move(src_loc));
+        wrapped_count += itr.second.second;
+    }
+
+    _call_stack->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, ""});
+
+    print_call_stack(*_call_stack);
+
+    delete _call_stack;
+    delete client_wrap_data;
+
+    if(wrapped_count == 0)
+    {
+        throw std::runtime_error{"intercept_table sample did not wrap HIP runtime API table"};
+    }
+}
+
+template <size_t Idx, typename RetT, typename... Args>
+RetT (*underlying_function)(Args...) = nullptr;
+
+template <size_t Idx, typename RetT, typename... Args>
+RetT
+get_wrapper_function(Args... args)
+{
+    if(client_wrap_data)
+    {
+        if(client_wrap_data->at(Idx).second == 0)
+            std::clog << "First invocation of wrapped function: '"
+                      << client_wrap_data->at(Idx).first.function << "'...\n"
+                      << std::flush;
+
+        client_wrap_data->at(Idx).second += 1;
+    }
+
+    if(underlying_function<Idx, RetT, Args...>)
+        return underlying_function<Idx, RetT, Args...>(args...);
+    if constexpr(!std::is_void<RetT>::value) return RetT{};
+}
+
+template <size_t Idx, typename RetT, typename... Args>
+auto
+generate_wrapper(const char* name, uint32_t line, RetT (*func)(Args...))
+{
+    func_width = std::max(func_width, std::string_view{name}.length());
+    client_wrap_data->emplace(Idx, wrap_count_t{source_location{name, __FILE__, line, ""}, 0});
+
+    underlying_function<Idx, RetT, Args...> = func;
+    return &get_wrapper_function<Idx, RetT, Args...>;
+}
+
+#define GENERATE_WRAPPER(TABLE, FUNC)                                                              \
+    TABLE->FUNC##_fn = generate_wrapper<__COUNTER__>(#FUNC, __LINE__, TABLE->FUNC##_fn)
+
+void
+api_registration_callback(rocprofiler_intercept_table_t type,
+                          uint64_t                      lib_version,
+                          uint64_t                      lib_instance,
+                          void**                        tables,
+                          uint64_t                      num_tables,
+                          void*                         user_data)
+{
+    if(type != ROCPROFILER_HIP_RUNTIME_TABLE)
+        throw std::runtime_error{"unexpected library type: " +
+                                 std::to_string(static_cast<int>(type))};
+    if(lib_instance != 0) throw std::runtime_error{"multiple instances of HIP runtime library"};
+    if(num_tables != 1)
+        throw std::runtime_error{"expected only one table of type HipDispatchTable"};
+
+    auto* call_stack = static_cast<std::vector<client::source_location>*>(user_data);
+
+    uint32_t major = lib_version / 10000;
+    uint32_t minor = (lib_version % 10000) / 100;
+    uint32_t patch = lib_version % 100;
+
+    auto info = std::stringstream{};
+    info << client_id->name << " is using HIP runtime v" << major << "." << minor << "." << patch;
+
+    std::clog << info.str() << "\n" << std::flush;
+
+    call_stack->emplace_back(client::source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+
+    auto* hip_api_table = static_cast<HipDispatchTable*>(tables[0]);
+
+    // common API functions
+    GENERATE_WRAPPER(hip_api_table, hipGetDeviceCount);
+    GENERATE_WRAPPER(hip_api_table, hipSetDevice);
+    GENERATE_WRAPPER(hip_api_table, hipStreamCreate);
+    GENERATE_WRAPPER(hip_api_table, hipStreamDestroy);
+    GENERATE_WRAPPER(hip_api_table, hipStreamSynchronize);
+    GENERATE_WRAPPER(hip_api_table, hipDeviceSynchronize);
+    GENERATE_WRAPPER(hip_api_table, hipDeviceReset);
+    GENERATE_WRAPPER(hip_api_table, hipGetErrorString);
+    // kernel launch
+    GENERATE_WRAPPER(hip_api_table, hipExtLaunchKernel);
+    GENERATE_WRAPPER(hip_api_table, hipExtLaunchMultiKernelMultiDevice);
+    GENERATE_WRAPPER(hip_api_table, hipGraphLaunch);
+    GENERATE_WRAPPER(hip_api_table, hipLaunchByPtr);
+    GENERATE_WRAPPER(hip_api_table, hipLaunchCooperativeKernel);
+    GENERATE_WRAPPER(hip_api_table, hipLaunchCooperativeKernelMultiDevice);
+    GENERATE_WRAPPER(hip_api_table, hipLaunchHostFunc);
+    GENERATE_WRAPPER(hip_api_table, hipLaunchKernel);
+    GENERATE_WRAPPER(hip_api_table, hipModuleLaunchCooperativeKernel);
+    GENERATE_WRAPPER(hip_api_table, hipModuleLaunchCooperativeKernelMultiDevice);
+    GENERATE_WRAPPER(hip_api_table, hipModuleLaunchKernel);
+    GENERATE_WRAPPER(hip_api_table, hipExtModuleLaunchKernel);
+    GENERATE_WRAPPER(hip_api_table, hipHccModuleLaunchKernel);
+    // memcpy + memset
+    GENERATE_WRAPPER(hip_api_table, hipMemcpy);
+    GENERATE_WRAPPER(hip_api_table, hipMemcpyAsync);
+    GENERATE_WRAPPER(hip_api_table, hipMemset);
+    GENERATE_WRAPPER(hip_api_table, hipMemsetAsync);
+}
+}  // namespace
+
+void
+setup()
+{}
+
+void
+shutdown()
+{}
+}  // namespace client
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name = "ExampleTool";
+
+    // store client info
+    client::client_id = id;
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " (priority=" << priority << ") is using rocprofiler-sdk v" << major << "."
+         << minor << "." << patch << " (" << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    // demonstration of alternative way to get the version info
+    {
+        auto version_info = std::array<uint32_t, 3>{};
+        ROCPROFILER_CALL(
+            rocprofiler_get_version(&version_info.at(0), &version_info.at(1), &version_info.at(2)),
+            "failed to get version info");
+
+        if(std::array<uint32_t, 3>{major, minor, patch} != version_info)
+        {
+            throw std::runtime_error{"version info mismatch"};
+        }
+    }
+
+    // data passed around all the callbacks
+    auto* client_tool_data = new std::vector<client::source_location>{};
+
+    // add first entry
+    client_tool_data->emplace_back(
+        client::source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+
+    ROCPROFILER_CALL(
+        rocprofiler_at_intercept_table_registration(client::api_registration_callback,
+                                                    ROCPROFILER_HIP_RUNTIME_TABLE,
+                                                    static_cast<void*>(client_tool_data)),
+        "runtime api registration");
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            nullptr,
+                                            &client::tool_fini,
+                                            static_cast<void*>(client_tool_data)};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/intercept_table/client.hpp

@@ -0,0 +1,38 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#ifdef intercept_table_client_EXPORTS
+#    define CLIENT_API __attribute__((visibility("default")))
+#else
+#    define CLIENT_API
+#endif
+
+namespace client
+{
+void
+setup() CLIENT_API;
+
+void
+shutdown() CLIENT_API;
+}  // namespace client

projects/rocprofiler-sdk/samples/intercept_table/main.cpp

@@ -0,0 +1,236 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "hip/hip_runtime.h"
+
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <mutex>
+#include <random>
+#include <stdexcept>
+#include <thread>
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t                      = std::unique_lock<std::mutex>;
+auto               print_lock          = std::mutex{};
+size_t             nthreads            = 2;
+size_t             nitr                = 500;
+size_t             nsync               = 10;
+constexpr unsigned shared_mem_tile_dim = 32;
+
+void
+check_hip_error(void);
+
+void
+verify(int* in, int* out, int M, int N);
+}  // namespace
+
+__global__ void
+transpose_a(int* in, int* out, int M, int N);
+
+void
+run(int rank, int tid, hipStream_t stream, int argc, char** argv);
+
+int
+main(int argc, char** argv)
+{
+    int rank = 0;
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{argv[i]};
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            fprintf(stderr,
+                    "usage: transpose [NUM_THREADS (%zu)] [NUM_ITERATION (%zu)] "
+                    "[SYNC_EVERY_N_ITERATIONS (%zu)]\n",
+                    nthreads,
+                    nitr,
+                    nsync);
+            exit(EXIT_SUCCESS);
+        }
+    }
+    if(argc > 1) nthreads = atoll(argv[1]);
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    printf("[transpose] Number of threads: %zu\n", nthreads);
+    printf("[transpose] Number of iterations: %zu\n", nitr);
+    printf("[transpose] Syncing every %zu iterations\n", nsync);
+
+    // this is a temporary workaround in omnitrace when HIP + MPI is enabled
+    int ndevice = 0;
+    int devid   = rank;
+    HIP_API_CALL(hipGetDeviceCount(&ndevice));
+    printf("[transpose] Number of devices found: %i\n", ndevice);
+    if(ndevice > 0)
+    {
+        devid = rank % ndevice;
+        HIP_API_CALL(hipSetDevice(devid));
+        printf("[transpose] Rank %i assigned to device %i\n", rank, devid);
+    }
+    if(rank == devid && rank < ndevice)
+    {
+        std::vector<std::thread> _threads{};
+        std::vector<hipStream_t> _streams(nthreads);
+        for(size_t i = 0; i < nthreads; ++i)
+            HIP_API_CALL(hipStreamCreate(&_streams.at(i)));
+        for(size_t i = 1; i < nthreads; ++i)
+            _threads.emplace_back(run, rank, i, _streams.at(i), argc, argv);
+        run(rank, 0, _streams.at(0), argc, argv);
+        for(auto& itr : _threads)
+            itr.join();
+        for(size_t i = 0; i < nthreads; ++i)
+            HIP_API_CALL(hipStreamDestroy(_streams.at(i)));
+    }
+    HIP_API_CALL(hipDeviceSynchronize());
+    HIP_API_CALL(hipDeviceReset());
+
+    return 0;
+}
+
+__global__ void
+transpose_a(int* in, int* out, int M, int N)
+{
+    __shared__ int tile[shared_mem_tile_dim][shared_mem_tile_dim];
+
+    int idx = (blockIdx.y * blockDim.y + threadIdx.y) * M + blockIdx.x * blockDim.x + threadIdx.x;
+    tile[threadIdx.y][threadIdx.x] = in[idx];
+    __syncthreads();
+    idx      = (blockIdx.x * blockDim.x + threadIdx.y) * N + blockIdx.y * blockDim.y + threadIdx.x;
+    out[idx] = tile[threadIdx.x][threadIdx.y];
+}
+
+void
+run(int rank, int tid, hipStream_t stream, int argc, char** argv)
+{
+    unsigned int M = 4960 * 2;
+    unsigned int N = 4960 * 2;
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    auto_lock_t _lk{print_lock};
+    std::cout << "[transpose][" << rank << "][" << tid << "] M: " << M << " N: " << N << std::endl;
+    _lk.unlock();
+
+    std::default_random_engine         _engine{std::random_device{}() * (rank + 1) * (tid + 1)};
+    std::uniform_int_distribution<int> _dist{0, 1000};
+
+    size_t size       = sizeof(int) * M * N;
+    int*   inp_matrix = new int[size];
+    int*   out_matrix = new int[size];
+    for(size_t i = 0; i < M * N; i++)
+    {
+        inp_matrix[i] = _dist(_engine);
+        out_matrix[i] = 0;
+    }
+    int* in  = nullptr;
+    int* out = nullptr;
+
+    HIP_API_CALL(hipMalloc(&in, size));
+    HIP_API_CALL(hipMalloc(&out, size));
+    HIP_API_CALL(hipMemsetAsync(in, 0, size, stream));
+    HIP_API_CALL(hipMemsetAsync(out, 0, size, stream));
+    HIP_API_CALL(hipMemcpyAsync(in, inp_matrix, size, hipMemcpyHostToDevice, stream));
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    dim3 grid(M / 32, N / 32, 1);
+    dim3 block(32, 32, 1);  // transpose_a
+
+    auto t1 = std::chrono::high_resolution_clock::now();
+    for(size_t i = 0; i < nitr; ++i)
+    {
+        transpose_a<<<grid, block, 0, stream>>>(in, out, M, N);
+        check_hip_error();
+        if(i % nsync == (nsync - 1)) HIP_API_CALL(hipStreamSynchronize(stream));
+    }
+    auto t2 = std::chrono::high_resolution_clock::now();
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipMemcpyAsync(out_matrix, out, size, hipMemcpyDeviceToHost, stream));
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+    float  GB   = (float) size * nitr * 2 / (1 << 30);
+
+    print_lock.lock();
+    std::cout << "[" << rank << "][" << tid << "] Runtime of transpose is " << time << " sec\n"
+              << "The average performance of transpose is " << GB / time << " GBytes/sec"
+              << std::endl;
+    print_lock.unlock();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    // cpu_transpose(matrix, out_matrix, M, N);
+    verify(inp_matrix, out_matrix, M, N);
+
+    HIP_API_CALL(hipFree(in));
+    HIP_API_CALL(hipFree(out));
+
+    delete[] inp_matrix;
+    delete[] out_matrix;
+}
+
+namespace
+{
+void
+check_hip_error(void)
+{
+    hipError_t err = hipGetLastError();
+    if(err != hipSuccess)
+    {
+        auto_lock_t _lk{print_lock};
+        std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
+        throw std::runtime_error("hip_api_call");
+    }
+}
+
+void
+verify(int* in, int* out, int M, int N)
+{
+    for(int i = 0; i < 10; i++)
+    {
+        int row = rand() % M;
+        int col = rand() % N;
+        if(in[row * N + col] != out[col * M + row])
+        {
+            auto_lock_t _lk{print_lock};
+            std::cout << "mismatch: " << row << ", " << col << " : " << in[row * N + col] << " | "
+                      << out[col * M + row] << "\n";
+        }
+    }
+}
+}  // namespace

projects/rocprofiler-sdk/samples/openmp_target/CMakeLists.txt

@@ -0,0 +1,96 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT OMP_TARGET_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(OMP_TARGET_COMPILER
+            "${amdclangpp_EXECUTABLE}"
+            CACHE FILEPATH "")
+    endif()
+endif()
+
+project(rocprofiler-sdk-samples-openmp-target LANGUAGES CXX)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+add_library(openmp-target-sample-client SHARED)
+target_sources(openmp-target-sample-client PRIVATE client.cpp client.hpp)
+target_link_libraries(
+    openmp-target-sample-client
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::samples-build-flags
+            rocprofiler-sdk::samples-common-library)
+
+set(DEFAULT_GPU_TARGETS "gfx906" "gfx908" "gfx90a" "gfx942" "gfx950" "gfx1100" "gfx1101"
+                        "gfx1102")
+
+set(OPENMP_GPU_TARGETS
+    "${DEFAULT_GPU_TARGETS}"
+    CACHE STRING "GPU targets to compile for")
+
+set(ROCPROFILER_MEMCHECK_TYPES "ThreadSanitizer" "AddressSanitizer"
+                               "UndefinedBehaviorSanitizer")
+
+if(ROCPROFILER_MEMCHECK AND ROCPROFILER_MEMCHECK IN_LIST ROCPROFILER_MEMCHECK_TYPES)
+    set(IS_DISABLED ON)
+else()
+    set(IS_DISABLED OFF)
+endif()
+
+find_package(Threads REQUIRED)
+find_package(rocprofiler-sdk-roctx REQUIRED)
+
+# disable when GPU-0 is navi2, navi3, and navi4
+list(GET rocprofiler-sdk-samples-gfx-info 0 openmp-tools-gpu-0-gfx-info)
+if("${openmp-tools-gpu-0-gfx-info}" MATCHES "^gfx(10|11|12)[0-9][0-9]$")
+    set(IS_DISABLED ON)
+endif()
+
+add_executable(openmp-target-sample)
+target_sources(openmp-target-sample PRIVATE main.cpp)
+target_link_libraries(
+    openmp-target-sample PRIVATE Threads::Threads
+                                 rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
+target_compile_options(openmp-target-sample PRIVATE -fopenmp)
+target_link_options(openmp-target-sample PRIVATE -fopenmp)
+
+foreach(_TARGET ${OPENMP_GPU_TARGETS})
+    target_compile_options(openmp-target-sample PRIVATE --offload-arch=${_TARGET})
+    target_link_options(openmp-target-sample PRIVATE --offload-arch=${_TARGET})
+endforeach()
+
+include(rocprofiler-sdk-custom-compilation)
+rocprofiler_sdk_custom_compilation(TARGET openmp-target-sample
+                                   COMPILER ${OMP_TARGET_COMPILER})
+
+rocprofiler_samples_get_preload_env(PRELOAD_ENV openmp-target-sample-client)
+rocprofiler_samples_get_ld_library_path_env(
+    LIBRARY_PATH_ENV rocprofiler-sdk-roctx::rocprofiler-sdk-roctx-shared-library)
+
+set(openmp-target-sample-env
+    ${PRELOAD_ENV} ${LIBRARY_PATH_ENV} "OMP_NUM_THREADS=2" "OMP_TARGET_OFFLOAD=mandatory"
+    "OMP_DISPLAY_ENV=1" "ROCR_VISIBLE_DEVICES=0")
+
+add_test(NAME openmp-target-sample COMMAND $<TARGET_FILE:openmp-target-sample>)
+
+set_tests_properties(
+    openmp-target-sample
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "samples;openmp-target"
+               ENVIRONMENT
+               "${openmp-target-sample-env}"
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               DISABLED
+               "${IS_DISABLED}")

projects/rocprofiler-sdk/samples/openmp_target/README.md

@@ -0,0 +1,15 @@
+# OMPT tool tracing
+
+## Services
+
+- OMPT tracing.
+- CodeObject tracing.
+- Marker API (Core, Name).
+
+## Properties
+
+- Configures tool for callback tracing.
+- Configures tool for buffer tracing.
+- Sets up callstack for tracing kind names and tracing operation names.
+- Create a specialized (throw-away) context for handling ROCTx profiler pause and resume.
+- Demonstrates the use of the `ompt_data_t*` fields from OMPT.

projects/rocprofiler-sdk/samples/openmp_target/client.cpp

@@ -0,0 +1,655 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+/**
+ * @file samples/ompt/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include "client.hpp"
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include "common/call_stack.hpp"
+#include "common/defines.hpp"
+#include "common/filesystem.hpp"
+#include "common/name_info.hpp"
+
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <ratio>
+#include <string>
+#include <string_view>
+#include <unordered_set>
+#include <vector>
+namespace client
+{
+namespace
+{
+using common::call_stack_t;
+using common::callback_name_info;
+using common::source_location;
+using kernel_symbol_data_t = rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t;
+using kernel_symbol_map_t  = std::unordered_map<rocprofiler_kernel_id_t, kernel_symbol_data_t>;
+
+rocprofiler_client_id_t*      client_id        = nullptr;
+rocprofiler_client_finalize_t client_fini_func = nullptr;
+auto                          client_ctx       = rocprofiler_context_id_t{};
+auto                          cb_name_info     = common::get_callback_tracing_names();
+auto                          bf_name_info     = common::get_buffer_tracing_names();
+auto                          client_buffer    = rocprofiler_buffer_id_t{};
+auto                          client_kernels   = kernel_symbol_map_t{};
+auto                          call_stack_mtx   = std::mutex{};
+
+auto
+get_call_stack_lock()
+{
+    return std::unique_lock<std::mutex>{call_stack_mtx};
+}
+
+void
+print_call_stack(const call_stack_t& _call_stack)
+{
+    common::print_call_stack("openmp_target_trace.log", _call_stack);
+}
+
+void
+tool_tracing_ctrl_callback(rocprofiler_callback_tracing_record_t record,
+                           rocprofiler_user_data_t*,
+                           void* client_data)
+{
+    auto* ctx = static_cast<rocprofiler_context_id_t*>(client_data);
+
+    if(record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER &&
+       record.kind == ROCPROFILER_CALLBACK_TRACING_MARKER_CONTROL_API &&
+       record.operation == ROCPROFILER_MARKER_CONTROL_API_ID_roctxProfilerPause)
+    {
+        ROCPROFILER_CALL(rocprofiler_stop_context(*ctx), "pausing client context");
+    }
+    else if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT &&
+            record.kind == ROCPROFILER_CALLBACK_TRACING_MARKER_CONTROL_API &&
+            record.operation == ROCPROFILER_MARKER_CONTROL_API_ID_roctxProfilerResume)
+    {
+        ROCPROFILER_CALL(rocprofiler_start_context(*ctx), "resuming client context");
+    }
+}
+
+void
+tool_callback_tracing_callback(rocprofiler_callback_tracing_record_t record,
+                               rocprofiler_user_data_t*              user_data,
+                               void*                                 callback_data)
+{
+    assert(callback_data != nullptr);
+
+    if(record.kind == ROCPROFILER_CALLBACK_TRACING_HSA_CORE_API)
+    {
+        if(record.operation == ROCPROFILER_HSA_CORE_API_ID_hsa_queue_destroy)
+        {
+            // skip hsa_queue_destroy for now, it tries to print the queue after it is destroyed
+            return;
+        }
+    }
+    else if(record.kind == ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT &&
+            record.operation == ROCPROFILER_CODE_OBJECT_LOAD)
+    {
+        if(record.phase == ROCPROFILER_CALLBACK_PHASE_UNLOAD)
+        {
+            // flush the buffer to ensure that any lookups for the client kernel names for the code
+            // object are completed
+            auto flush_status = rocprofiler_flush_buffer(client_buffer);
+            if(flush_status != ROCPROFILER_STATUS_ERROR_BUFFER_BUSY)
+                ROCPROFILER_CALL(flush_status, "buffer flush");
+        }
+    }
+    else if(record.kind == ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT &&
+            record.operation == ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER)
+    {
+        auto* data = static_cast<kernel_symbol_data_t*>(record.payload);
+        if(record.phase == ROCPROFILER_CALLBACK_PHASE_LOAD)
+        {
+            client_kernels.emplace(data->kernel_id, *data);
+        }
+        else if(record.phase == ROCPROFILER_CALLBACK_PHASE_UNLOAD)
+        {
+            client_kernels.erase(data->kernel_id);
+        }
+    }
+    else if(record.kind == ROCPROFILER_CALLBACK_TRACING_OMPT)
+    {
+        // demonstrate the use of the ompt_data_t* fields from OMPT
+        // The client has its own version of those fields as well as an interface to the
+        // ompt API entry points.
+        auto* data = static_cast<rocprofiler_callback_tracing_ompt_data_t*>(record.payload);
+
+        if(record.operation == ROCPROFILER_OMPT_ID_parallel_begin)
+        {
+            // set the parallel_data value
+            auto& args                = data->args.parallel_begin;
+            args.parallel_data->value = record.correlation_id.internal;
+        }
+        else if(record.operation == ROCPROFILER_OMPT_ID_parallel_end)
+        {
+            // set the parallel_data value
+            auto& args                = data->args.parallel_end;
+            args.parallel_data->value = 0;
+        }
+        else if(record.operation == ROCPROFILER_OMPT_ID_thread_begin)
+        {
+            // set the thread_data value
+            auto& args              = data->args.thread_begin;
+            args.thread_data->value = record.thread_id;
+        }
+        else if(record.operation == ROCPROFILER_OMPT_ID_thread_end)
+        {
+            // set the thread_data value
+            auto& args              = data->args.thread_end;
+            args.thread_data->value = 0;
+        }
+        else if(record.operation == ROCPROFILER_OMPT_ID_implicit_task)
+        {
+            auto& args = data->args.implicit_task;
+            // set the task_data value
+            if(record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER)
+                args.task_data->value = record.correlation_id.internal;
+            else if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT)
+                args.task_data->value = 0;
+            else
+                assert(0);
+        }
+        else if(record.operation == ROCPROFILER_OMPT_ID_target_emi)
+        {
+            auto& args = data->args.target_emi;
+            // set the target_data value
+            if(record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER)
+                args.target_data->value = record.correlation_id.internal;
+            else if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT)
+                args.target_data->value = 0;
+            else
+                assert(0);
+        }
+        else if(record.operation == ROCPROFILER_OMPT_ID_target_data_op_emi)
+        {
+            auto& args = data->args.target_data_op_emi;
+            // set the host_op_id value
+            if(record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER)
+                args.host_op_id->value = record.correlation_id.internal;
+            else if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT)
+                args.host_op_id->value = 0;
+            else
+                assert(0);
+        }
+        else if(record.operation == ROCPROFILER_OMPT_ID_target_submit_emi)
+        {
+            // set the host_op_id value
+            auto& args = data->args.target_submit_emi;
+            if(record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER)
+                args.host_op_id->value = record.correlation_id.internal;
+            else if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT)
+                args.host_op_id->value = 0;
+            else
+                assert(0);
+        }
+    }
+
+    auto     now = std::chrono::steady_clock::now().time_since_epoch().count();
+    uint64_t dt  = 0;
+    if(record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER)
+        user_data->value = now;
+    else if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT)
+        dt = (now - user_data->value);
+
+    const char* name = nullptr;
+    rocprofiler_query_callback_tracing_kind_operation_name(
+        record.kind, record.operation, &name, nullptr);
+
+    auto info = std::stringstream{};
+    info << std::left << "tid=" << record.thread_id << ", cid=" << std::setw(3)
+         << record.correlation_id.internal << ", kind=" << std::setw(2) << record.kind
+         << ", operation=" << std::setw(3) << record.operation << ", phase=" << record.phase
+         << ", dt_nsec=" << std::setw(8) << dt << ", name=" << name;
+
+    auto info_data_cb = [](rocprofiler_callback_tracing_kind_t,
+                           int,
+                           uint32_t          arg_num,
+                           const void* const arg_value_addr,
+                           int32_t           indirection_count,
+                           const char*       arg_type,
+                           const char*       arg_name,
+                           const char*       arg_value_str,
+                           int32_t           dereference_count,
+                           void*             cb_data) -> int {
+        auto& dss = *static_cast<std::stringstream*>(cb_data);
+        dss << ((arg_num == 0) ? "(" : ", ");
+        dss << arg_num << ": " << arg_name << "=" << arg_value_str;
+        (void) arg_value_addr;
+        (void) arg_type;
+        (void) indirection_count;
+        (void) dereference_count;
+        return 0;
+    };
+
+    int32_t max_deref = 1;
+    if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT)
+        // not for PHASE_NONE
+        max_deref = 2;
+    auto info_data = std::stringstream{};
+    if(record.kind != ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT)
+    {
+        ROCPROFILER_CALL(rocprofiler_iterate_callback_tracing_kind_operation_args(
+                             record, info_data_cb, max_deref, static_cast<void*>(&info_data)),
+                         "Failure iterating trace operation args");
+    }
+
+    auto info_data_str = info_data.str();
+    if(!info_data_str.empty()) info << " " << info_data_str << ")";
+
+    auto* call_stack_v = static_cast<call_stack_t*>(callback_data);
+    auto  _lk          = get_call_stack_lock();
+    call_stack_v->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+}
+
+void
+tool_buffered_tracing_callback(rocprofiler_context_id_t      context,
+                               rocprofiler_buffer_id_t       buffer_id,
+                               rocprofiler_record_header_t** headers,
+                               size_t                        num_headers,
+                               void*                         user_data,
+                               uint64_t                      drop_count)
+{
+    assert(drop_count == 0 && "drop count should be zero for lossless policy");
+
+    auto* call_stack_v = static_cast<call_stack_t*>(user_data);
+
+    for(size_t i = 0; i < num_headers; ++i)
+    {
+        auto* header = headers[i];
+
+        if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING &&
+           header->kind == ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH)
+        {
+            auto* record =
+                static_cast<rocprofiler_buffer_tracing_kernel_dispatch_record_t*>(header->payload);
+
+            auto info = std::stringstream{};
+
+            auto dt = (record->end_timestamp - record->start_timestamp);
+            info << std::left << "tid=" << record->thread_id << ", cid=" << std::setw(3)
+                 << record->correlation_id.internal << ", kind=" << std::setw(2) << record->kind
+                 << ", operation=" << std::setw(3) << record->operation << ", phase= "
+                 << ", dt_nsec=" << std::setw(8) << dt
+                 << ", agent_id=" << record->dispatch_info.agent_id.handle
+                 << ", queue_id=" << record->dispatch_info.queue_id.handle
+                 << ", kernel_id=" << record->dispatch_info.kernel_id
+                 << ", kernel=" << client_kernels.at(record->dispatch_info.kernel_id).kernel_name
+                 << ", start=" << record->start_timestamp << ", stop=" << record->end_timestamp
+                 << ", private_segment_size=" << record->dispatch_info.private_segment_size
+                 << ", group_segment_size=" << record->dispatch_info.group_segment_size
+                 << ", workgroup_size=(" << record->dispatch_info.workgroup_size.x << ","
+                 << record->dispatch_info.workgroup_size.y << ","
+                 << record->dispatch_info.workgroup_size.z << "), grid_size=("
+                 << record->dispatch_info.grid_size.x << "," << record->dispatch_info.grid_size.y
+                 << "," << record->dispatch_info.grid_size.z << ")";
+
+            if(record->start_timestamp > record->end_timestamp)
+                throw std::runtime_error("kernel dispatch: start > end");
+
+            auto _lk = get_call_stack_lock();
+            call_stack_v->emplace_back(
+                source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+        }
+        else if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING &&
+                header->kind == ROCPROFILER_BUFFER_TRACING_MEMORY_COPY)
+        {
+            auto* record =
+                static_cast<rocprofiler_buffer_tracing_memory_copy_record_t*>(header->payload);
+
+            auto info = std::stringstream{};
+
+            auto dt = (record->end_timestamp - record->start_timestamp);
+            info << std::left << "tid=" << record->thread_id << ", cid=" << std::setw(3)
+                 << record->correlation_id.internal << ", kind=" << std::setw(2) << record->kind
+                 << ", operation=" << std::setw(3) << record->operation << ", phase= "
+                 << ", dt_nsec=" << std::setw(8) << dt
+                 << ", src_agent_id=" << record->src_agent_id.handle
+                 << ", dst_agent_id=" << record->dst_agent_id.handle
+                 << ", direction=" << record->operation << ", start=" << record->start_timestamp
+                 << ", stop=" << record->end_timestamp
+                 << ", name=" << bf_name_info.at(record->kind, record->operation);
+
+            if(record->start_timestamp > record->end_timestamp)
+                throw std::runtime_error("memory copy: start > end");
+
+            auto _lk = get_call_stack_lock();
+            call_stack_v->emplace_back(
+                source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+        }
+        else if(header->category == ROCPROFILER_BUFFER_CATEGORY_TRACING &&
+                header->kind == ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY)
+        {
+            auto* record =
+                static_cast<rocprofiler_buffer_tracing_scratch_memory_record_t*>(header->payload);
+
+            auto info = std::stringstream{};
+
+            auto _elapsed =
+                std::chrono::duration_cast<std::chrono::duration<double, std::micro>>(
+                    std::chrono::nanoseconds{record->end_timestamp - record->start_timestamp})
+                    .count();
+
+            auto dt = (record->end_timestamp - record->start_timestamp);
+            info << std::left << "tid=" << record->thread_id << ", cid=" << std::setw(3)
+                 << record->correlation_id.internal << ", kind=" << std::setw(2) << record->kind
+                 << ", operation=" << std::setw(3) << record->operation << ", phase= "
+                 << ", dt_nsec=" << std::setw(8) << dt << ", agent_id=" << record->agent_id.handle
+                 << ", queue_id=" << record->queue_id.handle << ", thread_id=" << record->thread_id
+                 << ", elapsed=" << std::setprecision(3) << std::fixed << _elapsed
+                 << " usec, flags=" << record->flags
+                 << ", name=" << bf_name_info.at(record->kind, record->operation);
+
+            auto _lk = get_call_stack_lock();
+            call_stack_v->emplace_back(
+                source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+        }
+        else
+        {
+            auto _msg = std::stringstream{};
+            _msg << "unexpected rocprofiler_record_header_t category + kind: (" << header->category
+                 << " + " << header->kind << ")";
+            throw std::runtime_error{_msg.str()};
+        }
+    }
+
+    (void) context;
+    (void) buffer_id;
+}
+
+void
+tool_control_init(rocprofiler_context_id_t& primary_ctx)
+{
+    // Create a specialized (throw-away) context for handling ROCTx profiler pause and resume.
+    // A separate context is used because if the context that is associated with roctxProfilerPause
+    // disabled that same context, a call to roctxProfilerResume would be ignored because the
+    // context that enables the callback for that API call is disabled.
+    auto cntrl_ctx = rocprofiler_context_id_t{};
+    ROCPROFILER_CALL(rocprofiler_create_context(&cntrl_ctx), "control context creation failed");
+
+    // enable callback marker tracing with only the pause/resume operations
+    ROCPROFILER_CALL(rocprofiler_configure_callback_tracing_service(
+                         cntrl_ctx,
+                         ROCPROFILER_CALLBACK_TRACING_MARKER_CONTROL_API,
+                         nullptr,
+                         0,
+                         tool_tracing_ctrl_callback,
+                         &primary_ctx),
+                     "callback tracing service failed to configure");
+
+    // start the context so that it is always active
+    ROCPROFILER_CALL(rocprofiler_start_context(cntrl_ctx), "start of control context");
+}
+
+int
+tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
+{
+    assert(tool_data != nullptr);
+
+    auto* call_stack_v = static_cast<call_stack_t*>(tool_data);
+
+    call_stack_v->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, ""});
+
+    for(const auto& itr : cb_name_info)
+    {
+        auto name_idx = std::stringstream{};
+        name_idx << " [" << std::setw(3) << itr.value << "]";
+        call_stack_v->emplace_back(
+            source_location{"rocprofiler_callback_tracing_kind_names          " + name_idx.str(),
+                            __FILE__,
+                            __LINE__,
+                            std::string{itr.name}});
+
+        for(auto [didx, ditr] : itr.items())
+        {
+            auto operation_idx = std::stringstream{};
+            operation_idx << " [" << std::setw(3) << didx << "]";
+            call_stack_v->emplace_back(source_location{
+                "rocprofiler_callback_tracing_kind_operation_names" + operation_idx.str(),
+                __FILE__,
+                __LINE__,
+                std::string{"- "} + std::string{*ditr}});
+        }
+    }
+
+    for(const auto& itr : bf_name_info)
+    {
+        auto name_idx = std::stringstream{};
+        name_idx << " [" << std::setw(3) << itr.value << "]";
+        call_stack_v->emplace_back(
+            source_location{"rocprofiler_buffer_tracing_kind_names          " + name_idx.str(),
+                            __FILE__,
+                            __LINE__,
+                            std::string{itr.name}});
+
+        for(auto [didx, ditr] : itr.items())
+        {
+            auto operation_idx = std::stringstream{};
+            operation_idx << " [" << std::setw(3) << didx << "]";
+            call_stack_v->emplace_back(source_location{
+                "rocprofiler_buffer_tracing_kind_operation_names" + operation_idx.str(),
+                __FILE__,
+                __LINE__,
+                std::string{"- "} + std::string{*ditr}});
+        }
+    }
+
+    client_fini_func = fini_func;
+
+    ROCPROFILER_CALL(rocprofiler_create_context(&client_ctx), "context creation failed");
+
+    // enable the control
+    tool_control_init(client_ctx);
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_tracing_service(client_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT,
+                                                       nullptr,
+                                                       0,
+                                                       tool_callback_tracing_callback,
+                                                       tool_data),
+        "callback tracing service failed to configure");
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_tracing_service(client_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_OMPT,
+                                                       nullptr,
+                                                       0,
+                                                       tool_callback_tracing_callback,
+                                                       tool_data),
+        "callback tracing service failed to configure")
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_tracing_service(client_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_MARKER_CORE_API,
+                                                       nullptr,
+                                                       0,
+                                                       tool_callback_tracing_callback,
+                                                       tool_data),
+        "callback tracing service failed to configure");
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_tracing_service(client_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_MARKER_NAME_API,
+                                                       nullptr,
+                                                       0,
+                                                       tool_callback_tracing_callback,
+                                                       tool_data),
+        "callback tracing service failed to configure");
+
+    constexpr auto buffer_size_bytes      = 4096;
+    constexpr auto buffer_watermark_bytes = buffer_size_bytes - (buffer_size_bytes / 8);
+
+    ROCPROFILER_CALL(rocprofiler_create_buffer(client_ctx,
+                                               buffer_size_bytes,
+                                               buffer_watermark_bytes,
+                                               ROCPROFILER_BUFFER_POLICY_LOSSLESS,
+                                               tool_buffered_tracing_callback,
+                                               tool_data,
+                                               &client_buffer),
+                     "buffer creation");
+
+    for(auto itr : {ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH,
+                    ROCPROFILER_BUFFER_TRACING_MEMORY_COPY,
+                    ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY})
+    {
+        ROCPROFILER_CALL(rocprofiler_configure_buffer_tracing_service(
+                             client_ctx, itr, nullptr, 0, client_buffer),
+                         "buffer tracing service configure");
+    }
+
+    int valid_ctx = 0;
+    ROCPROFILER_CALL(rocprofiler_context_is_valid(client_ctx, &valid_ctx),
+                     "failure checking context validity");
+    if(valid_ctx == 0)
+    {
+        // notify rocprofiler that initialization failed
+        // and all the contexts, buffers, etc. created
+        // should be ignored
+        return -1;
+    }
+    ROCPROFILER_CALL(rocprofiler_start_context(client_ctx), "rocprofiler context start failed");
+
+    // no errors
+    return 0;
+}
+
+void
+tool_fini(void* tool_data)
+{
+    assert(tool_data != nullptr);
+
+    auto* _call_stack = static_cast<call_stack_t*>(tool_data);
+    auto  _lk         = get_call_stack_lock();
+    _call_stack->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, ""});
+
+    print_call_stack(*_call_stack);
+
+    delete _call_stack;
+}
+}  // namespace
+
+void
+setup()
+{}
+
+void
+shutdown()
+{
+    if(client_id) client_fini_func(*client_id);
+}
+
+void
+start()
+{
+    ROCPROFILER_CALL(rocprofiler_start_context(client_ctx), "rocprofiler context start failed");
+}
+
+void
+stop()
+{
+    int status = 0;
+    ROCPROFILER_CALL(rocprofiler_is_initialized(&status), "failed to retrieve init status");
+    if(status != 0)
+    {
+        ROCPROFILER_CALL(rocprofiler_stop_context(client_ctx), "rocprofiler context stop failed");
+    }
+}
+}  // namespace client
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name = "ExampleTool";
+
+    // store client info
+    client::client_id = id;
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " (priority=" << priority << ") is using rocprofiler-sdk v" << major << "."
+         << minor << "." << patch << " (" << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    // demonstration of alternative way to get the version info
+    {
+        auto version_info = std::array<uint32_t, 3>{};
+        ROCPROFILER_CALL(
+            rocprofiler_get_version(&version_info.at(0), &version_info.at(1), &version_info.at(2)),
+            "failed to get version info");
+
+        if(std::array<uint32_t, 3>{major, minor, patch} != version_info)
+        {
+            throw std::runtime_error{"version info mismatch"};
+        }
+    }
+
+    // data passed around all the callbacks
+    auto* client_tool_data = new std::vector<client::source_location>{};
+
+    // add first entry
+    client_tool_data->emplace_back(
+        client::source_location{__FUNCTION__, __FILE__, __LINE__, info.str()});
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &client::tool_init,
+                                            &client::tool_fini,
+                                            static_cast<void*>(client_tool_data)};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/openmp_target/client.hpp

@@ -0,0 +1,44 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#ifdef openmp_target_sample_client_EXPORTS
+#    define CLIENT_API __attribute__((visibility("default")))
+#else
+#    define CLIENT_API
+#endif
+
+namespace client
+{
+void
+setup() CLIENT_API;
+
+void
+shutdown() CLIENT_API;
+
+void
+start() CLIENT_API;
+
+void
+stop() CLIENT_API;
+}  // namespace client

projects/rocprofiler-sdk/samples/openmp_target/main.cpp

@@ -0,0 +1,161 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// #include "client.hpp"
+
+#include <rocprofiler-sdk-roctx/roctx.h>
+
+#include <math.h>
+#include <stdio.h>
+
+constexpr float  EPS_FLOAT  = 1.0e-7f;
+constexpr double EPS_DOUBLE = 1.0e-15;
+
+#pragma omp declare target
+template <typename T>
+T
+mul(T a, T b)
+{
+    T c;
+    c = a * b;
+    return c;
+}
+#pragma omp end declare target
+
+template <typename T>
+void
+vmul(T* a, T* b, T* c, int N)
+{
+#pragma omp target           map(to : a [0:N], b [0:N]) map(from : c [0:N])
+#pragma omp teams distribute parallel for
+    for(int i = 0; i < N; i++)
+    {
+        c[i] = mul(a[i], b[i]);
+    }
+}
+
+int
+main()
+{
+    // client::setup();
+    auto range_id = roctxRangeStart("main");
+
+    constexpr int N = 100000;
+    int           a_i[N], b_i[N], c_i[N], validate_i[N];
+    float         a_f[N], b_f[N], c_f[N], validate_f[N];
+    double        a_d[N], b_d[N], c_d[N], validate_d[N];
+    int           N_errors = 0;
+    bool          flag     = false;
+
+    roctxMark("initialization");
+
+#pragma omp parallel for
+    for(int i = 0; i < N; ++i)
+    {
+        a_f[i] = a_i[i] = i + 1;
+        b_f[i] = b_i[i] = i + 2;
+        a_d[i]          = a_i[i];
+        b_d[i]          = b_i[i];
+        validate_i[i]   = a_i[i] * b_i[i];
+        validate_f[i]   = a_f[i] * b_f[i];
+        validate_d[i]   = a_d[i] * b_d[i];
+    }
+
+    vmul(a_i, b_i, c_i, N);
+    vmul(a_f, b_f, c_f, N);
+
+    auto tid = roctx_thread_id_t{};
+    // get the thread id recognized by rocprofiler-sdk from roctx
+    roctxGetThreadId(&tid);
+    // pause API tracing
+    roctxProfilerPause(tid);
+
+    // we don't expect to see the third vmul
+    vmul(a_d, b_d, c_d, N);
+
+    // resume API tracing
+    roctxProfilerResume(tid);
+
+    for(int i = 0; i < N; i++)
+    {
+        if(c_i[i] != validate_i[i])
+        {
+            ++N_errors;
+            //       print 1st bad index
+            if(!flag)
+            {
+                printf(
+                    "First fail: c_i[%d](%d) != validate_i[%d](%d)\n", i, c_i[i], i, validate_i[i]);
+                flag = true;
+            }
+        }
+    }
+    flag = false;
+    for(int i = 0; i < N; i++)
+    {
+        if(fabs(c_f[i] - validate_f[i]) > EPS_FLOAT)
+        {
+            ++N_errors;
+            //      print 1st bad index
+            if(!flag)
+            {
+                printf("First fail: c_f[%d](%f) != validate_f[%d](%f)\n",
+                       i,
+                       static_cast<double>(c_f[i]),
+                       i,
+                       static_cast<double>(validate_f[i]));
+                flag = true;
+            }
+        }
+    }
+    flag = false;
+    for(int i = 0; i < N; i++)
+    {
+        if(fabs(c_d[i] - validate_d[i]) > EPS_DOUBLE)
+        {
+            ++N_errors;
+            //      print 1st bad index
+            if(!flag)
+            {
+                printf(
+                    "First fail: c_d[%d](%f) != validate_d[%d](%f)\n", i, c_d[i], i, validate_d[i]);
+                flag = true;
+            }
+        }
+    }
+    if(N_errors == 0)
+    {
+        printf("Success\n");
+        return 0;
+    }
+    else
+    {
+        printf("Total %d failures\n", N_errors);
+        printf("Fail\n");
+        return 1;
+    }
+
+    roctxRangeStop(range_id);
+
+    // client::stop();
+    // client::shutdown();
+}

projects/rocprofiler-sdk/samples/pc_sampling/CMakeLists.txt

@@ -0,0 +1,70 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+if(NOT CMAKE_HIP_COMPILER)
+    find_program(
+        amdclangpp_EXECUTABLE
+        NAMES amdclang++
+        HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
+        PATH_SUFFIXES bin llvm/bin NO_CACHE)
+    mark_as_advanced(amdclangpp_EXECUTABLE)
+
+    if(amdclangpp_EXECUTABLE)
+        set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
+    endif()
+endif()
+
+project(rocprofiler-sdk-samples-pc-sampling LANGUAGES CXX HIP)
+
+foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+    if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
+        set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
+    endif()
+endforeach()
+
+find_package(rocprofiler-sdk REQUIRED)
+
+add_library(pc-sampling-client SHARED)
+target_sources(pc-sampling-client PRIVATE client.cpp pcs.hpp pcs.cpp utils.hpp utils.cpp)
+target_link_libraries(
+    pc-sampling-client
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler-sdk::samples-build-flags
+            rocprofiler-sdk::samples-common-library)
+
+set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
+find_package(Threads REQUIRED)
+
+add_executable(pc-sampling)
+target_sources(pc-sampling PRIVATE main.cpp)
+target_link_libraries(
+    pc-sampling
+    PRIVATE pc-sampling-client Threads::Threads rocprofiler-sdk::samples-build-flags
+            rocprofiler-sdk::samples-common-library)
+
+rocprofiler_samples_get_preload_env(PRELOAD_ENV pc-sampling-client)
+rocprofiler_samples_get_ld_library_path_env(LIBRARY_PATH_ENV)
+
+# Check if PC sampling is disabled and whether we should disable the test
+rocprofiler_sdk_pc_sampling_disabled(IS_PC_SAMPLING_DISABLED)
+
+set(pc-sampling-env ${PRELOAD_ENV} ${LIBRARY_PATH_ENV})
+
+add_test(NAME pc-sampling COMMAND $<TARGET_FILE:pc-sampling>)
+
+set_tests_properties(
+    pc-sampling
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "samples;pc-sampling"
+               ENVIRONMENT
+               "${pc-sampling-env}"
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               SKIP_REGULAR_EXPRESSION
+               "PC sampling unavailable"
+               DISABLED
+               "${IS_PC_SAMPLING_DISABLED}")

projects/rocprofiler-sdk/samples/pc_sampling/README.md

@@ -0,0 +1,12 @@
+# PC sampling service
+
+## Services
+
+- PC sampling stochastic method
+
+## Properties
+
+- Iterate through all gpu agents that supports PC sampling.
+- Iterate through the supported configuration for that agent.
+- The `configure_pc_sampling_prefer_stochastic` function is responsible for configuring PC sampling on a given GPU agent. It attempts to select a stochastic sampling configuration if available, falling back to a host-trap configuration otherwise.
+- `rocprofiler_pc_sampling_callback` function processes PC sampling records delivered by the profiler. It validates the records, determines their type, and delegates the printing of their details to the appropriate print_sample function.

projects/rocprofiler-sdk/samples/pc_sampling/client.cpp

@@ -0,0 +1,237 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+/**
+ * @file samples/pc_sampling_library/client.cpp
+ *
+ * @brief Example rocprofiler client (tool)
+ */
+
+#include "pcs.hpp"
+#include "utils.hpp"
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include "common/defines.hpp"
+#include "common/filesystem.hpp"
+
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <regex>
+#include <string>
+#include <vector>
+
+namespace client
+{
+namespace
+{
+rocprofiler_client_id_t*      client_id        = nullptr;
+rocprofiler_client_finalize_t client_fini_func = nullptr;
+rocprofiler_context_id_t      client_ctx{0};
+
+int
+tool_init(rocprofiler_client_finalize_t fini_func, void* /*tool_data*/)
+{
+    client_fini_func = fini_func;
+
+    // Initialize necessary data structures
+    pcs::init();
+
+    client::pcs::find_all_gpu_agents_supporting_pc_sampling();
+
+    if(client::pcs::gpu_agents.empty())
+    {
+        *utils::get_output_stream() << "No availabe gpu agents supporting PC sampling" << std::endl;
+        // Emit the message to explicitly skip the sample.
+        std::cerr << "PC sampling unavailable" << std::endl;
+        // Exit with no error if none of the GPUs support PC sampling.
+        exit(0);
+    }
+
+    // The relations assumed:
+    // - One context for all gpu agents
+    // - a buffer per agent
+    // - a callback thread per buffer
+    // - a pc sampling service per agent/buffer
+
+    ROCPROFILER_CHECK(rocprofiler_create_context(&client_ctx));
+
+    auto* buff_ids_vec = pcs::get_pc_sampling_buffer_ids();
+
+    for(auto& gpu_agent : pcs::gpu_agents)
+    {
+        // creating a buffer that will hold pc sampling information
+        rocprofiler_buffer_policy_t drop_buffer_action = ROCPROFILER_BUFFER_POLICY_LOSSLESS;
+        auto                        buffer_id          = rocprofiler_buffer_id_t{};
+        ROCPROFILER_CHECK(rocprofiler_create_buffer(client_ctx,
+                                                    client::pcs::BUFFER_SIZE_BYTES,
+                                                    client::pcs::WATERMARK,
+                                                    drop_buffer_action,
+                                                    client::pcs::rocprofiler_pc_sampling_callback,
+                                                    nullptr,
+                                                    &buffer_id));
+
+        client::pcs::configure_pc_sampling_prefer_stochastic(
+            gpu_agent.get(), client_ctx, buffer_id);
+
+        // One helper thread per GPU agent's buffer.
+        auto client_agent_thread = rocprofiler_callback_thread_t{};
+        ROCPROFILER_CHECK(rocprofiler_create_callback_thread(&client_agent_thread));
+
+        ROCPROFILER_CHECK(rocprofiler_assign_callback_thread(buffer_id, client_agent_thread));
+
+        buff_ids_vec->emplace_back(buffer_id);
+    }
+
+    int valid_ctx = 0;
+    ROCPROFILER_CHECK(rocprofiler_context_is_valid(client_ctx, &valid_ctx));
+    if(valid_ctx == 0)
+    {
+        // notify rocprofiler that initialization failed
+        // and all the contexts, buffers, etc. created
+        // should be ignored
+        return -1;
+    }
+
+    // Start PC sampling
+    ROCPROFILER_CHECK(rocprofiler_start_context(client_ctx));
+
+    return 0;
+}
+
+void
+tool_fini(void* /*tool_data*/)
+{
+    if(client_id)
+    {
+        // Assert the context is inactive.
+        int state = -1;
+        ROCPROFILER_CHECK(rocprofiler_context_is_active(client_ctx, &state))
+        assert(state == 0);
+
+        // No need to stop the context, since it has been stopped implicitly by the rocprofiler-SDK.
+        for(auto buff_id : *pcs::get_pc_sampling_buffer_ids())
+        {
+            // Flush the buffer explicitly
+            ROCPROFILER_CHECK(rocprofiler_flush_buffer(buff_id));
+            // Destroying the buffer
+            rocprofiler_status_t status = rocprofiler_destroy_buffer(buff_id);
+            if(status == ROCPROFILER_STATUS_ERROR_BUFFER_BUSY)
+            {
+                *utils::get_output_stream()
+                    << "The buffer is busy, so we cannot destroy it at the moment." << std::endl;
+            }
+            else
+            {
+                ROCPROFILER_CHECK(status);
+            }
+        }
+    }
+
+    // deallocation
+    pcs::fini();
+}
+
+}  // namespace
+
+// forward declaration
+void
+setup();
+
+void
+setup()
+{
+    if(int status = 0;
+       rocprofiler_is_initialized(&status) == ROCPROFILER_STATUS_SUCCESS && status == 0)
+    {
+        ROCPROFILER_CHECK(rocprofiler_force_configure(&rocprofiler_configure));
+    }
+}
+
+void
+shutdown()
+{}
+
+}  // namespace client
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t                 version,
+                      const char*              runtime_version,
+                      uint32_t                 priority,
+                      rocprofiler_client_id_t* id)
+{
+    // only activate if main tool
+    if(priority > 0) return nullptr;
+
+    // set the client name
+    id->name = "PCSamplingExampleTool";
+
+    // store client info
+    client::client_id = id;
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " is using rocprofiler v" << major << "." << minor << "." << patch << " ("
+         << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    std::ostream* output_stream = nullptr;
+    std::string   filename      = "pc_sampling.log";
+    if(auto* outfile = getenv("ROCPROFILER_SAMPLE_OUTPUT_FILE"); outfile) filename = outfile;
+    if(filename == "stdout")
+        output_stream = &std::cout;
+    else if(filename == "stderr")
+        output_stream = &std::cerr;
+    else
+        output_stream = new std::ofstream{filename};
+
+    client::utils::get_output_stream() = output_stream;
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &client::tool_init,
+                                            &client::tool_fini,
+                                            static_cast<void*>(output_stream)};
+
+    // return pointer to configure data
+    return &cfg;
+}

projects/rocprofiler-sdk/samples/pc_sampling/main.cpp

@@ -0,0 +1,415 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "common/defines.hpp"
+#include "hip/hip_runtime.h"
+
+#include <libgen.h>
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+#include <mutex>
+#include <random>
+#include <sstream>
+#include <stdexcept>
+#include <thread>
+
+#define HIP_API_CALL(CALL)                                                                         \
+    {                                                                                              \
+        hipError_t error_ = (CALL);                                                                \
+        if(error_ != hipSuccess)                                                                   \
+        {                                                                                          \
+            auto _hip_api_print_lk = auto_lock_t{print_lock};                                      \
+            fprintf(stderr,                                                                        \
+                    "%s:%d :: HIP error : %s\n",                                                   \
+                    __FILE__,                                                                      \
+                    __LINE__,                                                                      \
+                    hipGetErrorString(error_));                                                    \
+            throw std::runtime_error("hip_api_call");                                              \
+        }                                                                                          \
+    }
+
+namespace
+{
+using auto_lock_t                      = std::unique_lock<std::mutex>;
+auto               print_lock          = std::mutex{};
+size_t             nthread_per_device  = 2;
+size_t             nitr                = 500;
+size_t             nsync               = 10;
+constexpr unsigned shared_mem_tile_dim = 32;
+
+void
+check_hip_error(void);
+
+void
+verify(int* in, int* out, int M, int N);
+}  // namespace
+
+__global__ void
+transpose(const int* in, int* out, int M, int N);
+
+void
+run(int rank, int tid, int devid, int argc, char** argv);
+
+void
+run_transpose(int rank, int tid, hipStream_t stream, int argc, char** argv);
+
+void
+run_migrate(int rank, int tid, hipStream_t stream, int, char** argv);
+
+void
+run_scratch(int rank, int tid, hipStream_t stream, int argc, char** argv);
+
+int
+main(int argc, char** argv)
+{
+    auto* exe_name = basename(argv[0]);
+
+    int rank = 0;
+    for(int i = 1; i < argc; ++i)
+    {
+        auto _arg = std::string{argv[i]};
+        if(_arg == "?" || _arg == "-h" || _arg == "--help")
+        {
+            fprintf(stderr,
+                    "usage: %s [NUM_THREADS_PER_DEVICE (%zu)] [NUM_ITERATION (%zu)] "
+                    "[SYNC_EVERY_N_ITERATIONS (%zu)]\n",
+                    exe_name,
+                    nthread_per_device,
+                    nitr,
+                    nsync);
+            exit(EXIT_SUCCESS);
+        }
+    }
+    if(argc > 1) nthread_per_device = atoll(argv[1]);
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    int ndevice = 0;
+    HIP_API_CALL(hipGetDeviceCount(&ndevice));
+
+    auto nthreads = (ndevice * nthread_per_device);
+
+    printf("[%s] Number of devices found: %i\n", exe_name, ndevice);
+    printf("[%s] Number of threads (per device): %zu\n", exe_name, nthread_per_device);
+    printf("[%s] Number of threads (total): %zu\n", exe_name, nthreads);
+    printf("[%s] Number of iterations: %zu\n", exe_name, nitr);
+    printf("[%s] Syncing every %zu iterations\n", exe_name, nsync);
+
+    {
+        auto _threads = std::vector<std::thread>{};
+        for(size_t i = 0; i < nthreads; ++i)
+            _threads.emplace_back(run, rank, i, i % ndevice, argc, argv);
+        for(auto& itr : _threads)
+            itr.join();
+    }
+
+    HIP_API_CALL(hipDeviceSynchronize());
+    HIP_API_CALL(hipDeviceReset());
+
+    return 0;
+}
+
+__global__ void
+transpose(const int* in, int* out, int M, int N)
+{
+    __shared__ int tile[shared_mem_tile_dim][shared_mem_tile_dim];
+
+    int idx = (blockIdx.y * blockDim.y + threadIdx.y) * M + blockIdx.x * blockDim.x + threadIdx.x;
+    tile[threadIdx.y][threadIdx.x] = in[idx];
+    __syncthreads();
+    idx      = (blockIdx.x * blockDim.x + threadIdx.y) * N + blockIdx.y * blockDim.y + threadIdx.x;
+    out[idx] = tile[threadIdx.x][threadIdx.y];
+}
+
+template <typename Tp>
+__global__ void
+test_page_migrate(Tp* data, Tp val)
+{
+    int idx = (blockIdx.x * blockDim.x) + threadIdx.x;
+    data[idx] += val;
+}
+
+__global__ void
+test_kern_large(uint64_t* output)
+{
+    uint64_t result = 0;
+    int      test[4000];
+    memset(test, 5, 4000);
+    for(int& i : test)
+    {
+        i = i + 7;
+        *output += i;
+        result += i;
+    }
+    *output ^= result;
+    *output ^= result;
+}
+
+__global__ void
+test_kern_medium(uint64_t* output)
+{
+    uint64_t result = 0;
+    int      test[175];
+    memset(test, 5, 175);
+    for(int& i : test)
+    {
+        i = i + 7;
+        *output += i;
+        result += i;
+    }
+    *output ^= result;
+    *output ^= result;
+}
+
+__global__ void
+test_kern_small(uint64_t* output)
+{
+    uint64_t result = 0;
+    int      test[2];
+    for(int& i : test)
+    {
+        i = i + 7;
+        *output += i;
+        result += i;
+    }
+    *output ^= result;
+    *output ^= result;
+}
+
+void
+run(int rank, int tid, int devid, int argc, char** argv)
+{
+    auto* stream = hipStream_t{};
+    HIP_API_CALL(hipSetDevice(devid));
+    HIP_API_CALL(hipStreamCreate(&stream));
+
+    run_migrate(rank, tid, stream, argc, argv);
+    run_scratch(rank, tid, stream, argc, argv);
+    run_transpose(rank, tid, stream, argc, argv);
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipStreamDestroy(stream));
+}
+
+void
+run_transpose(int rank, int tid, hipStream_t stream, int argc, char** argv)
+{
+    auto* exe_name = basename(argv[0]);
+
+    unsigned int M = 4960 * 2;
+    unsigned int N = 4960 * 2;
+    if(argc > 2) nitr = atoll(argv[2]);
+    if(argc > 3) nsync = atoll(argv[3]);
+
+    auto_lock_t _lk{print_lock};
+    std::cout << "[" << exe_name << "][transpose][" << rank << "][" << tid << "] M: " << M
+              << " N: " << N << std::endl;
+    _lk.unlock();
+
+    std::default_random_engine         _engine{std::random_device{}() * (rank + 1) * (tid + 1)};
+    std::uniform_int_distribution<int> _dist{0, 1000};
+
+    size_t size       = sizeof(int) * M * N;
+    int*   inp_matrix = new int[size];
+    int*   out_matrix = new int[size];
+    for(size_t i = 0; i < M * N; i++)
+    {
+        inp_matrix[i] = _dist(_engine);
+        out_matrix[i] = 0;
+    }
+    int* in  = nullptr;
+    int* out = nullptr;
+
+    HIP_API_CALL(hipMalloc(&in, size));
+    HIP_API_CALL(hipMalloc(&out, size));
+    HIP_API_CALL(hipMemsetAsync(in, 0, size, stream));
+    HIP_API_CALL(hipMemsetAsync(out, 0, size, stream));
+    HIP_API_CALL(hipMemcpyAsync(in, inp_matrix, size, hipMemcpyHostToDevice, stream));
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    dim3 grid(M / 32, N / 32, 1);
+    dim3 block(32, 32, 1);  // transpose
+
+    print_lock.lock();
+    printf("[%s][transpose][%i][%i] grid=(%i,%i,%i), block=(%i,%i,%i)\n",
+           exe_name,
+           rank,
+           tid,
+           grid.x,
+           grid.y,
+           grid.z,
+           block.x,
+           block.y,
+           block.z);
+    print_lock.unlock();
+
+    auto t1 = std::chrono::high_resolution_clock::now();
+    for(size_t i = 0; i < nitr; ++i)
+    {
+        transpose<<<grid, block, 0, stream>>>(in, out, M, N);
+        check_hip_error();
+        if(i % nsync == (nsync - 1)) HIP_API_CALL(hipStreamSynchronize(stream));
+    }
+    auto t2 = std::chrono::high_resolution_clock::now();
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipMemcpyAsync(out_matrix, out, size, hipMemcpyDeviceToHost, stream));
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+    float  GB   = (float) size * nitr * 2 / (1 << 30);
+
+    print_lock.lock();
+    std::cout << "[" << exe_name << "][transpose][" << rank << "][" << tid
+              << "] Runtime of transpose is " << time << " sec\n";
+    std::cout << "[" << exe_name << "][transpose][" << rank << "][" << tid
+              << "] The average performance of transpose is " << GB / time << " GBytes/sec"
+              << std::endl;
+    print_lock.unlock();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    // cpu_transpose(matrix, out_matrix, M, N);
+    verify(inp_matrix, out_matrix, M, N);
+
+    HIP_API_CALL(hipFree(in));
+    HIP_API_CALL(hipFree(out));
+
+    delete[] inp_matrix;
+    delete[] out_matrix;
+}
+
+void
+run_scratch(int rank, int tid, hipStream_t stream, int, char** argv)
+{
+    auto t1 = std::chrono::high_resolution_clock::now();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    const auto* exe_name = basename(argv[0]);
+
+    uint64_t* data_ptr = nullptr;
+    HIP_API_CALL(HIP_HOST_ALLOC_FUNC(&data_ptr, sizeof(uint64_t), 0));
+    *data_ptr = 0;
+
+    test_kern_small<<<1000, 1, 0, stream>>>(data_ptr);
+    test_kern_medium<<<1000, 1, 0, stream>>>(data_ptr);
+    test_kern_small<<<1000, 1, 0, stream>>>(data_ptr);
+    test_kern_large<<<1100, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    test_kern_small<<<1000, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    test_kern_medium<<<1000, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    test_kern_small<<<1000, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    test_kern_large<<<1100, 1, 0, stream>>>(data_ptr);
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    auto   t2   = std::chrono::high_resolution_clock::now();
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+
+    print_lock.lock();
+    std::cout << "[" << exe_name << "][scratch][" << rank << "][" << tid
+              << "] Runtime of scratch is " << time << " sec\n";
+    print_lock.unlock();
+}
+
+void
+run_migrate(int rank, int tid, hipStream_t stream, int, char** argv)
+{
+    using data_type            = uint64_t;
+    constexpr data_type init_v = 1;
+    constexpr data_type incr_v = 1;
+
+    auto t1 = std::chrono::high_resolution_clock::now();
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    const auto* exe_name  = basename(argv[0]);
+    auto        page_data = std::vector<data_type>(1024, 0);
+
+    HIP_API_CALL(hipHostRegister(
+        page_data.data(), page_data.size() * sizeof(data_type), hipHostRegisterDefault));
+
+    for(auto& itr : page_data)
+        itr = init_v;
+
+    test_page_migrate<<<1, 1024, 0, stream>>>(page_data.data(), incr_v);
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+
+    for(auto& itr : page_data)
+    {
+        auto diff = (itr - incr_v);
+        if(diff != init_v)
+        {
+            auto msg = std::stringstream{};
+            msg << "invalid diff: " << diff << ". expected: " << init_v;
+            throw std::runtime_error{msg.str()};
+        }
+    }
+
+    HIP_API_CALL(hipHostUnregister(page_data.data()));
+
+    auto   t2   = std::chrono::high_resolution_clock::now();
+    double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
+
+    print_lock.lock();
+    std::cout << "[" << exe_name << "][migrate][" << rank << "][" << tid
+              << "] Runtime of migrate is " << time << " sec\n";
+    print_lock.unlock();
+}
+
+namespace
+{
+void
+check_hip_error(void)
+{
+    hipError_t err = hipGetLastError();
+    if(err != hipSuccess)
+    {
+        auto_lock_t _lk{print_lock};
+        std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
+        throw std::runtime_error("hip_api_call");
+    }
+}
+
+void
+verify(int* in, int* out, int M, int N)
+{
+    for(int i = 0; i < 10; i++)
+    {
+        int row = rand() % M;
+        int col = rand() % N;
+        if(in[row * N + col] != out[col * M + row])
+        {
+            auto_lock_t _lk{print_lock};
+            std::cout << "mismatch: " << row << ", " << col << " : " << in[row * N + col] << " | "
+                      << out[col * M + row] << "\n";
+        }
+    }
+}
+}  // namespace

projects/rocprofiler-sdk/samples/pc_sampling/pcs.cpp

@@ -0,0 +1,441 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+// undefine NDEBUG so asserts are implemented
+#ifdef NDEBUG
+#    undef NDEBUG
+#endif
+
+#include "pcs.hpp"
+#include "utils.hpp"
+
+#include "common/defines.hpp"
+
+#include <cassert>
+#include <cstdio>
+#include <iomanip>
+#include <memory>
+#include <sstream>
+
+namespace client
+{
+namespace pcs
+{
+// TODO: Since this is used only within the `tool_init`,
+// we are safe using static constructor.
+// It would be nice to make this consistent with the `buffer_ids`.
+tool_agent_info_vec_t gpu_agents = {};
+// The reason for using raw pointers is the following.
+// Sometimes, statically created objects of the client::pcs
+// namespace might be freed prior to the `tool_fini`,
+// meaning `buffer_ids` become unusable inside `tool_fini`.
+// Instead, use raw pointers to control objects deallocation time.
+// TODO: The approach with exporting raw pointers outside of the
+// `pcs` namespace is a temporary solution.
+// Instead, it would be better to encapsulate `buffer_ids` inside the
+// `pcs` namespace and export functions for registering/flushing/destroying buffers.
+pc_sampling_buffer_id_vec_t* buffer_ids = nullptr;
+
+namespace
+{
+constexpr uint64_t host_trap_interval  = 10000;    // 10ms
+constexpr uint64_t stochastic_interval = 1048576;  // 2 ^ 20 cycles
+}  // namespace
+
+void
+init()
+{
+    buffer_ids = new pc_sampling_buffer_id_vec_t();
+}
+
+void
+fini()
+{
+    // Clear the data
+    buffer_ids->clear();
+    delete buffer_ids;
+    buffer_ids = nullptr;
+}
+
+pc_sampling_buffer_id_vec_t*
+get_pc_sampling_buffer_ids()
+{
+    return buffer_ids;
+}
+
+rocprofiler_status_t
+find_all_gpu_agents_supporting_pc_sampling_impl(rocprofiler_agent_version_t version,
+                                                const void**                agents,
+                                                size_t                      num_agents,
+                                                void*                       user_data)
+{
+    assert(version == ROCPROFILER_AGENT_INFO_VERSION_0);
+    // user_data represent the pointer to the array where gpu_agent will be stored
+    if(!user_data) return ROCPROFILER_STATUS_ERROR;
+
+    std::stringstream ss;
+
+    auto* _out_agents = static_cast<tool_agent_info_vec_t*>(user_data);
+    auto* _agents     = reinterpret_cast<const rocprofiler_agent_t**>(agents);
+    for(size_t i = 0; i < num_agents; i++)
+    {
+        if(_agents[i]->type == ROCPROFILER_AGENT_TYPE_GPU)
+        {
+            // Instantiate the tool_agent_info.
+            // Store pointer to the rocprofiler_agent_t and instatiate a vector of
+            // available configurations.
+            // Move the ownership to the _out_agents
+            auto tool_gpu_agent           = std::make_unique<tool_agent_info>();
+            tool_gpu_agent->agent_id      = _agents[i]->id;
+            tool_gpu_agent->avail_configs = std::make_unique<avail_configs_vec_t>();
+            tool_gpu_agent->agent         = _agents[i];
+            // Check if the GPU agent supports PC sampling. If so, add it to the
+            // output list `_out_agents`.
+            if(query_avail_configs_for_agent(tool_gpu_agent.get()))
+                _out_agents->push_back(std::move(tool_gpu_agent));
+        }
+
+        ss << "[" << __FUNCTION__ << "] " << _agents[i]->name << " :: "
+           << "id=" << _agents[i]->id.handle << ", "
+           << "type=" << _agents[i]->type << "\n";
+    }
+
+    *utils::get_output_stream() << ss.str() << "\n";
+
+    return ROCPROFILER_STATUS_SUCCESS;
+}
+
+void
+find_all_gpu_agents_supporting_pc_sampling()
+{
+    // This function returns the all gpu agents supporting some kind of PC sampling
+    ROCPROFILER_CHECK(
+        rocprofiler_query_available_agents(ROCPROFILER_AGENT_INFO_VERSION_0,
+                                           &find_all_gpu_agents_supporting_pc_sampling_impl,
+                                           sizeof(rocprofiler_agent_t),
+                                           static_cast<void*>(&gpu_agents)));
+}
+
+/**
+ * @brief The function queries available PC sampling configurations.
+ * If there is at least one available configuration, it returns true.
+ * Otherwise, this function returns false to indicate the agent does
+ * not support PC sampling.
+ */
+bool
+query_avail_configs_for_agent(tool_agent_info* agent_info)
+{
+    // Clear the available configurations vector
+    agent_info->avail_configs->clear();
+
+    auto cb = [](const rocprofiler_pc_sampling_configuration_t* configs,
+                 size_t                                         num_config,
+                 void*                                          user_data) {
+        auto* avail_configs = static_cast<avail_configs_vec_t*>(user_data);
+        for(size_t i = 0; i < num_config; i++)
+        {
+            avail_configs->emplace_back(configs[i]);
+        }
+        return ROCPROFILER_STATUS_SUCCESS;
+    };
+
+    auto status = rocprofiler_query_pc_sampling_agent_configurations(
+        agent_info->agent_id, cb, agent_info->avail_configs.get());
+
+    std::stringstream ss;
+
+    if(status != ROCPROFILER_STATUS_SUCCESS)
+    {
+        // The query operation failed, so consider the PC sampling is unsupported at the agent.
+        // This can happen if the PC sampling service is invoked within the ROCgdb.
+        ss << "Querying PC sampling capabilities failed with status=" << status
+           << " :: " << rocprofiler_get_status_string(status) << "\n";
+        *utils::get_output_stream() << ss.str() << "\n";
+        return false;
+    }
+    else if(agent_info->avail_configs->empty())
+    {
+        // No available configuration at the moment, so mark the PC sampling as unsupported.
+        return false;
+    }
+
+    ss << "The agent with the id: " << agent_info->agent_id.handle << " supports the "
+       << agent_info->avail_configs->size() << " configurations: "
+       << "\n";
+    size_t ind = 0;
+    for(auto& cfg : *agent_info->avail_configs)
+    {
+        ss << "(" << ++ind << ".) "
+           << "method: " << cfg.method << ", "
+           << "unit: " << cfg.unit << ", "
+           << "min_interval: " << cfg.min_interval << ", "
+           << "max_interval: " << cfg.max_interval << ", "
+           << "flags: " << std::hex << cfg.flags << std::dec
+           << ((cfg.flags == ROCPROFILER_PC_SAMPLING_CONFIGURATION_FLAGS_INTERVAL_POW2)
+                   ? " (an interval value must be power of 2)"
+                   : "")
+           << "\n";
+    }
+
+    *utils::get_output_stream() << ss.str() << std::flush;
+
+    return true;
+}
+
+void
+configure_pc_sampling_prefer_stochastic(tool_agent_info*         agent_info,
+                                        rocprofiler_context_id_t context_id,
+                                        rocprofiler_buffer_id_t  buffer_id)
+{
+    auto   stochastic_picked = false;
+    int    failures          = 10;
+    size_t interval          = 0;
+    do
+    {
+        // Update the list of available configurations
+        auto success = query_avail_configs_for_agent(agent_info);
+        if(!success)
+        {
+            // An error occured while querying PC sampling capabilities,
+            // so avoid trying configuring PC sampling service.
+            // Instead return false to indicated a failure.
+            ROCPROFILER_CHECK(ROCPROFILER_STATUS_ERROR);
+        }
+
+        const rocprofiler_pc_sampling_configuration_t* first_host_trap_config  = nullptr;
+        const rocprofiler_pc_sampling_configuration_t* first_stochastic_config = nullptr;
+        // Search until encountering on the stochastic configuration, if any.
+        // Otherwise, use the host trap config
+        for(auto const& cfg : *agent_info->avail_configs)
+        {
+            if(cfg.method == ROCPROFILER_PC_SAMPLING_METHOD_STOCHASTIC)
+            {
+                first_stochastic_config = &cfg;
+                stochastic_picked       = true;
+                break;
+            }
+            else if(!first_host_trap_config &&
+                    cfg.method == ROCPROFILER_PC_SAMPLING_METHOD_HOST_TRAP)
+            {
+                first_host_trap_config = &cfg;
+            }
+        }
+
+        // Check if the stochastic config is found. Use host trap config otherwise.
+        const rocprofiler_pc_sampling_configuration_t* picked_cfg =
+            (first_stochastic_config != nullptr) ? first_stochastic_config : first_host_trap_config;
+
+        if(picked_cfg->min_interval == picked_cfg->max_interval)
+        {
+            // Another process already configured PC sampling, so use the intreval it set up.
+            interval = picked_cfg->min_interval;
+        }
+        else
+        {
+            interval = stochastic_picked ? stochastic_interval : host_trap_interval;
+        }
+
+        auto status = rocprofiler_configure_pc_sampling_service(context_id,
+                                                                agent_info->agent_id,
+                                                                picked_cfg->method,
+                                                                picked_cfg->unit,
+                                                                interval,
+                                                                buffer_id,
+                                                                0);
+        if(status == ROCPROFILER_STATUS_SUCCESS)
+        {
+            *utils::get_output_stream()
+                << ">>> We chose " << (stochastic_picked ? "stochastic" : "Host-Trap")
+                << " PC sampling with the interval: " << interval << " "
+                << (stochastic_picked ? "clock-cycles" : "micro seconds")
+                << " on the agent: " << agent_info->agent->id.handle << "\n";
+            return;
+        }
+        else if(status != ROCPROFILER_STATUS_ERROR_NOT_AVAILABLE)
+        {
+            ROCPROFILER_CHECK(status);
+        }
+        // status ==  ROCPROFILER_STATUS_ERROR_NOT_AVAILABLE
+        // means another process P2 already configured PC sampling.
+        // Query available configurations again and receive the configurations picked by P2.
+        // However, if P2 destroys PC sampling service after query function finished,
+        // but before the `rocprofiler_configure_pc_sampling_service` is called,
+        // then the `rocprofiler_configure_pc_sampling_service` will fail again.
+        // The process P1 executing this loop can spin wait (starve) if it is unlucky enough
+        // to always be interuppted by some other process P2 that creates/destroys
+        // PC sampling service on the same device while P1 is executing the code
+        // after the `query_avail_configs_for_agent` and
+        // before the `rocprofiler_configure_pc_sampling_service`.
+        // This should happen very rarely, but just to be sure, we introduce a counter `failures`
+        // that will allow certain amount of failures to process P1.
+    } while(--failures);
+
+    // The process failed too many times configuring PC sampling,
+    // report this to user;
+    ROCPROFILER_CHECK(ROCPROFILER_STATUS_ERROR);
+}
+
+template <typename PcSamplingRecordT>
+void
+print_sample_common_fields(std::ostream& os, const PcSamplingRecordT* pc_sample)
+{
+    os << "(code_obj_id, offset): (" << pc_sample->pc.code_object_id << ", 0x" << std::hex
+       << pc_sample->pc.code_object_offset << "), "
+       << "timestamp: " << std::dec << pc_sample->timestamp << ", "
+       << "exec: " << std::hex << std::setw(16) << pc_sample->exec_mask << ", "
+       << "workgroup_id_(x=" << std::dec << std::setw(5) << pc_sample->workgroup_id.x << ", "
+       << "y=" << std::setw(5) << pc_sample->workgroup_id.y << ", "
+       << "z=" << std::setw(5) << pc_sample->workgroup_id.z << "), "
+       << "wave_in_group: " << std::setw(2) << static_cast<unsigned int>(pc_sample->wave_in_group)
+       << ", "
+       << "chiplet: " << std::setw(2) << static_cast<unsigned int>(pc_sample->hw_id.chiplet) << ", "
+       << "dispatch_id: " << std::setw(7) << pc_sample->dispatch_id << ","
+       << "correlation: {internal=" << std::setw(7) << pc_sample->correlation_id.internal << ", "
+       << "external=" << std::setw(5) << pc_sample->correlation_id.external.value << "}, ";
+}
+
+void
+print_sample(std::ostream& os, const rocprofiler_pc_sampling_record_host_trap_v0_t* sample)
+{
+    print_sample_common_fields(os, sample);
+    os << "\n";
+}
+
+void
+print_sample(std::ostream& os, const rocprofiler_pc_sampling_record_stochastic_v0_t* sample)
+{
+    print_sample_common_fields(os, sample);
+
+    if(sample->wave_issued)
+    {
+        auto* inst_c_str = rocprofiler_get_pc_sampling_instruction_type_name(
+            static_cast<rocprofiler_pc_sampling_instruction_type_t>(sample->inst_type));
+        utils::pcs_assert(inst_c_str != nullptr, "Invalid instruction type");
+        os << "wave issued " << std::string(inst_c_str) << " instruction, ";
+    }
+    else
+    {
+        auto* reason_c_str = rocprofiler_get_pc_sampling_instruction_not_issued_reason_name(
+            static_cast<rocprofiler_pc_sampling_instruction_not_issued_reason_t>(
+                sample->snapshot.reason_not_issued));
+        utils::pcs_assert(reason_c_str != nullptr, "Invalid not issued reason");
+        os << "wave is stalled due to: " << std::string(reason_c_str) << " reason, ";
+    }
+
+    auto snapshot = sample->snapshot;
+    os << "two VALU instructions issued: " << static_cast<unsigned int>(snapshot.dual_issue_valu)
+       << ", ";
+
+    os << "arbiter state: {pipe issued: ("
+       << "VALU: " << static_cast<unsigned int>(snapshot.arb_state_issue_valu) << ", "
+       << "MATRIX: " << static_cast<unsigned int>(snapshot.arb_state_issue_matrix) << ", "
+       << "LDS: " << static_cast<unsigned int>(snapshot.arb_state_issue_lds) << ", "
+       << "LDS_DIRECT: " << static_cast<unsigned int>(snapshot.arb_state_issue_lds_direct) << ", "
+       << "SCALAR: " << static_cast<unsigned int>(snapshot.arb_state_issue_scalar) << ", "
+       << "TEX: " << static_cast<unsigned int>(snapshot.arb_state_issue_vmem_tex) << ", "
+       << "FLAT: " << static_cast<unsigned int>(snapshot.arb_state_issue_flat) << ", "
+       << "EXPORT: " << static_cast<unsigned int>(snapshot.arb_state_issue_exp) << ", "
+       << "MISC: " << static_cast<unsigned int>(snapshot.arb_state_issue_misc) << "), "
+       << "pipe stalled: ("
+       << "VALU: " << static_cast<unsigned int>(snapshot.arb_state_stall_valu) << ", "
+       << "MATRIX: " << static_cast<unsigned int>(snapshot.arb_state_stall_matrix) << ", "
+       << "LDS: " << static_cast<unsigned int>(snapshot.arb_state_stall_lds) << ", "
+       << "LDS_DIRECT: " << static_cast<unsigned int>(snapshot.arb_state_stall_lds_direct) << ", "
+       << "SCALAR: " << static_cast<unsigned int>(snapshot.arb_state_stall_scalar) << ", "
+       << "TEX: " << static_cast<unsigned int>(snapshot.arb_state_stall_vmem_tex) << ", "
+       << "FLAT: " << static_cast<unsigned int>(snapshot.arb_state_stall_flat) << ", "
+       << "EXPORT: " << static_cast<unsigned int>(snapshot.arb_state_stall_exp) << ", "
+       << "MISC: " << static_cast<unsigned int>(snapshot.arb_state_stall_misc) << ")}";
+
+    os << "\n";
+}
+
+void
+print_sample(std::ostream& os, const rocprofiler_pc_sampling_record_invalid_t* /*sample*/)
+{
+    os << "Invalid sample detected.\n";
+}
+
+void
+rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
+                                 rocprofiler_buffer_id_t /*buffer_id*/,
+                                 rocprofiler_record_header_t** headers,
+                                 size_t                        num_headers,
+                                 void* /*data*/,
+                                 uint64_t drop_count)
+{
+    std::stringstream ss;
+    ss << "The number of delivered samples is: " << num_headers << ", "
+       << "while the number of dropped samples is: " << drop_count << "\n";
+
+    for(size_t i = 0; i < num_headers; i++)
+    {
+        auto* cur_header = headers[i];
+
+        if(cur_header == nullptr)
+        {
+            throw std::runtime_error{
+                "rocprofiler provided a null pointer to header. this should never happen"};
+        }
+        else if(cur_header->hash !=
+                rocprofiler_record_header_compute_hash(cur_header->category, cur_header->kind))
+        {
+            throw std::runtime_error{"rocprofiler_record_header_t (category | kind) != hash"};
+        }
+        else if(cur_header->category == ROCPROFILER_BUFFER_CATEGORY_PC_SAMPLING)
+        {
+            if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_HOST_TRAP_V0_SAMPLE)
+            {
+                auto* pc_sample = static_cast<rocprofiler_pc_sampling_record_host_trap_v0_t*>(
+                    cur_header->payload);
+
+                print_sample(ss, pc_sample);
+            }
+            else if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_STOCHASTIC_V0_SAMPLE)
+            {
+                auto* pc_sample = static_cast<rocprofiler_pc_sampling_record_stochastic_v0_t*>(
+                    cur_header->payload);
+
+                print_sample(ss, pc_sample);
+            }
+            else if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_INVALID_SAMPLE)
+            {
+                auto* pc_sample =
+                    static_cast<rocprofiler_pc_sampling_record_invalid_t*>(cur_header->payload);
+
+                print_sample(ss, pc_sample);
+            }
+            else
+            {
+                assert(false);
+            }
+        }
+        else
+        {
+            throw std::runtime_error{"unexpected rocprofiler_record_header_t category + kind"};
+        }
+    }
+
+    *utils::get_output_stream() << ss.str() << "\n";
+}
+}  // namespace pcs
+}  // namespace client

projects/rocprofiler-sdk/samples/pc_sampling/pcs.hpp

@@ -0,0 +1,91 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <rocprofiler-sdk/fwd.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#include <memory>
+#include <vector>
+
+namespace client
+{
+namespace pcs
+{
+constexpr size_t BUFFER_SIZE_BYTES = 8192;
+constexpr size_t WATERMARK         = (BUFFER_SIZE_BYTES / 4);
+
+struct tool_agent_info;
+using avail_configs_vec_t         = std::vector<rocprofiler_pc_sampling_configuration_t>;
+using tool_agent_info_vec_t       = std::vector<std::unique_ptr<tool_agent_info>>;
+using pc_sampling_buffer_id_vec_t = std::vector<rocprofiler_buffer_id_t>;
+
+struct tool_agent_info
+{
+    rocprofiler_agent_id_t               agent_id;
+    std::unique_ptr<avail_configs_vec_t> avail_configs;
+    const rocprofiler_agent_t*           agent;
+};
+
+// GPU agents supporting some kind of PC sampling.
+// Note that for some of these agent, the corresponding context might be invalid,
+// meaning we were not able to enable PC sampling service.
+// Check the `tool_init` for more information.
+extern tool_agent_info_vec_t gpu_agents;
+
+// Must be called first (prior to any other function from this namespace)
+void
+init();
+
+// Must be called at the end of the `tool_fini`
+void
+fini();
+
+// Ids of the buffers used as containers for PC sampling records
+pc_sampling_buffer_id_vec_t*
+get_pc_sampling_buffer_ids();
+
+void
+find_all_gpu_agents_supporting_pc_sampling();
+
+/**
+ * @brief The return value indicates if the agent supports PC sampling.
+ * Check the implementation for more info.
+ */
+bool
+query_avail_configs_for_agent(tool_agent_info* agent_info);
+
+void
+configure_pc_sampling_prefer_stochastic(tool_agent_info*         agent_info,
+                                        rocprofiler_context_id_t context_id,
+                                        rocprofiler_buffer_id_t  buffer_id);
+
+void
+rocprofiler_pc_sampling_callback(rocprofiler_context_id_t      context_id,
+                                 rocprofiler_buffer_id_t       buffer_id,
+                                 rocprofiler_record_header_t** headers,
+                                 size_t                        num_headers,
+                                 void*                         data,
+                                 uint64_t                      drop_count);
+}  // namespace pcs
+}  // namespace client

projects/rocprofiler-sdk/samples/pc_sampling/utils.cpp

@@ -0,0 +1,51 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "utils.hpp"
+
+namespace client
+{
+namespace utils
+{
+std::ostream*&
+get_output_stream()
+{
+    // The output strea is initially unitialized
+    static std::ostream* _v = nullptr;
+    return _v;
+}
+
+/**
+ * @brief Shows @p error_msg and aborts if @p condition is false.
+ *
+ */
+void
+pcs_assert(bool condition, std::string_view error_msg)
+{
+    if(!condition)
+    {
+        std::cerr << "PC Sampling Assertion Error: " << error_msg << "\n";
+        abort();
+    }
+}
+}  // namespace utils
+}  // namespace client

projects/rocprofiler-sdk/samples/pc_sampling/utils.hpp

@@ -0,0 +1,39 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <rocprofiler-sdk/fwd.h>
+
+#include <iostream>
+
+namespace client
+{
+namespace utils
+{
+std::ostream*&
+get_output_stream();
+
+void
+pcs_assert(bool condition, std::string_view error_msg);
+}  // namespace utils
+}  // namespace client