Callback based handler for counter collection (#506)

* Callback based handler for counter collection * source formatting (clang-format v11) (#507) Co-authored-by: bwelton <bwelton@users.noreply.github.com> * cmake formatting (cmake-format) (#508) Co-authored-by: bwelton <bwelton@users.noreply.github.com> * Doc fix * Minor doc fix * More doc fixes * More doc fixes * More doc fixes * Update CI * Changes to the API per comments * Mutex exception for HSA * source formatting (clang-format v11) (#511) Co-authored-by: bwelton <bwelton@users.noreply.github.com> * Doc fix --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: bwelton <bwelton@users.noreply.github.com> [ROCm/rocprofiler-sdk commit: 3638351b4c]
2024-02-19 15:55:21 -08:00
parent 56d1023338
commit cb6e12ab3a
9 changed files with 494 additions and 16 deletions
@@ -35,7 +35,7 @@ jobs:

    steps:
      - id: get_build_number
-        run: echo "LATEST_BUILD_NUMBER=581" >> $GITHUB_OUTPUT
+        run: echo "LATEST_BUILD_NUMBER=$(wget -qO- 'http://rocm-ci.amd.com/job/compute-rocm-dkms-component-staging-profiler/lastSuccessfulBuild/buildNumber')" >> $GITHUB_OUTPUT

  # run: echo "LATEST_BUILD_NUMBER=$(wget -qO- 'http://rocm-ci.amd.com/job/compute-rocm-dkms-component-staging-profiler/lastSuccessfulBuild/buildNumber')" >> $GITHUB_OUTPUT
  # Changed job name from mi200-ubuntu to vega20-ubuntu
@@ -54,6 +54,34 @@ set_tests_properties(
        FAIL_REGULAR_EXPRESSION
        "threw an exception")

+add_library(counter-collection-callback-client SHARED)
+target_sources(counter-collection-callback-client PRIVATE callback_client.cpp client.hpp)
+target_link_libraries(
+    counter-collection-callback-client
+    PUBLIC rocprofiler::samples-build-flags
+    PRIVATE rocprofiler-sdk::rocprofiler-sdk rocprofiler::samples-common-library)
+
+set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
+add_executable(counter-collection-callback)
+target_sources(counter-collection-callback PRIVATE main.cpp)
+target_link_libraries(counter-collection-callback
+                      PRIVATE counter-collection-callback-client Threads::Threads)
+
+add_test(NAME counter-collection-callback
+         COMMAND $<TARGET_FILE:counter-collection-callback>)
+
+set_tests_properties(
+    counter-collection-callback
+    PROPERTIES
+        TIMEOUT
+        600
+        LABELS
+        "samples"
+        ENVIRONMENT
+        "${ROCPROFILER_MEMCHECK_PRELOAD_ENV};HSA_TOOLS_LIB=$<TARGET_FILE:rocprofiler::rocprofiler-shared-library>"
+        FAIL_REGULAR_EXPRESSION
+        "threw an exception")
+
 add_library(counter-collection-functional-counter-client SHARED)
 target_sources(counter-collection-functional-counter-client
               PRIVATE print_functional_counters.cpp client.hpp)
@@ -0,0 +1,246 @@
+// MIT License
+//
+// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "client.hpp"
+
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <mutex>
+#include <set>
+#include <shared_mutex>
+#include <sstream>
+#include <unordered_map>
+#include <vector>
+
+#include <rocprofiler-sdk/registration.h>
+#include <rocprofiler-sdk/rocprofiler.h>
+
+#define ROCPROFILER_CALL(result, msg)                                                              \
+    {                                                                                              \
+        rocprofiler_status_t CHECKSTATUS = result;                                                 \
+        if(CHECKSTATUS != ROCPROFILER_STATUS_SUCCESS)                                              \
+        {                                                                                          \
+            std::string status_msg = rocprofiler_get_status_string(CHECKSTATUS);                   \
+            std::cerr << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg            \
+                      << " failed with error code " << CHECKSTATUS << ": " << status_msg           \
+                      << std::endl;                                                                \
+            std::stringstream errmsg{};                                                            \
+            errmsg << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " << msg " failure ("  \
+                   << status_msg << ")";                                                           \
+            throw std::runtime_error(errmsg.str());                                                \
+        }                                                                                          \
+    }
+
+int
+start()
+{
+    return 1;
+}
+
+namespace
+{
+rocprofiler_context_id_t&
+get_client_ctx()
+{
+    static rocprofiler_context_id_t ctx;
+    return ctx;
+}
+
+void
+record_callback(rocprofiler_queue_id_t,
+                rocprofiler_agent_id_t,
+                rocprofiler_correlation_id_t,
+                uint64_t,
+                void*                         callback_data_args,
+                size_t                        record_count,
+                rocprofiler_record_counter_t* record_data)
+{
+    std::stringstream ss;
+    for(size_t i = 0; i < record_count; ++i)
+    {
+        ss << "(Id: " << record_data[i].id << " Value [D]: " << record_data[i].counter_value
+           << " Corr_Id: " << record_data[i].corr_id.internal << "),";
+    }
+    auto* output_stream = static_cast<std::ostream*>(callback_data_args);
+    if(!output_stream) throw std::runtime_error{"nullptr to output stream"};
+    *output_stream << "[" << __FUNCTION__ << "] " << ss.str() << "\n";
+}
+
+/**
+ * Callback from rocprofiler when an kernel dispatch is enqueued into the HSA queue.
+ * rocprofiler_profile_config_id_t* is a return to specify what counters to collect
+ * for this dispatch (dispatch_packet). This example function creates a profile
+ * to collect the counter SQ_WAVES for all kernel dispatch packets.
+ */
+void
+dispatch_callback(rocprofiler_queue_id_t /*queue_id*/,
+                  const rocprofiler_agent_t* agent,
+                  rocprofiler_correlation_id_t /*correlation_id*/,
+                  const hsa_kernel_dispatch_packet_t* /*dispatch_packet*/,
+                  uint64_t /*kernel_id*/,
+                  void* /*callback_data_args*/,
+                  rocprofiler_profile_config_id_t* config)
+{
+    /**
+     * This simple example uses the same profile counter set for all agents.
+     * We store this in a cache to prevent constructing many identical profile counter
+     * sets. We first check the cache to see if we have already constructed a counter"
+     * set for the agent. If we have, return it. Otherwise, construct a new profile counter
+     * set.
+     */
+    static std::shared_mutex                                             m_mutex       = {};
+    static std::unordered_map<uint64_t, rocprofiler_profile_config_id_t> profile_cache = {};
+
+    auto search_cache = [&]() {
+        if(auto pos = profile_cache.find(agent->id.handle); pos != profile_cache.end())
+        {
+            *config = pos->second;
+            return true;
+        }
+        return false;
+    };
+
+    {
+        auto rlock = std::shared_lock{m_mutex};
+        if(search_cache()) return;
+    }
+
+    auto wlock = std::unique_lock{m_mutex};
+    if(search_cache()) return;
+
+    // Counters we want to collect (here its SQ_WAVES)
+    std::set<std::string> counters_to_collect = {"SQ_WAVES"};
+    // GPU Counter IDs
+    std::vector<rocprofiler_counter_id_t> gpu_counters;
+
+    // Iterate through the agents and get the counters available on that agent
+    ROCPROFILER_CALL(rocprofiler_iterate_agent_supported_counters(
+                         agent->id,
+                         [](rocprofiler_agent_id_t,
+                            rocprofiler_counter_id_t* counters,
+                            size_t                    num_counters,
+                            void*                     user_data) {
+                             std::vector<rocprofiler_counter_id_t>* vec =
+                                 static_cast<std::vector<rocprofiler_counter_id_t>*>(user_data);
+                             for(size_t i = 0; i < num_counters; i++)
+                             {
+                                 vec->push_back(counters[i]);
+                             }
+                             return ROCPROFILER_STATUS_SUCCESS;
+                         },
+                         static_cast<void*>(&gpu_counters)),
+                     "Could not fetch supported counters");
+
+    std::vector<rocprofiler_counter_id_t> collect_counters;
+    // Look for the counters contained in counters_to_collect in gpu_counters
+    for(auto& counter : gpu_counters)
+    {
+        const char* name;
+        size_t      size;
+        ROCPROFILER_CALL(rocprofiler_query_counter_name(counter, &name, &size),
+                         "Could not query name");
+        if(counters_to_collect.count(std::string(name)) > 0)
+        {
+            std::clog << "Counter: " << counter.handle << " " << name << "\n";
+            collect_counters.push_back(counter);
+        }
+    }
+
+    // Create a colleciton profile for the counters
+    rocprofiler_profile_config_id_t profile;
+    ROCPROFILER_CALL(rocprofiler_create_profile_config(
+                         agent->id, collect_counters.data(), collect_counters.size(), &profile),
+                     "Could not construct profile cfg");
+
+    profile_cache.emplace(agent->id.handle, profile);
+    // Return the profile to collect those counters for this dispatch
+    *config = profile;
+}
+
+int
+tool_init(rocprofiler_client_finalize_t, void* user_data)
+{
+    ROCPROFILER_CALL(rocprofiler_create_context(&get_client_ctx()), "context creation failed");
+
+    ROCPROFILER_CALL(rocprofiler_configure_callback_dispatch_profile_counting_service(
+                         get_client_ctx(), dispatch_callback, nullptr, record_callback, user_data),
+                     "Could not setup counting service");
+    ROCPROFILER_CALL(rocprofiler_start_context(get_client_ctx()), "start context");
+
+    // no errors
+    return 0;
+}
+
+void
+tool_fini(void* user_data)
+{
+    std::clog << "In tool fini\n";
+    rocprofiler_stop_context(get_client_ctx());
+
+    auto* output_stream = static_cast<std::ostream*>(user_data);
+    *output_stream << std::flush;
+    if(output_stream != &std::cout && output_stream != &std::cerr) delete output_stream;
+}
+}  // namespace
+
+extern "C" rocprofiler_tool_configure_result_t*
+rocprofiler_configure(uint32_t    version,
+                      const char* runtime_version,
+                      uint32_t,
+                      rocprofiler_client_id_t* id)
+{
+    // set the client name
+    id->name = "CounterClientSample";
+
+    // compute major/minor/patch version info
+    uint32_t major = version / 10000;
+    uint32_t minor = (version % 10000) / 100;
+    uint32_t patch = version % 100;
+
+    // generate info string
+    auto info = std::stringstream{};
+    info << id->name << " is using rocprofiler-sdk v" << major << "." << minor << "." << patch
+         << " (" << runtime_version << ")";
+
+    std::clog << info.str() << std::endl;
+
+    std::ostream* output_stream = nullptr;
+    std::string   filename      = "counter_collection.log";
+    if(auto* outfile = getenv("ROCPROFILER_SAMPLE_OUTPUT_FILE"); outfile) filename = outfile;
+    if(filename == "stdout")
+        output_stream = &std::cout;
+    else if(filename == "stderr")
+        output_stream = &std::cerr;
+    else
+        output_stream = new std::ofstream{filename};
+
+    // create configure data
+    static auto cfg =
+        rocprofiler_tool_configure_result_t{sizeof(rocprofiler_tool_configure_result_t),
+                                            &tool_init,
+                                            &tool_fini,
+                                            static_cast<void*>(output_stream)};
+
+    // return pointer to configure data
+    return &cfg;
+}
@@ -94,6 +94,49 @@ rocprofiler_configure_buffered_dispatch_profile_counting_service(
    rocprofiler_buffer_id_t                          buffer_id,
    rocprofiler_profile_counting_dispatch_callback_t callback,
    void*                                            callback_data_args);
+
+/**
+ * @brief Counting record callback. This is a callback is invoked when the kernel
+ *        execution is complete and contains the counter profile data requested in
+ *        @ref rocprofiler_profile_counting_dispatch_callback_t. Only used with
+ *        @ref rocprofiler_configure_callback_dispatch_profile_counting_service.
+ *
+ * @param [in] queue_id           Queue the kernel dispatch packet is being enqueued onto
+ * @param [in] agent              Agent of this queue
+ * @param [in] correlation_id     Correlation ID for this dispatch
+ * @param [in] kernel_id          Kernel identifier
+ * @param [in] callback_data_args Callback supplied via buffered_dispatch_profile_counting_service
+ * @param [in] record_count       Number of counter records.
+ * @param [in] record_data        Counter record data.
+ */
+typedef void (*rocprofiler_profile_counting_record_callback_t)(
+    rocprofiler_queue_id_t        queue_id,
+    rocprofiler_agent_id_t        agent,
+    rocprofiler_correlation_id_t  correlation_id,
+    uint64_t                      kernel_id,
+    void*                         callback_data_args,
+    size_t                        record_count,
+    rocprofiler_record_counter_t* record_data);
+
+/**
+ * @brief Configure buffered dispatch profile Counting Service.
+ *        Collects the counters in dispatch packets and calls a callback
+ *        with the counters collected during that dispatch.
+ *
+ * @param [in] context_id context id
+ * @param [in] dispatch_callback callback to perform when dispatch is enqueued
+ * @param [in] dispatch_callback_args callback data for dispatch callback
+ * @param [in] record_callback  Record callback for completed profile data
+ * @param [in] record_callback_args Callback args for record callback
+ * @return ::rocprofiler_status_t
+ */
+rocprofiler_status_t ROCPROFILER_API
+rocprofiler_configure_callback_dispatch_profile_counting_service(
+    rocprofiler_context_id_t                         context_id,
+    rocprofiler_profile_counting_dispatch_callback_t dispatch_callback,
+    void*                                            dispatch_callback_args,
+    rocprofiler_profile_counting_record_callback_t   record_callback,
+    void*                                            record_callback_args);
 /** @} */

 ROCPROFILER_EXTERN_C_FINI
@@ -22,6 +22,7 @@

 #include "lib/rocprofiler-sdk/counters/core.hpp"

+#include "lib/common/container/small_vector.hpp"
 #include "lib/common/synchronized.hpp"
 #include "lib/rocprofiler-sdk/agent.hpp"
 #include "lib/rocprofiler-sdk/aql/helpers.hpp"
@@ -74,7 +75,9 @@ public:
    static bool configure_dispatch(rocprofiler_context_id_t                         context_id,
                                   rocprofiler_buffer_id_t                          buffer,
                                   rocprofiler_profile_counting_dispatch_callback_t callback,
-                                   void*                                            callback_args)
+                                   void*                                            callback_args,
+                                   rocprofiler_profile_counting_record_callback_t   record_callback,
+                                   void* record_callback_args)
    {
        auto* ctx_p = rocprofiler::context::get_mutable_registered_context(context_id);
        if(!ctx_p) return false;
@@ -90,11 +93,16 @@ public:
        auto& cb = *ctx.counter_collection->callbacks.emplace_back(
            std::make_shared<counter_callback_info>());

-        cb.user_cb          = callback;
-        cb.callback_args    = callback_args;
-        cb.context          = context_id;
-        cb.buffer           = buffer;
-        cb.internal_context = ctx_p;
+        cb.user_cb       = callback;
+        cb.callback_args = callback_args;
+        cb.context       = context_id;
+        if(buffer.handle != 0)
+        {
+            cb.buffer = buffer;
+        }
+        cb.internal_context     = ctx_p;
+        cb.record_callback      = record_callback;
+        cb.record_callback_args = record_callback_args;

        return true;
    }
@@ -317,7 +325,7 @@ queue_cb(const std::shared_ptr<counter_callback_info>&                   info,
 */
 void
 completed_cb(const std::shared_ptr<counter_callback_info>& info,
-             const hsa::Queue&,
+             const hsa::Queue&                             queue,
             hsa::rocprofiler_packet,
             const hsa::Queue::queue_info_session_t& session,
             inst_pkt_t&                             pkts)
@@ -359,12 +367,13 @@ completed_cb(const std::shared_ptr<counter_callback_info>& info,
        }
    });

-    if(!info->buffer) return;
+    common::container::small_vector<rocprofiler_record_counter_t, 128> out;
+    rocprofiler::buffer::instance*                                     buf = nullptr;

-    std::vector<rocprofiler_record_counter_t> out;
-    rocprofiler::buffer::instance*            buf = nullptr;
-
-    buf = CHECK_NOTNULL(buffer::get_buffer(info->buffer->handle));
+    if(info->buffer)
+    {
+        buf = CHECK_NOTNULL(buffer::get_buffer(info->buffer->handle));
+    }

    auto _corr_id_v =
        rocprofiler_correlation_id_t{.internal = 0, .external = context::null_user_data};
@@ -388,9 +397,24 @@ completed_cb(const std::shared_ptr<counter_callback_info>& info,
        for(auto& val : *ret)
        {
            val.corr_id = _corr_id_v;
-            buf->emplace(ROCPROFILER_BUFFER_CATEGORY_COUNTERS, 0, val);
+            if(buf)
+                buf->emplace(ROCPROFILER_BUFFER_CATEGORY_COUNTERS, 0, val);
+            else
+                out.push_back(val);
        }
    }
+
+    if(!out.empty())
+    {
+        CHECK(info->record_callback);
+        info->record_callback(queue.get_id(),
+                              queue.get_agent().get_rocp_agent()->id,
+                              _corr_id_v,
+                              session.kernel_id,
+                              info->record_callback_args,
+                              out.size(),
+                              out.data());
+    }
 }

 void
@@ -451,7 +475,24 @@ configure_buffered_dispatch(rocprofiler_context_id_t                         con
                            rocprofiler_profile_counting_dispatch_callback_t callback,
                            void*                                            callback_args)
 {
-    return get_controller().configure_dispatch(context_id, buffer, callback, callback_args);
+    CHECK_NE(buffer.handle, 0);
+    return get_controller().configure_dispatch(
+        context_id, buffer, callback, callback_args, nullptr, nullptr);
+}
+
+bool
+configure_callback_dispatch(rocprofiler_context_id_t                         context_id,
+                            rocprofiler_profile_counting_dispatch_callback_t callback,
+                            void*                                            callback_data_args,
+                            rocprofiler_profile_counting_record_callback_t   record_callback,
+                            void*                                            record_callback_args)
+{
+    return get_controller().configure_dispatch(context_id,
+                                               {.handle = 0},
+                                               callback,
+                                               callback_data_args,
+                                               record_callback,
+                                               record_callback_args);
 }

 }  // namespace counters
@@ -84,6 +84,9 @@ struct counter_callback_info
    // Buffer to use for storing counter data. Used if callback is not set.
    std::optional<rocprofiler_buffer_id_t> buffer;

+    rocprofiler_profile_counting_record_callback_t record_callback;
+    void*                                          record_callback_args;
+
    // Facilitates the return of an AQL Packet to the profile config that constructed it.
    rocprofiler::common::Synchronized<
        std::unordered_map<rocprofiler::hsa::AQLPacket*, std::shared_ptr<profile_config>>>
@@ -109,6 +112,13 @@ configure_buffered_dispatch(rocprofiler_context_id_t                         con
                            rocprofiler_profile_counting_dispatch_callback_t callback,
                            void*                                            callback_args);

+bool
+configure_callback_dispatch(rocprofiler_context_id_t                         context_id,
+                            rocprofiler_profile_counting_dispatch_callback_t callback,
+                            void*                                            callback_data_args,
+                            rocprofiler_profile_counting_record_callback_t   record_callback,
+                            void*                                            record_callback_args);
+
 void
 start_context(const context::context*);

@@ -222,6 +222,16 @@ null_buffered_callback(rocprofiler_context_id_t,
                       uint64_t)
 {}

+void
+null_record_callback(rocprofiler_queue_id_t,
+                     rocprofiler_agent_id_t,
+                     rocprofiler_correlation_id_t,
+                     uint64_t,
+                     void*,
+                     size_t,
+                     rocprofiler_record_counter_t*)
+{}
+
 }  // namespace

 TEST(core, check_packet_generation)
@@ -528,7 +538,7 @@ TEST(core, destroy_counter_profile)
    registration::finalize();
 }

-TEST(core, start_stop_ctx)
+TEST(core, start_stop_buffered_ctx)
 {
    ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS);
    test_init();
@@ -606,6 +616,75 @@ TEST(core, start_stop_ctx)
    registration::finalize();
 }

+TEST(core, start_stop_callback_ctx)
+{
+    registration::init_logging();
+    registration::set_init_status(-1);
+    context::push_client(1);
+
+    ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS);
+    test_init();
+
+    ROCPROFILER_CALL(rocprofiler_create_context(&get_client_ctx()), "context creation failed");
+
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_dispatch_profile_counting_service(get_client_ctx(),
+                                                                         null_dispatch_callback,
+                                                                         (void*) 0x12345,
+                                                                         null_record_callback,
+                                                                         (void*) 0x54321),
+        "Could not setup counting service");
+    ROCPROFILER_CALL(rocprofiler_start_context(get_client_ctx()), "start context");
+
+    /**
+     * Check that the context was actually started
+     */
+    auto* ctx_p = context::get_mutable_registered_context(get_client_ctx());
+    ASSERT_TRUE(ctx_p);
+    auto& ctx = *ctx_p;
+
+    ASSERT_TRUE(ctx.counter_collection);
+    ASSERT_EQ(ctx.counter_collection->callbacks.size(), 1);
+    EXPECT_EQ(ctx.counter_collection->callbacks.at(0)->user_cb, null_dispatch_callback);
+    EXPECT_EQ(ctx.counter_collection->callbacks.at(0)->callback_args, (void*) 0x12345);
+    EXPECT_EQ(ctx.counter_collection->callbacks.at(0)->record_callback, null_record_callback);
+    EXPECT_EQ(ctx.counter_collection->callbacks.at(0)->record_callback_args, (void*) 0x54321);
+    EXPECT_EQ(ctx.counter_collection->callbacks.at(0)->context.handle, get_client_ctx().handle);
+
+    bool found = false;
+    ctx.counter_collection->enabled.rlock([&](const auto& data) { found = data; });
+    EXPECT_TRUE(found);
+
+    found = false;
+    hsa::get_queue_controller().iterate_callbacks([&](auto cid, const auto&) {
+        if(cid == ctx.counter_collection->callbacks.at(0)->queue_id)
+        {
+            found = true;
+        }
+    });
+    EXPECT_TRUE(found);
+
+    /**
+     * Check if context can be disabled correctly
+     */
+    ROCPROFILER_CALL(rocprofiler_stop_context(get_client_ctx()), "stop context");
+
+    found = false;
+    hsa::get_queue_controller().iterate_callbacks([&](auto cid, const auto&) {
+        if(cid == ctx.counter_collection->callbacks.at(0)->queue_id)
+        {
+            found = true;
+        }
+    });
+    EXPECT_FALSE(found);
+    found = false;
+    ctx.counter_collection->enabled.rlock([&](const auto& data) { found = data; });
+    EXPECT_FALSE(found);
+
+    registration::set_init_status(1);
+    context::pop_client(1);
+}
+
 TEST(core, public_api_iterate_agents)
 {
    ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS);
@@ -53,4 +53,33 @@ rocprofiler_configure_buffered_dispatch_profile_counting_service(
               ? ROCPROFILER_STATUS_SUCCESS
               : ROCPROFILER_STATUS_ERROR;
 }
+
+/**
+ * @brief Configure buffered dispatch profile Counting Service.
+ *        Collects the counters in dispatch packets and calls a callback
+ *        with the counters collected during that dispatch.
+ *
+ * @param [in] context_id context id
+ * @param [in] dispatch_callback callback to perform when dispatch is enqueued
+ * @param [in] dispatch_callback_args callback data for dispatch callback
+ * @param [in] record_callback  Record callback for completed profile data
+ * @param [in] record_callback_args Callback args for record callback
+ * @return ::rocprofiler_status_t
+ */
+rocprofiler_status_t ROCPROFILER_API
+rocprofiler_configure_callback_dispatch_profile_counting_service(
+    rocprofiler_context_id_t                         context_id,
+    rocprofiler_profile_counting_dispatch_callback_t dispatch_callback,
+    void*                                            dispatch_callback_args,
+    rocprofiler_profile_counting_record_callback_t   record_callback,
+    void*                                            record_callback_args)
+{
+    return rocprofiler::counters::configure_callback_dispatch(context_id,
+                                                              dispatch_callback,
+                                                              dispatch_callback_args,
+                                                              record_callback,
+                                                              record_callback_args)
+               ? ROCPROFILER_STATUS_SUCCESS
+               : ROCPROFILER_STATUS_ERROR;
+}
 }
@@ -10,6 +10,8 @@ race:libamdhip64.so

 # data race arising from hsa runtime
 race:libhsa-runtime64.so
+# unlock of an unlocked mutex (or by a wrong thread)
+mutex:libhsa-runtime64.so

 # unlock of an unlocked mutex (or by a wrong thread)
 mutex:librocm_smi64.so