Async memory copy callback tracing + memory copy size (#791)

* Async memory copy tracing update

- rocprofiler_buffer_tracing_memory_copy_record_t: thread_id and bytes
- support ROCPROFILER_CALLBACK_TRACING_MEMORY_COPY
- init_public_api_struct can fully construct

* Testing for callback async copy tracing

tests/async-copy-tracing/conftest.py

@@ -3,6 +3,8 @@
 import json
 import pytest
 
+from rocprofiler_sdk.pytest_utils.dotdict import dotdict
+
 
 def pytest_addoption(parser):
     parser.addoption(
@@ -17,4 +19,4 @@ def pytest_addoption(parser):
 def input_data(request):
     filename = request.config.getoption("--input")
     with open(filename, "r") as inp:
-        return json.load(inp)
+        return dotdict(json.load(inp))

tests/async-copy-tracing/validate.py

@@ -9,7 +9,7 @@ def node_exists(name, data, min_len=1):
     assert name in data
     assert data[name] is not None
     if isinstance(data[name], (list, tuple, dict, set)):
-        assert len(data[name]) >= min_len
+        assert len(data[name]) >= min_len, f"{name}:\n{data}"
 
 
 def test_data_structure(input_data):
@@ -38,10 +38,11 @@ def test_data_structure(input_data):
     node_exists("hip_api_traces", sdk_data["callback_records"], 0)
     node_exists("marker_api_traces", sdk_data["callback_records"])
     node_exists("kernel_dispatches", sdk_data["callback_records"])
+    node_exists("memory_copies", sdk_data["callback_records"], 24)
 
     node_exists("names", sdk_data["buffer_records"])
     node_exists("kernel_dispatches", sdk_data["buffer_records"])
-    node_exists("memory_copies", sdk_data["buffer_records"], 4)
+    node_exists("memory_copies", sdk_data["buffer_records"], 12)
     node_exists("hsa_api_traces", sdk_data["buffer_records"])
     node_exists("hip_api_traces", sdk_data["buffer_records"], 0)
     node_exists("marker_api_traces", sdk_data["buffer_records"])
@@ -171,6 +172,10 @@ def test_external_correlation_ids(input_data):
             assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
             assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
 
+        for itr in sdk_data["callback_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
+
 
 def test_kernel_ids(input_data):
     data = input_data
@@ -205,7 +210,7 @@ def test_kernel_dispatch_ids(input_data):
     num_dispatches = len(sdk_data["buffer_records"]["kernel_dispatches"])
     num_cb_dispatches = len(sdk_data["callback_records"]["kernel_dispatches"])
 
-    assert num_cb_dispatches == (2 * num_dispatches)
+    assert num_cb_dispatches == (3 * num_dispatches)
 
     bf_seq_ids = []
     for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
@@ -218,7 +223,7 @@ def test_kernel_dispatch_ids(input_data):
     bf_seq_ids = sorted(bf_seq_ids)
     cb_seq_ids = sorted(cb_seq_ids)
 
-    assert (2 * len(bf_seq_ids)) == len(cb_seq_ids)
+    assert (3 * len(bf_seq_ids)) == len(cb_seq_ids)
 
     assert bf_seq_ids[0] == cb_seq_ids[0]
     assert bf_seq_ids[-1] == cb_seq_ids[-1]
@@ -230,7 +235,7 @@ def test_kernel_dispatch_ids(input_data):
     cb_seq_ids_uniq = get_uniq(cb_seq_ids)
 
     assert bf_seq_ids == bf_seq_ids_uniq
-    assert len(cb_seq_ids) == (2 * len(cb_seq_ids_uniq))
+    assert len(cb_seq_ids) == (3 * len(cb_seq_ids_uniq))
     assert len(bf_seq_ids) == num_dispatches
     assert len(bf_seq_ids_uniq) == num_dispatches
     assert len(cb_seq_ids_uniq) == num_dispatches
@@ -249,17 +254,39 @@ def test_async_copy_direction(input_data):
     async_dir_cnt = dict([(idx, 0) for idx in range(0, 5)])
     for itr in sdk_data["buffer_records"]["memory_copies"]:
         op_id = itr["operation"]
-        assert op_id > 1
-        assert op_id < 4
+        assert op_id > 1, f"{itr}"
+        assert op_id < 4, f"{itr}"
         async_dir_cnt[op_id] += 1
 
+    for itr in sdk_data["callback_records"]["memory_copies"]:
+        op_id = itr.operation
+        assert op_id > 1, f"{itr}"
+        assert op_id < 4, f"{itr}"
+        async_dir_cnt[op_id] += 1
+
+        phase = itr.phase
+        pitr = itr.payload
+
+        assert phase is not None, f"{itr}"
+        assert pitr is not None, f"{itr}"
+
+        if phase == 1:
+            assert pitr.start_timestamp == 0, f"{itr}"
+            assert pitr.end_timestamp == 0, f"{itr}"
+        elif phase == 2:
+            assert pitr.start_timestamp > 0, f"{itr}"
+            assert pitr.end_timestamp > 0, f"{itr}"
+            assert pitr.end_timestamp >= pitr.start_timestamp, f"{itr}"
+        else:
+            assert phase == 1 or phase == 2, f"{itr}"
+
     # in the transpose test which generates the input file,
     # two threads and the main thread (so three threads total)
     # each perform one H2D + one D2H memory copy
     assert async_dir_cnt[0] == 0
     assert async_dir_cnt[1] == 0
-    assert async_dir_cnt[2] == 6
-    assert async_dir_cnt[3] == 6
+    assert async_dir_cnt[2] == 6 * 3
+    assert async_dir_cnt[3] == 6 * 3
     assert async_dir_cnt[4] == 0
 
 

tests/common/hash.hpp

@@ -0,0 +1,44 @@
+// MIT License
+//
+// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <rocprofiler-sdk/agent.h>
+#include <rocprofiler-sdk/fwd.h>
+
+namespace std
+{
+template <typename Tp>
+struct hash;
+
+template <>
+struct hash<rocprofiler_agent_id_t>
+{
+    size_t operator()(rocprofiler_agent_id_t id) const { return id.handle; }
+};
+}  // namespace std
+
+inline bool
+operator==(rocprofiler_agent_id_t lhs, rocprofiler_agent_id_t rhs)
+{
+    return (lhs.handle == rhs.handle);
+}

tests/common/serialization.hpp

@@ -300,6 +300,18 @@ save(ArchiveT& ar, rocprofiler_callback_tracing_kernel_dispatch_data_t data)
     SAVE_DATA_FIELD(dispatch_info);
 }
 
+template <typename ArchiveT>
+void
+save(ArchiveT& ar, rocprofiler_callback_tracing_memory_copy_data_t data)
+{
+    SAVE_DATA_FIELD(size);
+    SAVE_DATA_FIELD(start_timestamp);
+    SAVE_DATA_FIELD(end_timestamp);
+    SAVE_DATA_FIELD(dst_agent_id);
+    SAVE_DATA_FIELD(src_agent_id);
+    SAVE_DATA_FIELD(bytes);
+}
+
 template <typename ArchiveT>
 void
 save(ArchiveT& ar, rocprofiler_profile_counting_dispatch_data_t data)
@@ -395,11 +407,13 @@ save(ArchiveT& ar, rocprofiler_buffer_tracing_memory_copy_record_t data)
     SAVE_DATA_FIELD(size);
     SAVE_DATA_FIELD(kind);
     SAVE_DATA_FIELD(operation);
+    SAVE_DATA_FIELD(thread_id);
     SAVE_DATA_FIELD(correlation_id);
     SAVE_DATA_FIELD(start_timestamp);
     SAVE_DATA_FIELD(end_timestamp);
     SAVE_DATA_FIELD(dst_agent_id);
     SAVE_DATA_FIELD(src_agent_id);
+    SAVE_DATA_FIELD(bytes);
 }
 
 template <typename ArchiveT>

tests/kernel-tracing/validate.py

@@ -265,7 +265,7 @@ def test_kernel_dispatch_ids(input_data):
     num_dispatches = len(sdk_data["buffer_records"]["kernel_dispatches"])
     num_cb_dispatches = len(sdk_data["callback_records"]["kernel_dispatches"])
 
-    assert num_cb_dispatches == (2 * num_dispatches)
+    assert num_cb_dispatches == (3 * num_dispatches)
 
     bf_seq_ids = []
     for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
@@ -278,7 +278,7 @@ def test_kernel_dispatch_ids(input_data):
     bf_seq_ids = sorted(bf_seq_ids)
     cb_seq_ids = sorted(cb_seq_ids)
 
-    assert (2 * len(bf_seq_ids)) == len(cb_seq_ids)
+    assert (3 * len(bf_seq_ids)) == len(cb_seq_ids)
 
     assert bf_seq_ids[0] == cb_seq_ids[0]
     assert bf_seq_ids[-1] == cb_seq_ids[-1]
@@ -290,7 +290,7 @@ def test_kernel_dispatch_ids(input_data):
     cb_seq_ids_uniq = get_uniq(cb_seq_ids)
 
     assert bf_seq_ids == bf_seq_ids_uniq
-    assert len(cb_seq_ids) == (2 * len(cb_seq_ids_uniq))
+    assert len(cb_seq_ids) == (3 * len(cb_seq_ids_uniq))
     assert len(bf_seq_ids) == num_dispatches
     assert len(bf_seq_ids_uniq) == num_dispatches
     assert len(cb_seq_ids_uniq) == num_dispatches

tests/tools/json-tool.cpp

@@ -33,6 +33,7 @@
 
 #include "common/defines.hpp"
 #include "common/filesystem.hpp"
+#include "common/hash.hpp"
 #include "common/perfetto.hpp"
 #include "common/serialization.hpp"
 
@@ -240,20 +241,6 @@ rocprofiler_client_finalize_t client_fini_func = nullptr;
 callback_name_info
 get_callback_tracing_names()
 {
-    static const auto supported = std::unordered_set<rocprofiler_callback_tracing_kind_t>{
-        ROCPROFILER_CALLBACK_TRACING_HSA_CORE_API,
-        ROCPROFILER_CALLBACK_TRACING_HSA_AMD_EXT_API,
-        ROCPROFILER_CALLBACK_TRACING_HSA_IMAGE_EXT_API,
-        ROCPROFILER_CALLBACK_TRACING_HSA_FINALIZE_EXT_API,
-        ROCPROFILER_CALLBACK_TRACING_HIP_RUNTIME_API,
-        ROCPROFILER_CALLBACK_TRACING_HIP_COMPILER_API,
-        ROCPROFILER_CALLBACK_TRACING_MARKER_CORE_API,
-        ROCPROFILER_CALLBACK_TRACING_MARKER_CONTROL_API,
-        ROCPROFILER_CALLBACK_TRACING_MARKER_NAME_API,
-        ROCPROFILER_CALLBACK_TRACING_SCRATCH_MEMORY,
-        ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT,
-    };
-
     auto cb_name_info = callback_name_info{};
     //
     // callback for each kind operation
@@ -262,14 +249,11 @@ get_callback_tracing_names()
         [](rocprofiler_callback_tracing_kind_t kindv, uint32_t operation, void* data_v) {
             auto* name_info_v = static_cast<callback_name_info*>(data_v);
 
-            if(supported.count(kindv) > 0)
-            {
-                const char* name = nullptr;
-                ROCPROFILER_CALL(rocprofiler_query_callback_tracing_kind_operation_name(
-                                     kindv, operation, &name, nullptr),
-                                 "query buffer tracing kind operation name");
-                if(name) name_info_v->operation_names[kindv][operation] = name;
-            }
+            const char* name = nullptr;
+            ROCPROFILER_CALL(rocprofiler_query_callback_tracing_kind_operation_name(
+                                 kindv, operation, &name, nullptr),
+                             "query buffer tracing kind operation name");
+            if(name) name_info_v->operation_names[kindv][operation] = name;
             return 0;
         };
 
@@ -284,12 +268,8 @@ get_callback_tracing_names()
                          "query buffer tracing kind operation name");
         if(name) name_info_v->kind_names[kind] = name;
 
-        if(supported.count(kind) > 0)
-        {
-            ROCPROFILER_CALL(rocprofiler_iterate_callback_tracing_kind_operations(
-                                 kind, tracing_kind_operation_cb, static_cast<void*>(data)),
-                             "iterating buffer tracing kind operations");
-        }
+        rocprofiler_iterate_callback_tracing_kind_operations(
+            kind, tracing_kind_operation_cb, static_cast<void*>(data));
         return 0;
     };
 
@@ -326,14 +306,11 @@ get_buffer_tracing_names()
         [](rocprofiler_buffer_tracing_kind_t kindv, uint32_t operation, void* data_v) {
             auto* name_info_v = static_cast<buffer_name_info*>(data_v);
 
-            if(supported.count(kindv) > 0)
-            {
-                const char* name = nullptr;
-                ROCPROFILER_CALL(rocprofiler_query_buffer_tracing_kind_operation_name(
-                                     kindv, operation, &name, nullptr),
-                                 "query buffer tracing kind operation name");
-                if(name) name_info_v->operation_names[kindv][operation] = name;
-            }
+            const char* name = nullptr;
+            ROCPROFILER_CALL(rocprofiler_query_buffer_tracing_kind_operation_name(
+                                 kindv, operation, &name, nullptr),
+                             "query buffer tracing kind operation name");
+            if(name) name_info_v->operation_names[kindv][operation] = name;
             return 0;
         };
 
@@ -348,12 +325,8 @@ get_buffer_tracing_names()
                          "query buffer tracing kind operation name");
         if(name) name_info_v->kind_names[kind] = name;
 
-        if(supported.count(kind) > 0)
-        {
-            ROCPROFILER_CALL(rocprofiler_iterate_buffer_tracing_kind_operations(
-                                 kind, tracing_kind_operation_cb, static_cast<void*>(data)),
-                             "iterating buffer tracing kind operations");
-        }
+        rocprofiler_iterate_buffer_tracing_kind_operations(
+            kind, tracing_kind_operation_cb, static_cast<void*>(data));
         return 0;
     };
 
@@ -517,6 +490,21 @@ struct kernel_dispatch_callback_record_t
     }
 };
 
+struct memory_copy_callback_record_t
+{
+    uint64_t                                        timestamp = 0;
+    rocprofiler_callback_tracing_record_t           record    = {};
+    rocprofiler_callback_tracing_memory_copy_data_t payload   = {};
+
+    template <typename ArchiveT>
+    void save(ArchiveT& ar) const
+    {
+        ar(cereal::make_nvp("timestamp", timestamp));
+        cereal::save(ar, record);
+        ar(cereal::make_nvp("payload", payload));
+    }
+};
+
 struct scratch_memory_callback_record_t
 {
     uint64_t                                           timestamp = 0;
@@ -608,6 +596,7 @@ auto counter_collection_bf_records = std::deque<profile_counting_record>{};
 auto hip_api_cb_records            = std::deque<hip_api_callback_record_t>{};
 auto scratch_memory_cb_records     = std::deque<scratch_memory_callback_record_t>{};
 auto kernel_dispatch_cb_records    = std::deque<kernel_dispatch_callback_record_t>{};
+auto memory_copy_cb_records        = std::deque<memory_copy_callback_record_t>{};
 
 int
 set_external_correlation_id(rocprofiler_thread_id_t                            thr_id,
@@ -812,6 +801,14 @@ tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
         kernel_dispatch_cb_records.emplace_back(
             kernel_dispatch_callback_record_t{ts, record, *data});
     }
+    else if(record.kind == ROCPROFILER_CALLBACK_TRACING_MEMORY_COPY)
+    {
+        auto* data = static_cast<rocprofiler_callback_tracing_memory_copy_data_t*>(record.payload);
+
+        static auto _mutex = std::mutex{};
+        auto        _lk    = std::unique_lock<std::mutex>{_mutex};
+        memory_copy_cb_records.emplace_back(memory_copy_callback_record_t{ts, record, *data});
+    }
     else
     {
         throw std::runtime_error{"unsupported callback kind"};
@@ -822,7 +819,7 @@ auto hsa_api_bf_records         = std::deque<rocprofiler_buffer_tracing_hsa_api_
 auto marker_api_bf_records      = std::deque<rocprofiler_buffer_tracing_marker_api_record_t>{};
 auto hip_api_bf_records         = std::deque<rocprofiler_buffer_tracing_hip_api_record_t>{};
 auto kernel_dispatch_bf_records = std::deque<rocprofiler_buffer_tracing_kernel_dispatch_record_t>{};
-auto memory_copy_records        = std::deque<rocprofiler_buffer_tracing_memory_copy_record_t>{};
+auto memory_copy_bf_records     = std::deque<rocprofiler_buffer_tracing_memory_copy_record_t>{};
 auto scratch_memory_records     = std::deque<rocprofiler_buffer_tracing_scratch_memory_record_t>{};
 auto page_migration_records     = std::deque<rocprofiler_buffer_tracing_page_migration_record_t>{};
 auto corr_id_retire_records =
@@ -900,7 +897,7 @@ tool_tracing_buffered(rocprofiler_context_id_t /*context*/,
                 auto* record =
                     static_cast<rocprofiler_buffer_tracing_memory_copy_record_t*>(header->payload);
 
-                memory_copy_records.emplace_back(*record);
+                memory_copy_bf_records.emplace_back(*record);
             }
             else if(header->kind == ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY)
             {
@@ -1006,7 +1003,8 @@ rocprofiler_context_id_t code_object_ctx              = {};
 rocprofiler_context_id_t hsa_api_buffered_ctx         = {};
 rocprofiler_context_id_t hip_api_buffered_ctx         = {};
 rocprofiler_context_id_t marker_api_buffered_ctx      = {};
-rocprofiler_context_id_t memory_copy_ctx              = {};
+rocprofiler_context_id_t memory_copy_callback_ctx     = {};
+rocprofiler_context_id_t memory_copy_buffered_ctx     = {};
 rocprofiler_context_id_t counter_collection_ctx       = {};
 rocprofiler_context_id_t scratch_memory_ctx           = {};
 rocprofiler_context_id_t corr_id_retire_ctx           = {};
@@ -1030,11 +1028,12 @@ auto contexts = std::unordered_map<std::string_view, rocprofiler_context_id_t*>{
     {"MARKER_API_CALLBACK", &marker_api_callback_ctx},
     {"CODE_OBJECT", &code_object_ctx},
     {"KERNEL_DISPATCH_CALLBACK", &kernel_dispatch_callback_ctx},
+    {"MEMORY_COPY_CALLBACK", &memory_copy_callback_ctx},
     {"HSA_API_BUFFERED", &hsa_api_buffered_ctx},
     {"HIP_API_BUFFERED", &hip_api_buffered_ctx},
     {"MARKER_API_BUFFERED", &marker_api_buffered_ctx},
     {"KERNEL_DISPATCH_BUFFERED", &kernel_dispatch_buffered_ctx},
-    {"MEMORY_COPY", &memory_copy_ctx},
+    {"MEMORY_COPY_BUFFERED", &memory_copy_buffered_ctx},
     {"PAGE_MIGRATION", &page_migration_ctx},
     {"COUNTER_COLLECTION", &counter_collection_ctx},
     {"SCRATCH_MEMORY", &scratch_memory_ctx},
@@ -1051,7 +1050,8 @@ auto buffers = std::array<rocprofiler_buffer_id_t*, 9>{&hsa_api_buffered_buffer,
                                                        &counter_collection_buffer,
                                                        &corr_id_retire_buffer};
 
-auto agents = std::vector<rocprofiler_agent_t>{};
+auto agents     = std::vector<rocprofiler_agent_t>{};
+auto agents_map = std::unordered_map<rocprofiler_agent_id_t, rocprofiler_agent_t>{};
 
 rocprofiler_timestamp_t init_time             = 0;
 rocprofiler_timestamp_t fini_time             = 0;
@@ -1085,6 +1085,9 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
                                            const_cast<void*>(static_cast<const void*>(&agents))),
         "query available agents");
 
+    for(auto itr : agents)
+        agents_map.emplace(itr.id, itr);
+
     auto* call_stack_v = static_cast<call_stack_t*>(tool_data);
 
     call_stack_v->emplace_back(source_location{__FUNCTION__, __FILE__, __LINE__, ""});
@@ -1154,8 +1157,8 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
                                                        nullptr),
         "marker name api tracing service configure");
 
-    auto kernel_dispatch_cb_ops =
-        make_array<rocprofiler_tracing_operation_t>(ROCPROFILER_KERNEL_DISPATCH_ENQUEUE);
+    auto kernel_dispatch_cb_ops = make_array<rocprofiler_tracing_operation_t>(
+        ROCPROFILER_KERNEL_DISPATCH_ENQUEUE, ROCPROFILER_KERNEL_DISPATCH_COMPLETE);
 
     ROCPROFILER_CALL(
         rocprofiler_configure_callback_tracing_service(kernel_dispatch_callback_ctx,
@@ -1166,6 +1169,15 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
                                                        nullptr),
         "kernel dispatch callback tracing service configure");
 
+    ROCPROFILER_CALL(
+        rocprofiler_configure_callback_tracing_service(memory_copy_callback_ctx,
+                                                       ROCPROFILER_CALLBACK_TRACING_MEMORY_COPY,
+                                                       nullptr,
+                                                       0,
+                                                       tool_tracing_callback,
+                                                       nullptr),
+        "memory copy callback tracing service configure");
+
     ROCPROFILER_CALL(
         rocprofiler_configure_callback_tracing_service(scratch_memory_ctx,
                                                        ROCPROFILER_CALLBACK_TRACING_SCRATCH_MEMORY,
@@ -1214,7 +1226,7 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
                                                &kernel_dispatch_buffer),
                      "buffer creation");
 
-    ROCPROFILER_CALL(rocprofiler_create_buffer(memory_copy_ctx,
+    ROCPROFILER_CALL(rocprofiler_create_buffer(memory_copy_buffered_ctx,
                                                buffer_size,
                                                watermark,
                                                ROCPROFILER_BUFFER_POLICY_LOSSLESS,
@@ -1310,7 +1322,7 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
         "buffer tracing service for kernel dispatch configure");
 
     ROCPROFILER_CALL(
-        rocprofiler_configure_buffer_tracing_service(memory_copy_ctx,
+        rocprofiler_configure_buffer_tracing_service(memory_copy_buffered_ctx,
                                                      ROCPROFILER_BUFFER_TRACING_MEMORY_COPY,
                                                      nullptr,
                                                      0,
@@ -1495,8 +1507,9 @@ tool_fini(void* tool_data)
               << ", marker_api_callback_records=" << marker_api_cb_records.size()
               << ", scratch_memory_callback_records=" << scratch_memory_cb_records.size()
               << ", kernel_dispatch_callback_records=" << kernel_dispatch_cb_records.size()
+              << ", memory_copy_callback_records=" << memory_copy_cb_records.size()
               << ", kernel_dispatch_bf_records=" << kernel_dispatch_bf_records.size()
-              << ", memory_copy_records=" << memory_copy_records.size()
+              << ", memory_copy_bf_records=" << memory_copy_bf_records.size()
               << ", scratch_memory_records=" << scratch_memory_records.size()
               << ", page_migration=" << page_migration_records.size()
               << ", hsa_api_bf_records=" << hsa_api_bf_records.size()
@@ -1593,6 +1606,7 @@ write_json(call_stack_t* _call_stack)
             json_ar(cereal::make_nvp("marker_api_traces", marker_api_cb_records));
             json_ar(cereal::make_nvp("scratch_memory_traces", scratch_memory_cb_records));
             json_ar(cereal::make_nvp("kernel_dispatches", kernel_dispatch_cb_records));
+            json_ar(cereal::make_nvp("memory_copies", memory_copy_cb_records));
         } catch(std::exception& e)
         {
             std::cerr << "[" << getpid() << "][" << __FUNCTION__
@@ -1607,7 +1621,7 @@ write_json(call_stack_t* _call_stack)
         {
             json_ar(cereal::make_nvp("names", buffer_name_info));
             json_ar(cereal::make_nvp("kernel_dispatches", kernel_dispatch_bf_records));
-            json_ar(cereal::make_nvp("memory_copies", memory_copy_records));
+            json_ar(cereal::make_nvp("memory_copies", memory_copy_bf_records));
             json_ar(cereal::make_nvp("scratch_memory_traces", scratch_memory_records));
             json_ar(cereal::make_nvp("page_migration", page_migration_records));
             json_ar(cereal::make_nvp("hsa_api_traces", hsa_api_bf_records));
@@ -1682,7 +1696,7 @@ write_perfetto()
         for(auto itr : marker_api_bf_records)
             tids.emplace(itr.thread_id);
 
-        for(auto itr : memory_copy_records)
+        for(auto itr : memory_copy_bf_records)
         {
             agent_ids.emplace(itr.dst_agent_id.handle);
             agent_ids.emplace(itr.src_agent_id.handle);
@@ -1853,7 +1867,7 @@ write_perfetto()
                             itr.end_timestamp);
         }
 
-        for(auto itr : memory_copy_records)
+        for(auto itr : memory_copy_bf_records)
         {
             auto& name  = buffer_name_info.operation_names.at(itr.kind).at(itr.operation);
             auto& track = agent_tracks.at(itr.dst_agent_id.handle);
@@ -1870,9 +1884,9 @@ write_perfetto()
                               "operation",
                               itr.operation,
                               "src_agent",
-                              itr.src_agent_id.handle,
+                              agents_map.at(itr.src_agent_id).logical_node_id,
                               "dst_agent",
-                              itr.dst_agent_id.handle);
+                              agents_map.at(itr.dst_agent_id).logical_node_id);
             TRACE_EVENT_END(rocprofiler::trait::name<rocprofiler::category::memory_copy>::value,
                             track,
                             itr.end_timestamp,
@@ -1913,7 +1927,7 @@ write_perfetto()
                 "kind",
                 itr.kind,
                 "agent",
-                info.agent_id.handle,
+                agents_map.at(info.agent_id).logical_node_id,
                 "corr_id",
                 itr.correlation_id.internal,
                 "queue",