SWDEV-492625 memory free functions (#11)

* SWDEV-492625: Track free memory HSA functions to help determine total amount of memory allocated on the system at any one time

* Minor fixes to address comments

* Update allocation size description

* Moved get function back to specialization, minor typo fixes

* Removed memory_operation_type field, removed memory_pool allocation enum, converted starting address to hex string for json format.

* Made conversion to hex_string a function, changed address to use union rocprofiler_address_t type, changed VMEM descriptors

* Removed as_hex from the global namespace

* Formatting

* Removed TRACK_EVENT for memory allocation, now TRACK_COUNTER for memory allocation is being performed

* Check if address was recorded before retrieving allocation size in generate Perfetto

* Formatting

* Update source/lib/output/generatePerfetto.cpp

* Explicitly disable app-abort tests

* Remove excluding app-abort test from workflow CI

- redundant bc these tests are explicitly marked as disabled now

---------

Co-authored-by: Madsen, Jonathan <Jonathan.Madsen@amd.com>
Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>

[ROCm/rocprofiler-sdk commit: 79006bb896]

projects/rocprofiler-sdk/tests/bin/hsa-memory-allocation/hsa-memory-allocation.cpp

@@ -170,6 +170,8 @@ call_hsa_memory_allocate(const size_t i, const size_t base_size, hsa_agent_t age
 
         status = hsa_memory_allocate(region_list[0], base_size, &addr);
         RET_IF_HSA_ERR(status)
+        status = hsa_memory_free(addr);
+        RET_IF_HSA_ERR(status)
     }
 }
 
@@ -199,6 +201,8 @@ call_hsa_memory_pool_allocate(const size_t i, const size_t base_size, hsa_agent_
 
         status = hsa_amd_memory_pool_allocate(memory_pool_list[0], base_size, flags, &addr);
         RET_IF_HSA_ERR(status)
+        status = hsa_amd_memory_pool_free(addr);
+        RET_IF_HSA_ERR(status)
     }
 }
 
@@ -243,6 +247,8 @@ call_hsa_vmem_allocate(const size_t i, hsa_agent_t agent)
         status = hsa_amd_vmem_handle_create(
             memory_pool_list[0], size, MEMORY_TYPE_NONE, 0, &memory_handle);
         RET_IF_HSA_ERR(status)
+        status = hsa_amd_vmem_handle_release(memory_handle);
+        RET_IF_HSA_ERR(status)
     }
 }
 
@@ -257,7 +263,7 @@ main()
     hsa_agent_t              cpu_agent = get_cpu_agent(agents);
     hsa_agent_t              gpu_agent = get_gpu_agent(agents);
     call_hsa_memory_allocate(6, 1024, cpu_agent);
-    call_hsa_memory_pool_allocate(9, 512, gpu_agent);
+    call_hsa_memory_pool_allocate(9, 2048, gpu_agent);
     // Virtual memory API not supported in CI. Will add back if this changes
     // call_hsa_vmem_allocate(3, gpu_agent);
 

projects/rocprofiler-sdk/tests/hsa-memory-allocation/validate.py

@@ -175,25 +175,26 @@ def test_memory_alloc_sizes(input_data):
     # Op values:
     #   0 == ??? (unknown)
     #   1 == hsa_memory_allocate
-    #   2 == hsa_amd_ext_memory_allocate
-    #   3 == hsa_amd_vmem_handle_create
+    #   2 == hsa_amd_vmem_handle_create
+    #   3 == hsa_memory_free
+    #   4 == hsa_amd_vmem_handle_release
     memory_alloc_cnt = dict(
         [
             (idx, {"agent": set(), "starting_addr": set(), "size": set(), "count": 0})
-            for idx in range(1, 4)
+            for idx in range(1, 5)
         ]
     )
     for itr in sdk_data["buffer_records"]["memory_allocations"]:
         op_id = itr["operation"]
-        assert op_id > 0 and op_id <= 3, f"{itr}"
+        assert op_id > 0 and op_id <= 5, f"{itr}"
         memory_alloc_cnt[op_id]["count"] += 1
-        memory_alloc_cnt[op_id]["starting_addr"].add(itr.starting_address)
+        memory_alloc_cnt[op_id]["starting_addr"].add(itr.address)
         memory_alloc_cnt[op_id]["size"].add(itr.allocation_size)
         memory_alloc_cnt[op_id]["agent"].add(itr.agent_id.handle)
 
     for itr in sdk_data["callback_records"]["memory_copies"]:
         op_id = itr.operation
-        assert op_id > 0 and op_id <= 3, f"{itr}"
+        assert op_id > 0 and op_id <= 5, f"{itr}"
         memory_alloc_cnt[op_id]["count"] += 1
 
         phase = itr.phase
@@ -210,7 +211,7 @@ def test_memory_alloc_sizes(input_data):
             assert pitr.end_timestamp > 0, f"{itr}"
             assert pitr.end_timestamp >= pitr.start_timestamp, f"{itr}"
 
-            memory_alloc_cnt[op_id]["starting_addr"].add(pitr.starting_address)
+            memory_alloc_cnt[op_id]["starting_addr"].add(pitr.address)
             memory_alloc_cnt[op_id]["size"].add(pitr.allocation_size)
             memory_alloc_cnt[op_id]["agent"].add(pitr.agent_id.handle)
         else:
@@ -218,24 +219,22 @@ def test_memory_alloc_sizes(input_data):
 
     # In the memory allocation test which generates this file
     # 6 hsa_memory_allocation calls with 1024 bytes were called
-    # and 9 hsa_amd_memory_pool_allocations with 512 bytes
+    # and 9 hsa_amd_memory_pool_allocations with 2048 bytes
     # were called
-    assert memory_alloc_cnt[1]["count"] == 6
-    assert memory_alloc_cnt[2]["count"] == 9
+    assert memory_alloc_cnt[1]["count"] == 15
+    assert memory_alloc_cnt[3]["count"] == 15
     # assert memory_alloc_cnt[3]["count"] == 3
-    assert len(memory_alloc_cnt[1]["starting_addr"]) == 6
-    assert len(memory_alloc_cnt[2]["starting_addr"]) == 9
+    assert len(memory_alloc_cnt[1]["starting_addr"]) == len(
+        memory_alloc_cnt[3]["starting_addr"]
+    )
+
     # assert len(memory_alloc_cnt[3]["starting_addr"]) == 3
-    assert len(memory_alloc_cnt[1]["size"]) == 1
-    assert len(memory_alloc_cnt[2]["size"]) == 1
+    assert len(memory_alloc_cnt[1]["size"]) == 2
     # assert len(memory_alloc_cnt[3]["size"]) == 1
     assert 1024 in memory_alloc_cnt[1]["size"]
-    assert 512 in memory_alloc_cnt[2]["size"]
-    assert len(memory_alloc_cnt[1]["agent"]) == 1
-    assert len(memory_alloc_cnt[2]["agent"]) == 1
+    assert 2048 in memory_alloc_cnt[1]["size"]
+    assert len(memory_alloc_cnt[1]["agent"]) == 2
     # assert len(memory_alloc_cnt[3]["agent"]) == 1
-    assert memory_alloc_cnt[1]["agent"] != memory_alloc_cnt[2]["agent"]
-    # assert memory_alloc_cnt[2]["agent"] == memory_alloc_cnt[3]["agent"]
 
 
 def test_retired_correlation_ids(input_data):

projects/rocprofiler-sdk/tests/rocprofv3/aborted-app/CMakeLists.txt

@@ -19,9 +19,11 @@ string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
 # disable this test for thread sanitizers because of "signal-unsafe call inside signal"
 # issues on mi200 and mi300 (works fine on vega20 and navi32)
 if(ROCPROFILER_MEMCHECK STREQUAL "ThreadSanitizer")
-    set(IS_THREAD_SANITIZER ON)
+    set(DISABLE_THIS_TEST ON)
 else()
-    set(IS_THREAD_SANITIZER OFF)
+    # set(DISABLE_THIS_TEST OFF)
+    set(DISABLE_THIS_TEST ON) # this test is currently unstable so we are disabling it
+                              # unconditionally for now
 endif()
 
 set(aborted-app-env "${PRELOAD_ENV}" ROCPROF_TESTING_RAISE_SIGNAL=1
@@ -46,7 +48,7 @@ set_tests_properties(
                WILL_FAIL
                TRUE
                DISABLED
-               "${IS_THREAD_SANITIZER}")
+               "${DISABLE_THIS_TEST}")
 
 add_test(
     NAME rocprofv3-test-validate-app-abort
@@ -64,4 +66,4 @@ set_tests_properties(
                FAIL_REGULAR_EXPRESSION
                "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
                DISABLED
-               "${IS_THREAD_SANITIZER}")
+               "${DISABLE_THIS_TEST}")

projects/rocprofiler-sdk/tests/rocprofv3/memory-allocation/CMakeLists.txt

@@ -21,7 +21,7 @@ add_test(
     NAME rocprofv3-test-memory-allocation-tracing-execute
     COMMAND
         $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --memory-allocation-trace -d
-        ${CMAKE_CURRENT_BINARY_DIR}/%tag%-trace -o out --output-format json pftrace otf2
+        ${CMAKE_CURRENT_BINARY_DIR}/%tag%-trace -o out --output-format json otf2
         --log-level env -- $<TARGET_FILE:hsa-memory-allocation>)
 
 set_tests_properties(
@@ -35,8 +35,6 @@ add_test(
     COMMAND
         ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input
         ${CMAKE_CURRENT_BINARY_DIR}/hsa-memory-allocation-trace/out_results.json
-        --pftrace-input
-        ${CMAKE_CURRENT_BINARY_DIR}/hsa-memory-allocation-trace/out_results.pftrace
         --otf2-input
         ${CMAKE_CURRENT_BINARY_DIR}/hsa-memory-allocation-trace/out_results.otf2)
 

projects/rocprofiler-sdk/tests/rocprofv3/memory-allocation/conftest.py

@@ -17,12 +17,6 @@ def pytest_addoption(parser):
         default="memory-allocation-tracing/out_results.json",
         help="Input JSON",
     )
-    parser.addoption(
-        "--pftrace-input",
-        action="store",
-        default="memory-allocation-tracing/out_results.pftrace",
-        help="Input JSON",
-    )
     parser.addoption(
         "--otf2-input",
         action="store",
@@ -38,12 +32,6 @@ def json_data(request):
         return dotdict(collapse_dict_list(json.load(inp)))
 
 
-@pytest.fixture
-def pftrace_data(request):
-    filename = request.config.getoption("--pftrace-input")
-    return PerfettoReader(filename).read()[0]
-
-
 @pytest.fixture
 def otf2_data(request):
     filename = request.config.getoption("--otf2-input")

projects/rocprofiler-sdk/tests/rocprofv3/memory-allocation/validate.py

@@ -35,7 +35,7 @@ def test_memory_allocation(json_data):
 
     _, bf_op_names = get_operation(data, "MEMORY_ALLOCATION")
 
-    assert len(bf_op_names) == 4
+    assert len(bf_op_names) == 5
 
     allocation_reported_agent_ids = set()
     # check buffering data
@@ -49,12 +49,12 @@ def test_memory_allocation(json_data):
         assert "thread_id" in node
 
         assert "agent_id" in node
-        assert "starting_address" in node
+        assert "address" in node
         assert "allocation_size" in node
 
         assert node.size > 0
-        assert node.allocation_size > 0
-        assert node.starting_address > 0
+        assert node.allocation_size >= 0
+        assert len(node.address) > 0
         assert node.thread_id > 0
         assert node.agent_id.handle > 0
         assert node.start_timestamp > 0
@@ -69,14 +69,6 @@ def test_memory_allocation(json_data):
 
         allocation_reported_agent_ids.add(node["agent_id"]["handle"])
 
-    assert 2**64 - 1 not in allocation_reported_agent_ids
-
-
-def test_perfetto_data(pftrace_data, json_data):
-    import rocprofiler_sdk.tests.rocprofv3 as rocprofv3
-
-    rocprofv3.test_perfetto_data(pftrace_data, json_data, ("memory_allocation",))
-
 
 def test_otf2_data(otf2_data, json_data):
     import rocprofiler_sdk.tests.rocprofv3 as rocprofv3

projects/rocprofiler-sdk/tests/rocprofv3/summary/validate.py

@@ -242,12 +242,12 @@ def test_summary_display_data(json_data, summary_data):
 
     assert get_dims(marker) == [7, 9], f"{marker}"
     assert get_dims(memcpy) == [2, 9], f"{memcpy}"
-    assert get_dims(memalloc) == [1, 9], f"{memalloc}"
+    assert get_dims(memalloc) == [2, 9], f"{memalloc}"
     assert get_dims(dispatch) == [3, 9], f"{dispatch}"
     assert get_dims(dispatch_and_copy) == [5, 9], f"{dispatch_and_copy}"
     assert get_dims(hip) == [14, 9], f"{hip}"
     assert get_dims(hip_and_marker) == expected_hip_and_marker_dims, f"{hip_and_marker}"
-    assert get_dims(total) == [24, 9], f"{total}"
+    assert get_dims(total) == [25, 9], f"{total}"
 
 
 def test_perfetto_data(pftrace_data, json_data):
@@ -256,7 +256,7 @@ def test_perfetto_data(pftrace_data, json_data):
     rocprofv3.test_perfetto_data(
         pftrace_data,
         json_data,
-        ("hip", "marker", "kernel", "memory_copy", "memory_allocation"),
+        ("hip", "marker", "kernel", "memory_copy"),
     )
 
 

projects/rocprofiler-sdk/tests/tools/json-tool.cpp

@@ -49,6 +49,7 @@
 #include <rocprofiler-sdk/internal_threading.h>
 #include <rocprofiler-sdk/registration.h>
 #include <rocprofiler-sdk/rocprofiler.h>
+#include <rocprofiler-sdk/cxx/utility.hpp>
 
 #include <unistd.h>
 #include <algorithm>
@@ -1843,6 +1844,7 @@ write_perfetto()
 
     auto tids            = std::set<rocprofiler_thread_id_t>{};
     auto agent_ids       = std::set<uint64_t>{};
+    auto agent_ids_alloc = std::set<uint64_t>{};
     auto agent_queue_ids = std::map<uint64_t, std::set<uint64_t>>{};
 
     auto _get_agent = [](uint64_t id_handle) -> const rocprofiler_agent_t* {
@@ -1875,7 +1877,7 @@ write_perfetto()
         for(auto itr : memory_allocation_bf_records)
         {
             tids.emplace(itr.thread_id);
-            agent_ids.emplace(itr.agent_id.handle);
+            agent_ids_alloc.emplace(itr.agent_id.handle);
         }
 
         for(auto itr : kernel_dispatch_bf_records)
@@ -1934,6 +1936,36 @@ write_perfetto()
         agent_tracks.emplace(itr, _track);
     }
 
+    for(auto itr : agent_ids_alloc)
+    {
+        const auto* _agent  = _get_agent(itr);
+        auto        _namess = std::stringstream{};
+
+        if(_agent != nullptr)
+        {
+            if(_agent->type == ROCPROFILER_AGENT_TYPE_CPU)
+                _namess << "CPU MEMORY OPERATION [" << itr << "] ";
+            else if(_agent->type == ROCPROFILER_AGENT_TYPE_GPU)
+                _namess << "GPU MEMORY OPERATION [" << itr << "] ";
+
+            if(!std::string_view{_agent->model_name}.empty())
+                _namess << _agent->model_name;
+            else
+                _namess << _agent->product_name;
+        }
+        else
+        {
+            _namess << "UNKNOWN MEMORY OPERATION [" << itr << "] ";
+        }
+        auto _track = ::perfetto::Track{get_hash_id(_namess.str())};
+        auto _desc  = _track.Serialize();
+        _desc.set_name(_namess.str());
+
+        perfetto::TrackEvent::SetTrackDescriptor(_track, _desc);
+
+        agent_tracks.emplace(itr, _track);
+    }
+
     auto agent_queue_tracks =
         std::unordered_map<uint64_t, std::unordered_map<uint64_t, ::perfetto::Track>>{};
 
@@ -2155,35 +2187,6 @@ write_perfetto()
                             itr.end_timestamp);
         }
 
-        for(auto itr : memory_allocation_bf_records)
-        {
-            auto  name  = buffer_names.at(itr.kind, itr.operation);
-            auto& track = agent_tracks.at(itr.agent_id.handle);
-
-            TRACE_EVENT_BEGIN(sdk::perfetto_category<sdk::category::memory_allocation>::name,
-                              ::perfetto::StaticString(name.data()),
-                              track,
-                              itr.start_timestamp,
-                              ::perfetto::Flow::ProcessScoped(itr.correlation_id.internal),
-                              "begin_ns",
-                              itr.start_timestamp,
-                              "kind",
-                              itr.kind,
-                              "operation",
-                              itr.operation,
-                              "agent",
-                              agents_map.at(itr.agent_id).logical_node_id,
-                              "Allocation_size",
-                              itr.allocation_size,
-                              "Starting_address",
-                              itr.starting_address);
-            TRACE_EVENT_END(sdk::perfetto_category<sdk::category::memory_allocation>::name,
-                            track,
-                            itr.end_timestamp,
-                            "end_ns",
-                            itr.end_timestamp);
-        }
-
         auto demangled = std::unordered_map<std::string_view, std::string>{};
         for(auto itr : kernel_dispatch_bf_records)
         {