Fix async memory copy validation tests (#182)

* Fix async copy validation test

- make the async copy tracing test work regardless of however many HSA memory copies the HIP memory copy decomposes into

* Fix rocprofv3 memory copy tests

* Fix compilation support for hipGraphBatchMemOpNodeGetParams

* Fix rocprofv3-test-summary-*-validate

---------

Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>

source/lib/rocprofiler-sdk/hip/details/format.hpp

@@ -187,9 +187,6 @@ struct formatter<hipGraphNodeType> : rocprofiler::hip::details::base_formatter
             ROCP_SDK_HIP_FORMAT_CASE_STMT(hipGraphNodeType, MemcpyToSymbol);
             ROCP_SDK_HIP_FORMAT_CASE_STMT(hipGraphNodeType, Empty);
             ROCP_SDK_HIP_FORMAT_CASE_STMT(hipGraphNodeType, Count);
-#if HIP_VERSION >= 60400000
-            ROCP_SDK_HIP_FORMAT_CASE_STMT(hipGraphNodeType, BatchMemOp);
-#endif
             ROCP_SDK_HIP_FORMAT_DFLT_CASE(hipGraphNodeType);
         }
         return fmt::format_to(ctx.out(), "Unknown");

tests/async-copy-tracing/validate.py

@@ -274,18 +274,25 @@ def test_async_copy_direction(input_data):
     #   2 == H2D (host to device)
     #   3 == D2H (device to host)
     #   4 == D2D (device to device)
-    async_dir_cnt = dict([(idx, 0) for idx in range(0, 5)])
-    for itr in sdk_data["buffer_records"]["memory_copies"]:
-        op_id = itr["operation"]
-        assert op_id > 1, f"{itr}"
-        assert op_id < 4, f"{itr}"
-        async_dir_cnt[op_id] += 1
-
-    for itr in sdk_data["callback_records"]["memory_copies"]:
+    default_async_dir_cnt = dict([(idx, 0) for idx in range(0, 5)])
+    thread_async_dir_cnt = {}
+    for itr in sdk_data.buffer_records.memory_copies:
+        tid = itr.thread_id
+        if tid not in thread_async_dir_cnt.keys():
+            thread_async_dir_cnt[tid] = default_async_dir_cnt
         op_id = itr.operation
         assert op_id > 1, f"{itr}"
         assert op_id < 4, f"{itr}"
-        async_dir_cnt[op_id] += 1
+        thread_async_dir_cnt[tid][op_id] += 1
+
+    for itr in sdk_data.callback_records.memory_copies:
+        tid = itr.thread_id
+        if tid not in thread_async_dir_cnt.keys():
+            thread_async_dir_cnt[tid] = default_async_dir_cnt
+        op_id = itr.operation
+        assert op_id > 1, f"{itr}"
+        assert op_id < 4, f"{itr}"
+        thread_async_dir_cnt[tid][op_id] += 1
 
         phase = itr.phase
         pitr = itr.payload
@@ -304,13 +311,27 @@ def test_async_copy_direction(input_data):
             assert phase == 1 or phase == 2, f"{itr}"
 
     # in the transpose test which generates the input file,
-    # two threads and the main thread (so three threads total)
-    # each perform one H2D + one D2H memory copy
-    assert async_dir_cnt[0] == 0
-    assert async_dir_cnt[1] == 0
-    assert async_dir_cnt[2] == 6 * 3
-    assert async_dir_cnt[3] == 6 * 3
-    assert async_dir_cnt[4] == 0
+    # two threads each perform one H2D + one D2H memory copy.
+    # there are at least two callback records (phase start +
+    # phase end) and one buffer record for each memory copy,
+    # i.e., at least 3 records per memory copy
+    assert len(thread_async_dir_cnt) == 2, f"{thread_async_dir_cnt}"
+    for tid, async_dir_cnt in thread_async_dir_cnt.items():
+        min_copy_records = 3
+        assert async_dir_cnt[0] == 0
+        assert async_dir_cnt[1] == 0
+        assert async_dir_cnt[2] >= min_copy_records, f"TID={tid}:\n\t{async_dir_cnt}"
+        assert async_dir_cnt[3] >= min_copy_records, f"TID={tid}:\n\t{async_dir_cnt}"
+        assert async_dir_cnt[4] == 0
+        # HIP memory copies may be decomposed into more than one
+        # memory copy at the HSA level so require it to be a multiple
+        # of min_copy_records
+        assert (
+            async_dir_cnt[2] % min_copy_records
+        ) == 0, f"TID={tid}:\n\t{async_dir_cnt}"
+        assert (
+            async_dir_cnt[3] % min_copy_records
+        ) == 0, f"TID={tid}:\n\t{async_dir_cnt}"
 
 
 def test_retired_correlation_ids(input_data):

tests/rocprofv3/kernel-rename/validate.py

@@ -161,7 +161,9 @@ def test_memory_copy_json_trace(json_data):
                 return agent
         return None
 
-    assert len(memory_copy_data) == 12
+    # two threads * two directions
+    assert len(memory_copy_data) >= (2 * 2), f"{memory_copy_data}"
+    assert (len(memory_copy_data) % (2 * 2)) == 0, f"{memory_copy_data}"
 
     for row in memory_copy_data:
         src_agent = get_agent(row["src_agent_id"])

tests/rocprofv3/summary/validate.py

@@ -137,7 +137,9 @@ def test_memory_copy_trace(json_data):
                 return agent
         return None
 
-    assert len(memory_copy_data) == 12
+    # two threads * two directions
+    assert len(memory_copy_data) >= (2 * 2), f"{memory_copy_data}"
+    assert (len(memory_copy_data) % (2 * 2)) == 0, f"{memory_copy_data}"
 
     for row in memory_copy_data:
         src_agent = get_agent(row["src_agent_id"])
@@ -209,7 +211,9 @@ def test_summary_data(json_data):
         elif itr.domain == "HIP_API":
             assert itr.stats.count >= 2130 and itr.stats.count <= 2150
         elif itr.domain == "MEMORY_COPY":
-            assert itr.stats.count == 12
+            # two threads + two memory copies (H2D + D2H).
+            # HIP may decompose memory copies into more than one HSA memory copy
+            assert itr.stats.count >= 4 and (itr.stats.count % 4) == 0
         elif itr.domain == "MEMORY_ALLOCATION":
             assert itr.stats.count >= 10 and itr.stats.count <= 30
         elif itr.domain == "MARKER_API":

tests/rocprofv3/tracing/validate.py

@@ -255,7 +255,9 @@ def test_memory_copy_json_trace(json_data):
                 return agent
         return None
 
-    assert len(memory_copy_data) == 2
+    # one threads * two directions
+    assert len(memory_copy_data) >= 2, f"{memory_copy_data}"
+    assert (len(memory_copy_data) % 2) == 0, f"{memory_copy_data}"
 
     def test_row(idx, direction):
         assert direction in ("MEMORY_COPY_HOST_TO_DEVICE", "MEMORY_COPY_DEVICE_TO_HOST")