From 7fcd80f7442b98d75c9dbf1bc7e9f376d8702a02 Mon Sep 17 00:00:00 2001 From: "Madsen, Jonathan" Date: Tue, 4 Feb 2025 02:41:19 -0600 Subject: [PATCH] Fix async memory copy validation tests (#182) * Fix async copy validation test - make the async copy tracing test work regardless of however many HSA memory copies the HIP memory copy decomposes into * Fix rocprofv3 memory copy tests * Fix compilation support for hipGraphBatchMemOpNodeGetParams * Fix rocprofv3-test-summary-*-validate --------- Co-authored-by: Jonathan R. Madsen --- .../rocprofiler-sdk/hip/details/format.hpp | 3 -- tests/async-copy-tracing/validate.py | 53 +++++++++++++------ tests/rocprofv3/kernel-rename/validate.py | 4 +- tests/rocprofv3/summary/validate.py | 8 ++- tests/rocprofv3/tracing/validate.py | 4 +- 5 files changed, 49 insertions(+), 23 deletions(-) diff --git a/source/lib/rocprofiler-sdk/hip/details/format.hpp b/source/lib/rocprofiler-sdk/hip/details/format.hpp index 7509fecf1a..9fb7635d13 100644 --- a/source/lib/rocprofiler-sdk/hip/details/format.hpp +++ b/source/lib/rocprofiler-sdk/hip/details/format.hpp @@ -187,9 +187,6 @@ struct formatter : rocprofiler::hip::details::base_formatter ROCP_SDK_HIP_FORMAT_CASE_STMT(hipGraphNodeType, MemcpyToSymbol); ROCP_SDK_HIP_FORMAT_CASE_STMT(hipGraphNodeType, Empty); ROCP_SDK_HIP_FORMAT_CASE_STMT(hipGraphNodeType, Count); -#if HIP_VERSION >= 60400000 - ROCP_SDK_HIP_FORMAT_CASE_STMT(hipGraphNodeType, BatchMemOp); -#endif ROCP_SDK_HIP_FORMAT_DFLT_CASE(hipGraphNodeType); } return fmt::format_to(ctx.out(), "Unknown"); diff --git a/tests/async-copy-tracing/validate.py b/tests/async-copy-tracing/validate.py index 46a711f32d..17daef6c6e 100644 --- a/tests/async-copy-tracing/validate.py +++ b/tests/async-copy-tracing/validate.py @@ -274,18 +274,25 @@ def test_async_copy_direction(input_data): # 2 == H2D (host to device) # 3 == D2H (device to host) # 4 == D2D (device to device) - async_dir_cnt = dict([(idx, 0) for idx in range(0, 5)]) - for itr in sdk_data["buffer_records"]["memory_copies"]: - op_id = itr["operation"] - assert op_id > 1, f"{itr}" - assert op_id < 4, f"{itr}" - async_dir_cnt[op_id] += 1 - - for itr in sdk_data["callback_records"]["memory_copies"]: + default_async_dir_cnt = dict([(idx, 0) for idx in range(0, 5)]) + thread_async_dir_cnt = {} + for itr in sdk_data.buffer_records.memory_copies: + tid = itr.thread_id + if tid not in thread_async_dir_cnt.keys(): + thread_async_dir_cnt[tid] = default_async_dir_cnt op_id = itr.operation assert op_id > 1, f"{itr}" assert op_id < 4, f"{itr}" - async_dir_cnt[op_id] += 1 + thread_async_dir_cnt[tid][op_id] += 1 + + for itr in sdk_data.callback_records.memory_copies: + tid = itr.thread_id + if tid not in thread_async_dir_cnt.keys(): + thread_async_dir_cnt[tid] = default_async_dir_cnt + op_id = itr.operation + assert op_id > 1, f"{itr}" + assert op_id < 4, f"{itr}" + thread_async_dir_cnt[tid][op_id] += 1 phase = itr.phase pitr = itr.payload @@ -304,13 +311,27 @@ def test_async_copy_direction(input_data): assert phase == 1 or phase == 2, f"{itr}" # in the transpose test which generates the input file, - # two threads and the main thread (so three threads total) - # each perform one H2D + one D2H memory copy - assert async_dir_cnt[0] == 0 - assert async_dir_cnt[1] == 0 - assert async_dir_cnt[2] == 6 * 3 - assert async_dir_cnt[3] == 6 * 3 - assert async_dir_cnt[4] == 0 + # two threads each perform one H2D + one D2H memory copy. + # there are at least two callback records (phase start + + # phase end) and one buffer record for each memory copy, + # i.e., at least 3 records per memory copy + assert len(thread_async_dir_cnt) == 2, f"{thread_async_dir_cnt}" + for tid, async_dir_cnt in thread_async_dir_cnt.items(): + min_copy_records = 3 + assert async_dir_cnt[0] == 0 + assert async_dir_cnt[1] == 0 + assert async_dir_cnt[2] >= min_copy_records, f"TID={tid}:\n\t{async_dir_cnt}" + assert async_dir_cnt[3] >= min_copy_records, f"TID={tid}:\n\t{async_dir_cnt}" + assert async_dir_cnt[4] == 0 + # HIP memory copies may be decomposed into more than one + # memory copy at the HSA level so require it to be a multiple + # of min_copy_records + assert ( + async_dir_cnt[2] % min_copy_records + ) == 0, f"TID={tid}:\n\t{async_dir_cnt}" + assert ( + async_dir_cnt[3] % min_copy_records + ) == 0, f"TID={tid}:\n\t{async_dir_cnt}" def test_retired_correlation_ids(input_data): diff --git a/tests/rocprofv3/kernel-rename/validate.py b/tests/rocprofv3/kernel-rename/validate.py index 6c21fd867c..f3550d2708 100644 --- a/tests/rocprofv3/kernel-rename/validate.py +++ b/tests/rocprofv3/kernel-rename/validate.py @@ -161,7 +161,9 @@ def test_memory_copy_json_trace(json_data): return agent return None - assert len(memory_copy_data) == 12 + # two threads * two directions + assert len(memory_copy_data) >= (2 * 2), f"{memory_copy_data}" + assert (len(memory_copy_data) % (2 * 2)) == 0, f"{memory_copy_data}" for row in memory_copy_data: src_agent = get_agent(row["src_agent_id"]) diff --git a/tests/rocprofv3/summary/validate.py b/tests/rocprofv3/summary/validate.py index 7558d1f3cc..6f757c7a6b 100644 --- a/tests/rocprofv3/summary/validate.py +++ b/tests/rocprofv3/summary/validate.py @@ -137,7 +137,9 @@ def test_memory_copy_trace(json_data): return agent return None - assert len(memory_copy_data) == 12 + # two threads * two directions + assert len(memory_copy_data) >= (2 * 2), f"{memory_copy_data}" + assert (len(memory_copy_data) % (2 * 2)) == 0, f"{memory_copy_data}" for row in memory_copy_data: src_agent = get_agent(row["src_agent_id"]) @@ -209,7 +211,9 @@ def test_summary_data(json_data): elif itr.domain == "HIP_API": assert itr.stats.count >= 2130 and itr.stats.count <= 2150 elif itr.domain == "MEMORY_COPY": - assert itr.stats.count == 12 + # two threads + two memory copies (H2D + D2H). + # HIP may decompose memory copies into more than one HSA memory copy + assert itr.stats.count >= 4 and (itr.stats.count % 4) == 0 elif itr.domain == "MEMORY_ALLOCATION": assert itr.stats.count >= 10 and itr.stats.count <= 30 elif itr.domain == "MARKER_API": diff --git a/tests/rocprofv3/tracing/validate.py b/tests/rocprofv3/tracing/validate.py index f7fbd70612..8f6f514c4e 100644 --- a/tests/rocprofv3/tracing/validate.py +++ b/tests/rocprofv3/tracing/validate.py @@ -255,7 +255,9 @@ def test_memory_copy_json_trace(json_data): return agent return None - assert len(memory_copy_data) == 2 + # one threads * two directions + assert len(memory_copy_data) >= 2, f"{memory_copy_data}" + assert (len(memory_copy_data) % 2) == 0, f"{memory_copy_data}" def test_row(idx, direction): assert direction in ("MEMORY_COPY_HOST_TO_DEVICE", "MEMORY_COPY_DEVICE_TO_HOST")